From 2c2779bdd5ac3faf9b45a60edac1ac2cd3971a7e Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Tue, 4 Nov 2025 13:58:21 -0500 Subject: [PATCH 1/6] invalidate submission when systemId is not found --- .../src/services/submission/processor.ts | 52 ++++++++++++++++++- .../src/utils/submissionResponseParser.ts | 26 +++++++++- .../src/utils/submissionUtils.ts | 19 +++++++ 3 files changed, 94 insertions(+), 3 deletions(-) diff --git a/packages/data-provider/src/services/submission/processor.ts b/packages/data-provider/src/services/submission/processor.ts index 7c733694..33230f52 100644 --- a/packages/data-provider/src/services/submission/processor.ts +++ b/packages/data-provider/src/services/submission/processor.ts @@ -19,11 +19,13 @@ import { getDictionarySchemaRelations, type SchemaChildNode } from '../../utils/ import { BadRequest } from '../../utils/errors.js'; import { convertRecordToString } from '../../utils/formatUtils.js'; import { parseRecordsToInsert } from '../../utils/recordsParser.js'; +import { createUnrecognizedFieldBatchError } from '../../utils/submissionResponseParser.js'; import { extractSchemaDataFromMergedDataRecords, filterDeletesFromUpdates, filterRelationsForPrimaryIdUpdate, findInvalidRecordErrorsBySchemaName, + foundEditSubbmittedData, groupSchemaErrorsByEntity, mapGroupedUpdateSubmissionData, mergeAndReferenceEntityData, @@ -421,6 +423,39 @@ const processor = (dependencies: BaseDependencies) => { dataValidated: dataMergedByEntityName, }); + // Check for records to be updated that its systemId was not found in the Submitted Data collection. + // Any error found will cause the submission to be marked as 'invalid' + Object.entries(submissionData.updates ?? {}).forEach(([entityName, recordsToUpdate]) => { + recordsToUpdate.forEach((submissionEditData, index) => { + const found = foundEditSubbmittedData(entityName, submissionEditData.systemId, dataMergedByEntityName); + + if (found) { + return; + } + + logger.error( + LOG_MODULE, + `Record with systemId '${submissionEditData.systemId}' not found in entity '${entityName}'`, + ); + + if (!submissionSchemaErrors.updates) { + submissionSchemaErrors.updates = {}; + } + + if (!submissionSchemaErrors.updates[entityName]) { + submissionSchemaErrors.updates[entityName] = []; + } + + submissionSchemaErrors.updates[entityName].push( + createUnrecognizedFieldBatchError({ + fieldName: 'systemId', + fieldValue: submissionEditData.systemId, + index, + }), + ); + }); + }); + if (_.isEmpty(submissionSchemaErrors)) { logger.info(LOG_MODULE, `No error found on data submission`); } else { @@ -474,7 +509,7 @@ const processor = (dependencies: BaseDependencies) => { // Parse file data const recordsParsed = records.map(convertRecordToString).map(parseToSchema(schema)); - const filesDataProcessed = await compareUpdatedData(recordsParsed); + const filesDataProcessed = await compareUpdatedData(recordsParsed, schema.name); const currentDictionary = await getDictionaryById(submission.dictionaryId); if (!currentDictionary) { @@ -560,11 +595,12 @@ const processor = (dependencies: BaseDependencies) => { /** * Processes a list of data records and compares them with previously submitted data. * @param {DataRecord[]} records An array of data records to be processed + * @param {string} schemaName The name of the schema associated with the records * @returns {Promise} An array of `SubmissionUpdateData` objects. Each object * contains the `systemId`, `old` data, and `new` data representing the differences * between the previously submitted data and the updated record. */ - const compareUpdatedData = async (records: DataRecord[]): Promise => { + const compareUpdatedData = async (records: DataRecord[], schemaName: string): Promise => { const { getSubmittedDataBySystemId } = submittedRepository(dependencies); const results: SubmissionUpdateData[] = []; @@ -576,6 +612,18 @@ const processor = (dependencies: BaseDependencies) => { const foundSubmittedData = await getSubmittedDataBySystemId(systemId); if (foundSubmittedData?.data) { + if (foundSubmittedData.entityName !== schemaName) { + logger.info( + LOG_MODULE, + `Entity name mismatch for system ID '${systemId}': expected '${schemaName}', found '${foundSubmittedData.entityName}'`, + ); + results.push({ + systemId: systemId, + old: {}, + new: {}, + }); + return; + } const changeData = _.omit(record, 'systemId'); const diffData = computeDataDiff(foundSubmittedData.data, changeData); if (!_.isEmpty(diffData.old) && !_.isEmpty(diffData.new)) { diff --git a/packages/data-provider/src/utils/submissionResponseParser.ts b/packages/data-provider/src/utils/submissionResponseParser.ts index 97d74097..2876eeb9 100644 --- a/packages/data-provider/src/utils/submissionResponseParser.ts +++ b/packages/data-provider/src/utils/submissionResponseParser.ts @@ -1,6 +1,30 @@ -import type { DataRecord } from '@overture-stack/lectern-client'; +import type { DataRecord, DictionaryValidationRecordErrorDetails } from '@overture-stack/lectern-client'; import type { SubmissionInsertData } from '@overture-stack/lyric-data-model/models'; +/** + * Creates an error object for unrecognized fields in a batch submission. + */ +export const createUnrecognizedFieldBatchError = ({ + fieldName, + fieldValue, + index, +}: { + fieldName: string; + fieldValue: string; + index: number; +}) => { + const errorDetails: DictionaryValidationRecordErrorDetails = { + fieldName, + fieldValue, + reason: 'UNRECOGNIZED_FIELD', + }; + + return { + index, + ...errorDetails, + }; +}; + export const createBatchResponse = (schemaName: string, records: DataRecord[]): SubmissionInsertData => { return { batchName: schemaName, records }; }; diff --git a/packages/data-provider/src/utils/submissionUtils.ts b/packages/data-provider/src/utils/submissionUtils.ts index e6d8e328..af09097a 100644 --- a/packages/data-provider/src/utils/submissionUtils.ts +++ b/packages/data-provider/src/utils/submissionUtils.ts @@ -41,6 +41,25 @@ import { SubmittedDataReference, } from './types.js'; +/** + * Checks whether a record exists within a collection of submitted data records marked for update. + * The lookup is performed by matching the given 'entityName' and 'systemId'. + * + * @Returns true if found, false otherwise + */ +export const foundEditSubbmittedData = ( + entityName: string, + systemId: string, + dataByEntityName: Record, +) => { + return ( + dataByEntityName[entityName]?.some( + (data) => + data.reference.type === MERGE_REFERENCE_TYPE.EDIT_SUBMITTED_DATA && data.reference.systemId === systemId, + ) ?? false + ); +}; + // export default utils; // Only "open", "valid", and "invalid" statuses are considered Active Submission const statusesAllowedToClose = [SUBMISSION_STATUS.OPEN, SUBMISSION_STATUS.VALID, SUBMISSION_STATUS.INVALID] as const; From 71374de17c487ddb13dfbb481ef1d0b6bf4c92a5 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Tue, 4 Nov 2025 14:13:40 -0500 Subject: [PATCH 2/6] add error when system id not found --- .../data-provider/src/services/submission/processor.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/data-provider/src/services/submission/processor.ts b/packages/data-provider/src/services/submission/processor.ts index 33230f52..41acac10 100644 --- a/packages/data-provider/src/services/submission/processor.ts +++ b/packages/data-provider/src/services/submission/processor.ts @@ -613,7 +613,7 @@ const processor = (dependencies: BaseDependencies) => { const foundSubmittedData = await getSubmittedDataBySystemId(systemId); if (foundSubmittedData?.data) { if (foundSubmittedData.entityName !== schemaName) { - logger.info( + logger.error( LOG_MODULE, `Entity name mismatch for system ID '${systemId}': expected '${schemaName}', found '${foundSubmittedData.entityName}'`, ); @@ -633,6 +633,13 @@ const processor = (dependencies: BaseDependencies) => { new: diffData.new, }); } + } else { + logger.error(LOG_MODULE, `No submitted data found for system ID '${systemId}'`); + results.push({ + systemId: systemId, + old: {}, + new: {}, + }); } return; }); From f0e6356f7642906de4f6615521a190ff27369c4c Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Tue, 4 Nov 2025 15:53:11 -0500 Subject: [PATCH 3/6] rename func and add unit tests --- .../src/services/submission/processor.ts | 4 +- .../src/utils/submissionUtils.ts | 37 +++++---- .../createUnrecognizedFieldBatchError.spec.ts | 51 ++++++++++++ .../submission/findEditSubmittedData.spec.ts | 79 +++++++++++++++++++ 4 files changed, 150 insertions(+), 21 deletions(-) create mode 100644 packages/data-provider/test/utils/submission/createUnrecognizedFieldBatchError.spec.ts create mode 100644 packages/data-provider/test/utils/submission/findEditSubmittedData.spec.ts diff --git a/packages/data-provider/src/services/submission/processor.ts b/packages/data-provider/src/services/submission/processor.ts index 41acac10..75499770 100644 --- a/packages/data-provider/src/services/submission/processor.ts +++ b/packages/data-provider/src/services/submission/processor.ts @@ -24,8 +24,8 @@ import { extractSchemaDataFromMergedDataRecords, filterDeletesFromUpdates, filterRelationsForPrimaryIdUpdate, + findEditSubmittedData, findInvalidRecordErrorsBySchemaName, - foundEditSubbmittedData, groupSchemaErrorsByEntity, mapGroupedUpdateSubmissionData, mergeAndReferenceEntityData, @@ -427,7 +427,7 @@ const processor = (dependencies: BaseDependencies) => { // Any error found will cause the submission to be marked as 'invalid' Object.entries(submissionData.updates ?? {}).forEach(([entityName, recordsToUpdate]) => { recordsToUpdate.forEach((submissionEditData, index) => { - const found = foundEditSubbmittedData(entityName, submissionEditData.systemId, dataMergedByEntityName); + const found = findEditSubmittedData(entityName, submissionEditData.systemId, dataMergedByEntityName); if (found) { return; diff --git a/packages/data-provider/src/utils/submissionUtils.ts b/packages/data-provider/src/utils/submissionUtils.ts index af09097a..41eac807 100644 --- a/packages/data-provider/src/utils/submissionUtils.ts +++ b/packages/data-provider/src/utils/submissionUtils.ts @@ -41,25 +41,6 @@ import { SubmittedDataReference, } from './types.js'; -/** - * Checks whether a record exists within a collection of submitted data records marked for update. - * The lookup is performed by matching the given 'entityName' and 'systemId'. - * - * @Returns true if found, false otherwise - */ -export const foundEditSubbmittedData = ( - entityName: string, - systemId: string, - dataByEntityName: Record, -) => { - return ( - dataByEntityName[entityName]?.some( - (data) => - data.reference.type === MERGE_REFERENCE_TYPE.EDIT_SUBMITTED_DATA && data.reference.systemId === systemId, - ) ?? false - ); -}; - // export default utils; // Only "open", "valid", and "invalid" statuses are considered Active Submission const statusesAllowedToClose = [SUBMISSION_STATUS.OPEN, SUBMISSION_STATUS.VALID, SUBMISSION_STATUS.INVALID] as const; @@ -96,6 +77,24 @@ export const extractSchemaDataFromMergedDataRecords = ( return _.mapValues(mergeDataRecordsByEntityName, (mappingArray) => mappingArray.map((o) => o.dataRecord)); }; +/** + * Checks whether a record exists within a collection of submitted data records marked for update. + * The lookup is performed by matching the given 'entityName' and 'systemId'. + * + * @Returns true if found, false otherwise + */ +export const findEditSubmittedData = ( + entityName: string, + systemId: string, + dataByEntityName: Record, +) => { + return ( + dataByEntityName[entityName]?.some( + (data) => + data.reference.type === MERGE_REFERENCE_TYPE.EDIT_SUBMITTED_DATA && data.reference.systemId === systemId, + ) ?? false + ); +}; /** * Finds and returns a list of invalid records based on a provided schema name. * diff --git a/packages/data-provider/test/utils/submission/createUnrecognizedFieldBatchError.spec.ts b/packages/data-provider/test/utils/submission/createUnrecognizedFieldBatchError.spec.ts new file mode 100644 index 00000000..5ad236aa --- /dev/null +++ b/packages/data-provider/test/utils/submission/createUnrecognizedFieldBatchError.spec.ts @@ -0,0 +1,51 @@ +import { expect } from 'chai'; +import { describe, it } from 'mocha'; + +import { createUnrecognizedFieldBatchError } from '../../../src/utils/submissionResponseParser.js'; + +describe('Submission Utils - Create Unrecognized Field Batch Error', () => { + it('should create an error object with correct structure and values', () => { + const result = createUnrecognizedFieldBatchError({ + fieldName: 'systemId', + fieldValue: 'DOESNOTEXIST', + index: 3, + }); + + expect(result).to.eql({ + index: 3, + fieldName: 'systemId', + fieldValue: 'DOESNOTEXIST', + reason: 'UNRECOGNIZED_FIELD', + }); + }); + + it('should handle zero index correctly', () => { + const result = createUnrecognizedFieldBatchError({ + fieldName: '', + fieldValue: '', + index: 0, + }); + + expect(result).to.eql({ + index: 0, + fieldName: '', + fieldValue: '', + reason: 'UNRECOGNIZED_FIELD', + }); + }); + + it('should handle empty values correctly', () => { + const result = createUnrecognizedFieldBatchError({ + fieldName: '', + fieldValue: '', + index: 2, + }); + + expect(result).to.eql({ + index: 2, + fieldName: '', + fieldValue: '', + reason: 'UNRECOGNIZED_FIELD', + }); + }); +}); diff --git a/packages/data-provider/test/utils/submission/findEditSubmittedData.spec.ts b/packages/data-provider/test/utils/submission/findEditSubmittedData.spec.ts new file mode 100644 index 00000000..28e8d45d --- /dev/null +++ b/packages/data-provider/test/utils/submission/findEditSubmittedData.spec.ts @@ -0,0 +1,79 @@ +import { expect } from 'chai'; +import { describe, it } from 'mocha'; + +import { findEditSubmittedData } from '../../../index.js'; +import { type DataRecordReference, MERGE_REFERENCE_TYPE } from '../../../src/utils/types.js'; + +const recordsByEntityName: Record = { + animals: [ + { + dataRecord: { name: 'Bird', color: 'blue' }, + reference: { + systemId: 'BB4546', + submissionId: 2, + index: 0, + type: MERGE_REFERENCE_TYPE.EDIT_SUBMITTED_DATA, + }, + }, + { + dataRecord: { name: 'Dinosaur', color: 'red' }, + reference: { + systemId: 'DINO8912', + submissionId: 2, + index: 1, + type: MERGE_REFERENCE_TYPE.EDIT_SUBMITTED_DATA, + }, + }, + ], + teams: [ + { + dataRecord: { title: 'Raptors' }, + reference: { + systemId: 'RPT5678', + submissionId: 2, + index: 3, + type: MERGE_REFERENCE_TYPE.EDIT_SUBMITTED_DATA, + }, + }, + { + dataRecord: { tile: 'Blue Jays' }, + reference: { + systemId: 'BJ1425', + submittedDataId: 45, + type: MERGE_REFERENCE_TYPE.SUBMITTED_DATA, + }, + }, + ], +}; + +describe('Submission Utils - Find Edited Submitted Data by systemId', () => { + it('should return true when matching entityName and systemId and marked to edit', () => { + const result = findEditSubmittedData('animals', 'BB4546', recordsByEntityName); + expect(result).to.be.true; + }); + + it('should return false when matching systemId but not entityName', () => { + const result = findEditSubmittedData('teams', 'BB4546', recordsByEntityName); + expect(result).to.be.false; + }); + + it('should return false when matching systemId and entityName but not marked to edit', () => { + const result = findEditSubmittedData('teams', 'BJ1425', recordsByEntityName); + expect(result).to.be.false; + }); + + it('should return false when systemId not found', () => { + const result = findEditSubmittedData('animals', 'DOESNOTEXISTS', recordsByEntityName); + expect(result).to.be.false; + }); + + it('should return false when entityName does not exist', () => { + const result = findEditSubmittedData('incorrect', 'DINO8912', recordsByEntityName); + expect(result).to.be.false; + }); + + it('should return false when there are no records to find', () => { + const result = findEditSubmittedData('animals', 'DINO8912', {}); + expect(result).to.be.false; + }); +}); From 1b3fd70ddc05b87eec63b012ca99a239b6d519cd Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Mon, 17 Nov 2025 15:41:53 -0500 Subject: [PATCH 4/6] change message for invalid value batch error --- .../src/services/submission/processor.ts | 4 +- .../src/utils/submissionResponseParser.ts | 15 +++++-- .../createUnrecognizedFieldBatchError.spec.ts | 41 +++++++++++++++---- 3 files changed, 48 insertions(+), 12 deletions(-) diff --git a/packages/data-provider/src/services/submission/processor.ts b/packages/data-provider/src/services/submission/processor.ts index 75499770..3a391426 100644 --- a/packages/data-provider/src/services/submission/processor.ts +++ b/packages/data-provider/src/services/submission/processor.ts @@ -19,7 +19,7 @@ import { getDictionarySchemaRelations, type SchemaChildNode } from '../../utils/ import { BadRequest } from '../../utils/errors.js'; import { convertRecordToString } from '../../utils/formatUtils.js'; import { parseRecordsToInsert } from '../../utils/recordsParser.js'; -import { createUnrecognizedFieldBatchError } from '../../utils/submissionResponseParser.js'; +import { createInvalidValueBatchError } from '../../utils/submissionResponseParser.js'; import { extractSchemaDataFromMergedDataRecords, filterDeletesFromUpdates, @@ -447,7 +447,7 @@ const processor = (dependencies: BaseDependencies) => { } submissionSchemaErrors.updates[entityName].push( - createUnrecognizedFieldBatchError({ + createInvalidValueBatchError({ fieldName: 'systemId', fieldValue: submissionEditData.systemId, index, diff --git a/packages/data-provider/src/utils/submissionResponseParser.ts b/packages/data-provider/src/utils/submissionResponseParser.ts index 2876eeb9..c140a776 100644 --- a/packages/data-provider/src/utils/submissionResponseParser.ts +++ b/packages/data-provider/src/utils/submissionResponseParser.ts @@ -2,9 +2,9 @@ import type { DataRecord, DictionaryValidationRecordErrorDetails } from '@overtu import type { SubmissionInsertData } from '@overture-stack/lyric-data-model/models'; /** - * Creates an error object for unrecognized fields in a batch submission. + * Creates an error object for invalid value in a batch submission. */ -export const createUnrecognizedFieldBatchError = ({ +export const createInvalidValueBatchError = ({ fieldName, fieldValue, index, @@ -16,7 +16,16 @@ export const createUnrecognizedFieldBatchError = ({ const errorDetails: DictionaryValidationRecordErrorDetails = { fieldName, fieldValue, - reason: 'UNRECOGNIZED_FIELD', + reason: 'INVALID_BY_RESTRICTION', + errors: [ + { + message: 'Value does not match any existing record.', + restriction: { + rule: true, + type: 'required', + }, + }, + ], }; return { diff --git a/packages/data-provider/test/utils/submission/createUnrecognizedFieldBatchError.spec.ts b/packages/data-provider/test/utils/submission/createUnrecognizedFieldBatchError.spec.ts index 5ad236aa..a145c1fd 100644 --- a/packages/data-provider/test/utils/submission/createUnrecognizedFieldBatchError.spec.ts +++ b/packages/data-provider/test/utils/submission/createUnrecognizedFieldBatchError.spec.ts @@ -1,11 +1,11 @@ import { expect } from 'chai'; import { describe, it } from 'mocha'; -import { createUnrecognizedFieldBatchError } from '../../../src/utils/submissionResponseParser.js'; +import { createInvalidValueBatchError } from '../../../src/utils/submissionResponseParser.js'; describe('Submission Utils - Create Unrecognized Field Batch Error', () => { it('should create an error object with correct structure and values', () => { - const result = createUnrecognizedFieldBatchError({ + const result = createInvalidValueBatchError({ fieldName: 'systemId', fieldValue: 'DOESNOTEXIST', index: 3, @@ -15,12 +15,21 @@ describe('Submission Utils - Create Unrecognized Field Batch Error', () => { index: 3, fieldName: 'systemId', fieldValue: 'DOESNOTEXIST', - reason: 'UNRECOGNIZED_FIELD', + reason: 'INVALID_BY_RESTRICTION', + errors: [ + { + message: 'Value does not match any existing record.', + restriction: { + rule: true, + type: 'required', + }, + }, + ], }); }); it('should handle zero index correctly', () => { - const result = createUnrecognizedFieldBatchError({ + const result = createInvalidValueBatchError({ fieldName: '', fieldValue: '', index: 0, @@ -30,12 +39,21 @@ describe('Submission Utils - Create Unrecognized Field Batch Error', () => { index: 0, fieldName: '', fieldValue: '', - reason: 'UNRECOGNIZED_FIELD', + reason: 'INVALID_BY_RESTRICTION', + errors: [ + { + message: 'Value does not match any existing record.', + restriction: { + rule: true, + type: 'required', + }, + }, + ], }); }); it('should handle empty values correctly', () => { - const result = createUnrecognizedFieldBatchError({ + const result = createInvalidValueBatchError({ fieldName: '', fieldValue: '', index: 2, @@ -45,7 +63,16 @@ describe('Submission Utils - Create Unrecognized Field Batch Error', () => { index: 2, fieldName: '', fieldValue: '', - reason: 'UNRECOGNIZED_FIELD', + reason: 'INVALID_BY_RESTRICTION', + errors: [ + { + message: 'Value does not match any existing record.', + restriction: { + rule: true, + type: 'required', + }, + }, + ], }); }); }); From 23f59b532446d542326a6b5dd3c60ef264a420e5 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Tue, 18 Nov 2025 10:52:28 -0500 Subject: [PATCH 5/6] custom submission errors type --- packages/data-model/src/models/submissions.ts | 29 ++++++- .../src/services/submission/processor.ts | 24 +++--- .../src/utils/submissionResponseParser.ts | 35 +-------- .../src/utils/submissionUtils.ts | 8 +- packages/data-provider/src/utils/types.ts | 6 +- .../createUnrecognizedFieldBatchError.spec.ts | 78 ------------------- 6 files changed, 48 insertions(+), 132 deletions(-) delete mode 100644 packages/data-provider/test/utils/submission/createUnrecognizedFieldBatchError.spec.ts diff --git a/packages/data-model/src/models/submissions.ts b/packages/data-model/src/models/submissions.ts index 9f1ce0ed..1b4d9161 100644 --- a/packages/data-model/src/models/submissions.ts +++ b/packages/data-model/src/models/submissions.ts @@ -1,7 +1,11 @@ import { relations } from 'drizzle-orm'; import { integer, jsonb, pgEnum, pgTable, serial, timestamp, varchar } from 'drizzle-orm/pg-core'; -import { type DataRecord, type DictionaryValidationRecordErrorDetails } from '@overture-stack/lectern-client'; +import { + type DataRecord, + type DataRecordValue, + type DictionaryValidationRecordErrorDetails, +} from '@overture-stack/lectern-client'; import { dictionaries } from './dictionaries.js'; import { dictionaryCategories } from './dictionary_categories.js'; @@ -33,6 +37,27 @@ export type SubmissionData = { deletes?: Record; }; +export type FieldDetails = { + fieldName: string; + fieldValue: DataRecordValue; +}; + +export type UnrecognizedValueReason = { + reason: 'UNRECOGNIZED_VALUE'; +}; + +export type RecordErrorInvalidValue = FieldDetails & UnrecognizedValueReason; + +export type SubmissionRecordErrorDetails = { + index: number; +} & (DictionaryValidationRecordErrorDetails | RecordErrorInvalidValue); + +export type SubmissionErrors = { + inserts?: Record; + updates?: Record; + deletes?: Record; +}; + export const submissions = pgTable('submissions', { id: serial('id').primaryKey(), data: jsonb('data').$type().notNull(), @@ -42,7 +67,7 @@ export const submissions = pgTable('submissions', { dictionaryId: integer('dictionary_id') .references(() => dictionaries.id) .notNull(), - errors: jsonb('errors').$type>>(), + errors: jsonb('errors').$type(), organization: varchar('organization').notNull(), status: submissionStatusEnum('status').notNull(), createdAt: timestamp('created_at').defaultNow(), diff --git a/packages/data-provider/src/services/submission/processor.ts b/packages/data-provider/src/services/submission/processor.ts index 49e8689b..bd2833d8 100644 --- a/packages/data-provider/src/services/submission/processor.ts +++ b/packages/data-provider/src/services/submission/processor.ts @@ -1,11 +1,13 @@ import * as _ from 'lodash-es'; -import { type DataRecord, DictionaryValidationRecordErrorDetails, type Schema } from '@overture-stack/lectern-client'; +import { type DataRecord, type Schema } from '@overture-stack/lectern-client'; import { Submission, SubmissionData, type SubmissionDeleteData, + type SubmissionErrors, type SubmissionInsertData, + type SubmissionRecordErrorDetails, type SubmissionUpdateData, SubmittedData, } from '@overture-stack/lyric-data-model/models'; @@ -19,7 +21,6 @@ import { getDictionarySchemaRelations, type SchemaChildNode } from '../../utils/ import { BadRequest } from '../../utils/errors.js'; import { convertRecordToString } from '../../utils/formatUtils.js'; import { parseRecordsToInsert } from '../../utils/recordsParser.js'; -import { createInvalidValueBatchError } from '../../utils/submissionResponseParser.js'; import { extractSchemaDataFromMergedDataRecords, filterDeletesFromUpdates, @@ -447,13 +448,14 @@ const processor = (dependencies: BaseDependencies) => { submissionSchemaErrors.updates[entityName] = []; } - submissionSchemaErrors.updates[entityName].push( - createInvalidValueBatchError({ - fieldName: 'systemId', - fieldValue: submissionEditData.systemId, - index, - }), - ); + const unrecodgnizedValueError: SubmissionRecordErrorDetails = { + fieldName: 'systemId', + fieldValue: submissionEditData.systemId, + index, + reason: 'UNRECOGNIZED_VALUE', + }; + + submissionSchemaErrors.updates[entityName].push(unrecodgnizedValueError); }); }); @@ -657,7 +659,7 @@ const processor = (dependencies: BaseDependencies) => { * @param {number} input.dictionaryId The Dictionary ID of the Submission * @param {SubmissionData} input.submissionData Data to be submitted grouped on inserts, updates and deletes * @param {number} input.idActiveSubmission ID of the Active Submission - * @param {Record>} input.schemaErrors Array of schemaErrors + * @param {SubmissionErrors} input.schemaErrors Array of schemaErrors * @param {string} input.username User updating the active submission * @returns {Promise} An Active Submission updated */ @@ -665,7 +667,7 @@ const processor = (dependencies: BaseDependencies) => { dictionaryId: number; submissionData: SubmissionData; idActiveSubmission: number; - schemaErrors: Record>; + schemaErrors: SubmissionErrors; username: string; }): Promise => { const { dictionaryId, submissionData, idActiveSubmission, schemaErrors, username } = input; diff --git a/packages/data-provider/src/utils/submissionResponseParser.ts b/packages/data-provider/src/utils/submissionResponseParser.ts index c140a776..97d74097 100644 --- a/packages/data-provider/src/utils/submissionResponseParser.ts +++ b/packages/data-provider/src/utils/submissionResponseParser.ts @@ -1,39 +1,6 @@ -import type { DataRecord, DictionaryValidationRecordErrorDetails } from '@overture-stack/lectern-client'; +import type { DataRecord } from '@overture-stack/lectern-client'; import type { SubmissionInsertData } from '@overture-stack/lyric-data-model/models'; -/** - * Creates an error object for invalid value in a batch submission. - */ -export const createInvalidValueBatchError = ({ - fieldName, - fieldValue, - index, -}: { - fieldName: string; - fieldValue: string; - index: number; -}) => { - const errorDetails: DictionaryValidationRecordErrorDetails = { - fieldName, - fieldValue, - reason: 'INVALID_BY_RESTRICTION', - errors: [ - { - message: 'Value does not match any existing record.', - restriction: { - rule: true, - type: 'required', - }, - }, - ], - }; - - return { - index, - ...errorDetails, - }; -}; - export const createBatchResponse = (schemaName: string, records: DataRecord[]): SubmissionInsertData => { return { batchName: schemaName, records }; }; diff --git a/packages/data-provider/src/utils/submissionUtils.ts b/packages/data-provider/src/utils/submissionUtils.ts index c098650e..8b8a1081 100644 --- a/packages/data-provider/src/utils/submissionUtils.ts +++ b/packages/data-provider/src/utils/submissionUtils.ts @@ -5,7 +5,6 @@ import { type DataRecord, Dictionary as SchemasDictionary, DictionaryValidationError, - DictionaryValidationRecordErrorDetails, parse, Schema, TestResult, @@ -15,6 +14,7 @@ import { type Submission, SubmissionData, type SubmissionDeleteData, + type SubmissionErrors, type SubmissionInsertData, type SubmissionUpdateData, type SubmittedData, @@ -237,15 +237,15 @@ export const filterRelationsForPrimaryIdUpdate = ( * @param {object} input * @param {TestResult} input.resultValidation * @param {Record} input.dataValidated - * @returns {Record>} + * @returns {SubmissionErrors} */ export const groupSchemaErrorsByEntity = (input: { resultValidation: TestResult; dataValidated: Record; -}): Record> => { +}): SubmissionErrors => { const { resultValidation, dataValidated } = input; - const submissionSchemaErrors: Record> = {}; + const submissionSchemaErrors: SubmissionErrors = {}; if (resultValidation.valid) { return {}; } diff --git a/packages/data-provider/src/utils/types.ts b/packages/data-provider/src/utils/types.ts index b34d2bd4..35484ee3 100644 --- a/packages/data-provider/src/utils/types.ts +++ b/packages/data-provider/src/utils/types.ts @@ -4,7 +4,6 @@ import { type DataRecord, type DataRecordValue, Dictionary as SchemasDictionary, - DictionaryValidationRecordErrorDetails, type Schema, } from '@overture-stack/lectern-client'; import { @@ -15,6 +14,7 @@ import { Submission, SubmissionData, type SubmissionDeleteData, + type SubmissionErrors, type SubmissionUpdateData, type SubmittedData, } from '@overture-stack/lyric-data-model/models'; @@ -229,7 +229,7 @@ export type SubmissionResponse = { data: SubmissionData; dictionary: DictionaryActiveSubmission; dictionaryCategory: CategoryActiveSubmission; - errors: Record> | null; + errors: SubmissionErrors | null; organization: string; status: SubmissionStatus | null; createdAt: string | null; @@ -258,7 +258,7 @@ export type SubmissionSummaryRepository = { data: SubmissionData; dictionary: Pick; dictionaryCategory: Pick; - errors: Record> | null; + errors: SubmissionErrors | null; organization: string | null; status: SubmissionStatus | null; createdAt: Date | null; diff --git a/packages/data-provider/test/utils/submission/createUnrecognizedFieldBatchError.spec.ts b/packages/data-provider/test/utils/submission/createUnrecognizedFieldBatchError.spec.ts deleted file mode 100644 index a145c1fd..00000000 --- a/packages/data-provider/test/utils/submission/createUnrecognizedFieldBatchError.spec.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { expect } from 'chai'; -import { describe, it } from 'mocha'; - -import { createInvalidValueBatchError } from '../../../src/utils/submissionResponseParser.js'; - -describe('Submission Utils - Create Unrecognized Field Batch Error', () => { - it('should create an error object with correct structure and values', () => { - const result = createInvalidValueBatchError({ - fieldName: 'systemId', - fieldValue: 'DOESNOTEXIST', - index: 3, - }); - - expect(result).to.eql({ - index: 3, - fieldName: 'systemId', - fieldValue: 'DOESNOTEXIST', - reason: 'INVALID_BY_RESTRICTION', - errors: [ - { - message: 'Value does not match any existing record.', - restriction: { - rule: true, - type: 'required', - }, - }, - ], - }); - }); - - it('should handle zero index correctly', () => { - const result = createInvalidValueBatchError({ - fieldName: '', - fieldValue: '', - index: 0, - }); - - expect(result).to.eql({ - index: 0, - fieldName: '', - fieldValue: '', - reason: 'INVALID_BY_RESTRICTION', - errors: [ - { - message: 'Value does not match any existing record.', - restriction: { - rule: true, - type: 'required', - }, - }, - ], - }); - }); - - it('should handle empty values correctly', () => { - const result = createInvalidValueBatchError({ - fieldName: '', - fieldValue: '', - index: 2, - }); - - expect(result).to.eql({ - index: 2, - fieldName: '', - fieldValue: '', - reason: 'INVALID_BY_RESTRICTION', - errors: [ - { - message: 'Value does not match any existing record.', - restriction: { - rule: true, - type: 'required', - }, - }, - ], - }); - }); -}); From 44334c1fa752722fb7d9f2e2eb261b93a5846b4d Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Tue, 18 Nov 2025 11:18:27 -0500 Subject: [PATCH 6/6] refactoring function --- .../src/utils/submissionUtils.ts | 64 ++++++++++--------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/packages/data-provider/src/utils/submissionUtils.ts b/packages/data-provider/src/utils/submissionUtils.ts index 8b8a1081..1b51e355 100644 --- a/packages/data-provider/src/utils/submissionUtils.ts +++ b/packages/data-provider/src/utils/submissionUtils.ts @@ -245,45 +245,47 @@ export const groupSchemaErrorsByEntity = (input: { }): SubmissionErrors => { const { resultValidation, dataValidated } = input; - const submissionSchemaErrors: SubmissionErrors = {}; if (resultValidation.valid) { return {}; } + + const submissionSchemaErrors: SubmissionErrors = {}; resultValidation.details.forEach((dictionaryValidationError) => { const entityName = dictionaryValidationError.schemaName; - if (dictionaryValidationError.reason === 'INVALID_RECORDS') { - const validationErrors = dictionaryValidationError.invalidRecords; - - const hasErrorByIndex = groupErrorsByIndex(validationErrors); - - if (!_.isEmpty(hasErrorByIndex)) { - Object.entries(hasErrorByIndex).map(([indexBasedOnCrossSchemas, schemaValidationErrors]) => { - const mapping = dataValidated[entityName][Number(indexBasedOnCrossSchemas)]; - if (determineIfIsSubmission(mapping.reference)) { - const submissionIndex = mapping.reference.index; - const actionType = - mapping.reference.type === MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA ? 'inserts' : 'updates'; - - const mutableSchemaValidationErrors = schemaValidationErrors.map((errors) => { - return { - ...errors, - index: submissionIndex, - }; - }); - - if (!submissionSchemaErrors[actionType]) { - submissionSchemaErrors[actionType] = {}; - } + if (dictionaryValidationError.reason !== 'INVALID_RECORDS') { + return; + } - if (!submissionSchemaErrors[actionType][entityName]) { - submissionSchemaErrors[actionType][entityName] = []; - } + const groupedErrorsByIndex = groupErrorsByIndex(dictionaryValidationError.invalidRecords); - submissionSchemaErrors[actionType][entityName].push(...mutableSchemaValidationErrors); - } - }); - } + if (!groupedErrorsByIndex || Object.keys(groupedErrorsByIndex).length === 0) { + return; } + + Object.entries(groupedErrorsByIndex).forEach(([indexBasedOnCrossSchemas, schemaValidationErrors]) => { + const mapping = dataValidated[entityName][Number(indexBasedOnCrossSchemas)]; + if (!determineIfIsSubmission(mapping.reference)) { + return; + } + + const submissionIndex = mapping.reference.index; + const actionType = mapping.reference.type === MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA ? 'inserts' : 'updates'; + + const mutableSchemaValidationErrors = schemaValidationErrors.map((errors) => ({ + ...errors, + index: submissionIndex, + })); + + if (!submissionSchemaErrors[actionType]) { + submissionSchemaErrors[actionType] = {}; + } + + if (!submissionSchemaErrors[actionType][entityName]) { + submissionSchemaErrors[actionType][entityName] = []; + } + + submissionSchemaErrors[actionType][entityName].push(...mutableSchemaValidationErrors); + }); }); return submissionSchemaErrors; };