diff --git a/packages/data-model/src/models/submissions.ts b/packages/data-model/src/models/submissions.ts index 9f1ce0ed..1b4d9161 100644 --- a/packages/data-model/src/models/submissions.ts +++ b/packages/data-model/src/models/submissions.ts @@ -1,7 +1,11 @@ import { relations } from 'drizzle-orm'; import { integer, jsonb, pgEnum, pgTable, serial, timestamp, varchar } from 'drizzle-orm/pg-core'; -import { type DataRecord, type DictionaryValidationRecordErrorDetails } from '@overture-stack/lectern-client'; +import { + type DataRecord, + type DataRecordValue, + type DictionaryValidationRecordErrorDetails, +} from '@overture-stack/lectern-client'; import { dictionaries } from './dictionaries.js'; import { dictionaryCategories } from './dictionary_categories.js'; @@ -33,6 +37,27 @@ export type SubmissionData = { deletes?: Record; }; +export type FieldDetails = { + fieldName: string; + fieldValue: DataRecordValue; +}; + +export type UnrecognizedValueReason = { + reason: 'UNRECOGNIZED_VALUE'; +}; + +export type RecordErrorInvalidValue = FieldDetails & UnrecognizedValueReason; + +export type SubmissionRecordErrorDetails = { + index: number; +} & (DictionaryValidationRecordErrorDetails | RecordErrorInvalidValue); + +export type SubmissionErrors = { + inserts?: Record; + updates?: Record; + deletes?: Record; +}; + export const submissions = pgTable('submissions', { id: serial('id').primaryKey(), data: jsonb('data').$type().notNull(), @@ -42,7 +67,7 @@ export const submissions = pgTable('submissions', { dictionaryId: integer('dictionary_id') .references(() => dictionaries.id) .notNull(), - errors: jsonb('errors').$type>>(), + errors: jsonb('errors').$type(), organization: varchar('organization').notNull(), status: submissionStatusEnum('status').notNull(), createdAt: timestamp('created_at').defaultNow(), diff --git a/packages/data-provider/src/services/submission/processor.ts b/packages/data-provider/src/services/submission/processor.ts index 229ea0ea..bd2833d8 100644 --- a/packages/data-provider/src/services/submission/processor.ts +++ b/packages/data-provider/src/services/submission/processor.ts @@ -1,11 +1,13 @@ import * as _ from 'lodash-es'; -import { type DataRecord, DictionaryValidationRecordErrorDetails, type Schema } from '@overture-stack/lectern-client'; +import { type DataRecord, type Schema } from '@overture-stack/lectern-client'; import { Submission, SubmissionData, type SubmissionDeleteData, + type SubmissionErrors, type SubmissionInsertData, + type SubmissionRecordErrorDetails, type SubmissionUpdateData, SubmittedData, } from '@overture-stack/lyric-data-model/models'; @@ -23,6 +25,7 @@ import { extractSchemaDataFromMergedDataRecords, filterDeletesFromUpdates, filterRelationsForPrimaryIdUpdate, + findEditSubmittedData, findInvalidRecordErrorsBySchemaName, groupSchemaErrorsByEntity, isSubmissionActive, @@ -422,6 +425,40 @@ const processor = (dependencies: BaseDependencies) => { dataValidated: dataMergedByEntityName, }); + // Check for records to be updated that its systemId was not found in the Submitted Data collection. + // Any error found will cause the submission to be marked as 'invalid' + Object.entries(submissionData.updates ?? {}).forEach(([entityName, recordsToUpdate]) => { + recordsToUpdate.forEach((submissionEditData, index) => { + const found = findEditSubmittedData(entityName, submissionEditData.systemId, dataMergedByEntityName); + + if (found) { + return; + } + + logger.error( + LOG_MODULE, + `Record with systemId '${submissionEditData.systemId}' not found in entity '${entityName}'`, + ); + + if (!submissionSchemaErrors.updates) { + submissionSchemaErrors.updates = {}; + } + + if (!submissionSchemaErrors.updates[entityName]) { + submissionSchemaErrors.updates[entityName] = []; + } + + const unrecodgnizedValueError: SubmissionRecordErrorDetails = { + fieldName: 'systemId', + fieldValue: submissionEditData.systemId, + index, + reason: 'UNRECOGNIZED_VALUE', + }; + + submissionSchemaErrors.updates[entityName].push(unrecodgnizedValueError); + }); + }); + if (_.isEmpty(submissionSchemaErrors)) { logger.info(LOG_MODULE, `No error found on data submission`); } else { @@ -475,7 +512,7 @@ const processor = (dependencies: BaseDependencies) => { // Parse file data const recordsParsed = records.map(convertRecordToString).map(parseToSchema(schema)); - const filesDataProcessed = await compareUpdatedData(recordsParsed); + const filesDataProcessed = await compareUpdatedData(recordsParsed, schema.name); const currentDictionary = await getDictionaryById(submission.dictionaryId); if (!currentDictionary) { @@ -561,11 +598,12 @@ const processor = (dependencies: BaseDependencies) => { /** * Processes a list of data records and compares them with previously submitted data. * @param {DataRecord[]} records An array of data records to be processed + * @param {string} schemaName The name of the schema associated with the records * @returns {Promise} An array of `SubmissionUpdateData` objects. Each object * contains the `systemId`, `old` data, and `new` data representing the differences * between the previously submitted data and the updated record. */ - const compareUpdatedData = async (records: DataRecord[]): Promise => { + const compareUpdatedData = async (records: DataRecord[], schemaName: string): Promise => { const { getSubmittedDataBySystemId } = submittedRepository(dependencies); const results: SubmissionUpdateData[] = []; @@ -577,6 +615,18 @@ const processor = (dependencies: BaseDependencies) => { const foundSubmittedData = await getSubmittedDataBySystemId(systemId); if (foundSubmittedData?.data) { + if (foundSubmittedData.entityName !== schemaName) { + logger.error( + LOG_MODULE, + `Entity name mismatch for system ID '${systemId}': expected '${schemaName}', found '${foundSubmittedData.entityName}'`, + ); + results.push({ + systemId: systemId, + old: {}, + new: {}, + }); + return; + } const changeData = _.omit(record, 'systemId'); const diffData = computeDataDiff(foundSubmittedData.data, changeData); if (!_.isEmpty(diffData.old) && !_.isEmpty(diffData.new)) { @@ -586,6 +636,13 @@ const processor = (dependencies: BaseDependencies) => { new: diffData.new, }); } + } else { + logger.error(LOG_MODULE, `No submitted data found for system ID '${systemId}'`); + results.push({ + systemId: systemId, + old: {}, + new: {}, + }); } return; }); @@ -602,7 +659,7 @@ const processor = (dependencies: BaseDependencies) => { * @param {number} input.dictionaryId The Dictionary ID of the Submission * @param {SubmissionData} input.submissionData Data to be submitted grouped on inserts, updates and deletes * @param {number} input.idActiveSubmission ID of the Active Submission - * @param {Record>} input.schemaErrors Array of schemaErrors + * @param {SubmissionErrors} input.schemaErrors Array of schemaErrors * @param {string} input.username User updating the active submission * @returns {Promise} An Active Submission updated */ @@ -610,7 +667,7 @@ const processor = (dependencies: BaseDependencies) => { dictionaryId: number; submissionData: SubmissionData; idActiveSubmission: number; - schemaErrors: Record>; + schemaErrors: SubmissionErrors; username: string; }): Promise => { const { dictionaryId, submissionData, idActiveSubmission, schemaErrors, username } = input; diff --git a/packages/data-provider/src/utils/submissionUtils.ts b/packages/data-provider/src/utils/submissionUtils.ts index a068921e..1b51e355 100644 --- a/packages/data-provider/src/utils/submissionUtils.ts +++ b/packages/data-provider/src/utils/submissionUtils.ts @@ -5,7 +5,6 @@ import { type DataRecord, Dictionary as SchemasDictionary, DictionaryValidationError, - DictionaryValidationRecordErrorDetails, parse, Schema, TestResult, @@ -15,6 +14,7 @@ import { type Submission, SubmissionData, type SubmissionDeleteData, + type SubmissionErrors, type SubmissionInsertData, type SubmissionUpdateData, type SubmittedData, @@ -80,6 +80,24 @@ export const extractSchemaDataFromMergedDataRecords = ( return _.mapValues(mergeDataRecordsByEntityName, (mappingArray) => mappingArray.map((o) => o.dataRecord)); }; +/** + * Checks whether a record exists within a collection of submitted data records marked for update. + * The lookup is performed by matching the given 'entityName' and 'systemId'. + * + * @Returns true if found, false otherwise + */ +export const findEditSubmittedData = ( + entityName: string, + systemId: string, + dataByEntityName: Record, +) => { + return ( + dataByEntityName[entityName]?.some( + (data) => + data.reference.type === MERGE_REFERENCE_TYPE.EDIT_SUBMITTED_DATA && data.reference.systemId === systemId, + ) ?? false + ); +}; /** * Finds and returns a list of invalid records based on a provided schema name. * @@ -219,53 +237,55 @@ export const filterRelationsForPrimaryIdUpdate = ( * @param {object} input * @param {TestResult} input.resultValidation * @param {Record} input.dataValidated - * @returns {Record>} + * @returns {SubmissionErrors} */ export const groupSchemaErrorsByEntity = (input: { resultValidation: TestResult; dataValidated: Record; -}): Record> => { +}): SubmissionErrors => { const { resultValidation, dataValidated } = input; - const submissionSchemaErrors: Record> = {}; if (resultValidation.valid) { return {}; } + + const submissionSchemaErrors: SubmissionErrors = {}; resultValidation.details.forEach((dictionaryValidationError) => { const entityName = dictionaryValidationError.schemaName; - if (dictionaryValidationError.reason === 'INVALID_RECORDS') { - const validationErrors = dictionaryValidationError.invalidRecords; - - const hasErrorByIndex = groupErrorsByIndex(validationErrors); - - if (!_.isEmpty(hasErrorByIndex)) { - Object.entries(hasErrorByIndex).map(([indexBasedOnCrossSchemas, schemaValidationErrors]) => { - const mapping = dataValidated[entityName][Number(indexBasedOnCrossSchemas)]; - if (determineIfIsSubmission(mapping.reference)) { - const submissionIndex = mapping.reference.index; - const actionType = - mapping.reference.type === MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA ? 'inserts' : 'updates'; - - const mutableSchemaValidationErrors = schemaValidationErrors.map((errors) => { - return { - ...errors, - index: submissionIndex, - }; - }); - - if (!submissionSchemaErrors[actionType]) { - submissionSchemaErrors[actionType] = {}; - } + if (dictionaryValidationError.reason !== 'INVALID_RECORDS') { + return; + } - if (!submissionSchemaErrors[actionType][entityName]) { - submissionSchemaErrors[actionType][entityName] = []; - } + const groupedErrorsByIndex = groupErrorsByIndex(dictionaryValidationError.invalidRecords); - submissionSchemaErrors[actionType][entityName].push(...mutableSchemaValidationErrors); - } - }); - } + if (!groupedErrorsByIndex || Object.keys(groupedErrorsByIndex).length === 0) { + return; } + + Object.entries(groupedErrorsByIndex).forEach(([indexBasedOnCrossSchemas, schemaValidationErrors]) => { + const mapping = dataValidated[entityName][Number(indexBasedOnCrossSchemas)]; + if (!determineIfIsSubmission(mapping.reference)) { + return; + } + + const submissionIndex = mapping.reference.index; + const actionType = mapping.reference.type === MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA ? 'inserts' : 'updates'; + + const mutableSchemaValidationErrors = schemaValidationErrors.map((errors) => ({ + ...errors, + index: submissionIndex, + })); + + if (!submissionSchemaErrors[actionType]) { + submissionSchemaErrors[actionType] = {}; + } + + if (!submissionSchemaErrors[actionType][entityName]) { + submissionSchemaErrors[actionType][entityName] = []; + } + + submissionSchemaErrors[actionType][entityName].push(...mutableSchemaValidationErrors); + }); }); return submissionSchemaErrors; }; diff --git a/packages/data-provider/src/utils/types.ts b/packages/data-provider/src/utils/types.ts index b34d2bd4..35484ee3 100644 --- a/packages/data-provider/src/utils/types.ts +++ b/packages/data-provider/src/utils/types.ts @@ -4,7 +4,6 @@ import { type DataRecord, type DataRecordValue, Dictionary as SchemasDictionary, - DictionaryValidationRecordErrorDetails, type Schema, } from '@overture-stack/lectern-client'; import { @@ -15,6 +14,7 @@ import { Submission, SubmissionData, type SubmissionDeleteData, + type SubmissionErrors, type SubmissionUpdateData, type SubmittedData, } from '@overture-stack/lyric-data-model/models'; @@ -229,7 +229,7 @@ export type SubmissionResponse = { data: SubmissionData; dictionary: DictionaryActiveSubmission; dictionaryCategory: CategoryActiveSubmission; - errors: Record> | null; + errors: SubmissionErrors | null; organization: string; status: SubmissionStatus | null; createdAt: string | null; @@ -258,7 +258,7 @@ export type SubmissionSummaryRepository = { data: SubmissionData; dictionary: Pick; dictionaryCategory: Pick; - errors: Record> | null; + errors: SubmissionErrors | null; organization: string | null; status: SubmissionStatus | null; createdAt: Date | null; diff --git a/packages/data-provider/test/utils/submission/findEditSubmittedData.spec.ts b/packages/data-provider/test/utils/submission/findEditSubmittedData.spec.ts new file mode 100644 index 00000000..28e8d45d --- /dev/null +++ b/packages/data-provider/test/utils/submission/findEditSubmittedData.spec.ts @@ -0,0 +1,79 @@ +import { expect } from 'chai'; +import { describe, it } from 'mocha'; + +import { findEditSubmittedData } from '../../../index.js'; +import { type DataRecordReference, MERGE_REFERENCE_TYPE } from '../../../src/utils/types.js'; + +const recordsByEntityName: Record = { + animals: [ + { + dataRecord: { name: 'Bird', color: 'blue' }, + reference: { + systemId: 'BB4546', + submissionId: 2, + index: 0, + type: MERGE_REFERENCE_TYPE.EDIT_SUBMITTED_DATA, + }, + }, + { + dataRecord: { name: 'Dinosaur', color: 'red' }, + reference: { + systemId: 'DINO8912', + submissionId: 2, + index: 1, + type: MERGE_REFERENCE_TYPE.EDIT_SUBMITTED_DATA, + }, + }, + ], + teams: [ + { + dataRecord: { title: 'Raptors' }, + reference: { + systemId: 'RPT5678', + submissionId: 2, + index: 3, + type: MERGE_REFERENCE_TYPE.EDIT_SUBMITTED_DATA, + }, + }, + { + dataRecord: { tile: 'Blue Jays' }, + reference: { + systemId: 'BJ1425', + submittedDataId: 45, + type: MERGE_REFERENCE_TYPE.SUBMITTED_DATA, + }, + }, + ], +}; + +describe('Submission Utils - Find Edited Submitted Data by systemId', () => { + it('should return true when matching entityName and systemId and marked to edit', () => { + const result = findEditSubmittedData('animals', 'BB4546', recordsByEntityName); + expect(result).to.be.true; + }); + + it('should return false when matching systemId but not entityName', () => { + const result = findEditSubmittedData('teams', 'BB4546', recordsByEntityName); + expect(result).to.be.false; + }); + + it('should return false when matching systemId and entityName but not marked to edit', () => { + const result = findEditSubmittedData('teams', 'BJ1425', recordsByEntityName); + expect(result).to.be.false; + }); + + it('should return false when systemId not found', () => { + const result = findEditSubmittedData('animals', 'DOESNOTEXISTS', recordsByEntityName); + expect(result).to.be.false; + }); + + it('should return false when entityName does not exist', () => { + const result = findEditSubmittedData('incorrect', 'DINO8912', recordsByEntityName); + expect(result).to.be.false; + }); + + it('should return false when there are no records to find', () => { + const result = findEditSubmittedData('animals', 'DINO8912', {}); + expect(result).to.be.false; + }); +});