Merge pull request #186 from mcode/develop

Dtphelan1 · web-flow · commit 4239fcfab26c · 2022-12-16T15:05:00.000-05:00
v2.1.0
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "mcode-extraction-framework",
-  "version": "2.0.1",
+  "version": "2.1.0",
   "description": "",
   "contributors": [
     "Julia Afeltra <jafeltra@mitre.org>",
@@ -29,7 +29,7 @@
     "fhir-crud-client": "^1.2.2",
     "fhirpath": "2.1.5",
     "lodash": "^4.17.21",
-    "moment": "^2.29.3",
+    "moment": "^2.29.4",
     "nodemailer": "^6.7.2",
     "winston": "^3.2.1"
   },
diff --git a/src/extractors/CSVCancerDiseaseStatusExtractor.js b/src/extractors/CSVCancerDiseaseStatusExtractor.js
@@ -36,7 +36,7 @@ class CSVCancerDiseaseStatusExtractor extends BaseCSVExtractor {
         status: observationStatus || 'final',
         value: {
           code: diseaseStatusCode,
-          system: 'http://snomed.info/sct',
+          system: diseaseStatusCode.includes('USCRS') ? 'http://hl7.org/fhir/us/mcode/CodeSystem/snomed-requested-cs' : 'http://snomed.info/sct',
           display: diseaseStatusText || getDiseaseStatusDisplay(diseaseStatusCode, this.implementation),
         },
         subject: {
diff --git a/src/helpers/csvParsingUtils.js b/src/helpers/csvParsingUtils.js
@@ -44,6 +44,8 @@ const DEFAULT_OPTIONS = {
   columns: (header) => header.map((column) => stringNormalizer(column)),
   // https://csv.js.org/parse/options/bom/
   bom: true,
+  // https://csv.js.org/parse/options/trim/
+  trim: true,
   // https://csv.js.org/parse/options/skip_empty_lines/
   skip_empty_lines: true,
   // NOTE: This will skip any records with empty values, not just skip the empty values themselves
@@ -57,9 +59,17 @@ function csvParse(csvData, options = {}) {
   return parse(csvData, { ...DEFAULT_OPTIONS, ...options });
 }
 
+function getCSVHeader(csvData) {
+  return parse(csvData, {
+    bom: true,
+    trim: true,
+    to: 1,
+  })[0].map((h) => h.toLowerCase());
+}
 
 module.exports = {
   stringNormalizer,
   normalizeEmptyValues,
   csvParse,
+  getCSVHeader,
 };
diff --git a/src/helpers/csvValidator.js b/src/helpers/csvValidator.js
@@ -3,11 +3,10 @@ const logger = require('./logger');
 
 // Validates csvData against the csvSchema
 // Uses the csvFileIdentifier in logs for readability
-function validateCSV(csvFileIdentifier, csvSchema, csvData) {
+function validateCSV(csvFileIdentifier, csvSchema, csvData, headers) {
   let isValid = true;
 
   // Check headers
-  const headers = Object.keys(csvData[0]).map((h) => h.toLowerCase());
   const schemaDiff = _.difference(csvSchema.headers.map((h) => h.name.toLowerCase()), headers);
   const fileDiff = _.difference(headers, csvSchema.headers.map((h) => h.name.toLowerCase()));
 
diff --git a/src/helpers/lookups/diseaseStatusLookup.js b/src/helpers/lookups/diseaseStatusLookup.js
@@ -1,12 +1,17 @@
 const { createInvertedLookup, createLowercaseLookup } = require('../lookupUtils');
 
-// Code mapping is based on current values at https://www.hl7.org/fhir/us/mcode/2021May/ValueSet-mcode-condition-status-trend-vs.html
+// Code mapping is based on current values at https://hl7.org/fhir/us/mcode/ValueSet-mcode-condition-status-trend-vs.html
+// along with legacy codes included at https://www.hl7.org/fhir/us/mcode/2021May/ValueSet-mcode-condition-status-trend-vs.html
 const mcodeDiseaseStatusTextToCodeLookup = {
-  'No abnormality detected (finding)': '281900007',
+  'No abnormality detected (finding)': '281900007', // No longer in the Vs, included for backwards compatibility
   'Patient condition improved (finding)': '268910001',
   'Patient\'s condition stable (finding)': '359746009',
   'Patient\'s condition worsened (finding)': '271299001',
   'Patient condition undetermined (finding)': '709137006',
+  // TODO: These are placeholder codes representing codes that are requested additions to the SNOMED vocabulary
+  // They will likely need to be updated in future versions of mCODE
+  'Cancer in complete remission(finding)': 'USCRS-352236',
+  'Cancer in partial remission (finding)': 'USCRS-352237',
 };
 const mcodeDiseaseStatusCodeToTextLookup = createInvertedLookup(mcodeDiseaseStatusTextToCodeLookup);
 
diff --git a/src/modules/CSVFileModule.js b/src/modules/CSVFileModule.js
@@ -2,14 +2,16 @@ const fs = require('fs');
 const moment = require('moment');
 const logger = require('../helpers/logger');
 const { validateCSV } = require('../helpers/csvValidator');
-const { csvParse, stringNormalizer, normalizeEmptyValues } = require('../helpers/csvParsingUtils');
+const { csvParse, stringNormalizer, normalizeEmptyValues, getCSVHeader } = require('../helpers/csvParsingUtils');
 
 class CSVFileModule {
   constructor(csvFilePath, unalterableColumns, parserOptions) {
     // Parse then normalize the data
-    const parsedData = csvParse(fs.readFileSync(csvFilePath), parserOptions);
+    const csvData = fs.readFileSync(csvFilePath);
+    const parsedData = csvParse(csvData, parserOptions);
     this.filePath = csvFilePath;
     this.data = normalizeEmptyValues(parsedData, unalterableColumns);
+    this.header = getCSVHeader(csvData);
   }
 
   async get(key, value, fromDate, toDate) {
@@ -32,7 +34,7 @@ class CSVFileModule {
   async validate(csvSchema) {
     if (csvSchema) {
       logger.info(`Validating CSV file for ${this.filePath}`);
-      return validateCSV(this.filePath, csvSchema, this.data);
+      return validateCSV(this.filePath, csvSchema, this.data, this.header);
     }
     logger.warn(`No CSV schema provided for ${this.filePath}`);
     return true;
diff --git a/src/modules/CSVURLModule.js b/src/modules/CSVURLModule.js
@@ -2,7 +2,7 @@ const axios = require('axios');
 const moment = require('moment');
 const logger = require('../helpers/logger');
 const { validateCSV } = require('../helpers/csvValidator');
-const { csvParse, stringNormalizer, normalizeEmptyValues } = require('../helpers/csvParsingUtils');
+const { csvParse, stringNormalizer, normalizeEmptyValues, getCSVHeader } = require('../helpers/csvParsingUtils');
 
 class CSVURLModule {
   constructor(url, unalterableColumns, parserOptions) {
@@ -28,6 +28,7 @@ class CSVURLModule {
       const parsedData = csvParse(csvData, this.parserOptions);
       logger.debug('CSV Data parsing successful');
       this.data = normalizeEmptyValues(parsedData, this.unalterableColumns);
+      this.header = getCSVHeader(csvData);
     }
   }
 
@@ -55,7 +56,7 @@ class CSVURLModule {
 
     if (csvSchema) {
       this.data = normalizeEmptyValues(this.data, this.unalterableColumns);
-      return validateCSV(this.url, csvSchema, this.data);
+      return validateCSV(this.url, csvSchema, this.data, this.header);
     }
     logger.warn(`No CSV schema provided for data found at ${this.url}`);
     return true;
diff --git a/test/extractors/fixtures/csv-cancer-disease-status-bundle.json b/test/extractors/fixtures/csv-cancer-disease-status-bundle.json
@@ -3,10 +3,10 @@
   "type": "collection",
   "entry": [
     {
-      "fullUrl": "urn:uuid:4b9e9b5a8db529782cd9e89b68c6a3fac408d2195f025691a3f2850cab18057f",
+      "fullUrl": "urn:uuid:e8293a0d18fb20d6b1749009032a26f2d34d8418369c87c8345f6c988fa0fc33",
       "resource": {
         "resourceType": "Observation",
-        "id": "4b9e9b5a8db529782cd9e89b68c6a3fac408d2195f025691a3f2850cab18057f",
+        "id": "e8293a0d18fb20d6b1749009032a26f2d34d8418369c87c8345f6c988fa0fc33",
         "meta": {
           "profile": [
             "http://hl7.org/fhir/us/mcode/StructureDefinition/mcode-cancer-disease-status"
@@ -47,9 +47,9 @@
         "valueCodeableConcept": {
           "coding": [
             {
-              "system": "http://snomed.info/sct",
-              "code": "268910001",
-              "display": "Patient condition improved (finding)"
+              "system": "http://hl7.org/fhir/us/mcode/CodeSystem/snomed-requested-cs",
+              "code": "USCRS-352236",
+              "display": "Cancer in complete remission(finding)"
             }
           ]
         },
diff --git a/test/extractors/fixtures/csv-cancer-disease-status-module-response.json b/test/extractors/fixtures/csv-cancer-disease-status-module-response.json
@@ -2,7 +2,7 @@
   {
     "mrn": "mrn-1",
     "conditionid": "cond-1",
-    "diseasestatuscode": "268910001",
+    "diseasestatuscode": "USCRS-352236",
     "dateofobservation": "2019-12-02",
     "evidence": "363679005|252416005",
     "observationstatus": "amended"
diff --git a/test/helpers/csvValidator.test.js b/test/helpers/csvValidator.test.js
@@ -83,26 +83,30 @@ const schema = {
 
 describe('csvValidator', () => {
   test('simple data validates', () => {
-    expect(validateCSV('', schema, SIMPLE_DATA)).toBe(true);
+    expect(validateCSV('', schema, SIMPLE_DATA, Object.keys(SIMPLE_DATA[0]))).toBe(true);
   });
 
   test('data missing required value does not validate', () => {
-    expect(validateCSV('', schema, SIMPLE_DATA_MISSING_REQUIRED_VALUE)).toBe(false);
+    expect(validateCSV('', schema, SIMPLE_DATA_MISSING_REQUIRED_VALUE, Object.keys(SIMPLE_DATA_MISSING_REQUIRED_VALUE[0]).map((h) => h.toLowerCase()))).toBe(false);
   });
 
   test('data missing required header does not validate', () => {
-    expect(validateCSV('', schema, SIMPLE_DATA_MISSING_HEADER)).toBe(false);
+    expect(validateCSV('', schema, SIMPLE_DATA_MISSING_HEADER, Object.keys(SIMPLE_DATA_MISSING_HEADER[0]).map((h) => h.toLowerCase()))).toBe(false);
   });
 
   test('data with erroneous column should still validate', () => {
-    expect(validateCSV('', schema, SIMPLE_DATA_EXTRA_COLUMNS)).toBe(true);
+    expect(validateCSV('', schema, SIMPLE_DATA_EXTRA_COLUMNS, Object.keys(SIMPLE_DATA_EXTRA_COLUMNS[0]).map((h) => h.toLowerCase()))).toBe(true);
   });
 
   test('data missing an optional column should still validate', () => {
-    expect(validateCSV('', schema, SIMPLE_DATA_MISSING_OPTIONAL_COLUMN)).toBe(true);
+    expect(validateCSV('', schema, SIMPLE_DATA_MISSING_OPTIONAL_COLUMN, Object.keys(SIMPLE_DATA_MISSING_OPTIONAL_COLUMN[0]).map((h) => h.toLowerCase()))).toBe(true);
   });
 
   test('data with different casing in the column header should still validate', () => {
-    expect(validateCSV('', schema, SIMPLE_DATA_DIFFERENT_CASING)).toBe(true);
+    expect(validateCSV('', schema, SIMPLE_DATA_DIFFERENT_CASING, Object.keys(SIMPLE_DATA_DIFFERENT_CASING[0]).map((h) => h.toLowerCase()))).toBe(true);
+  });
+
+  test('data with only the header but no rows should still validate', () => {
+    expect(validateCSV('', schema, [], ['header1', 'header2', 'header3'])).toBe(true);
   });
 });
diff --git a/test/helpers/diseaseStatusUtils.test.js b/test/helpers/diseaseStatusUtils.test.js
@@ -13,6 +13,8 @@ const mcodeDiseaseStatusTextToCodeLookup = {
   'Patient\'s condition stable (finding)': '359746009',
   'Patient\'s condition worsened (finding)': '271299001',
   'Patient condition undetermined (finding)': '709137006',
+  'Cancer in complete remission(finding)': 'USCRS-352236',
+  'Cancer in partial remission (finding)': 'USCRS-352237',
 };
 
 // Code mapping is based on initial values still in use by icare implementors

Original file line number	Diff line number	Diff line change
`@@ -2,7 +2,7 @@`
`2`	`2`	`{`
`3`	`3`	`"mrn": "mrn-1",`
`4`	`4`	`"conditionid": "cond-1",`
`5`		`- "diseasestatuscode": "268910001",`
	`5`	`+ "diseasestatuscode": "USCRS-352236",`
`6`	`6`	`"dateofobservation": "2019-12-02",`
`7`	`7`	`"evidence": "363679005\|252416005",`
`8`	`8`	`"observationstatus": "amended"`