Merge pull request #175 from mcode/config-csv-options

Passthrough csv-parse options supported in the config

Author: dmendelowitz

README.md

@@ -199,6 +199,22 @@ cat -v <file.csv>
 
 If there is an unexpected symbol at the beginning of the file, then there may be a byte order marker that needs to be removed.
 
+#### Troubleshooting Additional Errors
+The mCODE Extraction Framework uses the node `csv-parse` library to parse specified CSV files. [Parsing options for the `csv-parse` library](https://csv.js.org/parse/options/) can be included in the configuration file within the `commonExtractorArgs.csvParse.options` section. For example, the following configuration will pass the `to` option to the `csv-parse` module, causing the mCODE Extraction Framework to only read CSV files up to the specified line number: 
+
+```
+"commonExtractorArgs": {
+    "dataDirectory": "/Users/*****/Documents/dataDirectory",
+    "csvParse": {
+      "options": {
+        "to": 3
+      }
+    }
+  },
+```
+
+**Note:** The mCODE Extraction Framework enables the `bom`, `skip_empty_lines`, and `skip_lines_with_empty_values` options by default, including these options in the configuration file will cause these default options to be overwritten.
+
 ## Terminology and Architecture
 
 This framework consists of three key components: Extractors, Modules and Templates. Below is, in order:

src/application/app.js

@@ -31,7 +31,8 @@ async function mcodeApp(Client, fromDate, toDate, config, pathToRunLogs, debug,
 
   // Parse CSV for list of patient mrns
   const dataDirectory = config.commonExtractorArgs && config.commonExtractorArgs.dataDirectory;
-  const patientIds = parsePatientIds(config.patientIdCsvPath, dataDirectory);
+  const parserOptions = config.commonExtractorArgs && config.commonExtractorArgs.csvParse && config.commonExtractorArgs.csvParse.options ? config.commonExtractorArgs.csvParse.options : {};
+  const patientIds = parsePatientIds(config.patientIdCsvPath, dataDirectory, parserOptions);
 
   // Get RunInstanceLogger for recording new runs and inferring dates from previous runs
   const runLogger = allEntries ? null : new RunInstanceLogger(pathToRunLogs);

src/extractors/BaseCSVExtractor.js

@@ -6,26 +6,27 @@ const logger = require('../helpers/logger');
 
 class BaseCSVExtractor extends Extractor {
   constructor({
-    filePath, url, fileName, dataDirectory, csvSchema, unalterableColumns,
+    filePath, url, fileName, dataDirectory, csvSchema, unalterableColumns, csvParse,
   }) {
     super();
     this.unalterableColumns = unalterableColumns || [];
     this.csvSchema = csvSchema;
+    this.parserOptions = csvParse && csvParse.options ? csvParse.options : {};
     if (url) {
       logger.debug('Found url argument; creating a CSVURLModule with the provided url');
       this.url = url;
-      this.csvModule = new CSVURLModule(this.url, this.unalterableColumns);
+      this.csvModule = new CSVURLModule(this.url, this.unalterableColumns, this.parserOptions);
     } else if (fileName && dataDirectory) {
       if (!path.isAbsolute(dataDirectory)) throw new Error('dataDirectory is not an absolutePath, it needs to be.');
       this.filePath = path.join(dataDirectory, fileName);
       logger.debug(
         'Found fileName and dataDirectory arguments; creating a CSVFileModule with the provided dataDirectory and fileName',
       );
-      this.csvModule = new CSVFileModule(this.filePath, this.unalterableColumns);
+      this.csvModule = new CSVFileModule(this.filePath, this.unalterableColumns, this.parserOptions);
     } else if (filePath) {
       logger.debug('Found filePath argument; creating a CSVFileModule with the provided filePath');
       this.filePath = filePath;
-      this.csvModule = new CSVFileModule(this.filePath, this.unalterableColumns);
+      this.csvModule = new CSVFileModule(this.filePath, this.unalterableColumns, this.parserOptions);
     } else {
       logger.debug(
         'Could not instantiate a CSVExtractor with the provided constructor args',

src/extractors/CSVAdverseEventExtractor.js

@@ -69,8 +69,10 @@ function formatData(adverseEventData, patientId) {
 }
 
 class CSVAdverseEventExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url, fileName, dataDirectory }) {
-    super({ filePath, url, fileName, dataDirectory });
+  constructor({
+    filePath, url, fileName, dataDirectory, csvParse,
+  }) {
+    super({ filePath, url, fileName, dataDirectory, csvParse });
   }
 
   async getAdverseEventData(mrn) {

src/extractors/CSVCTCAdverseEventExtractor.js

@@ -109,8 +109,10 @@ function formatData(adverseEventData, patientId) {
 }
 
 class CSVCTCAdverseEventExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url, fileName, dataDirectory }) {
-    super({ filePath, url, fileName, dataDirectory });
+  constructor({
+    filePath, url, fileName, dataDirectory, csvParse,
+  }) {
+    super({ filePath, url, fileName, dataDirectory, csvParse });
   }
 
   async getAdverseEventData(mrn) {

src/extractors/CSVCancerDiseaseStatusExtractor.js

@@ -9,9 +9,9 @@ const { CSVCancerDiseaseStatusSchema } = require('../helpers/schemas/csv');
 
 class CSVCancerDiseaseStatusExtractor extends BaseCSVExtractor {
   constructor({
-    filePath, url, fileName, dataDirectory, implementation,
+    filePath, url, fileName, dataDirectory, csvParse, implementation,
   }) {
-    super({ filePath, url, fileName, dataDirectory, csvSchema: CSVCancerDiseaseStatusSchema });
+    super({ filePath, url, fileName, dataDirectory, csvSchema: CSVCancerDiseaseStatusSchema, csvParse });
     this.implementation = implementation;
   }
 

src/extractors/CSVCancerRelatedMedicationAdministrationExtractor.js

@@ -46,8 +46,10 @@ function formatData(medicationData, patientId) {
 }
 
 class CSVCancerRelatedMedicationAdministrationExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url, fileName, dataDirectory }) {
-    super({ filePath, url, fileName, dataDirectory });
+  constructor({
+    filePath, url, fileName, dataDirectory, csvParse,
+  }) {
+    super({ filePath, url, fileName, dataDirectory, csvParse });
   }
 
   async getMedicationData(mrn) {

src/extractors/CSVCancerRelatedMedicationRequestExtractor.js

@@ -48,8 +48,10 @@ function formatData(medicationData, patientId) {
 }
 
 class CSVCancerRelatedMedicationRequestExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url, fileName, dataDirectory }) {
-    super({ filePath, url, fileName, dataDirectory });
+  constructor({
+    filePath, url, fileName, dataDirectory, csvParse,
+  }) {
+    super({ filePath, url, fileName, dataDirectory, csvParse });
   }
 
   async getMedicationData(mrn) {

src/extractors/CSVClinicalTrialInformationExtractor.js

@@ -9,9 +9,9 @@ const { CSVClinicalTrialInformationSchema } = require('../helpers/schemas/csv');
 
 class CSVClinicalTrialInformationExtractor extends BaseCSVExtractor {
   constructor({
-    filePath, url, fileName, dataDirectory, clinicalSiteID, clinicalSiteSystem,
+    filePath, url, fileName, dataDirectory, csvParse, clinicalSiteID, clinicalSiteSystem,
   }) {
-    super({ filePath, url, fileName, dataDirectory, csvSchema: CSVClinicalTrialInformationSchema });
+    super({ filePath, url, fileName, dataDirectory, csvSchema: CSVClinicalTrialInformationSchema, csvParse });
     if (!clinicalSiteID) logger.warn(`${this.constructor.name} expects a value for clinicalSiteID but got ${clinicalSiteID}`);
     this.clinicalSiteID = clinicalSiteID;
     this.clinicalSiteSystem = clinicalSiteSystem;

src/extractors/CSVConditionExtractor.js

@@ -49,8 +49,10 @@ function formatData(conditionData, patientId) {
 }
 
 class CSVConditionExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url, fileName, dataDirectory }) {
-    super({ filePath, url, fileName, dataDirectory, csvSchema: CSVConditionSchema });
+  constructor({
+    filePath, url, fileName, dataDirectory, csvParse,
+  }) {
+    super({ filePath, url, fileName, dataDirectory, csvSchema: CSVConditionSchema, csvParse });
   }
 
   async getConditionData(mrn) {