Merge pull request #2403 from AllenInstitute/bugfix/2403-speedup-get-all-files-recursively

GetAllFilesRecursivelyFromPath: Speed it up with custom extension

Author: t-b

Packages/Conversion/MIES_MassExperimentProcessing.ipf

@@ -296,17 +296,16 @@ Function StartMultiExperimentProcessWrapper()
 	outputFolder = S_Path
 	ASSERT(V_flag, "Invalid path")
 
-	files = GetAllFilesRecursivelyFromPath("MultiExperimentInputFolder", extension = ".pxp")
+	WAVE/Z files = GetAllFilesRecursivelyFromPath("MultiExperimentInputFolder", regex = "(?i)\.pxp$")
 
-	// 16: Case-insensitive alphanumeric sort that sorts wave0 and wave9 before wave10.
-	// ...
-	// 64: Ignore + and - in the alphanumeric sort so that "Text-09" sorts before "Text-10". Set options to 80 or 81.
-	files = SortList(files, FILE_LIST_SEP, 80)
-
-	WAVE/Z/T inputPXPs = ListToTextWave(files, FILE_LIST_SEP)
+	if(WaveExists(files))
+		Sort/A=2 files, files
+	else
+		Make/FREE/T/N=0 files
+	endif
 
 	jsonID = JSON_New()
-	JSON_AddWave(jsonID, "/inputFiles", inputPXPs)
+	JSON_AddWave(jsonID, "/inputFiles", files)
 	JSON_AddString(jsonID, "/inputFolder", inputFolder)
 	JSON_AddString(jsonID, "/outputFolder", outputFolder)
 	JSON_AddVariable(jsonID, "/index", 0)

Packages/MIES/MIES_AnalysisBrowser.ipf

@@ -2649,19 +2649,24 @@ static Function AB_AddExperimentEntries(string win, WAVE/T entries)
 			symbPath = GetUniqueSymbolicPath()
 			NewPath/O/Q/Z $symbPath, entry
 			if(GetCheckBoxState(win, "check_load_pxp"))
-				pxpList = GetAllFilesRecursivelyFromPath(symbPath, extension = ".pxp")
-				uxpList = GetAllFilesRecursivelyFromPath(symbPath, extension = ".uxp")
-			else
-				pxpList = ""
-				uxpList = ""
+				WAVE/Z/T pxps = GetAllFilesRecursivelyFromPath(symbPath, regex = "(?i)\.(pxp|uxp)$")
 			endif
 			if(GetCheckBoxState(win, "check_load_nwb"))
-				nwbList = GetAllFilesRecursivelyFromPath(symbPath, extension = ".nwb")
-			else
-				nwbList = ""
+				WAVE/Z/T nwbs = GetAllFilesRecursivelyFromPath(symbPath, regex = "(?i)\.nwb$")
 			endif
 			KillPath/Z $symbPath
-			WAVE/T fileList = ListToTextWave(SortList(pxpList + uxpList + nwbList, FILE_LIST_SEP), FILE_LIST_SEP)
+
+			if(WaveExists(pxps) && WaveExists(nwbs))
+				Concatenate/NP=(ROWS)/FREE/T {nwbs, pxps}, fileList
+			elseif(WaveExists(pxps))
+				WAVE/T fileList = pxps
+			elseif(WaveExists(nwbs))
+				WAVE/T fileList = nwbs
+			else
+				break
+			endif
+
+			Sort fileList, fileList
 		elseif(FileExists(entry))
 			Make/FREE/T fileList = {entry}
 		else

Packages/MIES/MIES_CheckInstallation.ipf

@@ -82,13 +82,13 @@ static Function CHI_CheckITCXOPVersion(STRUCT CHI_InstallationState &state)
 End
 
 /// @brief Search list for matches of item and print the results
-static Function CHI_CheckXOP(string &list, string item, string name, STRUCT CHI_InstallationState &state, [string expectedHash])
+static Function CHI_CheckXOP(WAVE/T list, string item, string name, STRUCT CHI_InstallationState &state, [string expectedHash])
 
 	variable numMatches, i, hashMatches
-	string matches, fileVersion, filepath, existingHash, hashMsg
+	string fileVersion, filepath, existingHash, hashMsg
 
-	matches    = ListMatch(list, "*" + item, FILE_LIST_SEP)
-	numMatches = ItemsInList(matches, FILE_LIST_SEP)
+	WAVE/Z/T matches = GrepTextWave(list, "(?i)" + item)
+	numMatches = WaveExists(matches) ? DimSize(matches, ROWS) : 0
 
 	if(numMatches > 1)
 		if(CheckIfPathsRefIdenticalFiles(matches))
@@ -106,7 +106,7 @@ static Function CHI_CheckXOP(string &list, string item, string name, STRUCT CHI_
 			state.numErrors += 1
 			break
 		case 1:
-			filepath    = StringFromList(0, matches, FILE_LIST_SEP)
+			filepath    = matches[0]
 			fileVersion = GetFileVersion(filepath)
 			if(ParamIsDefault(expectedHash))
 				printf "%s: Found version %s (Nice!)\r", name, fileVersion
@@ -119,10 +119,10 @@ static Function CHI_CheckXOP(string &list, string item, string name, STRUCT CHI_
 			endif
 			break
 		default:
-			printf "%s: Found multiple versions in \"%s\" (Might create problems)\r", name, matches
+			printf "%s: Found multiple versions in \"%s\" (Might create problems)\r", name, TextWaveToList(matches, ", ", trailSep = 0)
 			printf "%s: Duplicates are:\r", name
 			for(i = 0; i < numMatches; i += 1)
-				filepath    = StringFromList(i, matches, FILE_LIST_SEP)
+				filepath    = matches[i]
 				fileVersion = GetFileVersion(filepath)
 				if(ParamIsDefault(expectedHash))
 					printf "%s: Found version %s\r", name, fileVersion
@@ -224,26 +224,32 @@ End
 Function CHI_CheckInstallation()
 
 	string symbPath, allFiles, path, extName, info, igorBuild
-	string allFilesSystem, allFilesUser, listOfXOPs
 	variable aslrEnabled, archBits, installedWithHW
 
 	symbPath = GetUniqueSymbolicPath()
 	extName  = GetIgorExtensionFolderName()
 
 	path = SpecialDirPath("Igor Pro User Files", 0, 0, 0) + extName
 	NewPath/Q/O $symbPath, path
-	allFilesUser = GetAllFilesRecursivelyFromPath(symbPath)
+	WAVE/Z/T allFilesUser = GetAllFilesRecursivelyFromPath(symbPath, regex = "(?i)\.xop$", resolveAliases = 1)
 
 	path = SpecialDirPath("Igor Application", 0, 0, 0) + extName
 	NewPath/Q/O $symbPath, path
-	allFilesSystem = GetAllFilesRecursivelyFromPath(symbPath)
+	WAVE/Z/T allFilesSystem = GetAllFilesRecursivelyFromPath(symbPath, regex = "(?i)\.xop$", resolveAliases = 1)
 
 	KillPath $symbPath
 
-	listOfXOPs = ListMatch(allFilesUser + FILE_LIST_SEP + allFilesSystem, "*.xop", FILE_LIST_SEP)
-	WAVE/T list       = ListToTextWave(listOfXOPs, FILE_LIST_SEP)
-	WAVE/T listNoDups = GetUniqueEntries(list)
-	listOfXOPs = TextWaveToList(listNoDups, FILE_LIST_SEP)
+	if(WaveExists(allFilesUser) && WaveExists(allFilesSystem))
+		Concatenate/T/FREE/NP=(ROWS) {allFilesUser, allFilesSystem}, listWithDuplicates
+	elseif(WaveExists(allFilesUser))
+		WAVE/T listWithDuplicates = allFilesUser
+	elseif(WaveExists(allFilesSystem))
+		WAVE/T listWithDuplicates = allFilesSystem
+	else
+		Make/FREE/N=0/T listWithDuplicates
+	endif
+
+	WAVE/T listOfXOPs = GetUniqueEntries(listWithDuplicates)
 
 	STRUCT CHI_InstallationState state
 	CHI_InitInstallationState(state)

Packages/MIES/MIES_Configuration.ipf

@@ -297,25 +297,26 @@ End
 /// @brief Return a text wave with absolute paths to the JSON configuration files
 static Function/WAVE CONF_GetConfigFiles([string customIPath])
 
-	string settingsPath, fileList
+	string settingsPath
 
 	if(ParamIsDefault(customIPath))
 		settingsPath = CONF_GetSettingsPath(CONF_AUTO_LOADER_GLOBAL)
 	else
 		settingsPath = customIPath
 	endif
-	fileList = GetAllFilesRecursivelyFromPath(settingsPath, extension = ".json")
 
-	if(IsEmpty(fileList) && !ParamIsDefault(customIPath))
+	WAVE/Z/T fileList = GetAllFilesRecursivelyFromPath(settingsPath, regex = "(?i)\.json$")
+
+	if(!WaveExists(fileList) && !ParamIsDefault(customIPath))
 		settingsPath = CONF_GetSettingsPath(CONF_AUTO_LOADER_USER)
-		fileList     = GetAllFilesRecursivelyFromPath(settingsPath, extension = ".json")
+		WAVE/Z/T fileList = GetAllFilesRecursivelyFromPath(settingsPath, regex = "(?i)\.json$")
 	endif
 
-	if(IsEmpty(fileList))
+	if(!WaveExists(fileList))
 		return $""
 	endif
 
-	return ListToTextWave(fileList, FILE_LIST_SEP)
+	return fileList
 End
 
 /// @brief Automatically loads all *.json files from MIES Settings folder and opens and restores the corresponding windows

Packages/MIES/MIES_DAEphys.ipf

@@ -4711,8 +4711,7 @@ End
 
 static Function DAP_LoadBuiltinStimsets()
 
-	string symbPath, stimset, files, filename
-	variable i, numEntries
+	string symbPath, filename
 
 	symbPath = GetUniqueSymbolicPath()
 	NewPath/Q $symbPath, GetFolder(FunctionPath("")) + ":Stimsets"
@@ -4723,10 +4722,13 @@ static Function DAP_LoadBuiltinStimsets()
 		return NaN
 	endif
 
-	files      = GetAllFilesRecursivelyFromPath(symbPath, extension = ".nwb")
-	numEntries = ItemsInList(files, FILE_LIST_SEP)
-	for(i = 0; i < numEntries; i += 1)
-		filename = StringFromList(i, files, FILE_LIST_SEP)
+	WAVE/Z/T files = GetAllFilesRecursivelyFromPath(symbPath, regex = "(?i)\.nwb$")
+
+	if(!WaveExists(files))
+		return NaN
+	endif
+
+	for(filename : files)
 		NWB_LoadAllStimsets(filename = filename, overwrite = 1, loadOnlyBuiltins = 1)
 	endfor
 

Packages/MIES/MIES_DebugPanel.ipf

@@ -75,7 +75,7 @@ End
 
 static Function DP_FillDebugPanelWaves()
 
-	string symbPath, path, allProcFiles
+	string symbPath, path
 
 	WAVE/T listWave    = GetDebugPanelListWave()
 	WAVE   listSelWave = GetDebugPanelListSelWave()
@@ -84,15 +84,17 @@ static Function DP_FillDebugPanelWaves()
 
 	path = FunctionPath("") + "::"
 	NewPath/Q/O $symbPath, path
-	allProcFiles = GetAllFilesRecursivelyFromPath(symbPath, extension = ".ipf")
+	WAVE/Z allProcFilesMIES = GetAllFilesRecursivelyFromPath(symbPath, regex = "(?i)\.ipf$")
+	ASSERT(WaveExists(allProcFilesMIES), "Expected some MIES procedure files")
 
 	path += ":IPNWB"
 	NewPath/Q/O $symbPath, path
-	allProcFiles = AddListItem(allProcFiles, GetAllFilesRecursivelyFromPath(symbPath, extension = ".ipf"), FILE_LIST_SEP)
+	WAVE/Z allProcFilesIPNWB = GetAllFilesRecursivelyFromPath(symbPath, regex = "(?i)\.ipf$")
+	ASSERT(WaveExists(allProcFilesIPNWB), "Expected some IPNWB procedure files")
 
 	KillPath $symbPath
 
-	WAVE/T list = ListToTextWave(allProcFiles, FILE_LIST_SEP)
+	Concatenate/FREE/NP=(ROWS)/T {allProcFilesMIES, allProcFilesIPNWB}, list
 	// remove path components
 	list = GetFile(list[p])
 	// remove non mies files

Packages/MIES/MIES_MiesUtilities_Uploads.ipf

@@ -182,12 +182,10 @@ Function UploadCrashDumps()
 
 	diagSymbPath = GetSymbolicPathForDiagnosticsDirectory()
 
-	WAVE/T files = ListTotextWave(GetAllFilesRecursivelyFromPath(diagSymbPath, extension = ".dmp"), FILE_LIST_SEP)
-	WAVE/T logs  = ListTotextWave(GetAllFilesRecursivelyFromPath(diagSymbPath, extension = ".txt"), FILE_LIST_SEP)
-	numFiles = DimSize(files, ROWS)
-	numLogs  = DimSize(logs, ROWS)
+	WAVE/Z/T files = GetAllFilesRecursivelyFromPath(diagSymbPath, regex = "(?i)\.dmp$")
+	WAVE/Z/T logs  = GetAllFilesRecursivelyFromPath(diagSymbPath, regex = "(?i)\.txt$")
 
-	if(!numFiles && !numLogs)
+	if(!WaveExists(files) && !WaveExists(logs))
 		return NaN
 	endif