Skip to content

Commit 43a1433

Browse files
t-bt-b
committed
GetAllFilesRecursivelyFromPath: Speed it up with custom extension
We used to always perform alias/shortcut handling even when we only look for certain file suffixes. But we only have to try following these if they can be part of the returned list. So for the common case of Windows OS and a extension which does not match .lnk we can skip the alias resolving. When we do have aliases we also first call GetFileFolderInfo and only if we have an alias ResolveAlias again, skipping another GetFileFolderInfo for the case when the pointed to file is not an alias. This allows to gather 100k files on a 1Gb/s network link in under 60s. ```igorpro Function Dostuff() Newpath/O testPath "Y:" beginfunctionProfiling(testTime = 50000) string result = getAllfilesRecursivelyFromPath("testPath", extension = ".mp3") endfunctionProfiling() printf "Number of files: %d\r", ItemsInList(result, "|") End ``` gives ``` Total time: 54.4173, Time in Function code: 54.3973 (100%) Top function percentages: Function MIES_Utilities_File.ipf GetAllFilesRecursivelyFromPath: 99% Annotated Top Functions: ******************************************************************************************* Function: MIES_Utilities_File.ipf MIES_Utilities_File.ipf#GetAllFilesRecursivelyFromPath; Percent total 99% ******************************************************************************************* [00] |Function/S GetAllFilesRecursivelyFromPath(string pathName, [string extension]) [00] | [00] | string fileOrPath, folders, subFolderPathName, fileName [00] | string files, allFilesList [00] | string allFiles = "" [00]* | string foldersFromAlias = "" [00] | variable err, isWindows [00] | [00] |#ifdef WINDOWS [00] | isWindows = 1 [00] |#endif [00] | [01]* | PathInfo $pathName [00] | ASSERT(V_flag, "Given symbolic path does not exist") [00] | [00] | if(ParamIsDefault(extension)) [00] | extension = "????" [00] | endif [00] | [00] | AssertOnAndClearRTError() [70]******* | allFilesList = IndexedFile($pathName, -1, extension, "????", FILE_LIST_SEP); err = GetRTError(1) [00] | [00]* | if(isWindows && cmpstr(extension, "????") && cmpstr(extension, ".lnk")) [00] | allFiles = AddPrefixToEachListItem(S_path, allFilesList, sep = FILE_LIST_SEP) [00] | else [00] | // try to resolve aliases/shortcuts when we can have them [00] | WAVE/T allFilesInDir = ListToTextWave(allFilesList, FILE_LIST_SEP) [00] | for(fileName : allFilesInDir) [00] | [00] | GetFileFolderInfo/P=$pathName/Q/Z fileName [00] | [00] | if(V_Flag != 0) [00] | // error querying file [00] | continue [00] | endif [00] | [00] | if(V_IsAliasShortcut) [00] | fileOrPath = ResolveAlias(fileName, pathName = pathName) [00] | [00] | // redo the check [00] | GetFileFolderInfo/P=$pathName/Q/Z fileOrPath [00] | [00] | if(V_Flag != 0) [00] | // error querying file/folder pointed to by alias [00] | continue [00] | endif [00] | endif [00] | [00] | if(V_isFile) [00] | allFiles = AddListItem(S_path, allFiles, FILE_LIST_SEP, Inf) [00] | elseif(V_isFolder) [00] | foldersFromAlias = AddListItem(S_path, foldersFromAlias, FILE_LIST_SEP, Inf) [00] | else [00] | ASSERT(0, "Unexpected file type") [00] | endif [00] | endfor [00] | endif [00] | [00] | AssertOnAndClearRTError() [04]* | folders = IndexedDir($pathName, -1, 1, FILE_LIST_SEP); err = GetRTError(1) [00] | folders = folders + foldersFromAlias [00]* | WAVE/T wFolders = ListToTextWave(folders, FILE_LIST_SEP) [00]* | for(folder : wFolders) [00] | [00] | subFolderPathName = GetUniqueSymbolicPath() [00] | [12]* | NewPath/Q/O $subFolderPathName, folder [00]* | files = GetAllFilesRecursivelyFromPath(subFolderPathName, extension = extension) [11]* | KillPath/Z $subFolderPathName [00] | [00]* | if(!isEmpty(files)) [01]* | allFiles = AddListItem(RemoveEnding(files, FILE_LIST_SEP), allFiles, FILE_LIST_SEP, Inf) [00] | endif [00] | endfor [00] | [00] | return allFiles [00] |End ``` Close #2402
1 parent d9b94ff commit 43a1433

File tree

2 files changed

+80
-22
lines changed

2 files changed

+80
-22
lines changed

‎Packages/MIES/MIES_Utilities_File.ipf‎

Lines changed: 45 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,11 @@ Function/S GetAllFilesRecursivelyFromPath(string pathName, [string extension])
237237
string files, allFilesList
238238
string allFiles = ""
239239
string foldersFromAlias = ""
240-
variable err
240+
variable err, isWindows
241+
242+
#ifdef WINDOWS
243+
isWindows = 1
244+
#endif // WINDOWS
241245

242246
PathInfo $pathName
243247
ASSERT(V_flag, "Given symbolic path does not exist")
@@ -248,27 +252,47 @@ Function/S GetAllFilesRecursivelyFromPath(string pathName, [string extension])
248252

249253
AssertOnAndClearRTError()
250254
allFilesList = IndexedFile($pathName, -1, extension, "????", FILE_LIST_SEP); err = GetRTError(1)
251-
WAVE/T allFilesInDir = ListToTextWave(allFilesList, FILE_LIST_SEP)
252-
for(fileName : allFilesInDir)
253-
254-
fileOrPath = ResolveAlias(fileName, pathName = pathName)
255-
256-
if(isEmpty(fileOrPath))
257-
// invalid shortcut, try next file
258-
continue
259-
endif
260-
261-
GetFileFolderInfo/P=$pathName/Q/Z fileOrPath
262-
ASSERT(!V_Flag, "Error in GetFileFolderInfo")
263255

264-
if(V_isFile)
265-
allFiles = AddListItem(S_path, allFiles, FILE_LIST_SEP, Inf)
266-
elseif(V_isFolder)
267-
foldersFromAlias = AddListItem(S_path, foldersFromAlias, FILE_LIST_SEP, Inf)
268-
else
269-
ASSERT(0, "Unexpected file type")
270-
endif
271-
endfor
256+
if(isWindows && cmpstr(extension, "????") && cmpstr(extension, ".lnk"))
257+
allFiles = AddPrefixToEachListItem(S_path, allFilesList, sep = FILE_LIST_SEP)
258+
else
259+
// try to resolve aliases/shortcuts when we can have them
260+
WAVE/T allFilesInDir = ListToTextWave(allFilesList, FILE_LIST_SEP)
261+
for(fileName : allFilesInDir)
262+
263+
GetFileFolderInfo/P=$pathName/Q/Z fileName
264+
265+
if(V_Flag != 0)
266+
// error querying file
267+
continue
268+
endif
269+
270+
if(V_IsAliasShortcut)
271+
fileOrPath = ResolveAlias(fileName, pathName = pathName)
272+
273+
if(IsEmpty(fileOrPath))
274+
// dead alias
275+
continue
276+
endif
277+
278+
// redo the check
279+
GetFileFolderInfo/P=$pathName/Q/Z fileOrPath
280+
281+
if(V_Flag != 0)
282+
// error querying file/folder pointed to by alias
283+
continue
284+
endif
285+
endif
286+
287+
if(V_isFile)
288+
allFiles = AddListItem(S_path, allFiles, FILE_LIST_SEP, Inf)
289+
elseif(V_isFolder)
290+
foldersFromAlias = AddListItem(S_path, foldersFromAlias, FILE_LIST_SEP, Inf)
291+
else
292+
ASSERT(0, "Unexpected file type")
293+
endif
294+
endfor
295+
endif
272296

273297
AssertOnAndClearRTError()
274298
folders = IndexedDir($pathName, -1, 1, FILE_LIST_SEP); err = GetRTError(1)

‎Packages/tests/Basic/UTF_Utils_File.ipf‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,14 +123,20 @@ End
123123

124124
static Function TestGetAllFilesRecursivelyFromPath()
125125

126-
string folder, symbPath, list
126+
string folder, symbPath, list, cmd
127127

128128
folder = GetFolder(FunctionPath("")) + "testFolder:"
129129

130130
symbPath = GetUniqueSymbolicPath()
131131
NewPath/Q/O/C/Z $symbPath, folder
132132
CHECK(!V_Flag)
133133

134+
// start with a fresh folder
135+
DeleteFolder/P=$symbPath/Z
136+
137+
NewPath/Q/O/C/Z $symbPath, folder
138+
CHECK(!V_Flag)
139+
134140
CreateFolderOnDisk(folder + "b:")
135141
CreateFolderOnDisk(folder + "c:")
136142

@@ -148,6 +154,34 @@ static Function TestGetAllFilesRecursivelyFromPath()
148154
result[] = RemovePrefix(result[p], start = folder)
149155
CHECK_EQUAL_TEXTWAVES(result, {"file.txt", "b:file1.txt", "c:file2.txt"})
150156

157+
list = GetAllFilesRecursivelyFromPath(symbPath)
158+
WAVE/T result = ListToTextWave(list, FILE_LIST_SEP)
159+
result[] = RemovePrefix(result[p], start = folder)
160+
// alias.txt.lnk points to file.txt
161+
CHECK_EQUAL_TEXTWAVES(result, {"file.txt", "file.txt", "b:file1.txt", "c:file2.txt"})
162+
163+
// shortcut to non-existing file (created above)
164+
CHECK(!V_flag)
165+
DeleteFile/P=$symbPath "file.txt"
166+
CHECK(!V_flag)
167+
168+
list = GetAllFilesRecursivelyFromPath(symbPath)
169+
WAVE/T result = ListToTextWave(list, FILE_LIST_SEP)
170+
result[] = RemovePrefix(result[p], start = folder)
171+
// file.txt is not included as alias.txt.lnk is invalid
172+
CHECK_EQUAL_TEXTWAVES(result, {"b:file1.txt", "c:file2.txt"})
173+
174+
// shortcut to non-existing folder
175+
CreateAliasShortcut/Z/P=$symbPath/D "b" as "someFolder"
176+
CHECK(!V_flag)
177+
DeleteFolder/P=$symbPath/Z "b"
178+
CHECK(!V_flag)
179+
180+
list = GetAllFilesRecursivelyFromPath(symbPath)
181+
WAVE/T result = ListToTextWave(list, FILE_LIST_SEP)
182+
result[] = RemovePrefix(result[p], start = folder)
183+
CHECK_EQUAL_TEXTWAVES(result, {"c:file2.txt"})
184+
151185
// no matches
152186
list = GetAllFilesRecursivelyFromPath(symbPath, extension = ".abc")
153187
CHECK_EMPTY_STR(list)

0 commit comments

Comments
 (0)