From 6182afea7ffc4db2a7ef9712ff696e0da4b510c3 Mon Sep 17 00:00:00 2001 From: Christian Zuellig Date: Wed, 4 Dec 2024 08:57:34 +0100 Subject: [PATCH 1/6] DebugNative-PnPDirectSource --- src/lib/PnP.Framework/PnP.Framework.csproj | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lib/PnP.Framework/PnP.Framework.csproj b/src/lib/PnP.Framework/PnP.Framework.csproj index 49eebe7b9..e38993a6f 100644 --- a/src/lib/PnP.Framework/PnP.Framework.csproj +++ b/src/lib/PnP.Framework/PnP.Framework.csproj @@ -246,7 +246,6 @@ - @@ -286,4 +285,8 @@ + + + + From 29d2843a312316c299008912bc203a9a5126f3eb Mon Sep 17 00:00:00 2001 From: Christian Zuellig Date: Fri, 6 Dec 2024 14:08:55 +0100 Subject: [PATCH 2/6] create Folders by hierarchy level to batch requests for speed --- .../ObjectHandlers/ObjectListInstance.cs | 382 +++++++++++++++++- 1 file changed, 377 insertions(+), 5 deletions(-) diff --git a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectListInstance.cs b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectListInstance.cs index 75e3ce31d..2058d4a27 100644 --- a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectListInstance.cs +++ b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectListInstance.cs @@ -13,6 +13,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Linq.Expressions; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; @@ -357,11 +358,15 @@ private void ProcessFolders(Web web, TokenParser parser, PnPMonitoredScope scope list.SiteList.Update(); web.Context.ExecuteQueryRetry(); - var rootFolder = list.SiteList.RootFolder; - foreach (var folder in list.TemplateList.Folders) - { - CreateFolderInList(list, rootFolder, folder, parser, scope); - } + var existingFolderItems = LoadAllExistingFolderListItems(list.SiteList, scope); + var mappedFolders = MapExistingFolderToTemplateFolders(list,existingFolderItems, parser, scope); + CreateFolderInListV2(list, mappedFolders, parser, scope); + + //var rootFolder = list.SiteList.RootFolder; + //foreach (var folder in list.TemplateList.Folders) + //{ + // CreateFolderInList(list, rootFolder, folder, parser, scope); + //} // Restore the value of EnableFolderCreation to what it was before if the value is different if (list.SiteList.EnableFolderCreation != enableFolderCreationPreviousValue) @@ -373,6 +378,373 @@ private void ProcessFolders(Web web, TokenParser parser, PnPMonitoredScope scope } } + private void CreateFolderInListV2(ListInfo list, Dictionary mappedFolders, TokenParser parser, PnPMonitoredScope scope) + { + // Determine the folder name, parsing any token + list.SiteList.ParentWeb.EnsureProperties(w => w.ServerRelativeUrl); + list.SiteList.EnsureProperties(l => l.RootFolder); + + // Handle root folder property bag + if (list.TemplateList.PropertyBagEntries != null && list.TemplateList.PropertyBagEntries.Count > 0) + { + foreach (var p in list.TemplateList.PropertyBagEntries) + { + list.SiteList.RootFolder.Properties[parser.ParseString(p.Key)] = parser.ParseString(p.Value); + } + list.SiteList.RootFolder.Update(); + } + + var ParentWebServerRelativeUrlLength = list.SiteList.ParentWeb.ServerRelativeUrl.Length; + + //do we need to load list contenttypes + var listContentTypeOrdered = new List(); + if (mappedFolders.Values.Any(f=>!string.IsNullOrWhiteSpace(f.folderTemplate.ContentTypeID))) + { + list.SiteList.Context.Load(list.SiteList, p => p.ContentTypes.Include(c => c.StringId)); + list.SiteList.Context.ExecuteQueryRetry(); + listContentTypeOrdered = list.SiteList.ContentTypes.OrderBy(p => p.StringId.Length).ToList(); + } + + foreach (var folderLevel in mappedFolders.GroupBy(f => f.Value.level).OrderBy(g => g.Key)) + { + //same level but can have different parents + var loadedPathsInLevel = new List(); + #region ****** Create and Load or Load existing Folder + foreach (var folder in folderLevel) + { + loadedPathsInLevel.Add(folder.Key); + var pathParts = folder.Key.Split('/'); + var parentFolderPath = string.Join("/", pathParts.Take(pathParts.Length - 1)); + if (folder.Value.FolderListItem == null) + { + //needs to be created + if (pathParts.Length == 1) + { + folder.Value.SPFolder = list.SiteList.RootFolder.Folders.AddUsingPath(ResourcePath.FromDecodedUrl(pathParts.Last()), new FolderCollectionAddParameters() { Overwrite = true }); + list.SiteList.Context.Load(folder.Value.SPFolder, f => f.Name, f => f.UniqueId, f => f.Folders, f => f.ServerRelativeUrl, f => f.ListItemAllFields, f => f.Properties); + } + else + { + folder.Value.SPFolder = mappedFolders[parentFolderPath].SPFolder.Folders.AddUsingPath(ResourcePath.FromDecodedUrl(pathParts.Last()), new FolderCollectionAddParameters() { Overwrite = true }); + list.SiteList.Context.Load(folder.Value.SPFolder, f => f.Name, f => f.UniqueId, f => f.Folders, f => f.ServerRelativeUrl, f => f.ListItemAllFields, f => f.Properties); + } + } + else + { + //needs to be loaded + if (Guid.TryParse(mappedFolders[folder.Key].FolderListItem["UniqueId"].ToString(), out var uniqueId)) + { + mappedFolders[folder.Key].SPFolder = list.SiteList.ParentWeb.GetFolderById(uniqueId); + list.SiteList.Context.Load(mappedFolders[folder.Key].SPFolder, f => f.Name, f => f.UniqueId, f => f.Folders, f => f.ServerRelativeUrl, f => f.ListItemAllFields, f => f.Properties); + } + } + } + #endregion //****** Create and Load or Load existing Folder + if (list.SiteList.Context.HasPendingRequest) + { + list.SiteList.Context.ExecuteQuery(); + foreach (var path in loadedPathsInLevel) + { + var flatFolder = mappedFolders[path]; + parser.AddToken(new FileUniqueIdToken(list.SiteList.ParentWeb, flatFolder.SPFolder.ServerRelativeUrl.Substring(ParentWebServerRelativeUrlLength).TrimStart("/".ToCharArray()), mappedFolders[path].SPFolder.UniqueId)); + parser.AddToken(new FileUniqueIdEncodedToken(list.SiteList.ParentWeb, flatFolder.SPFolder.ServerRelativeUrl.Substring(ParentWebServerRelativeUrlLength).TrimStart("/".ToCharArray()), mappedFolders[path].SPFolder.UniqueId)); + + #region ****** Set ContentType + if (!string.IsNullOrWhiteSpace(flatFolder.folderTemplate.ContentTypeID)) + { + var ct = listContentTypeOrdered.FirstOrDefault(c => c.StringId.StartsWith(flatFolder.folderTemplate.ContentTypeID)); + var currentFolderItem = flatFolder.SPFolder.ListItemAllFields; + var needToUpdateCT = currentFolderItem.FieldExists("ContentTypeId") == false || (currentFolderItem.FieldExists("ContentTypeId") && !ct.StringId.Equals(currentFolderItem["ContentTypeId"].ToString())); + + if (needToUpdateCT) + { + currentFolderItem["ContentTypeId"] = ct.StringId; + + if (flatFolder.folderTemplate.ContentTypeID.StartsWith(BuiltInContentTypeId.DocumentSet, StringComparison.InvariantCultureIgnoreCase)) + { + currentFolderItem["HTML_x0020_File_x0020_Type"] = "Sharepoint.DocumentSet"; + flatFolder.SPFolder.Properties["docset_LastRefresh"] = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss"); + flatFolder.SPFolder.Properties["vti_contenttypeorder"] = string.Join(",", list.SiteList.ContentTypes.ToList().Where(c => c.StringId.StartsWith(BuiltInContentTypeId.Document + "00"))?.Select(c => c.StringId)); + } + currentFolderItem.UpdateOverwriteVersion(); + flatFolder.SPFolder.Update(); + list.SiteList.Context.Load(flatFolder.SPFolder.ListItemAllFields); //if we change ContentType we can have additional fields + + //Channel Folder special treatment only on first Folder Level + if (folderLevel.Key == 0) + { + try + { + list.SiteList.Context.ExecuteQueryRetry(); + } + catch (ServerException srex) + { + //Handle Error To update this folder, go to the channel in Microsoft Teams + if (srex.ServerErrorCode == -2130575223) + { + scope.LogWarning($"ContentType on folder '{path}' can not be changed '{srex.Message}'"); + WriteMessage($"ContentType on folder '{path}' can not be changed '{srex.Message}'", ProvisioningMessageType.Warning); + } + else + throw; + } + } + } + } + #endregion //****** Set ContentType + } + if (folderLevel.Key > 0 && list.SiteList.Context.HasPendingRequest) + { + list.SiteList.Context.ExecuteQueryRetry(); + } + + #region ****** Set Property Fields of Folder + foreach (var path in loadedPathsInLevel) + { + var flatFolder = mappedFolders[path]; + //Set Property Fields of Folder in order to handle for example OneNote Folders + if (flatFolder.folderTemplate.Properties != null && flatFolder.folderTemplate.Properties.Any(p => !p.Key.Equals("ContentTypeId"))) + { + var currentFolderItem = flatFolder.SPFolder.ListItemAllFields; + foreach (var p in flatFolder.folderTemplate.Properties.Where(p => !p.Key.Equals("ContentTypeId") && !p.Key.Equals("_ModerationStatus"))) + { + currentFolderItem[parser.ParseString(p.Key)] = parser.ParseString(p.Value); + } + currentFolderItem.UpdateOverwriteVersion(); + flatFolder.SPFolder.Update(); + + //Channel Folder special treatment only on first Folder Level + if (folderLevel.Key == 0) + { + + try + { + list.SiteList.Context.ExecuteQueryRetry(); + } + catch (ServerException srex) + { + //Handle Error To update this folder, go to the channel in Microsoft Teams + if (srex.ServerErrorCode == -2130575223) + { + scope.LogWarning($"Properties on folder '{path}' can not be changed '{srex.Message}'"); + WriteMessage($"Properties on folder '{path}' can not be changed '{srex.Message}'", ProvisioningMessageType.Warning); + } + else + throw; + } + } + } + } + if (folderLevel.Key > 0 && list.SiteList.Context.HasPendingRequest) + { + list.SiteList.Context.ExecuteQueryRetry(); + } + #endregion //****** Set Property Fields of Folder + + #region ****** Set Folder Security + foreach (var path in loadedPathsInLevel) + { + var flatFolder = mappedFolders[path]; + + // Handle current folder security + if (flatFolder.folderTemplate.Security != null && flatFolder.folderTemplate.Security.RoleAssignments.Count != 0) + { + var currentFolderItem = flatFolder.SPFolder.ListItemAllFields; + currentFolderItem.SetSecurity(parser, flatFolder.folderTemplate.Security, WriteMessage); + } + } + #endregion ****** Set Folder Security + + #region ****** Set Folder PropertyBag + foreach (var path in loadedPathsInLevel) + { + var flatFolder = mappedFolders[path]; + // Handle current folder property bags + if (flatFolder.folderTemplate.PropertyBagEntries != null && flatFolder.folderTemplate.PropertyBagEntries.Count > 0) + { + foreach (var p in flatFolder.folderTemplate.PropertyBagEntries) + { + flatFolder.SPFolder.Properties[parser.ParseString(p.Key)] = parser.ParseString(p.Value); + } + flatFolder.SPFolder.Update(); + list.SiteList.Context.Load(flatFolder.SPFolder); + + //Channel Folder special treatment only on first Folder Level + if (folderLevel.Key == 0) + { + try + { + list.SiteList.Context.ExecuteQueryRetry(); + } + catch (ServerException srex) + { + //Handle Error To update this folder, go to the channel in Microsoft Teams + if (srex.ServerErrorCode == -2130575223) + { + scope.LogWarning($"PropertyBagEntries on folder '{path}' can not be changed '{srex.Message}'"); + WriteMessage($"PropertyBagEntries on folder '{path}' can not be changed '{srex.Message}'", ProvisioningMessageType.Warning); + } + else + throw; + } + } + } + } + if (folderLevel.Key > 0 && list.SiteList.Context.HasPendingRequest) + { + list.SiteList.Context.ExecuteQueryRetry(); + } + #endregion ****** Set Folder PropertyBag + + #region ****** Set Moderation status of Folder + foreach (var path in loadedPathsInLevel) + { + var flatFolder = mappedFolders[path]; + //Doing it in a different request, because SharePoint doesn't allow to update properties at the same time with other properties + if (list.SiteList.EnableModeration && flatFolder.folderTemplate.Properties != null && flatFolder.folderTemplate.Properties.Any(p => p.Key.Equals("_ModerationStatus"))) + { + var currentFolderItem = flatFolder.SPFolder.ListItemAllFields; + var propertyValue = flatFolder.folderTemplate.Properties["_ModerationStatus"]; + currentFolderItem["_ModerationStatus"] = parser.ParseString(propertyValue); + + currentFolderItem.UpdateOverwriteVersion(); + flatFolder.SPFolder.Update(); + + //Channel Folder special treatment only on first Folder Level + if (folderLevel.Key == 0) + { + try + { + list.SiteList.Context.ExecuteQueryRetry(); + } + catch (ServerException srex) + { + //Handle Error To update this folder, go to the channel in Microsoft Teams + if (srex.ServerErrorCode == -2130575223) + { + scope.LogWarning($"Moderation status on folder '{path}' can not be changed '{srex.Message}'"); + WriteMessage($"Moderation status on folder '{path}' can not be changed '{srex.Message}'", ProvisioningMessageType.Warning); + } + else + throw; + } + } + } + } + if (folderLevel.Key > 0 && list.SiteList.Context.HasPendingRequest) + { + list.SiteList.Context.ExecuteQueryRetry(); + } + #endregion //****** Set Moderation status of Folder + } + } + } + + private Dictionary MapExistingFolderToTemplateFolders(ListInfo list, System.Collections.Generic.List existingFolderListItems, TokenParser parser, PnPMonitoredScope scope) + { + list.SiteList.EnsureProperties(l => l.RootFolder.ServerRelativeUrl); + var flatFolderList = FlatFolderList("", 0, list.TemplateList.Folders, parser, scope); + var foldersDestination = new Dictionary(); + foreach (var folder in flatFolderList) + { + foldersDestination.Add(folder.folderPath, folder); + } + var spListRootFolderPathLength = list.SiteList.RootFolder.ServerRelativeUrl.Length + 1; + foreach (var folder in existingFolderListItems) + { + var folderRelativePath = folder["FileRef"].ToString().Substring(spListRootFolderPathLength).Trim('/'); + if (foldersDestination.ContainsKey(folderRelativePath)) + { + foldersDestination[folderRelativePath].FolderListItem = folder; + } + } + + return foldersDestination; + } + + private System.Collections.Generic.List FlatFolderList(string parentfolderName, int level, Model.FolderCollection folders, TokenParser parser, PnPMonitoredScope scope) + { + var foldersSource = new System.Collections.Generic.List(); + foreach (var folder in folders) + { + var folderPath = ""; + if (string.IsNullOrWhiteSpace(parentfolderName)) + folderPath = parser.ParseString(folder.Name); + else + folderPath = $"{parentfolderName}/{parser.ParseString(folder.Name)}"; + + foldersSource.Add(new FlatFolder() { folderPath = folderPath, level = level, folderTemplate = folder }); + if (folder.Folders.Any()) + foldersSource.AddRange(FlatFolderList(folderPath, level + 1, folder.Folders, parser, scope)); + } + + return foldersSource; + } + + + private class FlatFolder + { + public string folderPath { get; set; } + + public int level { get; set; } + + public Model.Folder folderTemplate { get; set; } + + public ListItem FolderListItem { get; set; } + + public Folder SPFolder { get; set; } + } + + private System.Collections.Generic.List LoadAllExistingFolderListItems(List spList, PnPMonitoredScope scope) + { + System.Collections.Generic.List listItemColl = new System.Collections.Generic.List(); + + var camlQuery = new CamlQuery() + { + ViewXml = @" + + + + + + + + + + + + + 1 + + + + + 200 + " + }; + + do + { + var listItems = spList.GetItems(camlQuery); + spList.Context.Load(listItems); + + try + { + spList.Context.ExecuteQuery(); + listItemColl.AddRange(listItems); + camlQuery.ListItemCollectionPosition = listItems.ListItemCollectionPosition; + } + catch (Exception ex) + { + throw new InvalidOperationException("LoadAllExistingFolderListItems: Error on paging", ex); + } + } + while (camlQuery.ListItemCollectionPosition != null); + + return listItemColl; + } + + private void ProcessViews(Web web, TokenParser parser, PnPMonitoredScope scope, ListInfo listInfo) { var list = listInfo.TemplateList; From bcb065f1e04ca10a9616232c9961166b31bedb68 Mon Sep 17 00:00:00 2001 From: Christian Zuellig Date: Tue, 10 Dec 2024 11:15:58 +0100 Subject: [PATCH 3/6] remove direct link to pnp.core.sdk --- src/lib/PnP.Framework/PnP.Framework.csproj | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/lib/PnP.Framework/PnP.Framework.csproj b/src/lib/PnP.Framework/PnP.Framework.csproj index e38993a6f..49eebe7b9 100644 --- a/src/lib/PnP.Framework/PnP.Framework.csproj +++ b/src/lib/PnP.Framework/PnP.Framework.csproj @@ -246,6 +246,7 @@ + @@ -285,8 +286,4 @@ - - - - From 8870c0271c871e4d4e2aed8713012c60e30eab54 Mon Sep 17 00:00:00 2001 From: Christian Zuellig Date: Fri, 20 Dec 2024 17:55:33 +0100 Subject: [PATCH 4/6] improve failure log ignore docid_msft_ for RootFolder PropertyBag as it will fail to set these values --- .../ObjectHandlers/ObjectListInstance.cs | 73 +++++++++++++++++-- 1 file changed, 65 insertions(+), 8 deletions(-) diff --git a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectListInstance.cs b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectListInstance.cs index 2058d4a27..d7d84bf23 100644 --- a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectListInstance.cs +++ b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectListInstance.cs @@ -387,21 +387,38 @@ private void CreateFolderInListV2(ListInfo list, Dictionary // Handle root folder property bag if (list.TemplateList.PropertyBagEntries != null && list.TemplateList.PropertyBagEntries.Count > 0) { - foreach (var p in list.TemplateList.PropertyBagEntries) + foreach (var p in list.TemplateList.PropertyBagEntries.Where(pp => !pp.Key.StartsWith("docid_msft"))) { list.SiteList.RootFolder.Properties[parser.ParseString(p.Key)] = parser.ParseString(p.Value); } list.SiteList.RootFolder.Update(); + + try + { + list.SiteList.Context.ExecuteQueryRetry(); + } + catch (ServerException srex) + { + scope.LogWarning($"library '{list.SiteList.RootFolder.Name}' root folder properties can not be changed '{srex.Message}'"); + } } var ParentWebServerRelativeUrlLength = list.SiteList.ParentWeb.ServerRelativeUrl.Length; //do we need to load list contenttypes var listContentTypeOrdered = new List(); - if (mappedFolders.Values.Any(f=>!string.IsNullOrWhiteSpace(f.folderTemplate.ContentTypeID))) + if (mappedFolders.Values.Any(f => !string.IsNullOrWhiteSpace(f.folderTemplate.ContentTypeID))) { list.SiteList.Context.Load(list.SiteList, p => p.ContentTypes.Include(c => c.StringId)); - list.SiteList.Context.ExecuteQueryRetry(); + try + { + list.SiteList.Context.ExecuteQueryRetry(); + } + catch (ServerException srex) + { + scope.LogWarning($"library '{list.SiteList.RootFolder.Name}' failed to load contenttypes '{srex.Message}'"); + throw; + } listContentTypeOrdered = list.SiteList.ContentTypes.OrderBy(p => p.StringId.Length).ToList(); } @@ -442,7 +459,15 @@ private void CreateFolderInListV2(ListInfo list, Dictionary #endregion //****** Create and Load or Load existing Folder if (list.SiteList.Context.HasPendingRequest) { - list.SiteList.Context.ExecuteQuery(); + try + { + list.SiteList.Context.ExecuteQuery(); + } + catch (ServerException srex) + { + scope.LogWarning($"library '{list.SiteList.RootFolder.Name}' failed to load folders on level '{folderLevel.Key}' with error '{srex.Message}'"); + throw; + } foreach (var path in loadedPathsInLevel) { var flatFolder = mappedFolders[path]; @@ -495,7 +520,15 @@ private void CreateFolderInListV2(ListInfo list, Dictionary } if (folderLevel.Key > 0 && list.SiteList.Context.HasPendingRequest) { - list.SiteList.Context.ExecuteQueryRetry(); + try + { + list.SiteList.Context.ExecuteQuery(); + } + catch (ServerException srex) + { + scope.LogWarning($"library '{list.SiteList.RootFolder.Name}' failed to set contenttype on folder level '{folderLevel.Key}' with error '{srex.Message}'"); + throw; + } } #region ****** Set Property Fields of Folder @@ -537,7 +570,15 @@ private void CreateFolderInListV2(ListInfo list, Dictionary } if (folderLevel.Key > 0 && list.SiteList.Context.HasPendingRequest) { - list.SiteList.Context.ExecuteQueryRetry(); + try + { + list.SiteList.Context.ExecuteQuery(); + } + catch (ServerException srex) + { + scope.LogWarning($"library '{list.SiteList.RootFolder.Name}' failed to set properties on folder level '{folderLevel.Key}' with error '{srex.Message}'"); + throw; + } } #endregion //****** Set Property Fields of Folder @@ -592,7 +633,15 @@ private void CreateFolderInListV2(ListInfo list, Dictionary } if (folderLevel.Key > 0 && list.SiteList.Context.HasPendingRequest) { - list.SiteList.Context.ExecuteQueryRetry(); + try + { + list.SiteList.Context.ExecuteQuery(); + } + catch (ServerException srex) + { + scope.LogWarning($"library '{list.SiteList.RootFolder.Name}' failed to set propertybag on folder level '{folderLevel.Key}' with error '{srex.Message}'"); + throw; + } } #endregion ****** Set Folder PropertyBag @@ -633,7 +682,15 @@ private void CreateFolderInListV2(ListInfo list, Dictionary } if (folderLevel.Key > 0 && list.SiteList.Context.HasPendingRequest) { - list.SiteList.Context.ExecuteQueryRetry(); + try + { + list.SiteList.Context.ExecuteQuery(); + } + catch (ServerException srex) + { + scope.LogWarning($"library '{list.SiteList.RootFolder.Name}' failed to set _ModerationStatus on folder level '{folderLevel.Key}' with error '{srex.Message}'"); + throw; + } } #endregion //****** Set Moderation status of Folder } From 6d6c12d63fb389a768748122c73ac5c99fc9b8a4 Mon Sep 17 00:00:00 2001 From: Christian Zuellig Date: Wed, 8 Jan 2025 11:24:24 +0100 Subject: [PATCH 5/6] CreateFolderInListV2 currently does not support Generic-Lists --- .../ObjectHandlers/ObjectListInstance.cs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectListInstance.cs b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectListInstance.cs index d7d84bf23..800966c62 100644 --- a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectListInstance.cs +++ b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectListInstance.cs @@ -360,14 +360,19 @@ private void ProcessFolders(Web web, TokenParser parser, PnPMonitoredScope scope var existingFolderItems = LoadAllExistingFolderListItems(list.SiteList, scope); var mappedFolders = MapExistingFolderToTemplateFolders(list,existingFolderItems, parser, scope); - CreateFolderInListV2(list, mappedFolders, parser, scope); - - //var rootFolder = list.SiteList.RootFolder; - //foreach (var folder in list.TemplateList.Folders) - //{ - // CreateFolderInList(list, rootFolder, folder, parser, scope); - //} + if (list.SiteList.BaseType == BaseType.DocumentLibrary) + { + CreateFolderInListV2(list, mappedFolders, parser, scope); + } + else + { + var rootFolder = list.SiteList.RootFolder; + foreach (var folder in list.TemplateList.Folders) + { + CreateFolderInList(list, rootFolder, folder, parser, scope); + } + } // Restore the value of EnableFolderCreation to what it was before if the value is different if (list.SiteList.EnableFolderCreation != enableFolderCreationPreviousValue) { From 9163fbb12350321ad0198d51b563f96428f67ae9 Mon Sep 17 00:00:00 2001 From: christian zuellig Date: Tue, 24 Jun 2025 22:37:19 +0200 Subject: [PATCH 6/6] fix Load Folders when library has more than 5000 items --- .../ObjectHandlers/ObjectListInstance.cs | 81 +++++++++++++------ 1 file changed, 57 insertions(+), 24 deletions(-) diff --git a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectListInstance.cs b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectListInstance.cs index 800966c62..f9f466f36 100644 --- a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectListInstance.cs +++ b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectListInstance.cs @@ -13,7 +13,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Linq.Expressions; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; @@ -761,11 +760,31 @@ private System.Collections.Generic.List LoadAllExistingFolderListItems { System.Collections.Generic.List listItemColl = new System.Collections.Generic.List(); - var camlQuery = new CamlQuery() + var camlTopQuery = new CamlQuery() { ViewXml = @" - + + + + + + 1 + " + }; + var maxIdListItem = spList.GetItems(camlTopQuery); + spList.Context.Load(maxIdListItem); + spList.Context.ExecuteQuery(); + var maxIdVal = (int?)maxIdListItem?.FirstOrDefault()?["ID"]; + + if (maxIdVal == null) + { + return listItemColl; + } + //FSObjType is not indexed and will throw error if used in a query with more than 5000 items. As Workaround we limit the filtered items by using the Indexed ID field + var queryXML = @" + + @@ -774,39 +793,53 @@ private System.Collections.Generic.List LoadAllExistingFolderListItems - - - 1 - + + + {0} + {1} + + + + 1 + + - + - 200 - " - }; + 1000 + "; + + var startId = 0; do { - var listItems = spList.GetItems(camlQuery); - spList.Context.Load(listItems); - - try + var camlQueryFolder = new CamlQuery { - spList.Context.ExecuteQuery(); - listItemColl.AddRange(listItems); - camlQuery.ListItemCollectionPosition = listItems.ListItemCollectionPosition; - } - catch (Exception ex) + ViewXml = string.Format(queryXML, startId, startId + 5000), + }; + do { - throw new InvalidOperationException("LoadAllExistingFolderListItems: Error on paging", ex); + var listItems = spList.GetItems(camlQueryFolder); + spList.Context.Load(listItems); + + try + { + spList.Context.ExecuteQuery(); + listItemColl.AddRange(listItems); + camlQueryFolder.ListItemCollectionPosition = listItems.ListItemCollectionPosition; + } + catch (Exception ex) + { + throw new InvalidOperationException("LoadAllExistingFolderListItems: Error on paging", ex); + } } - } - while (camlQuery.ListItemCollectionPosition != null); + while (camlQueryFolder.ListItemCollectionPosition != null); + startId += 4999; //because of and we need to increase the startId by 4999, so in next query we will include ID 5000 in + } while (maxIdVal >= startId); return listItemColl; } - private void ProcessViews(Web web, TokenParser parser, PnPMonitoredScope scope, ListInfo listInfo) { var list = listInfo.TemplateList;