Improve reliability of source calendar retrieval by enabling retries for intermittent HTTP failures (#403)

Author: octogonz

Code.gs

@@ -145,6 +145,9 @@ var addedEvents = [];
 var modifiedEvents = [];
 var removedEvents = [];
 
+// Syncing logic can set this to true to cause the Google Apps Script "Executions" dashboard to report failure
+var reportOverallFailure = false;
+
 function startSync(){
   if (PropertiesService.getUserProperties().getProperty('LastRun') > 0 && (new Date().getTime() - PropertiesService.getUserProperties().getProperty('LastRun')) < 360000) {
     Logger.log("Another iteration is currently running! Exiting...");
@@ -249,4 +252,10 @@ function startSync(){
   }
   Logger.log("Sync finished!");
   PropertiesService.getUserProperties().setProperty('LastRun', 0);
+
+  if (reportOverallFailure) {
+    // Cause the Google Apps Script "Executions" dashboard to show a failure
+    // (the message text does not seem to be logged anywhere)
+    throw new Error('The sync operation produced errors. See log for details.');
+  }
 }

Helpers.gs

@@ -44,7 +44,7 @@ function formatDate(date) {
 
   const time = date.slice(11,16)
   const timeZone = date.slice(19)
-  
+
   return formattedDate + " at " + time + " (UTC" + (timeZone == "Z" ? "": timeZone) + ")"
 }
 
@@ -132,39 +132,45 @@ function fetchSourceCalendars(sourceCalendarURLs){
   for (var source of sourceCalendarURLs){
     var url = source[0].replace("webcal://", "https://");
     var colorId = source[1];
-    
-    callWithBackoff(function() {
-      var urlResponse = UrlFetchApp.fetch(url, { 'validateHttpsCertificates' : false, 'muteHttpExceptions' : true });
-      if (urlResponse.getResponseCode() == 200){
-        var icsContent = urlResponse.getContentText()
-        const icsRegex = RegExp("(BEGIN:VCALENDAR.*?END:VCALENDAR)", "s")
-        var urlContent = icsRegex.exec(icsContent);
-        if (urlContent == null){
-          // Microsoft Outlook has a bug that sometimes results in incorrectly formatted ics files. This tries to fix that problem.
-          // Add END:VEVENT for every BEGIN:VEVENT that's missing it
-          const veventRegex = /BEGIN:VEVENT(?:(?!END:VEVENT).)*?(?=.BEGIN|.END:VCALENDAR|$)/sg;
-          icsContent = icsContent.replace(veventRegex, (match) => match + "\nEND:VEVENT");
-
-          // Add END:VCALENDAR if missing
-          if (!icsContent.endsWith("END:VCALENDAR")){
-              icsContent += "\nEND:VCALENDAR";
-          }          
-          urlContent = icsRegex.exec(icsContent)
+
+    try {
+      callWithBackoff(function() {
+        var urlResponse = UrlFetchApp.fetch(url, { 'validateHttpsCertificates' : false, 'muteHttpExceptions' : true });
+        if (urlResponse.getResponseCode() == 200){
+          var icsContent = urlResponse.getContentText()
+          const icsRegex = RegExp("(BEGIN:VCALENDAR.*?END:VCALENDAR)", "s")
+          var urlContent = icsRegex.exec(icsContent);
           if (urlContent == null){
-            Logger.log("[ERROR] Incorrect ics/ical URL: " + url)
-            return
+            // Microsoft Outlook has a bug that sometimes results in incorrectly formatted ics files. This tries to fix that problem.
+            // Add END:VEVENT for every BEGIN:VEVENT that's missing it
+            const veventRegex = /BEGIN:VEVENT(?:(?!END:VEVENT).)*?(?=.BEGIN|.END:VCALENDAR|$)/sg;
+            icsContent = icsContent.replace(veventRegex, (match) => match + "\nEND:VEVENT");
+
+            // Add END:VCALENDAR if missing
+            if (!icsContent.endsWith("END:VCALENDAR")){
+                icsContent += "\nEND:VCALENDAR";
+            }
+            urlContent = icsRegex.exec(icsContent)
+            if (urlContent == null){
+              Logger.log("[ERROR] Incorrect ics/ical URL: " + url)
+              reportOverallFailure = true;
+              return
+            }
+            Logger.log("[WARNING] Microsoft is incorrectly formatting ics/ical at: " + url)
           }
-          Logger.log("[WARNING] Microsoft is incorrectly formatting ics/ical at: " + url)
+          result.push([urlContent[0], colorId]);
+          return;
         }
-        result.push([urlContent[0], colorId]);
-        return; 
-      }
-      else{ //Throw here to make callWithBackoff run again
-        throw "Error: Encountered HTTP error " + urlResponse.getResponseCode() + " when accessing " + url; 
-      }
-    }, defaultMaxRetries);
+        else{ //Throw here to make callWithBackoff run again
+          throw "Error: Encountered HTTP error " + urlResponse.getResponseCode() + " when accessing " + url;
+        }
+      }, defaultMaxRetries);
+    }
+    catch (e) {
+      reportOverallFailure = true;
+    }
   }
-  
+
   return result;
 }
 
@@ -242,7 +248,7 @@ function parseResponses(responses){
     });
   }
 
-  //No need to process calcelled events as they will be added to gcal's trash anyway
+  //No need to process cancelled events as they will be added to gcal's trash anyway
   result = result.filter(function(event){
     try{
       return (event.getFirstPropertyValue('status').toString().toLowerCase() != "cancelled");
@@ -436,7 +442,7 @@ function createEvent(event, calendarTz){
     if (organizerMail)
       newEvent.organizer.email = organizerMail.toString();
 
-    if (addOrganizerToTitle && organizerName){  
+    if (addOrganizerToTitle && organizerName){
         newEvent.summary = organizerName + ": " + newEvent.summary;
     }
   }
@@ -1090,7 +1096,14 @@ function sendSummary() {
 var backoffRecoverableErrors = [
   "service invoked too many times in a short time",
   "rate limit exceeded",
-  "internal error"];
+  "internal error",
+  "http error 403", // forbidden
+  "http error 408", // request timeout
+  "http error 423", // locked
+  "http error 500", // internal server error
+  "http error 503", // service unavailable
+  "http error 504"  // gateway timeout
+];
 function callWithBackoff(func, maxRetries) {
   var tries = 0;
   var result;
@@ -1102,10 +1115,7 @@ function callWithBackoff(func, maxRetries) {
     }
     catch(err){
       err = err.message  || err;
-      if ( err.includes("HTTP error") ) {
-        Logger.log(err);
-        return null;
-      } else if ( err.includes("is not a function")  || !backoffRecoverableErrors.some(function(e){
+      if ( err.includes("is not a function")  || !backoffRecoverableErrors.some(function(e){
               return err.toLowerCase().includes(e);
             }) ) {
         throw err;