Merge pull request #12 from IBM/saikumar1607-202109270454

Optional support for Persistent caching

Author: ibm-cloud-appconfiguration

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "^.secrets.baseline$|go.sum|examples/SampleApp/go.sum|vendor",
     "lines": null
   },
-  "generated_at": "2021-09-09T11:47:12Z",
+  "generated_at": "2021-09-27T04:56:24Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -70,7 +70,7 @@
         "hashed_secret": "d4c3d66fd0c38547a3c7a4c6bdc29c36911bc030",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 74,
+        "line_number": 75,
         "type": "Secret Keyword",
         "verified_result": null
       }
@@ -86,7 +86,7 @@
       }
     ]
   },
-  "version": "0.13.1+ibm.45.dss",
+  "version": "0.13.1+ibm.46.dss",
   "word_list": {
     "file": null,
     "hash": null

README.md

@@ -25,7 +25,7 @@ properties for distributed applications centrally.
 
 ## Installation
 
-The current version of this SDK: 0.2.0
+The current version of this SDK: 0.2.1
 
 There are a few different ways to download and install the IBM App Configuration Go SDK project for use by your Go
 application:
@@ -74,24 +74,46 @@ appConfiguration.SetContext(collectionId, environmentId)
 * collectionId: Id of the collection created in App Configuration service instance under the **Collections** section.
 * environmentId: Id of the environment created in App Configuration service instance under the **Environments** section.
 
-> Here, by default live update from the server is enabled. To turn off this mode see the [below section](#work-offline-with-local-configuration-file-optional)
+### (Optional)
 
-### Work offline with local configuration file (Optional)
+In order for your application and SDK to continue its operations even during the unlikely scenario of App Configuration
+service across your application restarts, you can configure the SDK to work using a persistent cache. The SDK uses the
+persistent cache to store the App Configuration data that will be available across your application restarts.
 
-You can also work offline with local configuration file and perform feature and property related operations.
+```go
+// 1. default (without persistent cache)
+appConfiguration.SetContext(collectionId, environmentId)
+
+// 2. with persistent cache
+appConfiguration.SetContext(collectionId, environmentId, AppConfiguration.ContextOptions{
+    PersistentCacheDirectory: "/var/lib/docker/volumes/",
+})
+```
+
+* PersistentCacheDirectory: Absolute path to a directory which has read & write permission for the user. The SDK will
+  create a file - `AppConfiguration.json` in the specified directory, and it will be used as the persistent cache to
+  store the App Configuration service information.
+
+When persistent cache is enabled, the SDK will keep the last known good configuration at the persistent cache. In the
+case of App Configuration server being unreachable, the latest configurations at the persistent cache is loaded to the
+application to continue working.
+
+### (Optional)
 
-After [`appConfiguration.Init("region", "guid", "apikey")`](#using-the-sdk), follow the below steps
+The SDK is also designed to serve configurations, perform feature flag & property evaluations without being connected to
+App Configuration service.
 
 ```go
-appConfiguration.SetContext("airlines-webapp", "dev", AppConfiguration.ContextOptions{
-  ConfigurationFile:       "saflights/flights.json",
-  LiveConfigUpdateEnabled: false
+appConfiguration.SetContext(collectionId, environmentId, AppConfiguration.ContextOptions{
+    BootstrapFile: "saflights/flights.json",
+    LiveConfigUpdateEnabled: false,
 })
 ```
 
-- ConfigurationFile: Path to the JSON file, which contains configuration details.
-- LiveConfigUpdateEnabled: Set this value to `false` if the new configuration values shouldn't be fetched from the
-  server. Make sure to provide a proper JSON file in the path. By default, this value is enabled.
+* BootstrapFile: Absolute path of the JSON file, which contains configuration details. Make sure to provide a proper
+  JSON file. You can generate this file using `ibmcloud ac config` command of the IBM Cloud App Configuration CLI.
+* LiveConfigUpdateEnabled: Live configuration update from the server. Set this value to `false` if the new configuration
+  values shouldn't be fetched from the server. By default, this value is enabled.
 
 ## Get single feature
 

examples/go.mod

@@ -3,6 +3,6 @@ module examples
 go 1.16
 
 require (
-	github.com/IBM/appconfiguration-go-sdk v0.2.0
+	github.com/IBM/appconfiguration-go-sdk v0.2.1
 	github.com/gorilla/mux v1.7.2
 )

lib/AppConfiguration.go

@@ -32,10 +32,12 @@ type AppConfiguration struct {
 	configurationHandlerInstance *ConfigurationHandler
 }
 
-// ContextOptions : Struct having configFilePath and liveUpdateFlag.
+// ContextOptions : Struct having PersistentCacheDirectory path, BootstrapFile (ConfigurationFile) path and LiveConfigUpdateEnabled flag.
 type ContextOptions struct {
-	ConfigurationFile       string
-	LiveConfigUpdateEnabled bool
+	PersistentCacheDirectory string
+	BootstrapFile            string
+	ConfigurationFile        string
+	LiveConfigUpdateEnabled  bool
 }
 
 var appConfigurationInstance *AppConfiguration
@@ -103,13 +105,20 @@ func (ac *AppConfiguration) SetContext(collectionID string, environmentID string
 	}
 	switch len(options) {
 	case 0:
-		ac.configurationHandlerInstance.SetContext(collectionID, environmentID, "", true)
+		ac.configurationHandlerInstance.SetContext(collectionID, environmentID, ContextOptions{
+			LiveConfigUpdateEnabled: true,
+		})
 	case 1:
-		if !options[0].LiveConfigUpdateEnabled && len(options[0].ConfigurationFile) == 0 {
-			log.Error(messages.ConfigurationFileNotFoundError)
+		var temp = options[0]
+		if len(temp.ConfigurationFile) > 0 && len(temp.BootstrapFile) == 0 {
+			temp.BootstrapFile = temp.ConfigurationFile
+			log.Info(messages.ContextOptionsParameterDeprecation)
+		}
+		if !temp.LiveConfigUpdateEnabled && len(temp.BootstrapFile) == 0 {
+			log.Error(messages.BootstrapFileNotFoundError)
 			return
 		}
-		ac.configurationHandlerInstance.SetContext(collectionID, environmentID, options[0].ConfigurationFile, options[0].LiveConfigUpdateEnabled)
+		ac.configurationHandlerInstance.SetContext(collectionID, environmentID, temp)
 	default:
 		log.Error(messages.IncorrectUsageOfContextOptions)
 		return

lib/ConfigurationHandler.go

@@ -17,21 +17,20 @@
 package lib
 
 import (
+	"bytes"
 	"encoding/json"
 	"errors"
-	"fmt"
-	"net/http"
-	"sync"
-	"time"
-
 	"github.com/IBM/appconfiguration-go-sdk/lib/internal/constants"
 	"github.com/IBM/appconfiguration-go-sdk/lib/internal/messages"
 	"github.com/IBM/appconfiguration-go-sdk/lib/internal/models"
 	"github.com/IBM/appconfiguration-go-sdk/lib/internal/utils"
-	"github.com/IBM/go-sdk-core/v5/core"
-
 	"github.com/IBM/appconfiguration-go-sdk/lib/internal/utils/log"
+	"github.com/IBM/go-sdk-core/v5/core"
 	"github.com/gorilla/websocket"
+	"net/http"
+	"path"
+	"sync"
+	"time"
 )
 
 type configurationUpdateListenerFunc func()
@@ -48,8 +47,10 @@ type ConfigurationHandler struct {
 	appConfig                   *AppConfiguration
 	cache                       *models.Cache
 	configurationUpdateListener configurationUpdateListenerFunc
-	configurationFile           string
+	persistentCacheDirectory    string
+	bootstrapFile               string
 	liveConfigUpdateEnabled     bool
+	persistentData              []byte
 	retryCount                  int
 	retryInterval               int64
 	socketConnection            *websocket.Conn
@@ -75,14 +76,15 @@ func (ch *ConfigurationHandler) Init(region, guid, apikey string) {
 }
 
 // SetContext : Set Context
-func (ch *ConfigurationHandler) SetContext(collectionID, environmentID, configurationFile string, liveConfigUpdateEnabled bool) {
+func (ch *ConfigurationHandler) SetContext(collectionID, environmentID string, options ContextOptions) {
 	ch.collectionID = collectionID
 	ch.environmentID = environmentID
 	ch.urlBuilder = utils.GetInstance()
 	ch.urlBuilder.Init(ch.collectionID, ch.environmentID, ch.region, ch.guid, ch.apikey, OverrideServerHost)
 	utils.GetMeteringInstance().Init(ch.guid, environmentID, collectionID)
-	ch.configurationFile = configurationFile
-	ch.liveConfigUpdateEnabled = liveConfigUpdateEnabled
+	ch.persistentCacheDirectory = options.PersistentCacheDirectory
+	ch.bootstrapFile = options.BootstrapFile
+	ch.liveConfigUpdateEnabled = options.LiveConfigUpdateEnabled
 	ch.isInitialized = true
 	ch.retryCount = 3
 	ch.retryInterval = 600
@@ -91,14 +93,31 @@ func (ch *ConfigurationHandler) loadData() {
 	if !ch.isInitialized {
 		log.Error(messages.ConfigurationHandlerInitError)
 	}
-	log.Debug(messages.LoadingData)
-	log.Debug(messages.CheckConfigurationFileProvided)
-	if len(ch.configurationFile) > 0 {
-		log.Debug(messages.ConfigurationFileProvided)
-		ch.getFileData(ch.configurationFile)
+	if len(ch.persistentCacheDirectory) > 0 {
+		ch.persistentData = utils.ReadFiles(path.Join(ch.persistentCacheDirectory, constants.ConfigurationFile))
+		if !bytes.Equal(ch.persistentData, []byte(`{}`)) {
+			// no updating the listener here. Only updating cache is enough
+			ch.saveInCache(ch.persistentData)
+		}
+	}
+	if len(ch.bootstrapFile) > 0 {
+		log.Debug(messages.BootstrapFileProvided)
+		if len(ch.persistentCacheDirectory) > 0 {
+			if bytes.Equal(ch.persistentData, []byte(`{}`)) {
+				bootstrapFileData := utils.ReadFiles(ch.bootstrapFile)
+				go utils.StoreFiles(string(bootstrapFileData), ch.persistentCacheDirectory)
+				ch.updateCacheAndListener(bootstrapFileData)
+			} else {
+				// update the only listener here. Because, cache is already updated above (line 100)
+				if ch.configurationUpdateListener != nil {
+					ch.configurationUpdateListener()
+				}
+			}
+		} else {
+			bootstrapFileData := utils.ReadFiles(ch.bootstrapFile)
+			ch.updateCacheAndListener(bootstrapFileData)
+		}
 	}
-	log.Debug(messages.LiveUpdateCheck)
-	log.Debug(fmt.Sprint(ch.liveConfigUpdateEnabled))
 	if ch.liveConfigUpdateEnabled {
 		ch.FetchConfigurationData()
 	}
@@ -112,11 +131,11 @@ func (ch *ConfigurationHandler) FetchConfigurationData() {
 		go ch.startWebSocket()
 	}
 }
-func (ch *ConfigurationHandler) saveInCache(data string) {
+func (ch *ConfigurationHandler) saveInCache(data []byte) {
 	ch.mu.Lock()
 	defer ch.mu.Unlock()
 	configResponse := models.ConfigResponse{}
-	err := json.Unmarshal([]byte(data), &configResponse)
+	err := json.Unmarshal(data, &configResponse)
 	if err != nil {
 		log.Error(messages.UnmarshalJSONErr, err)
 		return
@@ -140,6 +159,12 @@ func (ch *ConfigurationHandler) saveInCache(data string) {
 	models.SetCache(featureMap, propertyMap, segmentMap)
 	ch.cache = models.GetCacheInstance()
 }
+func (ch *ConfigurationHandler) updateCacheAndListener(data []byte) {
+	ch.saveInCache(data)
+	if ch.configurationUpdateListener != nil {
+		ch.configurationUpdateListener()
+	}
+}
 func (ch *ConfigurationHandler) fetchFromAPI() {
 	if ch.isInitialized {
 		ch.retryCount--
@@ -159,14 +184,24 @@ func (ch *ConfigurationHandler) fetchFromAPI() {
 		if response != nil && response.StatusCode >= 200 && response.StatusCode <= 299 {
 			if ch.liveConfigUpdateEnabled {
 				jsonData, _ := json.Marshal(response.Result)
-				ch.saveInCache(string(jsonData))
-				if ch.configurationUpdateListener != nil {
-					ch.configurationUpdateListener()
+				// asynchronously write the response to persistent volume, if enabled
+				if len(ch.persistentCacheDirectory) > 0 {
+					go utils.StoreFiles(string(jsonData), ch.persistentCacheDirectory)
 				}
+				// load the configurations in the response to cache maps
+				ch.updateCacheAndListener(jsonData)
 			}
 		} else {
 			if ch.retryCount > 0 {
-				log.Error(messages.ConfigAPIError)
+				if response != nil {
+					if response.Result != nil {
+						log.Error(response.Result)
+					} else {
+						log.Error(string(response.RawResult))
+					}
+				} else {
+					log.Error(messages.ConfigAPIError)
+				}
 				ch.fetchFromAPI()
 			} else {
 				ch.retryCount = 3
@@ -219,18 +254,6 @@ func (ch *ConfigurationHandler) startWebSocket() {
 			}
 		}
 	}()
-
-}
-
-func (ch *ConfigurationHandler) getFeatureActions(featureID string) (models.Feature, error) {
-	if ch.cache != nil && len(ch.cache.FeatureMap) > 0 {
-		if val, ok := ch.cache.FeatureMap[featureID]; ok {
-			return val, nil
-		}
-		log.Error(messages.InvalidFeatureID, featureID)
-		return models.Feature{}, errors.New(messages.ErrorInvalidFeatureID + featureID)
-	}
-	return models.Feature{}, errors.New(messages.ErrorInvalidFeatureID + featureID)
 }
 func (ch *ConfigurationHandler) getFeatures() (map[string]models.Feature, error) {
 	if ch.cache == nil {
@@ -243,21 +266,10 @@ func (ch *ConfigurationHandler) getFeature(featureID string) (models.Feature, er
 		if val, ok := ch.cache.FeatureMap[featureID]; ok {
 			return val, nil
 		}
-		return ch.getFeatureActions(featureID)
 	}
-	return ch.getFeatureActions(featureID)
-
-}
+	log.Error(messages.InvalidFeatureID, featureID)
+	return models.Feature{}, errors.New(messages.ErrorInvalidFeatureID + featureID)
 
-func (ch *ConfigurationHandler) getPropertyActions(propertyID string) (models.Property, error) {
-	if ch.cache != nil && len(ch.cache.PropertyMap) > 0 {
-		if val, ok := ch.cache.PropertyMap[propertyID]; ok {
-			return val, nil
-		}
-		log.Error(messages.InvalidPropertyID, propertyID)
-		return models.Property{}, errors.New(messages.ErrorInvalidPropertyID + propertyID)
-	}
-	return models.Property{}, errors.New(messages.ErrorInvalidPropertyID + propertyID)
 }
 func (ch *ConfigurationHandler) getProperties() (map[string]models.Property, error) {
 	if ch.cache == nil {
@@ -270,9 +282,9 @@ func (ch *ConfigurationHandler) getProperty(propertyID string) (models.Property,
 		if val, ok := ch.cache.PropertyMap[propertyID]; ok {
 			return val, nil
 		}
-		return ch.getPropertyActions(propertyID)
 	}
-	return ch.getPropertyActions(propertyID)
+	log.Error(messages.InvalidPropertyID, propertyID)
+	return models.Property{}, errors.New(messages.ErrorInvalidPropertyID + propertyID)
 }
 
 func (ch *ConfigurationHandler) registerConfigurationUpdateListener(chl configurationUpdateListenerFunc) {
@@ -287,23 +299,3 @@ func (ch *ConfigurationHandler) registerConfigurationUpdateListener(chl configur
 		log.Error(messages.CollectionIDError)
 	}
 }
-
-func (ch *ConfigurationHandler) getFileData(filePath string) {
-	data := utils.ReadFiles(filePath)
-	configResp := models.ConfigResponse{}
-	err := json.Unmarshal(data, &configResp)
-	if err != nil {
-		log.Error(messages.UnmarshalJSONErr, err)
-		return
-	}
-	log.Debug(fmt.Sprint(configResp))
-	out, err := json.Marshal(configResp)
-	if err != nil {
-		log.Error(messages.MarshalJSONErr, err)
-		return
-	}
-	ch.saveInCache(string(out))
-	if ch.configurationUpdateListener != nil {
-		ch.configurationUpdateListener()
-	}
-}