From 5ccfa9f8ad04805552ff4d9cc7f6cacb44f48993 Mon Sep 17 00:00:00 2001
From: Steven Zou <szou@vmware.com>
Date: Wed, 28 Nov 2018 22:23:03 +0800
Subject: [PATCH] Add retry with backoff when creating redis cache for chart
 server

Signed-off-by: Steven Zou <szou@vmware.com>
---
 src/chartserver/cache.go | 35 +++++++++++++++++++++++++++--------
 1 file changed, 27 insertions(+), 8 deletions(-)

diff --git a/src/chartserver/cache.go b/src/chartserver/cache.go
index 0e86f883d..32c681c58 100644
--- a/src/chartserver/cache.go
+++ b/src/chartserver/cache.go
@@ -3,6 +3,7 @@ package chartserver
 import (
 	"encoding/json"
 	"errors"
+	"math"
 	"time"
 
 	beego_cache "github.com/astaxie/beego/cache"
@@ -19,6 +20,7 @@ const (
 	cacheDriverMem      = "memory"
 	cacheDriverRedis    = "redis"
 	cacheCollectionName = "helm_chart_cache"
+	maxTry              = 10
 )
 
 // ChartCache is designed to cache some processed data for repeated accessing
@@ -162,15 +164,26 @@ func initCacheDriver(cacheConfig *ChartCacheConfig) beego_cache.Cache {
 		hlog.Info("Enable memory cache for chart caching")
 		return beego_cache.NewMemoryCache()
 	case cacheDriverRedis:
-		redisCache, err := beego_cache.NewCache(cacheDriverRedis, cacheConfig.Config)
-		if err != nil {
-			// Just logged
-			hlog.Errorf("Failed to initialize redis cache: %s", err)
-			return nil
-		}
+		// New with retry
+		count := 0
+		for {
+			count++
+			redisCache, err := beego_cache.NewCache(cacheDriverRedis, cacheConfig.Config)
+			if err != nil {
+				// Just logged
+				hlog.Errorf("Failed to initialize redis cache: %s", err)
 
-		hlog.Info("Enable redis cache for chart caching")
-		return redisCache
+				if count < maxTry {
+					<-time.After(time.Duration(backoff(count)) * time.Second)
+					continue
+				}
+
+				return nil
+			}
+
+			hlog.Info("Enable redis cache for chart caching")
+			return redisCache
+		}
 	default:
 		break
 	}
@@ -179,3 +192,9 @@ func initCacheDriver(cacheConfig *ChartCacheConfig) beego_cache.Cache {
 	hlog.Info("No cache is enabled for chart caching")
 	return nil
 }
+
+// backoff: fast->slow->fast
+func backoff(count int) int {
+	f := 5 - math.Abs((float64)(count)-5)
+	return (int)(math.Pow(2, f))
+}