GODT-1356 GODT-1302: Cache on disk concurency and API retries

- GODT-1302: Change maximum resty retries from 0 to 30
- GODT-1302: Make sure we are closing GetAttachmen io.ReadCloser on error
- GODT-1356: Do not use attachmentPool - it was useless anyway
- GODT-1356: Increase cache watcher limit to 10min
- GODT-1356: Start cache watcher right after start (do not wait first 10 min)
- GODT-1356: Limit number of buildJobs (memory allocation) in BuildAndCacheMessage
- Other: Pass context from job options (message builder) to fetcher (both message and attachments)
- Other: BuildJob contains same function as returned buildDone (proper map locking)

pkg/pmapi/client.go

@@ -84,7 +84,7 @@ func (c *client) r(ctx context.Context) (*resty.Request, error) {
 	return r, nil
 }
 
-// do executes fn and may repeate it in case "401 Unauthorized" error is returned.
+// do executes fn and may repeat execution in case of retry after "401 Unauthorized" error.
 // Note: fn may be called more than once.
 func (c *client) do(ctx context.Context, fn func(*resty.Request) (*resty.Response, error)) (*resty.Response, error) {
 	r, err := c.r(ctx)

pkg/pmapi/manager.go

@@ -66,6 +66,13 @@ func newManager(cfg Config) *manager {
 	m.rc.OnError(m.handleRequestFailure)
 
 	// Configure retry mechanism.
+	//
+	// SetRetryCount(30): The most probable value of Retry-After is 1s (max
+	// 10s). Retrying up to 30 times will most probably cause a delay of
+	// 30s. The worst case scenario is 5min which is OK for background
+	// requests. We shuold use context to control a foreground requests
+	// which should be finish or fail sooner.
+	m.rc.SetRetryCount(30)
 	m.rc.SetRetryMaxWaitTime(time.Minute)
 	m.rc.SetRetryAfter(catchRetryAfter)
 	m.rc.AddRetryCondition(shouldRetry)

pkg/pmapi/manager_test.go

@@ -34,6 +34,11 @@ const testForceUpgradeBody = `{
 	"Error":"Upgrade!"
 }`
 
+const testTooManyAPIRequests = `{
+	"Code":85131,
+	"Error":"Too many recent API requests"
+}`
+
 func TestHandleTooManyRequests(t *testing.T) {
 	var numCalls int
 
@@ -42,6 +47,8 @@ func TestHandleTooManyRequests(t *testing.T) {
 
 		if numCalls < 5 {
 			w.WriteHeader(http.StatusTooManyRequests)
+			w.Header().Set("content-type", "application/json;charset=utf-8")
+			fmt.Fprint(w, testTooManyAPIRequests)
 		} else {
 			w.WriteHeader(http.StatusOK)
 		}

pkg/pmapi/response.go

@@ -129,6 +129,10 @@ func isTooManyRequest(res *resty.Response) bool {
 }
 
 func isNoResponse(res *resty.Response, err error) bool {
+	// Do not retry TLS failures
+	if errors.Is(err, ErrTLSMismatch) {
+		return false
+	}
 	return res.RawResponse == nil && err != nil
 }