GODT-1158: Store full messages bodies on disk

- GODT-1158: simple on-disk cache in store
- GODT-1158: better member naming in event loop
- GODT-1158: create on-disk cache during bridge setup
- GODT-1158: better job options
- GODT-1158: rename GetLiteral to GetRFC822
- GODT-1158: rename events -> currentEvents
- GODT-1158: unlock cache per-user
- GODT-1158: clean up cache after logout
- GODT-1158: randomized encrypted cache passphrase
- GODT-1158: Opt out of on-disk cache in settings
- GODT-1158: free space in cache
- GODT-1158: make tests compile
- GODT-1158: optional compression
- GODT-1158: cache custom location
- GODT-1158: basic capacity checker
- GODT-1158: cache free space config
- GODT-1158: only unlock cache if pmapi client is unlocked as well
- GODT-1158: simple background sync worker
- GODT-1158: set size/bodystructure when caching message
- GODT-1158: limit store db update blocking with semaphore
- GODT-1158: dumb 10-semaphore
- GODT-1158: properly handle delete; remove bad bodystructure handling
- GODT-1158: hacky fix for caching after logout... baaaaad
- GODT-1158: cache worker
- GODT-1158: compute body structure lazily
- GODT-1158: cache size in store
- GODT-1158: notify cacher when adding to store
- GODT-1158: 15 second store cache watcher
- GODT-1158: enable cacher
- GODT-1158: better cache worker starting/stopping
- GODT-1158: limit cacher to less concurrency than disk cache
- GODT-1158: message builder prio + pchan pkg
- GODT-1158: fix pchan, use in message builder
- GODT-1158: no sem in cacher (rely on message builder prio)
- GODT-1158: raise priority of existing jobs when requested
- GODT-1158: pending messages in on-disk cache
- GODT-1158: WIP just a note about deleting messages from disk cache
- GODT-1158: pending wait when trying to write
- GODT-1158: pending.add to return bool
- GODT-1225: Headers in bodystructure are stored as bytes.
- GODT-1158: fixing header caching
- GODT-1158: don't cache in background
- GODT-1158: all concurrency set in settings
- GODT-1158: worker pools inside message builder
- GODT-1158: fix linter issues
- GODT-1158: remove completed builds from builder
- GODT-1158: remove builder pool
- GODT-1158: cacher defer job done properly
- GODT-1158: fix linter
- GODT-1299: Continue with bodystructure build if deserialization failed
- GODT-1324: Delete messages from the cache when they are deleted on the server
- GODT-1158: refactor cache tests
- GODT-1158: move builder to app/bridge
- GODT-1306: Migrate cache on disk when location is changed (and delete when disabled)

internal/imap/backend.go

@@ -37,21 +37,13 @@ import (
 	"time"
 
 	"github.com/ProtonMail/proton-bridge/internal/bridge"
+	"github.com/ProtonMail/proton-bridge/internal/config/settings"
 	"github.com/ProtonMail/proton-bridge/internal/events"
 	"github.com/ProtonMail/proton-bridge/pkg/listener"
-	"github.com/ProtonMail/proton-bridge/pkg/message"
 	"github.com/emersion/go-imap"
 	goIMAPBackend "github.com/emersion/go-imap/backend"
 )
 
-const (
-	// NOTE: Each fetch worker has its own set of attach workers so there can be up to 20*5=100 API requests at once.
-	// This is a reasonable limit to not overwhelm API while still maintaining as much parallelism as possible.
-	fetchWorkers  = 20 // In how many workers to fetch message (group list on IMAP).
-	attachWorkers = 5  // In how many workers to fetch attachments (for one message).
-	buildWorkers  = 20 // In how many workers to build messages.
-)
-
 type panicHandler interface {
 	HandlePanic()
 }
@@ -61,26 +53,32 @@ type imapBackend struct {
 	bridge        bridger
 	updates       *imapUpdates
 	eventListener listener.Listener
+	listWorkers   int
 
 	users       map[string]*imapUser
 	usersLocker sync.Locker
 
-	builder *message.Builder
-
 	imapCache     map[string]map[string]string
 	imapCachePath string
 	imapCacheLock *sync.RWMutex
 }
 
+type settingsProvider interface {
+	GetInt(string) int
+}
+
 // NewIMAPBackend returns struct implementing go-imap/backend interface.
 func NewIMAPBackend(
 	panicHandler panicHandler,
 	eventListener listener.Listener,
 	cache cacheProvider,
+	setting settingsProvider,
 	bridge *bridge.Bridge,
 ) *imapBackend { //nolint[golint]
 	bridgeWrap := newBridgeWrap(bridge)
-	backend := newIMAPBackend(panicHandler, cache, bridgeWrap, eventListener)
+
+	imapWorkers := setting.GetInt(settings.IMAPWorkers)
+	backend := newIMAPBackend(panicHandler, cache, bridgeWrap, eventListener, imapWorkers)
 
 	go backend.monitorDisconnectedUsers()
 
@@ -92,6 +90,7 @@ func newIMAPBackend(
 	cache cacheProvider,
 	bridge bridger,
 	eventListener listener.Listener,
+	listWorkers int,
 ) *imapBackend {
 	return &imapBackend{
 		panicHandler:  panicHandler,
@@ -102,10 +101,9 @@ func newIMAPBackend(
 		users:       map[string]*imapUser{},
 		usersLocker: &sync.Mutex{},
 
-		builder: message.NewBuilder(fetchWorkers, attachWorkers, buildWorkers),
-
 		imapCachePath: cache.GetIMAPCachePath(),
 		imapCacheLock: &sync.RWMutex{},
+		listWorkers:   listWorkers,
 	}
 }