GODT-1158: Store full messages bodies on disk

- GODT-1158: simple on-disk cache in store
- GODT-1158: better member naming in event loop
- GODT-1158: create on-disk cache during bridge setup
- GODT-1158: better job options
- GODT-1158: rename GetLiteral to GetRFC822
- GODT-1158: rename events -> currentEvents
- GODT-1158: unlock cache per-user
- GODT-1158: clean up cache after logout
- GODT-1158: randomized encrypted cache passphrase
- GODT-1158: Opt out of on-disk cache in settings
- GODT-1158: free space in cache
- GODT-1158: make tests compile
- GODT-1158: optional compression
- GODT-1158: cache custom location
- GODT-1158: basic capacity checker
- GODT-1158: cache free space config
- GODT-1158: only unlock cache if pmapi client is unlocked as well
- GODT-1158: simple background sync worker
- GODT-1158: set size/bodystructure when caching message
- GODT-1158: limit store db update blocking with semaphore
- GODT-1158: dumb 10-semaphore
- GODT-1158: properly handle delete; remove bad bodystructure handling
- GODT-1158: hacky fix for caching after logout... baaaaad
- GODT-1158: cache worker
- GODT-1158: compute body structure lazily
- GODT-1158: cache size in store
- GODT-1158: notify cacher when adding to store
- GODT-1158: 15 second store cache watcher
- GODT-1158: enable cacher
- GODT-1158: better cache worker starting/stopping
- GODT-1158: limit cacher to less concurrency than disk cache
- GODT-1158: message builder prio + pchan pkg
- GODT-1158: fix pchan, use in message builder
- GODT-1158: no sem in cacher (rely on message builder prio)
- GODT-1158: raise priority of existing jobs when requested
- GODT-1158: pending messages in on-disk cache
- GODT-1158: WIP just a note about deleting messages from disk cache
- GODT-1158: pending wait when trying to write
- GODT-1158: pending.add to return bool
- GODT-1225: Headers in bodystructure are stored as bytes.
- GODT-1158: fixing header caching
- GODT-1158: don't cache in background
- GODT-1158: all concurrency set in settings
- GODT-1158: worker pools inside message builder
- GODT-1158: fix linter issues
- GODT-1158: remove completed builds from builder
- GODT-1158: remove builder pool
- GODT-1158: cacher defer job done properly
- GODT-1158: fix linter
- GODT-1299: Continue with bodystructure build if deserialization failed
- GODT-1324: Delete messages from the cache when they are deleted on the server
- GODT-1158: refactor cache tests
- GODT-1158: move builder to app/bridge
- GODT-1306: Migrate cache on disk when location is changed (and delete when disabled)

internal/app/bridge/bridge.go

@@ -32,7 +32,9 @@ import (
 	"github.com/ProtonMail/proton-bridge/internal/frontend/types"
 	"github.com/ProtonMail/proton-bridge/internal/imap"
 	"github.com/ProtonMail/proton-bridge/internal/smtp"
+	"github.com/ProtonMail/proton-bridge/internal/store/cache"
 	"github.com/ProtonMail/proton-bridge/internal/updater"
+	"github.com/ProtonMail/proton-bridge/pkg/message"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
@@ -69,10 +71,21 @@ func New(base *base.Base) *cli.App {
 func run(b *base.Base, c *cli.Context) error { // nolint[funlen]
 	tlsConfig, err := loadTLSConfig(b)
 	if err != nil {
-		logrus.WithError(err).Fatal("Failed to load TLS config")
+		return err
 	}
-	bridge := bridge.New(b.Locations, b.Cache, b.Settings, b.SentryReporter, b.CrashHandler, b.Listener, b.CM, b.Creds, b.Updater, b.Versioner)
-	imapBackend := imap.NewIMAPBackend(b.CrashHandler, b.Listener, b.Cache, bridge)
+
+	cache, err := loadCache(b)
+	if err != nil {
+		return err
+	}
+
+	builder := message.NewBuilder(
+		b.Settings.GetInt(settings.FetchWorkers),
+		b.Settings.GetInt(settings.AttachmentWorkers),
+	)
+
+	bridge := bridge.New(b.Locations, b.Cache, b.Settings, b.SentryReporter, b.CrashHandler, b.Listener, cache, builder, b.CM, b.Creds, b.Updater, b.Versioner)
+	imapBackend := imap.NewIMAPBackend(b.CrashHandler, b.Listener, b.Cache, b.Settings, bridge)
 	smtpBackend := smtp.NewSMTPBackend(b.CrashHandler, b.Listener, b.Settings, bridge)
 
 	go func() {
@@ -233,3 +246,35 @@ func checkAndHandleUpdate(u types.Updater, f frontend.Frontend, autoUpdate bool)
 
 	f.NotifySilentUpdateInstalled()
 }
+
+// NOTE(GODT-1158): How big should in-memory cache be?
+// NOTE(GODT-1158): How to handle cache location migration if user changes custom path?
+func loadCache(b *base.Base) (cache.Cache, error) {
+	if !b.Settings.GetBool(settings.CacheEnabledKey) {
+		return cache.NewInMemoryCache(100 * (1 << 20)), nil
+	}
+
+	var compressor cache.Compressor
+
+	// NOTE(GODT-1158): If user changes compression setting we have to nuke the cache.
+	if b.Settings.GetBool(settings.CacheCompressionKey) {
+		compressor = &cache.GZipCompressor{}
+	} else {
+		compressor = &cache.NoopCompressor{}
+	}
+
+	var path string
+
+	if customPath := b.Settings.Get(settings.CacheLocationKey); customPath != "" {
+		path = customPath
+	} else {
+		path = b.Cache.GetDefaultMessageCacheDir()
+	}
+
+	return cache.NewOnDiskCache(path, compressor, cache.Options{
+		MinFreeAbs:      uint64(b.Settings.GetInt(settings.CacheMinFreeAbsKey)),
+		MinFreeRat:      b.Settings.GetFloat64(settings.CacheMinFreeRatKey),
+		ConcurrentRead:  b.Settings.GetInt(settings.CacheConcurrencyRead),
+		ConcurrentWrite: b.Settings.GetInt(settings.CacheConcurrencyWrite),
+	})
+}