GODT-1158: Store full messages bodies on disk

- GODT-1158: simple on-disk cache in store
- GODT-1158: better member naming in event loop
- GODT-1158: create on-disk cache during bridge setup
- GODT-1158: better job options
- GODT-1158: rename GetLiteral to GetRFC822
- GODT-1158: rename events -> currentEvents
- GODT-1158: unlock cache per-user
- GODT-1158: clean up cache after logout
- GODT-1158: randomized encrypted cache passphrase
- GODT-1158: Opt out of on-disk cache in settings
- GODT-1158: free space in cache
- GODT-1158: make tests compile
- GODT-1158: optional compression
- GODT-1158: cache custom location
- GODT-1158: basic capacity checker
- GODT-1158: cache free space config
- GODT-1158: only unlock cache if pmapi client is unlocked as well
- GODT-1158: simple background sync worker
- GODT-1158: set size/bodystructure when caching message
- GODT-1158: limit store db update blocking with semaphore
- GODT-1158: dumb 10-semaphore
- GODT-1158: properly handle delete; remove bad bodystructure handling
- GODT-1158: hacky fix for caching after logout... baaaaad
- GODT-1158: cache worker
- GODT-1158: compute body structure lazily
- GODT-1158: cache size in store
- GODT-1158: notify cacher when adding to store
- GODT-1158: 15 second store cache watcher
- GODT-1158: enable cacher
- GODT-1158: better cache worker starting/stopping
- GODT-1158: limit cacher to less concurrency than disk cache
- GODT-1158: message builder prio + pchan pkg
- GODT-1158: fix pchan, use in message builder
- GODT-1158: no sem in cacher (rely on message builder prio)
- GODT-1158: raise priority of existing jobs when requested
- GODT-1158: pending messages in on-disk cache
- GODT-1158: WIP just a note about deleting messages from disk cache
- GODT-1158: pending wait when trying to write
- GODT-1158: pending.add to return bool
- GODT-1225: Headers in bodystructure are stored as bytes.
- GODT-1158: fixing header caching
- GODT-1158: don't cache in background
- GODT-1158: all concurrency set in settings
- GODT-1158: worker pools inside message builder
- GODT-1158: fix linter issues
- GODT-1158: remove completed builds from builder
- GODT-1158: remove builder pool
- GODT-1158: cacher defer job done properly
- GODT-1158: fix linter
- GODT-1299: Continue with bodystructure build if deserialization failed
- GODT-1324: Delete messages from the cache when they are deleted on the server
- GODT-1158: refactor cache tests
- GODT-1158: move builder to app/bridge
- GODT-1306: Migrate cache on disk when location is changed (and delete when disabled)

internal/bridge/bridge.go

@@ -28,17 +28,17 @@ import (
 	"github.com/ProtonMail/proton-bridge/internal/constants"
 	"github.com/ProtonMail/proton-bridge/internal/metrics"
 	"github.com/ProtonMail/proton-bridge/internal/sentry"
+	"github.com/ProtonMail/proton-bridge/internal/store/cache"
 	"github.com/ProtonMail/proton-bridge/internal/updater"
 	"github.com/ProtonMail/proton-bridge/internal/users"
+	"github.com/ProtonMail/proton-bridge/pkg/message"
 	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
 
 	"github.com/ProtonMail/proton-bridge/pkg/listener"
 	logrus "github.com/sirupsen/logrus"
 )
 
-var (
-	log = logrus.WithField("pkg", "bridge") //nolint[gochecknoglobals]
-)
+var log = logrus.WithField("pkg", "bridge") //nolint[gochecknoglobals]
 
 type Bridge struct {
 	*users.Users
@@ -52,11 +52,13 @@ type Bridge struct {
 
 func New(
 	locations Locator,
-	cache Cacher,
-	s SettingsProvider,
+	cacheProvider CacheProvider,
+	setting SettingsProvider,
 	sentryReporter *sentry.Reporter,
 	panicHandler users.PanicHandler,
 	eventListener listener.Listener,
+	cache cache.Cache,
+	builder *message.Builder,
 	clientManager pmapi.Manager,
 	credStorer users.CredentialsStorer,
 	updater Updater,
@@ -64,7 +66,7 @@ func New(
 ) *Bridge {
 	// Allow DoH before starting the app if the user has previously set this setting.
 	// This allows us to start even if protonmail is blocked.
-	if s.GetBool(settings.AllowProxyKey) {
+	if setting.GetBool(settings.AllowProxyKey) {
 		clientManager.AllowProxy()
 	}
 
@@ -74,25 +76,25 @@ func New(
 		eventListener,
 		clientManager,
 		credStorer,
-		newStoreFactory(cache, sentryReporter, panicHandler, eventListener),
+		newStoreFactory(cacheProvider, sentryReporter, panicHandler, eventListener, cache, builder),
 	)
 
 	b := &Bridge{
 		Users: u,
 
 		locations:     locations,
-		settings:      s,
+		settings:      setting,
 		clientManager: clientManager,
 		updater:       updater,
 		versioner:     versioner,
 	}
 
-	if s.GetBool(settings.FirstStartKey) {
+	if setting.GetBool(settings.FirstStartKey) {
 		if err := b.SendMetric(metrics.New(metrics.Setup, metrics.FirstStart, metrics.Label(constants.Version))); err != nil {
 			logrus.WithError(err).Error("Failed to send metric")
 		}
 
-		s.SetBool(settings.FirstStartKey, false)
+		setting.SetBool(settings.FirstStartKey, false)
 	}
 
 	go b.heartbeat()

internal/bridge/store_factory.go

@@ -23,47 +23,65 @@ import (
 
 	"github.com/ProtonMail/proton-bridge/internal/sentry"
 	"github.com/ProtonMail/proton-bridge/internal/store"
+	"github.com/ProtonMail/proton-bridge/internal/store/cache"
 	"github.com/ProtonMail/proton-bridge/internal/users"
 	"github.com/ProtonMail/proton-bridge/pkg/listener"
+	"github.com/ProtonMail/proton-bridge/pkg/message"
 )
 
 type storeFactory struct {
-	cache          Cacher
+	cacheProvider  CacheProvider
 	sentryReporter *sentry.Reporter
 	panicHandler   users.PanicHandler
 	eventListener  listener.Listener
-	storeCache     *store.Cache
+	events         *store.Events
+	cache          cache.Cache
+	builder        *message.Builder
 }
 
 func newStoreFactory(
-	cache Cacher,
+	cacheProvider CacheProvider,
 	sentryReporter *sentry.Reporter,
 	panicHandler users.PanicHandler,
 	eventListener listener.Listener,
+	cache cache.Cache,
+	builder *message.Builder,
 ) *storeFactory {
 	return &storeFactory{
-		cache:          cache,
+		cacheProvider:  cacheProvider,
 		sentryReporter: sentryReporter,
 		panicHandler:   panicHandler,
 		eventListener:  eventListener,
-		storeCache:     store.NewCache(cache.GetIMAPCachePath()),
+		events:         store.NewEvents(cacheProvider.GetIMAPCachePath()),
+		cache:          cache,
+		builder:        builder,
 	}
 }
 
 // New creates new store for given user.
 func (f *storeFactory) New(user store.BridgeUser) (*store.Store, error) {
-	storePath := getUserStorePath(f.cache.GetDBDir(), user.ID())
-	return store.New(f.sentryReporter, f.panicHandler, user, f.eventListener, storePath, f.storeCache)
+	return store.New(
+		f.sentryReporter,
+		f.panicHandler,
+		user,
+		f.eventListener,
+		f.cache,
+		f.builder,
+		getUserStorePath(f.cacheProvider.GetDBDir(), user.ID()),
+		f.events,
+	)
 }
 
 // Remove removes all store files for given user.
 func (f *storeFactory) Remove(userID string) error {
-	storePath := getUserStorePath(f.cache.GetDBDir(), userID)
-	return store.RemoveStore(f.storeCache, storePath, userID)
+	return store.RemoveStore(
+		f.events,
+		getUserStorePath(f.cacheProvider.GetDBDir(), userID),
+		userID,
+	)
 }
 
 // getUserStorePath returns the file path of the store database for the given userID.
 func getUserStorePath(storeDir string, userID string) (path string) {
-	fileName := fmt.Sprintf("mailbox-%v.db", userID)
-	return filepath.Join(storeDir, fileName)
+	return filepath.Join(storeDir, fmt.Sprintf("mailbox-%v.db", userID))
 }

internal/bridge/types.go

@@ -28,7 +28,7 @@ type Locator interface {
 	ClearUpdates() error
 }
 
-type Cacher interface {
+type CacheProvider interface {
 	GetIMAPCachePath() string
 	GetDBDir() string
 }
@@ -38,6 +38,7 @@ type SettingsProvider interface {
 	Set(key string, value string)
 	GetBool(key string) bool
 	SetBool(key string, val bool)
+	GetInt(key string) int
 }
 
 type Updater interface {