GODT-1158: Store full messages bodies on disk

- GODT-1158: simple on-disk cache in store
- GODT-1158: better member naming in event loop
- GODT-1158: create on-disk cache during bridge setup
- GODT-1158: better job options
- GODT-1158: rename GetLiteral to GetRFC822
- GODT-1158: rename events -> currentEvents
- GODT-1158: unlock cache per-user
- GODT-1158: clean up cache after logout
- GODT-1158: randomized encrypted cache passphrase
- GODT-1158: Opt out of on-disk cache in settings
- GODT-1158: free space in cache
- GODT-1158: make tests compile
- GODT-1158: optional compression
- GODT-1158: cache custom location
- GODT-1158: basic capacity checker
- GODT-1158: cache free space config
- GODT-1158: only unlock cache if pmapi client is unlocked as well
- GODT-1158: simple background sync worker
- GODT-1158: set size/bodystructure when caching message
- GODT-1158: limit store db update blocking with semaphore
- GODT-1158: dumb 10-semaphore
- GODT-1158: properly handle delete; remove bad bodystructure handling
- GODT-1158: hacky fix for caching after logout... baaaaad
- GODT-1158: cache worker
- GODT-1158: compute body structure lazily
- GODT-1158: cache size in store
- GODT-1158: notify cacher when adding to store
- GODT-1158: 15 second store cache watcher
- GODT-1158: enable cacher
- GODT-1158: better cache worker starting/stopping
- GODT-1158: limit cacher to less concurrency than disk cache
- GODT-1158: message builder prio + pchan pkg
- GODT-1158: fix pchan, use in message builder
- GODT-1158: no sem in cacher (rely on message builder prio)
- GODT-1158: raise priority of existing jobs when requested
- GODT-1158: pending messages in on-disk cache
- GODT-1158: WIP just a note about deleting messages from disk cache
- GODT-1158: pending wait when trying to write
- GODT-1158: pending.add to return bool
- GODT-1225: Headers in bodystructure are stored as bytes.
- GODT-1158: fixing header caching
- GODT-1158: don't cache in background
- GODT-1158: all concurrency set in settings
- GODT-1158: worker pools inside message builder
- GODT-1158: fix linter issues
- GODT-1158: remove completed builds from builder
- GODT-1158: remove builder pool
- GODT-1158: cacher defer job done properly
- GODT-1158: fix linter
- GODT-1299: Continue with bodystructure build if deserialization failed
- GODT-1324: Delete messages from the cache when they are deleted on the server
- GODT-1158: refactor cache tests
- GODT-1158: move builder to app/bridge
- GODT-1306: Migrate cache on disk when location is changed (and delete when disabled)

internal/store/user_message.go

@@ -27,7 +27,6 @@ import (
 	"strings"
 
 	"github.com/ProtonMail/gopenpgp/v2/crypto"
-	pkgMsg "github.com/ProtonMail/proton-bridge/pkg/message"
 	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -154,11 +153,6 @@ func (store *Store) checkDraftTotalSize(message *pmapi.Message, attachments []*d
 		return false, err
 	}
 
-	msgSize := message.Size
-	if msgSize == 0 {
-		msgSize = int64(len(message.Body))
-	}
-
 	var attSize int64
 	for _, att := range attachments {
 		b, err := ioutil.ReadAll(att.encReader)
@@ -169,7 +163,7 @@ func (store *Store) checkDraftTotalSize(message *pmapi.Message, attachments []*d
 		att.encReader = bytes.NewBuffer(b)
 	}
 
-	return msgSize+attSize <= maxUpload, nil
+	return int64(len(message.Body))+attSize <= maxUpload, nil
 }
 
 func (store *Store) getDraftAction(message *pmapi.Message) int {
@@ -237,39 +231,6 @@ func (store *Store) txPutMessage(metaBucket *bolt.Bucket, onlyMeta *pmapi.Messag
 	return nil
 }
 
-func (store *Store) txPutBodyStructure(bsBucket *bolt.Bucket, msgID string, bs *pkgMsg.BodyStructure) error {
-	raw, err := bs.Serialize()
-	if err != nil {
-		return err
-	}
-	err = bsBucket.Put([]byte(msgID), raw)
-	if err != nil {
-		return errors.Wrap(err, "cannot put bodystructure bucket")
-	}
-	return nil
-}
-
-func (store *Store) txGetBodyStructure(bsBucket *bolt.Bucket, msgID string) (*pkgMsg.BodyStructure, error) {
-	raw := bsBucket.Get([]byte(msgID))
-	if len(raw) == 0 {
-		return nil, nil
-	}
-	return pkgMsg.DeserializeBodyStructure(raw)
-}
-
-func (store *Store) txIncreaseMsgBuildCount(b *bolt.Bucket, msgID string) (uint32, error) {
-	key := []byte(msgID)
-	count := uint32(0)
-
-	raw := b.Get(key)
-	if raw != nil {
-		count = btoi(raw)
-	}
-
-	count++
-	return count, b.Put(key, itob(count))
-}
-
 // createOrUpdateMessageEvent is helper to create only one message with
 // createOrUpdateMessagesEvent.
 func (store *Store) createOrUpdateMessageEvent(msg *pmapi.Message) error {
@@ -287,7 +248,7 @@ func (store *Store) createOrUpdateMessagesEvent(msgs []*pmapi.Message) error { /
 		b := tx.Bucket(metadataBucket)
 		for _, msg := range msgs {
 			clearNonMetadata(msg)
-			txUpdateMetadaFromDB(b, msg, store.log)
+			txUpdateMetadataFromDB(b, msg, store.log)
 		}
 		return nil
 	})
@@ -341,6 +302,11 @@ func (store *Store) createOrUpdateMessagesEvent(msgs []*pmapi.Message) error { /
 		return err
 	}
 
+	// Notify the cacher that it should start caching messages.
+	for _, msg := range msgs {
+		store.cacher.newJob(msg.ID)
+	}
+
 	return nil
 }
 
@@ -351,16 +317,12 @@ func clearNonMetadata(onlyMeta *pmapi.Message) {
 	onlyMeta.Attachments = nil
 }
 
-// txUpdateMetadaFromDB changes the the onlyMeta data.
+// txUpdateMetadataFromDB changes the the onlyMeta data.
 // If there is stored message in metaBucket the size, header and MIMEType are
 // not changed if already set. To change these:
 // * size must be updated by Message.SetSize
 // * contentType and header must be updated by Message.SetContentTypeAndHeader.
-func txUpdateMetadaFromDB(metaBucket *bolt.Bucket, onlyMeta *pmapi.Message, log *logrus.Entry) {
-	// Size attribute on the server is counting encrypted data. We need to compute
-	// "real" size of decrypted data. Negative values will be processed during fetch.
-	onlyMeta.Size = -1
-
+func txUpdateMetadataFromDB(metaBucket *bolt.Bucket, onlyMeta *pmapi.Message, log *logrus.Entry) {
 	msgb := metaBucket.Get([]byte(onlyMeta.ID))
 	if msgb == nil {
 		return
@@ -378,8 +340,7 @@ func txUpdateMetadaFromDB(metaBucket *bolt.Bucket, onlyMeta *pmapi.Message, log
 		return
 	}
 
-	// Keep already calculated size and content type.
-	onlyMeta.Size = stored.Size
+	// Keep content type.
 	onlyMeta.MIMEType = stored.MIMEType
 	if stored.Header != "" && stored.Header != "(No Header)" {
 		tmpMsg, err := mail.ReadMessage(
@@ -401,6 +362,12 @@ func (store *Store) deleteMessageEvent(apiID string) error {
 
 // deleteMessagesEvent deletes the message from metadata and all mailbox buckets.
 func (store *Store) deleteMessagesEvent(apiIDs []string) error {
+	for _, messageID := range apiIDs {
+		if err := store.cache.Rem(store.UserID(), messageID); err != nil {
+			logrus.WithError(err).Error("Failed to remove message from cache")
+		}
+	}
+
 	return store.db.Update(func(tx *bolt.Tx) error {
 		for _, apiID := range apiIDs {
 			if err := tx.Bucket(metadataBucket).Delete([]byte(apiID)); err != nil {