yesBad/proton-bridge
1
0
Fork 0
forked from Silverfish/proton-bridge
Code Actions Activity

Cache body structure in order to reduce network traffic

Browse Source
This commit is contained in:
Jakub
2020-12-30 10:06:11 +01:00
committed by Michal Horejsek
parent 516ca018d3
commit 8ab852277c
15 changed files with 219 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
internal/imap/mailbox_message.go
View File

@ -240,7 +240,8 @@ func (im *imapMailbox) getMessage(storeMessage storeMessageProvider, items []ima
msg.Envelope = message.GetEnvelope(m)
case imap.FetchBody, imap.FetchBodyStructure:
var structure *message.BodyStructure
if structure, _, err = im.getBodyStructure(storeMessage); err != nil {
structure, err = im.getBodyStructure(storeMessage)
if err != nil {
return
}
if msg.BodyStructure, err = structure.IMAPBodyStructure([]int{}); err != nil {
@ -264,7 +265,7 @@ func (im *imapMailbox) getMessage(storeMessage storeMessageProvider, items []ima
// on our part and we need to compute "real" size of decrypted data.
if m.Size <= 0 {
im.log.WithField("msgID", storeMessage.ID()).Trace("Size unknown - downloading body")
if _, _, err = im.getBodyStructure(storeMessage); err != nil {
if _, _, err = im.getBodyAndStructure(storeMessage); err != nil {
return
}
}
@ -299,7 +300,24 @@ func (im *imapMailbox) getLiteralForSection(itemSection imap.FetchItem, msg *ima
return nil
}
func (im *imapMailbox) getBodyStructure(storeMessage storeMessageProvider) (
func (im *imapMailbox) getBodyStructure(storeMessage storeMessageProvider) (bs *message.BodyStructure, err error) {
// Apple Mail requests body structure for all
// messages irregularly. We cache bodystructure in
// local database in order to not re-download all
// messages from server.
bs, err = storeMessage.GetBodyStructure()
if err != nil {
im.log.WithError(err).Debug("Fail to retrieve bodystructure from database")
}
if bs == nil {
if bs, _, err = im.getBodyAndStructure(storeMessage); err != nil {
return
}
}
return
}
func (im *imapMailbox) getBodyAndStructure(storeMessage storeMessageProvider) (
structure *message.BodyStructure,
bodyReader *bytes.Reader, err error,
) {
@ -324,6 +342,11 @@ func (im *imapMailbox) getBodyStructure(storeMessage storeMessageProvider) (
}
// Drafts can change and we don't want to cache them.
if !isMessageInDraftFolder(m) {
if err := storeMessage.SetBodyStructure(structure); err != nil {
im.log.WithError(err).
WithField("msgID", m.ID).
Warn("Cannot update bodystructure while building")
}
cache.SaveMail(id, body, structure)
}
bodyReader = bytes.NewReader(body)
@ -380,7 +403,7 @@ func (im *imapMailbox) getMessageBodySection(storeMessage storeMessageProvider,
}
} else {
// The rest of cases need download and decrypt.
structure, bodyReader, err = im.getBodyStructure(storeMessage)
structure, bodyReader, err = im.getBodyAndStructure(storeMessage)
if err != nil {
return
}

2
internal/imap/mailbox_messages.go
View File

@ -277,7 +277,7 @@ func (im *imapMailbox) SearchMessages(isUID bool, criteria *imap.SearchCriteria)
}
if criteria.Body != nil || criteria.Text != nil {
log.Warn("Body and Text criteria not applied.")
log.Warn("Body and Text criteria not applied")
}
var apiIDs []string

3
internal/imap/store.go
View File

@ -24,6 +24,7 @@ import (
"github.com/ProtonMail/gopenpgp/v2/crypto"
"github.com/ProtonMail/proton-bridge/internal/imap/uidplus"
"github.com/ProtonMail/proton-bridge/internal/store"
backendMessage "github.com/ProtonMail/proton-bridge/pkg/message"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
)
@ -99,6 +100,8 @@ type storeMessageProvider interface {
SetSize(int64) error
SetContentTypeAndHeader(string, mail.Header) error
SetBodyStructure(*backendMessage.BodyStructure) error
GetBodyStructure() (*backendMessage.BodyStructure, error)
}
type storeUserWrap struct {

2
internal/imap/updates.go
View File

@ -214,7 +214,7 @@ func (iu *imapUpdates) sendIMAPUpdate(update goIMAPBackend.Update, block bool) {
select {
case <-done:
case <-time.After(1 * time.Second):
log.Warn("IMAP update could not be delivered (timeout).")
log.Warn("IMAP update could not be delivered (timeout)")
return
}
}

29
internal/store/message.go
View File

@ -20,6 +20,7 @@ package store
import (
"net/mail"
backendMessage "github.com/ProtonMail/proton-bridge/pkg/message"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
bolt "go.etcd.io/bbolt"
)
@ -119,3 +120,31 @@ func (message *Message) SetContentTypeAndHeader(mimeType string, header mail.Hea
}
return message.store.db.Update(txUpdate)
}
// SetBodyStructure stores serialized body structure in database
func (message *Message) SetBodyStructure(bs *backendMessage.BodyStructure) error {
txUpdate := func(tx *bolt.Tx) error {
return message.store.txPutBodyStructure(
tx.Bucket(bodystructureBucket),
message.ID(), bs,
)
}
return message.store.db.Update(txUpdate)
}
// GetBodyStructure deserialize body structure from database. If body structure
// is not in database it returns nil error and nil body structure. If error
// occurs it returns nil body structure.
func (message *Message) GetBodyStructure() (bs *backendMessage.BodyStructure, err error) {
txRead := func(tx *bolt.Tx) error {
bs, err = message.store.txGetBodyStructure(
tx.Bucket(bodystructureBucket),
message.ID(),
)
return err
}
if err = message.store.db.View(txRead); err != nil {
return nil, err
}
return bs, nil
}

25
internal/store/store.go
View File

@ -71,16 +71,17 @@ var (
// * {messageID} -> uint32 imapUID
// * deleted_ids (can be missing or have no keys)
// * {messageID} -> true
metadataBucket = []byte("metadata") //nolint[gochecknoglobals]
countsBucket = []byte("counts") //nolint[gochecknoglobals]
addressInfoBucket = []byte("address_info") //nolint[gochecknoglobals]
addressModeBucket = []byte("address_mode") //nolint[gochecknoglobals]
syncStateBucket = []byte("sync_state") //nolint[gochecknoglobals]
mailboxesBucket = []byte("mailboxes") //nolint[gochecknoglobals]
imapIDsBucket = []byte("imap_ids") //nolint[gochecknoglobals]
apiIDsBucket = []byte("api_ids") //nolint[gochecknoglobals]
deletedIDsBucket = []byte("deleted_ids") //nolint[gochecknoglobals]
mboxVersionBucket = []byte("mailboxes_version") //nolint[gochecknoglobals]
metadataBucket = []byte("metadata") //nolint[gochecknoglobals]
bodystructureBucket = []byte("bodystructure") //nolint[gochecknoglobals]
countsBucket = []byte("counts") //nolint[gochecknoglobals]
addressInfoBucket = []byte("address_info") //nolint[gochecknoglobals]
addressModeBucket = []byte("address_mode") //nolint[gochecknoglobals]
syncStateBucket = []byte("sync_state") //nolint[gochecknoglobals]
mailboxesBucket = []byte("mailboxes") //nolint[gochecknoglobals]
imapIDsBucket = []byte("imap_ids") //nolint[gochecknoglobals]
apiIDsBucket = []byte("api_ids") //nolint[gochecknoglobals]
deletedIDsBucket = []byte("deleted_ids") //nolint[gochecknoglobals]
mboxVersionBucket = []byte("mailboxes_version") //nolint[gochecknoglobals]
// ErrNoSuchAPIID when mailbox does not have API ID.
ErrNoSuchAPIID = errors.New("no such api id") //nolint[gochecknoglobals]
@ -193,6 +194,10 @@ func openBoltDatabase(filePath string) (db *bolt.DB, err error) {
return
}
if _, err = tx.CreateBucketIfNotExists(bodystructureBucket); err != nil {
return
}
if _, err = tx.CreateBucketIfNotExists(countsBucket); err != nil {
return
}

21
internal/store/user_message.go
View File

@ -27,6 +27,7 @@ import (
"strings"
"github.com/ProtonMail/gopenpgp/v2/crypto"
backendMessage "github.com/ProtonMail/proton-bridge/pkg/message"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
@ -170,6 +171,26 @@ func (store *Store) txPutMessage(metaBucket *bolt.Bucket, onlyMeta *pmapi.Messag
return nil
}
func (store *Store) txPutBodyStructure(bsBucket *bolt.Bucket, msgID string, bs *backendMessage.BodyStructure) error {
raw, err := bs.Serialize()
if err != nil {
return err
}
err = bsBucket.Put([]byte(msgID), raw)
if err != nil {
return errors.Wrap(err, "cannot put bodystructure bucket")
}
return nil
}
func (store *Store) txGetBodyStructure(bsBucket *bolt.Bucket, msgID string) (*backendMessage.BodyStructure, error) {
raw := bsBucket.Get([]byte(msgID))
if len(raw) == 0 {
return nil, nil
}
return backendMessage.DeserializeBodyStructure(raw)
}
// createOrUpdateMessageEvent is helper to create only one message with
// createOrUpdateMessagesEvent.
func (store *Store) createOrUpdateMessageEvent(msg *pmapi.Message) error {