Files
proton-bridge/internal/store/mailbox_message.go
2021-05-26 14:48:46 +00:00

556 lines
18 KiB
Go

// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package store
import (
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
bolt "go.etcd.io/bbolt"
)
// ErrAllMailOpNotAllowed is error user when user tries to do unsupported
// operation on All Mail folder.
var ErrAllMailOpNotAllowed = errors.New("operation not allowed for 'All Mail' folder")
// GetMessage returns the `pmapi.Message` struct wrapped in `StoreMessage`
// tied to this mailbox.
func (storeMailbox *Mailbox) GetMessage(apiID string) (*Message, error) {
msg, err := storeMailbox.store.getMessageFromDB(apiID)
if err != nil {
return nil, err
}
return newStoreMessage(storeMailbox, msg), nil
}
// FetchMessage fetches the message with the given `apiID`, stores it in the database, and returns a new store message
// wrapping it.
func (storeMailbox *Mailbox) FetchMessage(apiID string) (*Message, error) {
msg, err := storeMailbox.client().GetMessage(exposeContextForIMAP(), apiID)
if err != nil {
return nil, err
}
return newStoreMessage(storeMailbox, msg), nil
}
func (storeMailbox *Mailbox) ImportMessage(enc []byte, seen bool, labelIDs []string, flags, time int64) (string, error) {
defer storeMailbox.pollNow()
if storeMailbox.labelID != pmapi.AllMailLabel {
labelIDs = append(labelIDs, storeMailbox.labelID)
}
importReqs := &pmapi.ImportMsgReq{
Metadata: &pmapi.ImportMetadata{
AddressID: storeMailbox.storeAddress.addressID,
Unread: pmapi.Boolean(!seen),
Flags: flags,
Time: time,
LabelIDs: labelIDs,
},
Message: append(enc, "\r\n"...),
}
res, err := storeMailbox.client().Import(exposeContextForIMAP(), pmapi.ImportMsgReqs{importReqs})
if err != nil {
return "", err
}
if len(res) == 0 {
return "", errors.New("no import response")
}
return res[0].MessageID, res[0].Error
}
// LabelMessages adds the label by calling an API.
// It has to be propagated to all the same messages in all mailboxes.
// The propagation is processed by the event loop.
func (storeMailbox *Mailbox) LabelMessages(apiIDs []string) error {
log.WithFields(logrus.Fields{
"messages": apiIDs,
"label": storeMailbox.labelID,
"mailbox": storeMailbox.Name,
}).Trace("Labeling messages")
// Edge case is want to untrash message by drag&drop to AllMail (to not
// have it in trash but to not delete message forever). IMAP move would
// work okay but some clients might use COPY&EXPUNGE or APPEND&EXPUNGE.
// In this case COPY or APPEND is noop because the message is already
// in All mail. The consequent EXPUNGE would delete message forever.
if storeMailbox.labelID == pmapi.AllMailLabel {
return ErrAllMailOpNotAllowed
}
defer storeMailbox.pollNow()
return storeMailbox.client().LabelMessages(exposeContextForIMAP(), apiIDs, storeMailbox.labelID)
}
// UnlabelMessages removes the label by calling an API.
// It has to be propagated to all the same messages in all mailboxes.
// The propagation is processed by the event loop.
func (storeMailbox *Mailbox) UnlabelMessages(apiIDs []string) error {
storeMailbox.log.WithField("messages", apiIDs).
Trace("Unlabeling messages")
if storeMailbox.labelID == pmapi.AllMailLabel {
return ErrAllMailOpNotAllowed
}
defer storeMailbox.pollNow()
return storeMailbox.client().UnlabelMessages(exposeContextForIMAP(), apiIDs, storeMailbox.labelID)
}
// MarkMessagesRead marks the message read by calling an API.
// It has to be propagated to metadata mailbox which is done by the event loop.
func (storeMailbox *Mailbox) MarkMessagesRead(apiIDs []string) error {
log.WithFields(logrus.Fields{
"messages": apiIDs,
"label": storeMailbox.labelID,
"mailbox": storeMailbox.Name,
}).Trace("Marking messages as read")
defer storeMailbox.pollNow()
// Before deleting a message, TB sets \Seen flag which causes an event update
// and thus a refresh of the message by deleting and creating it again.
// TB does not notice this and happily continues with next command to move
// the message to the Trash but the message does not exist anymore.
// Therefore we do not issue API update if the message is already read.
ids := []string{}
for _, apiID := range apiIDs {
if message, _ := storeMailbox.store.getMessageFromDB(apiID); message == nil || message.Unread {
ids = append(ids, apiID)
}
}
if len(ids) == 0 {
return nil
}
return storeMailbox.client().MarkMessagesRead(exposeContextForIMAP(), ids)
}
// MarkMessagesUnread marks the message unread by calling an API.
// It has to be propagated to metadata mailbox which is done by the event loop.
func (storeMailbox *Mailbox) MarkMessagesUnread(apiIDs []string) error {
log.WithFields(logrus.Fields{
"messages": apiIDs,
"label": storeMailbox.labelID,
"mailbox": storeMailbox.Name,
}).Trace("Marking messages as unread")
defer storeMailbox.pollNow()
return storeMailbox.client().MarkMessagesUnread(exposeContextForIMAP(), apiIDs)
}
// MarkMessagesStarred adds the Starred label by calling an API.
// It has to be propagated to all the same messages in all mailboxes.
// The propagation is processed by the event loop.
func (storeMailbox *Mailbox) MarkMessagesStarred(apiIDs []string) error {
log.WithFields(logrus.Fields{
"messages": apiIDs,
"label": storeMailbox.labelID,
"mailbox": storeMailbox.Name,
}).Trace("Marking messages as starred")
defer storeMailbox.pollNow()
return storeMailbox.client().LabelMessages(exposeContextForIMAP(), apiIDs, pmapi.StarredLabel)
}
// MarkMessagesUnstarred removes the Starred label by calling an API.
// It has to be propagated to all the same messages in all mailboxes.
// The propagation is processed by the event loop.
func (storeMailbox *Mailbox) MarkMessagesUnstarred(apiIDs []string) error {
log.WithFields(logrus.Fields{
"messages": apiIDs,
"label": storeMailbox.labelID,
"mailbox": storeMailbox.Name,
}).Trace("Marking messages as unstarred")
defer storeMailbox.pollNow()
return storeMailbox.client().UnlabelMessages(exposeContextForIMAP(), apiIDs, pmapi.StarredLabel)
}
// MarkMessagesDeleted adds local flag \Deleted. This is not propagated to API
// until RemoveDeleted is called.
func (storeMailbox *Mailbox) MarkMessagesDeleted(apiIDs []string) error {
log.WithFields(logrus.Fields{
"messages": apiIDs,
"label": storeMailbox.labelID,
"mailbox": storeMailbox.Name,
}).Trace("Marking messages as deleted")
if storeMailbox.labelID == pmapi.AllMailLabel {
return ErrAllMailOpNotAllowed
}
return storeMailbox.store.db.Update(func(tx *bolt.Tx) error {
return storeMailbox.txMarkMessagesAsDeleted(tx, apiIDs, true)
})
}
// MarkMessagesUndeleted removes local flag \Deleted. This is not propagated to
// API.
func (storeMailbox *Mailbox) MarkMessagesUndeleted(apiIDs []string) error {
log.WithFields(logrus.Fields{
"messages": apiIDs,
"label": storeMailbox.labelID,
"mailbox": storeMailbox.Name,
}).Trace("Marking messages as undeleted")
if storeMailbox.labelID == pmapi.AllMailLabel {
return ErrAllMailOpNotAllowed
}
return storeMailbox.store.db.Update(func(tx *bolt.Tx) error {
return storeMailbox.txMarkMessagesAsDeleted(tx, apiIDs, false)
})
}
// RemoveDeleted sends request to API to remove message from mailbox.
// If the mailbox is All Mail or All Sent, it does nothing.
// If the mailbox is Trash or Spam and message is not in any other mailbox, messages is deleted.
// In all other cases the message is only removed from the mailbox.
// If nil is passed, all messages with \Deleted flag are removed.
// In other cases only messages with \Deleted flag and included in the passed list.
func (storeMailbox *Mailbox) RemoveDeleted(apiIDs []string) error {
storeMailbox.log.Trace("Deleting messages")
deletedAPIIDs, err := storeMailbox.GetDeletedAPIIDs()
if err != nil {
return err
}
if apiIDs == nil {
apiIDs = deletedAPIIDs
} else {
filteredAPIIDs := []string{}
for _, apiID := range apiIDs {
found := false
for _, deletedAPIID := range deletedAPIIDs {
if apiID == deletedAPIID {
found = true
break
}
}
if found {
filteredAPIIDs = append(filteredAPIIDs, apiID)
}
}
apiIDs = filteredAPIIDs
}
if len(apiIDs) == 0 {
storeMailbox.log.Debug("List to expunge is empty")
return nil
}
defer storeMailbox.pollNow()
switch storeMailbox.labelID {
case pmapi.AllMailLabel, pmapi.AllSentLabel:
break
case pmapi.TrashLabel, pmapi.SpamLabel:
if err := storeMailbox.deleteFromTrashOrSpam(apiIDs); err != nil {
return err
}
case pmapi.DraftLabel:
storeMailbox.log.WithField("ids", apiIDs).Warn("Deleting drafts")
if err := storeMailbox.client().DeleteMessages(exposeContextForIMAP(), apiIDs); err != nil {
return err
}
default:
if err := storeMailbox.client().UnlabelMessages(exposeContextForIMAP(), apiIDs, storeMailbox.labelID); err != nil {
return err
}
}
return nil
}
// deleteFromTrashOrSpam will remove messages from API forever. If messages
// still has some custom label the message will not be deleted. Instead it will
// be removed from Trash or Spam.
func (storeMailbox *Mailbox) deleteFromTrashOrSpam(apiIDs []string) error {
l := storeMailbox.log.WithField("messages", apiIDs)
l.Trace("Deleting messages from trash")
messageIDsToDelete := []string{}
messageIDsToUnlabel := []string{}
for _, apiID := range apiIDs {
msg, err := storeMailbox.store.getMessageFromDB(apiID)
if err != nil {
return err
}
otherLabels := false
// If the message has any custom label, we don't want to delete it, only remove trash/spam label.
for _, label := range msg.LabelIDs {
if label != pmapi.SpamLabel && label != pmapi.TrashLabel && label != pmapi.AllMailLabel && label != pmapi.AllSentLabel && label != pmapi.DraftLabel && label != pmapi.AllDraftsLabel {
otherLabels = true
break
}
}
if otherLabels {
messageIDsToUnlabel = append(messageIDsToUnlabel, apiID)
} else {
messageIDsToDelete = append(messageIDsToDelete, apiID)
}
}
if len(messageIDsToUnlabel) > 0 {
if err := storeMailbox.client().UnlabelMessages(exposeContextForIMAP(), messageIDsToUnlabel, storeMailbox.labelID); err != nil {
l.WithError(err).Warning("Cannot unlabel before deleting")
}
}
if len(messageIDsToDelete) > 0 {
storeMailbox.log.WithField("ids", messageIDsToDelete).Warn("Deleting messages")
if err := storeMailbox.client().DeleteMessages(exposeContextForIMAP(), messageIDsToDelete); err != nil {
return err
}
}
return nil
}
func (storeMailbox *Mailbox) txSkipAndRemoveFromMailbox(tx *bolt.Tx, msg *pmapi.Message) (skipAndRemove bool) {
defer func() {
if skipAndRemove {
if err := storeMailbox.txDeleteMessage(tx, msg.ID); err != nil {
storeMailbox.log.WithError(err).Error("Cannot remove message")
}
}
}()
mode, err := storeMailbox.store.getAddressMode()
if err != nil {
log.WithError(err).Error("Could not determine address mode")
return
}
skipAndRemove = true
// If it's split mode and it shouldn't be under this address, it should be skipped and removed.
if mode == splitMode && storeMailbox.storeAddress.addressID != msg.AddressID {
return
}
// If the message belongs in this mailbox, don't skip/remove it.
for _, labelID := range msg.LabelIDs {
if labelID == storeMailbox.labelID {
skipAndRemove = false
return
}
}
return skipAndRemove
}
// txCreateOrUpdateMessages will delete, create or update message from mailbox.
func (storeMailbox *Mailbox) txCreateOrUpdateMessages(tx *bolt.Tx, msgs []*pmapi.Message) error { //nolint[funlen]
shouldSendMailboxUpdate := false
// Buckets are not initialized right away because it's a heavy operation.
// The best option is to get the same bucket only once and only when needed.
var apiBucket, imapBucket, deletedBucket *bolt.Bucket
// Collect updates to send them later, after possibly sending the status/EXISTS update.
updates := make([]func(), 0, len(msgs))
for _, msg := range msgs {
if storeMailbox.txSkipAndRemoveFromMailbox(tx, msg) {
continue
}
// Update message.
if apiBucket == nil {
apiBucket = storeMailbox.txGetAPIIDsBucket(tx)
}
// Draft bodies can change and bodies are not re-fetched by IMAP clients.
// Every change has to be a new message; we need to delete the old one and always recreate it.
if msg.Type == pmapi.MessageTypeDraft {
if err := storeMailbox.txDeleteMessage(tx, msg.ID); err != nil {
return errors.Wrap(err, "cannot delete old draft")
}
} else {
uidb := apiBucket.Get([]byte(msg.ID))
if uidb != nil {
if imapBucket == nil {
imapBucket = storeMailbox.txGetIMAPIDsBucket(tx)
}
seqNum, seqErr := storeMailbox.txGetSequenceNumberOfUID(imapBucket, uidb)
if deletedBucket == nil {
deletedBucket = storeMailbox.txGetDeletedIDsBucket(tx)
}
isMarkedAsDeleted := deletedBucket.Get([]byte(msg.ID)) != nil
if seqErr == nil {
storeMailbox.store.notifyUpdateMessage(
storeMailbox.storeAddress.address,
storeMailbox.labelName,
btoi(uidb),
seqNum,
msg,
isMarkedAsDeleted,
)
}
continue
}
}
// Create a new message.
if imapBucket == nil {
imapBucket = storeMailbox.txGetIMAPIDsBucket(tx)
}
uid, err := storeMailbox.txGetNextUID(imapBucket, true)
if err != nil {
return errors.Wrap(err, "cannot generate new UID")
}
uidb := itob(uid)
if err = imapBucket.Put(uidb, []byte(msg.ID)); err != nil {
return errors.Wrap(err, "cannot add to IMAP bucket")
}
if err = apiBucket.Put([]byte(msg.ID), uidb); err != nil {
return errors.Wrap(err, "cannot add to API bucket")
}
seqNum, err := storeMailbox.txGetSequenceNumberOfUID(imapBucket, uidb)
if err != nil {
return errors.Wrap(err, "cannot get sequence number from UID")
}
updates = append(updates, func() {
storeMailbox.store.notifyUpdateMessage(
storeMailbox.storeAddress.address,
storeMailbox.labelName,
uid,
seqNum,
msg,
false, // new message is never marked as deleted
)
})
shouldSendMailboxUpdate = true
}
if shouldSendMailboxUpdate {
if err := storeMailbox.txMailboxStatusUpdate(tx); err != nil {
return err
}
}
for _, update := range updates {
update()
}
return nil
}
// txDeleteMessage deletes the message from the mailbox bucket.
// and issues message delete and mailbox update changes to updates channel.
func (storeMailbox *Mailbox) txDeleteMessage(tx *bolt.Tx, apiID string) error {
apiBucket := storeMailbox.txGetAPIIDsBucket(tx)
apiIDb := []byte(apiID)
uidb := apiBucket.Get(apiIDb)
if uidb == nil {
return nil
}
imapBucket := storeMailbox.txGetIMAPIDsBucket(tx)
deletedBucket := storeMailbox.txGetDeletedIDsBucket(tx)
seqNum, seqNumErr := storeMailbox.txGetSequenceNumberOfUID(imapBucket, uidb)
if seqNumErr != nil {
storeMailbox.log.WithField("apiID", apiID).WithError(seqNumErr).Warn("Cannot get seqNum of deleting message")
}
if err := imapBucket.Delete(uidb); err != nil {
return errors.Wrap(err, "cannot delete from IMAP bucket")
}
if err := apiBucket.Delete(apiIDb); err != nil {
return errors.Wrap(err, "cannot delete from API bucket")
}
if err := deletedBucket.Delete(apiIDb); err != nil {
return errors.Wrap(err, "cannot delete from mark-as-deleted bucket")
}
if seqNumErr == nil {
storeMailbox.store.notifyDeleteMessage(
storeMailbox.storeAddress.address,
storeMailbox.labelName,
seqNum,
)
// Outlook for Mac has problems with sending an EXISTS after deleting
// messages, mostly after moving message to other folder. It causes
// Outlook to rebuild the whole mailbox. [RFC-3501] says it's not
// necessary to send an EXISTS response with the new value.
}
return nil
}
func (storeMailbox *Mailbox) txMailboxStatusUpdate(tx *bolt.Tx) error {
total, unread, unreadSeqNum, err := storeMailbox.txGetCounts(tx)
if err != nil {
return errors.Wrap(err, "cannot get counts for mailbox status update")
}
storeMailbox.store.notifyMailboxStatus(
storeMailbox.storeAddress.address,
storeMailbox.labelName,
total,
unread,
unreadSeqNum,
)
return nil
}
func (storeMailbox *Mailbox) txMarkMessagesAsDeleted(tx *bolt.Tx, apiIDs []string, markAsDeleted bool) error {
// Load all buckets before looping over apiIDs
metaBucket := tx.Bucket(metadataBucket)
apiBucket := storeMailbox.txGetAPIIDsBucket(tx)
uidBucket := storeMailbox.txGetIMAPIDsBucket(tx)
deletedBucket := storeMailbox.txGetDeletedIDsBucket(tx)
for _, apiID := range apiIDs {
if markAsDeleted {
if err := deletedBucket.Put([]byte(apiID), []byte{1}); err != nil {
return err
}
} else {
if err := deletedBucket.Delete([]byte(apiID)); err != nil {
return err
}
}
msg, err := storeMailbox.store.txGetMessageFromBucket(metaBucket, apiID)
if err != nil {
return err
}
uid, err := storeMailbox.txGetUIDFromBucket(apiBucket, apiID)
if err != nil {
return err
}
seqNum, err := storeMailbox.txGetSequenceNumberOfUID(uidBucket, itob(uid))
if err != nil {
return err
}
// In order to send flags in format
// S: * 2 FETCH (FLAGS (\Deleted \Seen))
storeMailbox.store.notifyUpdateMessage(
storeMailbox.storeAddress.address,
storeMailbox.labelName,
uid,
seqNum,
msg,
markAsDeleted,
)
}
return nil
}