yesBad/proton-bridge
1
0
Fork 0
forked from Silverfish/proton-bridge
Code Actions Activity

Implement deleted flag GODT-461

Browse Source
This commit is contained in:
Jakub
2020-08-25 07:16:13 +02:00
committed by Michal Horejsek
parent 803353e300
commit 66e04dd5ed
25 changed files with 396 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
internal/store/change.go
View File

@ -48,18 +48,26 @@ func (store *Store) imapNotice(address, notice string) {
store.imapSendUpdate(update)
}
func (store *Store) imapUpdateMessage(address, mailboxName string, uid, sequenceNumber uint32, msg *pmapi.Message) {
func (store *Store) imapUpdateMessage(
address, mailboxName string,
uid, sequenceNumber uint32,
msg *pmapi.Message, hasDeletedFlag bool,
) {
store.log.WithFields(logrus.Fields{
"address": address,
"mailbox": mailboxName,
"seqNum": sequenceNumber,
"uid": uid,
"flags": message.GetFlags(msg),
"deleted": hasDeletedFlag,
}).Trace("IDLE update")
update := new(imapBackend.MessageUpdate)
update.Update = imapBackend.NewUpdate(address, mailboxName)
update.Message = imap.NewMessage(sequenceNumber, []imap.FetchItem{imap.FetchFlags, imap.FetchUid})
update.Message.Flags = message.GetFlags(msg)
if hasDeletedFlag {
update.Message.Flags = append(update.Message.Flags, imap.DeletedFlag)
}
update.Message.Uid = uid
store.imapSendUpdate(update)
}
@ -114,10 +122,13 @@ func (store *Store) imapSendUpdate(update imapBackend.Update) {
return
}
done := update.Done()
go func() { store.imapUpdates <- update }()
select {
case <-done:
case <-time.After(1 * time.Second):
store.log.Error("Could not send IMAP update (timeout)")
return
case store.imapUpdates <- update:
}
}

11
internal/store/mailbox.go
View File

@ -238,6 +238,17 @@ func (storeMailbox *Mailbox) txGetAPIIDsBucket(tx *bolt.Tx) *bolt.Bucket {
return storeMailbox.txGetBucket(tx).Bucket(apiIDsBucket)
}
// txGetDeletedIDsBucket returns the bucket with messagesID marked as deleted
func (storeMailbox *Mailbox) txGetDeletedIDsBucket(tx *bolt.Tx) *bolt.Bucket {
// There should be no error since it _...returns an error if the bucket
// name is blank, or if the bucket name is too long._
bucket, err := storeMailbox.txGetBucket(tx).CreateBucketIfNotExists(deletedIDsBucket)
if err != nil || bucket == nil {
storeMailbox.log.WithError(err).Error("Cannot create or get bucket with deleted IDs.")
}
return bucket
}
// txGetBucket returns the bucket of mailbox containing mapping buckets.
func (storeMailbox *Mailbox) txGetBucket(tx *bolt.Tx) *bolt.Bucket {
return tx.Bucket(mailboxesBucket).Bucket(storeMailbox.getBucketName())

19
internal/store/mailbox_ids.go
View File

@ -129,7 +129,11 @@ func (storeMailbox *Mailbox) getUID(apiID string) (uid uint32, err error) {
}
func (storeMailbox *Mailbox) txGetUID(tx *bolt.Tx, apiID string) (uint32, error) {
b := storeMailbox.txGetAPIIDsBucket(tx)
return storeMailbox.txGetUIDFromBucket(storeMailbox.txGetAPIIDsBucket(tx), apiID)
}
// txGetUIDFromBucket expects pointer to API bucket.
func (storeMailbox *Mailbox) txGetUIDFromBucket(b *bolt.Bucket, apiID string) (uint32, error) {
v := b.Get([]byte(apiID))
if v == nil {
return 0, ErrNoSuchAPIID
@ -137,6 +141,19 @@ func (storeMailbox *Mailbox) txGetUID(tx *bolt.Tx, apiID string) (uint32, error)
return btoi(v), nil
}
// getUID returns IMAP UID in this mailbox for message ID.
func (storeMailbox *Mailbox) getDeletedAPIIDs() (apiIDs []string, err error) {
err = storeMailbox.db().Update(func(tx *bolt.Tx) error {
b := storeMailbox.txGetDeletedIDsBucket(tx)
c := b.Cursor()
for k, _ := c.First(); k != nil; k, _ = c.Next() {
apiIDs = append(apiIDs, string(k))
}
return nil
})
return
}
// getSequenceNumber returns IMAP sequence number in the mailbox for the message with the given API ID `apiID`.
func (storeMailbox *Mailbox) getSequenceNumber(apiID string) (seqNum uint32, err error) {
err = storeMailbox.db().View(func(tx *bolt.Tx) error {

193
internal/store/mailbox_message.go
View File

@ -24,6 +24,8 @@ import (
bolt "go.etcd.io/bbolt"
)
// ErrAllMailOpNotAllowed is error user when user tries to do unsupported
// operation on All Mail folder
var ErrAllMailOpNotAllowed = errors.New("operation not allowed for 'All Mail' folder")
// GetMessage returns the `pmapi.Message` struct wrapped in `StoreMessage`
@ -96,11 +98,8 @@ func (storeMailbox *Mailbox) LabelMessages(apiIDs []string) error {
// It has to be propagated to all the same messages in all mailboxes.
// The propagation is processed by the event loop.
func (storeMailbox *Mailbox) UnlabelMessages(apiIDs []string) error {
log.WithFields(logrus.Fields{
"messages": apiIDs,
"label": storeMailbox.labelID,
"mailbox": storeMailbox.Name,
}).Trace("Unlabeling messages")
storeMailbox.log.WithField("messages", apiIDs).
Trace("Unlabeling messages")
if storeMailbox.labelID == pmapi.AllMailLabel {
return ErrAllMailOpNotAllowed
}
@ -173,54 +172,57 @@ func (storeMailbox *Mailbox) MarkMessagesUnstarred(apiIDs []string) error {
return storeMailbox.client().UnlabelMessages(apiIDs, pmapi.StarredLabel)
}
// DeleteMessages deletes messages.
// If the mailbox is All Mail or All Sent, it does nothing.
// If the mailbox is Trash or Spam and message is not in any other mailbox, messages is deleted.
// In all other cases the message is only removed from the mailbox.
func (storeMailbox *Mailbox) DeleteMessages(apiIDs []string) error {
// MarkMessagesDeleted adds local flag \Deleted. This is not propagated to API
// until RemoveDeleted is called
func (storeMailbox *Mailbox) MarkMessagesDeleted(apiIDs []string) error {
log.WithFields(logrus.Fields{
"messages": apiIDs,
"label": storeMailbox.labelID,
"mailbox": storeMailbox.Name,
}).Trace("Deleting messages")
}).Trace("Marking messages as deleted")
return storeMailbox.store.db.Update(func(tx *bolt.Tx) error {
return storeMailbox.txMarkMessagesAsDeleted(tx, apiIDs, true)
})
}
// MarkMessagesUndeleted removes local flag \Deleted. This is not propagated to
// API.
func (storeMailbox *Mailbox) MarkMessagesUndeleted(apiIDs []string) error {
log.WithFields(logrus.Fields{
"messages": apiIDs,
"label": storeMailbox.labelID,
"mailbox": storeMailbox.Name,
}).Trace("Marking messages as undeleted")
return storeMailbox.store.db.Update(func(tx *bolt.Tx) error {
return storeMailbox.txMarkMessagesAsDeleted(tx, apiIDs, false)
})
}
// RemoveDeleted sends request to API to remove message from mailbox.
// If the mailbox is All Mail or All Sent, it does nothing.
// If the mailbox is Trash or Spam and message is not in any other mailbox, messages is deleted.
// In all other cases the message is only removed from the mailbox.
func (storeMailbox *Mailbox) RemoveDeleted() error {
storeMailbox.log.Trace("Deleting messages")
apiIDs, err := storeMailbox.getDeletedAPIIDs()
if err != nil {
return err
}
if len(apiIDs) == 0 {
storeMailbox.log.Debug("List to expunge is empty")
return nil
}
defer storeMailbox.pollNow()
switch storeMailbox.labelID {
case pmapi.AllMailLabel, pmapi.AllSentLabel:
break
case pmapi.TrashLabel, pmapi.SpamLabel:
messageIDsToDelete := []string{}
messageIDsToUnlabel := []string{}
for _, apiID := range apiIDs {
msg, err := storeMailbox.store.getMessageFromDB(apiID)
if err != nil {
return err
}
otherLabels := false
// If the message has any custom label, we don't want to delete it, only remove trash/spam label.
for _, label := range msg.LabelIDs {
if label != pmapi.SpamLabel && label != pmapi.TrashLabel && label != pmapi.AllMailLabel && label != pmapi.AllSentLabel && label != pmapi.DraftLabel && label != pmapi.AllDraftsLabel {
otherLabels = true
break
}
}
if otherLabels {
messageIDsToUnlabel = append(messageIDsToUnlabel, apiID)
} else {
messageIDsToDelete = append(messageIDsToDelete, apiID)
}
}
if len(messageIDsToUnlabel) > 0 {
if err := storeMailbox.client().UnlabelMessages(messageIDsToUnlabel, storeMailbox.labelID); err != nil {
log.WithError(err).Warning("Cannot unlabel before deleting")
}
}
if len(messageIDsToDelete) > 0 {
if err := storeMailbox.client().DeleteMessages(messageIDsToDelete); err != nil {
return err
}
if err := storeMailbox.deleteFromTrashOrSpam(apiIDs); err != nil {
return err
}
case pmapi.DraftLabel:
if err := storeMailbox.client().DeleteMessages(apiIDs); err != nil {
@ -234,6 +236,50 @@ func (storeMailbox *Mailbox) DeleteMessages(apiIDs []string) error {
return nil
}
// deleteFromTrashOrSpam will remove messages from API forever. If messages
// still has some custom label the message will not be deleted. Instead it will
// be removed from Trash or Spam.
func (storeMailbox *Mailbox) deleteFromTrashOrSpam(apiIDs []string) error {
l := storeMailbox.log.WithField("messages", apiIDs)
l.Trace("Deleting messages from trash")
messageIDsToDelete := []string{}
messageIDsToUnlabel := []string{}
for _, apiID := range apiIDs {
msg, err := storeMailbox.store.getMessageFromDB(apiID)
if err != nil {
return err
}
otherLabels := false
// If the message has any custom label, we don't want to delete it, only remove trash/spam label.
for _, label := range msg.LabelIDs {
if label != pmapi.SpamLabel && label != pmapi.TrashLabel && label != pmapi.AllMailLabel && label != pmapi.AllSentLabel && label != pmapi.DraftLabel && label != pmapi.AllDraftsLabel {
otherLabels = true
break
}
}
if otherLabels {
messageIDsToUnlabel = append(messageIDsToUnlabel, apiID)
} else {
messageIDsToDelete = append(messageIDsToDelete, apiID)
}
}
if len(messageIDsToUnlabel) > 0 {
if err := storeMailbox.client().UnlabelMessages(messageIDsToUnlabel, storeMailbox.labelID); err != nil {
l.WithError(err).Warning("Cannot unlabel before deleting")
}
}
if len(messageIDsToDelete) > 0 {
if err := storeMailbox.client().DeleteMessages(messageIDsToDelete); err != nil {
return err
}
}
return nil
}
func (storeMailbox *Mailbox) txSkipAndRemoveFromMailbox(tx *bolt.Tx, msg *pmapi.Message) (skipAndRemove bool) {
defer func() {
if skipAndRemove {
@ -273,7 +319,7 @@ func (storeMailbox *Mailbox) txCreateOrUpdateMessages(tx *bolt.Tx, msgs []*pmapi
// Buckets are not initialized right away because it's a heavy operation.
// The best option is to get the same bucket only once and only when needed.
var apiBucket, imapBucket *bolt.Bucket
var apiBucket, imapBucket, deletedBucket *bolt.Bucket
for _, msg := range msgs {
if storeMailbox.txSkipAndRemoveFromMailbox(tx, msg) {
continue
@ -292,12 +338,15 @@ func (storeMailbox *Mailbox) txCreateOrUpdateMessages(tx *bolt.Tx, msgs []*pmapi
}
} else {
uidb := apiBucket.Get([]byte(msg.ID))
if uidb != nil {
if imapBucket == nil {
imapBucket = storeMailbox.txGetIMAPIDsBucket(tx)
}
seqNum, seqErr := storeMailbox.txGetSequenceNumberOfUID(imapBucket, uidb)
if deletedBucket == nil {
deletedBucket = storeMailbox.txGetDeletedIDsBucket(tx)
}
isMarkedAsDeleted := deletedBucket.Get([]byte(msg.ID)) != nil
if seqErr == nil {
storeMailbox.store.imapUpdateMessage(
storeMailbox.storeAddress.address,
@ -305,6 +354,7 @@ func (storeMailbox *Mailbox) txCreateOrUpdateMessages(tx *bolt.Tx, msgs []*pmapi
btoi(uidb),
seqNum,
msg,
isMarkedAsDeleted,
)
}
continue
@ -338,6 +388,7 @@ func (storeMailbox *Mailbox) txCreateOrUpdateMessages(tx *bolt.Tx, msgs []*pmapi
uid,
seqNum,
msg,
false, // new message is never marked as deleted
)
shouldSendMailboxUpdate = true
}
@ -362,6 +413,7 @@ func (storeMailbox *Mailbox) txDeleteMessage(tx *bolt.Tx, apiID string) error {
}
imapBucket := storeMailbox.txGetIMAPIDsBucket(tx)
deletedBucket := storeMailbox.txGetDeletedIDsBucket(tx)
seqNum, seqNumErr := storeMailbox.txGetSequenceNumberOfUID(imapBucket, uidb)
if seqNumErr != nil {
@ -376,6 +428,10 @@ func (storeMailbox *Mailbox) txDeleteMessage(tx *bolt.Tx, apiID string) error {
return errors.Wrap(err, "cannot delete from API bucket")
}
if err := deletedBucket.Delete(apiIDb); err != nil {
return errors.Wrap(err, "cannot delete from mark-as-deleted bucket")
}
if seqNumErr == nil {
storeMailbox.store.imapDeleteMessage(
storeMailbox.storeAddress.address,
@ -404,3 +460,50 @@ func (storeMailbox *Mailbox) txMailboxStatusUpdate(tx *bolt.Tx) error {
)
return nil
}
func (storeMailbox *Mailbox) txMarkMessagesAsDeleted(tx *bolt.Tx, apiIDs []string, markAsDeleted bool) error {
// Load all buckets before looping over apiIDs
metaBucket := tx.Bucket(metadataBucket)
apiBucket := storeMailbox.txGetAPIIDsBucket(tx)
uidBucket := storeMailbox.txGetIMAPIDsBucket(tx)
deletedBucket := storeMailbox.txGetDeletedIDsBucket(tx)
for _, apiID := range apiIDs {
if markAsDeleted {
if err := deletedBucket.Put([]byte(apiID), []byte{1}); err != nil {
return err
}
} else {
if err := deletedBucket.Delete([]byte(apiID)); err != nil {
return err
}
}
msg, err := storeMailbox.store.txGetMessageFromBucket(metaBucket, apiID)
if err != nil {
return err
}
uid, err := storeMailbox.txGetUIDFromBucket(apiBucket, apiID)
if err != nil {
return err
}
seqNum, err := storeMailbox.txGetSequenceNumberOfUID(uidBucket, itob(uid))
if err != nil {
return err
}
// In order to send flags in format
// S: * 2 FETCH (FLAGS (\Deleted \Seen))
storeMailbox.store.imapUpdateMessage(
storeMailbox.storeAddress.address,
storeMailbox.labelName,
uid,
seqNum,
msg,
markAsDeleted,
)
}
return nil
}

15
internal/store/message.go
View File

@ -62,6 +62,21 @@ func (message *Message) Message() *pmapi.Message {
return message.msg
}
// IsMarkedDeleted returns true if message is marked as deleted for specific
// mailbox
func (message *Message) IsMarkedDeleted() bool {
isMarkedAsDeleted := false
err := message.storeMailbox.db().Update(func(tx *bolt.Tx) error {
isMarkedAsDeleted = message.storeMailbox.txGetDeletedIDsBucket(tx).Get([]byte(message.msg.ID)) != nil
return nil
})
if err != nil {
message.storeMailbox.log.WithError(err).Error("Not able to retrieve deleted mark, assuming false.")
return false
}
return isMarkedAsDeleted
}
// SetSize updates the information about size of decrypted message which can be
// used for IMAP. This should not trigger any IMAP update.
// NOTE: The size from the server corresponds to pure body bytes. Hence it

3
internal/store/store.go
View File

@ -70,6 +70,8 @@ var (
// * {imapUID} -> string messageID
// * api_ids
// * {messageID} -> uint32 imapUID
// * deleted_ids (can be missing or have no keys)
// * {messageID} -> true
metadataBucket = []byte("metadata") //nolint[gochecknoglobals]
countsBucket = []byte("counts") //nolint[gochecknoglobals]
addressInfoBucket = []byte("address_info") //nolint[gochecknoglobals]
@ -78,6 +80,7 @@ var (
mailboxesBucket = []byte("mailboxes") //nolint[gochecknoglobals]
imapIDsBucket = []byte("imap_ids") //nolint[gochecknoglobals]
apiIDsBucket = []byte("api_ids") //nolint[gochecknoglobals]
deletedIDsBucket = []byte("deleted_ids") //nolint[gochecknoglobals]
mboxVersionBucket = []byte("mailboxes_version") //nolint[gochecknoglobals]
// ErrNoSuchAPIID when mailbox does not have API ID.

4
internal/store/store_test_exports.go
View File

@ -106,11 +106,13 @@ func txDumpMailsFactory(tb assert.TestingT) func(tx *bolt.Tx) error {
err := mailboxes.ForEach(func(mboxName, mboxData []byte) error {
fmt.Println("mbox:", string(mboxName))
b := mailboxes.Bucket(mboxName).Bucket(imapIDsBucket)
deletedMailboxes := mailboxes.Bucket(mboxName).Bucket(deletedIDsBucket)
c := b.Cursor()
i := 0
for imapID, apiID := c.First(); imapID != nil; imapID, apiID = c.Next() {
i++
fmt.Println(" ", i, "imap", btoi(imapID), "api", string(apiID))
isDeleted := deletedMailboxes != nil && deletedMailboxes.Get(apiID) != nil
fmt.Println(" ", i, "imap", btoi(imapID), "api", string(apiID), "isDeleted", isDeleted)
data := metadata.Get(apiID)
if !assert.NotNil(tb, data) {
continue

4
internal/store/user_message.go
View File

@ -143,8 +143,10 @@ func (store *Store) getMessageFromDB(apiID string) (msg *pmapi.Message, err erro
}
func (store *Store) txGetMessage(tx *bolt.Tx, apiID string) (*pmapi.Message, error) {
b := tx.Bucket(metadataBucket)
return store.txGetMessageFromBucket(tx.Bucket(metadataBucket), apiID)
}
func (store *Store) txGetMessageFromBucket(b *bolt.Bucket, apiID string) (*pmapi.Message, error) {
msgb := b.Get([]byte(apiID))
if msgb == nil {
return nil, ErrNoSuchAPIID