From 75b788b793336edce94051d47edc7800e30e6b1c Mon Sep 17 00:00:00 2001 From: James Houlahan Date: Tue, 1 Nov 2022 17:11:07 +0100 Subject: [PATCH] GODT-1993: Use more efficient filtering for message deletion --- go.mod | 2 +- go.sum | 4 ++-- internal/user/imap.go | 31 ++++++++++++++++++------------- internal/user/smtp.go | 23 ++++++++++++----------- 4 files changed, 33 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index 8b1ab32c..a9d2b8a2 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/stretchr/testify v1.8.0 github.com/urfave/cli/v2 v2.20.3 github.com/vmihailenco/msgpack/v5 v5.3.5 - gitlab.protontech.ch/go/liteapi v0.38.1-0.20221101102120-060b4bbab844 + gitlab.protontech.ch/go/liteapi v0.39.2 go.uber.org/goleak v1.2.0 golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e golang.org/x/net v0.1.0 diff --git a/go.sum b/go.sum index 01d153e5..94dd1260 100644 --- a/go.sum +++ b/go.sum @@ -403,8 +403,8 @@ github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsr github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zclconf/go-cty v1.11.0 h1:726SxLdi2SDnjY+BStqB9J1hNp4+2WlzyXLuimibIe0= github.com/zclconf/go-cty v1.11.0/go.mod h1:s9IfD1LK5ccNMSWCVFCE2rJfHiZgi7JijgeWIMfhLvA= -gitlab.protontech.ch/go/liteapi v0.38.1-0.20221101102120-060b4bbab844 h1:HmWG1P2qhImVjx0mFMuwfBj6xmGjLzb3ZiUwe2wc8pg= -gitlab.protontech.ch/go/liteapi v0.38.1-0.20221101102120-060b4bbab844/go.mod h1:IM7ADWjgIL2hXopzx0WNamizEuMgM2QZl7QH12FNflk= +gitlab.protontech.ch/go/liteapi v0.39.2 h1:HWxuO6c9cnRAzpLaj2SrOD1jJEWVa4nAUH1TkVPA4t4= +gitlab.protontech.ch/go/liteapi v0.39.2/go.mod h1:IM7ADWjgIL2hXopzx0WNamizEuMgM2QZl7QH12FNflk= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= diff --git a/internal/user/imap.go b/internal/user/imap.go index 811cc0cd..479c6ca0 100644 --- a/internal/user/imap.go +++ b/internal/user/imap.go @@ -30,6 +30,7 @@ import ( "github.com/ProtonMail/proton-bridge/v2/internal/vault" "github.com/ProtonMail/proton-bridge/v2/pkg/message" "github.com/bradenaw/juniper/stream" + "github.com/bradenaw/juniper/xslices" "gitlab.protontech.ch/go/liteapi" "golang.org/x/exp/slices" ) @@ -308,26 +309,30 @@ func (conn *imapConnector) RemoveMessagesFromMailbox(ctx context.Context, messag } if mailboxID == liteapi.SpamLabel || mailboxID == liteapi.TrashLabel { - // check if messages are only in Trash and AllMail before they are permanently deleted. - var messagesToDelete []string + var metadata []liteapi.MessageMetadata - // GODT-1993 - Update to more efficient method. - for _, messageID := range messageIDs { - m, err := conn.client.GetMessage(ctx, string(messageID)) + // There's currently no limit on how many IDs we can filter on, + // but to be nice to API, let's chunk it by 150. + for _, messageIDs := range xslices.Chunk(messageIDs, 150) { + m, err := conn.client.GetMessageMetadata(ctx, liteapi.MessageFilter{ + ID: mapTo[imap.MessageID, string](messageIDs), + }) if err != nil { - return fmt.Errorf("failed to get message info") + return err } - if len(m.LabelIDs) == 1 && m.LabelIDs[0] == liteapi.AllMailLabel { - messagesToDelete = append(messagesToDelete, m.ID) - } + m = xslices.Filter(m, func(m liteapi.MessageMetadata) bool { + return len(m.LabelIDs) == 1 && m.LabelIDs[0] == liteapi.AllMailLabel + }) + + metadata = append(metadata, m...) } - if len(messagesToDelete) == 0 { - return nil + if err := conn.client.DeleteMessage(ctx, xslices.Map(metadata, func(m liteapi.MessageMetadata) string { + return m.ID + })...); err != nil { + return err } - - return conn.client.DeleteMessage(ctx, messagesToDelete...) } return nil diff --git a/internal/user/smtp.go b/internal/user/smtp.go index 39e0c367..0c7ca8f5 100644 --- a/internal/user/smtp.go +++ b/internal/user/smtp.go @@ -22,7 +22,6 @@ import ( "encoding/base64" "fmt" "net/mail" - "net/url" "runtime" "strings" @@ -143,15 +142,16 @@ func getParentID( //nolint:funlen // Try to find a parent ID in the internal references. for _, internal := range internal { - filter := url.Values{ - "ID": {internal}, - } + var addrID string if addrMode == vault.SplitMode { - filter["AddressID"] = []string{authAddrID} + addrID = authAddrID } - metadata, err := client.GetMessageMetadata(ctx, filter) + metadata, err := client.GetMessageMetadata(ctx, liteapi.MessageFilter{ + ID: []string{internal}, + AddressID: addrID, + }) if err != nil { return "", fmt.Errorf("failed to get message metadata: %w", err) } @@ -168,15 +168,16 @@ func getParentID( //nolint:funlen // If no parent was found, try to find it in the last external reference. // There can be multiple messages with the same external ID; in this case, we don't pick any parent. if parentID == "" && len(external) > 0 { - filter := url.Values{ - "ExternalID": {external[len(external)-1]}, - } + var addrID string if addrMode == vault.SplitMode { - filter["AddressID"] = []string{authAddrID} + addrID = authAddrID } - metadata, err := client.GetMessageMetadata(ctx, filter) + metadata, err := client.GetMessageMetadata(ctx, liteapi.MessageFilter{ + ExternalID: external[len(external)-1], + AddressID: addrID, + }) if err != nil { return "", fmt.Errorf("failed to get message metadata: %w", err) }