From ef85c8df24a72e0fc647adf0db3c82272f6e3c09 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Thu, 8 Oct 2020 15:03:03 +0200 Subject: [PATCH] Detect Gmail labels from All Mail mbox export --- Changelog.md | 3 + internal/transfer/message.go | 29 +++- internal/transfer/progress.go | 9 +- internal/transfer/progress_test.go | 12 +- internal/transfer/provider_eml_source.go | 4 +- internal/transfer/provider_imap_source.go | 4 +- internal/transfer/provider_mbox.go | 16 ++- .../transfer/provider_mbox_gmail_labels.go | 89 ++++++++++++ .../provider_mbox_gmail_labels_test.go | 125 ++++++++++++++++ internal/transfer/provider_mbox_source.go | 133 +++++++++++++----- internal/transfer/provider_mbox_test.go | 67 +++++++-- internal/transfer/provider_pmapi_source.go | 4 +- internal/transfer/provider_test.go | 8 +- internal/transfer/report.go | 6 +- internal/transfer/testdata/mbox/All Mail.mbox | 16 +++ internal/transfer/utils.go | 9 ++ 16 files changed, 461 insertions(+), 73 deletions(-) create mode 100644 internal/transfer/provider_mbox_gmail_labels.go create mode 100644 internal/transfer/provider_mbox_gmail_labels_test.go create mode 100644 internal/transfer/testdata/mbox/All Mail.mbox diff --git a/Changelog.md b/Changelog.md index 5c07a9be..032b6b4b 100644 --- a/Changelog.md +++ b/Changelog.md @@ -4,6 +4,9 @@ Changelog [format](http://keepachangelog.com/en/1.0.0/) ## Unreleased +### Added +* GODT-763 Detect Gmail labels from All Mail mbox export (using X-Gmail-Label header). + ### Fixed * GODT-749 Don't force PGP/Inline when sending plaintext messages. * GODT-764 Fix deadlock in integration tests for Import-Export. diff --git a/internal/transfer/message.go b/internal/transfer/message.go index fec7be38..dcec9288 100644 --- a/internal/transfer/message.go +++ b/internal/transfer/message.go @@ -29,17 +29,34 @@ type Message struct { ID string Unread bool Body []byte - Source Mailbox + Sources []Mailbox Targets []Mailbox } +// sourceNames returns array of source mailbox names. +func (msg Message) sourceNames() (names []string) { + for _, mailbox := range msg.Sources { + names = append(names, mailbox.Name) + } + return +} + +// targetNames returns array of target mailbox names. +func (msg Message) targetNames() (names []string) { + for _, mailbox := range msg.Targets { + names = append(names, mailbox.Name) + } + return +} + // MessageStatus holds status for message used by progress manager. type MessageStatus struct { - eventTime time.Time // Time of adding message to the process. - rule *Rule // Rule with source and target mailboxes. - SourceID string // Message ID at the source. - targetID string // Message ID at the target (if any). - bodyHash string // Hash of the message body. + eventTime time.Time // Time of adding message to the process. + sourceNames []string // Source mailbox names message is in. + SourceID string // Message ID at the source. + targetNames []string // Target mailbox names message is in. + targetID string // Message ID at the target (if any). + bodyHash string // Hash of the message body. exported bool imported bool diff --git a/internal/transfer/progress.go b/internal/transfer/progress.go index 57d05064..375eb37b 100644 --- a/internal/transfer/progress.go +++ b/internal/transfer/progress.go @@ -126,16 +126,17 @@ func (p *Progress) updateCount(mailbox string, count uint) { } // addMessage should be called as soon as there is ID of the message. -func (p *Progress) addMessage(messageID string, rule *Rule) { +func (p *Progress) addMessage(messageID string, sourceNames, targetNames []string) { p.lock.Lock() defer p.lock.Unlock() defer p.update() p.log.WithField("id", messageID).Trace("Message added") p.messageStatuses[messageID] = &MessageStatus{ - eventTime: time.Now(), - rule: rule, - SourceID: messageID, + eventTime: time.Now(), + sourceNames: sourceNames, + SourceID: messageID, + targetNames: targetNames, } } diff --git a/internal/transfer/progress_test.go b/internal/transfer/progress_test.go index 788f8e65..cdb4e024 100644 --- a/internal/transfer/progress_test.go +++ b/internal/transfer/progress_test.go @@ -48,21 +48,21 @@ func TestProgressAddingMessages(t *testing.T) { drainProgressUpdateChannel(&progress) // msg1 has no problem. - progress.addMessage("msg1", nil) + progress.addMessage("msg1", []string{}, []string{}) progress.messageExported("msg1", []byte(""), nil) progress.messageImported("msg1", "", nil) // msg2 has an import problem. - progress.addMessage("msg2", nil) + progress.addMessage("msg2", []string{}, []string{}) progress.messageExported("msg2", []byte(""), nil) progress.messageImported("msg2", "", errors.New("failed import")) // msg3 has an export problem. - progress.addMessage("msg3", nil) + progress.addMessage("msg3", []string{}, []string{}) progress.messageExported("msg3", []byte(""), errors.New("failed export")) // msg4 has an export problem and import is also called. - progress.addMessage("msg4", nil) + progress.addMessage("msg4", []string{}, []string{}) progress.messageExported("msg4", []byte(""), errors.New("failed export")) progress.messageImported("msg4", "", nil) @@ -92,7 +92,7 @@ func TestProgressFinish(t *testing.T) { progress.finish() r.Nil(t, progress.updateCh) - r.NotPanics(t, func() { progress.addMessage("msg", nil) }) + r.NotPanics(t, func() { progress.addMessage("msg", []string{}, []string{}) }) } func TestProgressFatalError(t *testing.T) { @@ -102,7 +102,7 @@ func TestProgressFatalError(t *testing.T) { progress.fatal(errors.New("fatal error")) r.Nil(t, progress.updateCh) - r.NotPanics(t, func() { progress.addMessage("msg", nil) }) + r.NotPanics(t, func() { progress.addMessage("msg", []string{}, []string{}) }) } func TestFailUnpauseAndStops(t *testing.T) { diff --git a/internal/transfer/provider_eml_source.go b/internal/transfer/provider_eml_source.go index 925ebe21..2576d49b 100644 --- a/internal/transfer/provider_eml_source.go +++ b/internal/transfer/provider_eml_source.go @@ -109,7 +109,7 @@ func (p *EMLProvider) exportMessages(rule *Rule, filePaths []string, progress *P // addMessage is called after time check to not report message // which should not be exported but any error from reading body // or parsing time is reported as an error. - progress.addMessage(filePath, rule) + progress.addMessage(filePath, msg.sourceNames(), msg.targetNames()) progress.messageExported(filePath, msg.Body, err) if err == nil { ch <- msg @@ -134,7 +134,7 @@ func (p *EMLProvider) exportMessage(rule *Rule, filePath string) (Message, error ID: filePath, Unread: false, Body: body, - Source: rule.SourceMailbox, + Sources: []Mailbox{rule.SourceMailbox}, Targets: rule.TargetMailboxes, }, nil } diff --git a/internal/transfer/provider_imap_source.go b/internal/transfer/provider_imap_source.go index e732dbf2..9f669a0c 100644 --- a/internal/transfer/provider_imap_source.go +++ b/internal/transfer/provider_imap_source.go @@ -124,7 +124,7 @@ func (p *IMAPProvider) loadMessagesInfo(rule *Rule, progress *Progress, uidValid uid: imapMessage.Uid, size: imapMessage.Size, } - progress.addMessage(id, rule) + progress.addMessage(id, []string{rule.SourceMailbox.Name}, rule.TargetMailboxNames()) } progress.callWrap(func() error { @@ -231,7 +231,7 @@ func (p *IMAPProvider) exportMessage(rule *Rule, id string, imapMessage *imap.Me ID: id, Unread: unread, Body: body, - Source: rule.SourceMailbox, + Sources: []Mailbox{rule.SourceMailbox}, Targets: rule.TargetMailboxes, } } diff --git a/internal/transfer/provider_mbox.go b/internal/transfer/provider_mbox.go index 0156fbc1..4c01a00a 100644 --- a/internal/transfer/provider_mbox.go +++ b/internal/transfer/provider_mbox.go @@ -49,11 +49,24 @@ func (p *MBOXProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, return nil, err } - mailboxes := []Mailbox{} + mailboxNames := []string{} for _, filePath := range filePaths { fileName := filepath.Base(filePath) mailboxName := strings.TrimSuffix(fileName, ".mbox") + mailboxNames = appendIfNew(mailboxNames, mailboxName) + labels, err := getGmailLabelsFromMboxFile(filepath.Join(p.root, filePath)) + if err != nil { + log.WithError(err).Error("Failed to get gmail labels from mbox file") + continue + } + for _, label := range labels { + mailboxNames = appendIfNew(mailboxNames, label) + } + } + + mailboxes := []Mailbox{} + for _, mailboxName := range mailboxNames { mailboxes = append(mailboxes, Mailbox{ ID: "", Name: mailboxName, @@ -61,6 +74,5 @@ func (p *MBOXProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, IsExclusive: false, }) } - return mailboxes, nil } diff --git a/internal/transfer/provider_mbox_gmail_labels.go b/internal/transfer/provider_mbox_gmail_labels.go new file mode 100644 index 00000000..19ebd365 --- /dev/null +++ b/internal/transfer/provider_mbox_gmail_labels.go @@ -0,0 +1,89 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "bufio" + "bytes" + "io" + "os" + "strings" +) + +const xGmailLabelsHeader = "X-Gmail-Labels" + +func getGmailLabelsFromMboxFile(filePath string) ([]string, error) { + f, err := os.Open(filePath) //nolint[gosec] + if err != nil { + return nil, err + } + return getGmailLabelsFromMboxReader(f) +} + +func getGmailLabelsFromMboxReader(f io.Reader) ([]string, error) { + allLabels := []string{} + + r := bufio.NewReader(f) + for { + b, isPrefix, err := r.ReadLine() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + if isPrefix { + for !isPrefix { + _, isPrefix, err = r.ReadLine() + if err != nil { + break + } + } + continue + } + if bytes.HasPrefix(b, []byte(xGmailLabelsHeader)) { + for _, label := range getGmailLabelsFromValue(string(b)) { + allLabels = appendIfNew(allLabels, label) + } + } + } + + return allLabels, nil +} + +func getGmailLabelsFromMessage(body []byte) ([]string, error) { + header, err := getMessageHeader(body) + if err != nil { + return nil, err + } + labels := header.Get(xGmailLabelsHeader) + return getGmailLabelsFromValue(labels), nil +} + +func getGmailLabelsFromValue(value string) []string { + value = strings.TrimPrefix(value, xGmailLabelsHeader+":") + labels := []string{} + for _, label := range strings.Split(value, ",") { + label = strings.TrimSpace(label) + if label == "" { + continue + } + labels = appendIfNew(labels, label) + } + return labels +} diff --git a/internal/transfer/provider_mbox_gmail_labels_test.go b/internal/transfer/provider_mbox_gmail_labels_test.go new file mode 100644 index 00000000..225ce20c --- /dev/null +++ b/internal/transfer/provider_mbox_gmail_labels_test.go @@ -0,0 +1,125 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +import ( + "fmt" + "strings" + "testing" + + r "github.com/stretchr/testify/require" +) + +func TestGetGmailLabelsFromMboxReader(t *testing.T) { + mboxFile := `From - Mon May 4 16:40:31 2020 +Subject: Test 1 +X-Gmail-Labels: Foo,Bar + +hello + +From - Mon May 4 16:40:31 2020 +Subject: Test 2 +X-Gmail-Labels: Foo , Baz + +hello + +From - Mon May 4 16:40:31 2020 +Subject: Test 3 +X-Gmail-Labels: , + +hello + +From - Mon May 4 16:40:31 2020 +Subject: Test 4 +X-Gmail-Labels: + +hello + +From - Mon May 4 16:40:31 2020 +Subject: Test 5 + +hello + +` + mboxReader := strings.NewReader(mboxFile) + labels, err := getGmailLabelsFromMboxReader(mboxReader) + r.NoError(t, err) + r.Equal(t, []string{"Foo", "Bar", "Baz"}, labels) +} + +func TestGetGmailLabelsFromMessage(t *testing.T) { + tests := []struct { + body string + wantLabels []string + }{ + {`Subject: One +X-Gmail-Labels: Foo,Bar + +Hello +`, []string{"Foo", "Bar"}}, + {`Subject: Two +X-Gmail-Labels: Foo , Bar , + +Hello +`, []string{"Foo", "Bar"}}, + {`Subject: Three +X-Gmail-Labels: , + +Hello +`, []string{}}, + {`Subject: Four +X-Gmail-Labels: + +Hello +`, []string{}}, + {`Subject: Five + +Hello +`, []string{}}, + } + for _, tc := range tests { + tc := tc + t.Run(fmt.Sprintf("%v", tc.body), func(t *testing.T) { + labels, err := getGmailLabelsFromMessage([]byte(tc.body)) + r.NoError(t, err) + r.Equal(t, tc.wantLabels, labels) + }) + } +} + +func TestGetGmailLabelsFromValue(t *testing.T) { + tests := []struct { + value string + wantLabels []string + }{ + {"Foo,Bar", []string{"Foo", "Bar"}}, + {" Foo , Bar ", []string{"Foo", "Bar"}}, + {" Foo , Bar , ", []string{"Foo", "Bar"}}, + {" Foo Bar ", []string{"Foo Bar"}}, + {" , ", []string{}}, + {" ", []string{}}, + {"", []string{}}, + } + for _, tc := range tests { + tc := tc + t.Run(fmt.Sprintf("%v", tc.value), func(t *testing.T) { + labels := getGmailLabelsFromValue(tc.value) + r.Equal(t, tc.wantLabels, labels) + }) + } +} diff --git a/internal/transfer/provider_mbox_source.go b/internal/transfer/provider_mbox_source.go index 5d4d6b0b..eaeb799c 100644 --- a/internal/transfer/provider_mbox_source.go +++ b/internal/transfer/provider_mbox_source.go @@ -34,7 +34,7 @@ func (p *MBOXProvider) TransferTo(rules transferRules, progress *Progress, ch ch log.Info("Started transfer from MBOX to channel") defer log.Info("Finished transfer from MBOX to channel") - filePathsPerFolder, err := p.getFilePathsPerFolder(rules) + filePathsPerFolder, err := p.getFilePathsPerFolder() if err != nil { progress.fatal(err) return @@ -45,31 +45,28 @@ func (p *MBOXProvider) TransferTo(rules transferRules, progress *Progress, ch ch } for folderName, filePaths := range filePathsPerFolder { - // No error guaranteed by getFilePathsPerFolder. - rule, _ := rules.getRuleBySourceMailboxName(folderName) + log.WithField("folder", folderName).Debug("Estimating folder counts") for _, filePath := range filePaths { if progress.shouldStop() { break } - p.updateCount(rule, progress, filePath) + p.updateCount(progress, filePath) } } progress.countsFinal() for folderName, filePaths := range filePathsPerFolder { - // No error guaranteed by getFilePathsPerFolder. - rule, _ := rules.getRuleBySourceMailboxName(folderName) - log.WithField("rule", rule).Debug("Processing rule") + log.WithField("folder", folderName).Debug("Processing folder") for _, filePath := range filePaths { if progress.shouldStop() { break } - p.transferTo(rule, progress, ch, filePath) + p.transferTo(rules, progress, ch, folderName, filePath) } } } -func (p *MBOXProvider) getFilePathsPerFolder(rules transferRules) (map[string][]string, error) { +func (p *MBOXProvider) getFilePathsPerFolder() (map[string][]string, error) { filePaths, err := getFilePathsWithSuffix(p.root, ".mbox") if err != nil { return nil, err @@ -79,18 +76,12 @@ func (p *MBOXProvider) getFilePathsPerFolder(rules transferRules) (map[string][] for _, filePath := range filePaths { fileName := filepath.Base(filePath) folder := strings.TrimSuffix(fileName, ".mbox") - _, err := rules.getRuleBySourceMailboxName(folder) - if err != nil { - log.WithField("msg", filePath).Trace("Mailbox skipped due to folder name") - continue - } - filePathsMap[folder] = append(filePathsMap[folder], filePath) } return filePathsMap, nil } -func (p *MBOXProvider) updateCount(rule *Rule, progress *Progress, filePath string) { +func (p *MBOXProvider) updateCount(progress *Progress, filePath string) { mboxReader := p.openMbox(progress, filePath) if mboxReader == nil { return @@ -107,10 +98,10 @@ func (p *MBOXProvider) updateCount(rule *Rule, progress *Progress, filePath stri } count++ } - progress.updateCount(rule.SourceMailbox.Name, uint(count)) + progress.updateCount(filePath, uint(count)) } -func (p *MBOXProvider) transferTo(rule *Rule, progress *Progress, ch chan<- Message, filePath string) { +func (p *MBOXProvider) transferTo(rules transferRules, progress *Progress, ch chan<- Message, folderName, filePath string) { mboxReader := p.openMbox(progress, filePath) if mboxReader == nil { return @@ -134,50 +125,122 @@ func (p *MBOXProvider) transferTo(rule *Rule, progress *Progress, ch chan<- Mess break } - msg, err := p.exportMessage(rule, id, msgReader) + msg, err := p.exportMessage(rules, folderName, id, msgReader) - // Read and check time in body only if the rule specifies it - // to not waste energy. - if err == nil && rule.HasTimeLimit() { - msgTime, msgTimeErr := getMessageTime(msg.Body) - if msgTimeErr != nil { - err = msgTimeErr - } else if !rule.isTimeInRange(msgTime) { - log.WithField("msg", id).Debug("Message skipped due to time") - continue - } + if err == nil && len(msg.Targets) == 0 { + // Here should be called progress.messageSkipped(id) once we have + // this feature, and following progress.updateCount can be removed. + continue } - // Counting only messages filtered by time to update count to correct total. count++ // addMessage is called after time check to not report message // which should not be exported but any error from reading body // or parsing time is reported as an error. - progress.addMessage(id, rule) + progress.addMessage(id, msg.sourceNames(), msg.targetNames()) progress.messageExported(id, msg.Body, err) if err == nil { ch <- msg } } - progress.updateCount(rule.SourceMailbox.Name, uint(count)) + progress.updateCount(filePath, uint(count)) } -func (p *MBOXProvider) exportMessage(rule *Rule, id string, msgReader io.Reader) (Message, error) { +func (p *MBOXProvider) exportMessage(rules transferRules, folderName, id string, msgReader io.Reader) (Message, error) { body, err := ioutil.ReadAll(msgReader) if err != nil { return Message{}, errors.Wrap(err, "failed to read message") } + msgRules := p.getMessageRules(rules, folderName, id, body) + sources := p.getMessageSources(msgRules) + targets := p.getMessageTargets(msgRules, id, body) return Message{ ID: id, Unread: false, Body: body, - Source: rule.SourceMailbox, - Targets: rule.TargetMailboxes, + Sources: sources, + Targets: targets, }, nil } +func (p *MBOXProvider) getMessageRules(rules transferRules, folderName, id string, body []byte) []*Rule { + msgRules := []*Rule{} + + folderRule, err := rules.getRuleBySourceMailboxName(folderName) + if err != nil { + log.WithField("msg", id).WithField("source", folderName).Debug("Message skipped due to source") + } else { + msgRules = append(msgRules, folderRule) + } + + gmailLabels, err := getGmailLabelsFromMessage(body) + if err != nil { + log.WithError(err).Error("Failed to get gmail labels, ") + } else { + for _, label := range gmailLabels { + rule, err := rules.getRuleBySourceMailboxName(label) + if err != nil { + log.WithField("msg", id).WithField("source", label).Debug("Message skipped due to source") + continue + } + msgRules = append(msgRules, rule) + } + } + + return msgRules +} + +func (p *MBOXProvider) getMessageSources(msgRules []*Rule) []Mailbox { + sources := []Mailbox{} + for _, rule := range msgRules { + sources = append(sources, rule.SourceMailbox) + } + return sources +} + +func (p *MBOXProvider) getMessageTargets(msgRules []*Rule, id string, body []byte) []Mailbox { + targets := []Mailbox{} + haveExclusiveMailbox := false + for _, rule := range msgRules { + // Read and check time in body only if the rule specifies it + // to not waste energy. + if rule.HasTimeLimit() { + msgTime, err := getMessageTime(body) + if err != nil { + log.WithError(err).Error("Failed to parse time, time check skipped") + } else if !rule.isTimeInRange(msgTime) { + log.WithField("msg", id).WithField("source", rule.SourceMailbox.Name).Debug("Message skipped due to time") + continue + } + } + for _, newTarget := range rule.TargetMailboxes { + // msgRules is sorted. The first rule is based on the folder name, + // followed by the order from X-Gmail-Labels. The rule based on + // the folder name should have priority for exclusive target. + if newTarget.IsExclusive && haveExclusiveMailbox { + continue + } + found := false + for _, target := range targets { + if target.Hash() == newTarget.Hash() { + found = true + break + } + } + if found { + continue + } + if newTarget.IsExclusive { + haveExclusiveMailbox = true + } + targets = append(targets, newTarget) + } + } + return targets +} + func (p *MBOXProvider) openMbox(progress *Progress, mboxPath string) *mbox.Reader { mboxPath = filepath.Join(p.root, mboxPath) mboxFile, err := os.Open(mboxPath) //nolint[gosec] diff --git a/internal/transfer/provider_mbox_test.go b/internal/transfer/provider_mbox_test.go index d9145644..815a4095 100644 --- a/internal/transfer/provider_mbox_test.go +++ b/internal/transfer/provider_mbox_test.go @@ -35,25 +35,28 @@ func newTestMBOXProvider(path string) *MBOXProvider { } func TestMBOXProviderMailboxes(t *testing.T) { - provider := newTestMBOXProvider("") - tests := []struct { + provider *MBOXProvider includeEmpty bool wantMailboxes []Mailbox }{ - {true, []Mailbox{ + {newTestMBOXProvider(""), true, []Mailbox{ + {Name: "All Mail"}, {Name: "Foo"}, + {Name: "Bar"}, {Name: "Inbox"}, }}, - {false, []Mailbox{ + {newTestMBOXProvider(""), false, []Mailbox{ + {Name: "All Mail"}, {Name: "Foo"}, + {Name: "Bar"}, {Name: "Inbox"}, }}, } for _, tc := range tests { tc := tc t.Run(fmt.Sprintf("%v", tc.includeEmpty), func(t *testing.T) { - mailboxes, err := provider.Mailboxes(tc.includeEmpty, false) + mailboxes, err := tc.provider.Mailboxes(tc.includeEmpty, false) r.NoError(t, err) r.Equal(t, tc.wantMailboxes, mailboxes) }) @@ -67,14 +70,26 @@ func TestMBOXProviderTransferTo(t *testing.T) { defer rulesClose() setupMBOXRules(rules) - testTransferTo(t, rules, provider, []string{ + msgs := testTransferTo(t, rules, provider, []string{ + "All Mail.mbox:1", + "All Mail.mbox:2", "Foo.mbox:1", "Inbox.mbox:1", }) + got := map[string][]string{} + for _, msg := range msgs { + got[msg.ID] = msg.targetNames() + } + r.Equal(t, map[string][]string{ + "All Mail.mbox:1": {"Archive", "Foo"}, // Bar is not in rules. + "All Mail.mbox:2": {"Archive", "Foo"}, + "Foo.mbox:1": {"Foo"}, + "Inbox.mbox:1": {"Inbox"}, + }, got) } func TestMBOXProviderTransferFrom(t *testing.T) { - dir, err := ioutil.TempDir("", "eml") + dir, err := ioutil.TempDir("", "mbox") r.NoError(t, err) defer os.RemoveAll(dir) //nolint[errcheck] @@ -94,7 +109,7 @@ func TestMBOXProviderTransferFrom(t *testing.T) { } func TestMBOXProviderTransferFromTo(t *testing.T) { - dir, err := ioutil.TempDir("", "eml") + dir, err := ioutil.TempDir("", "mbox") r.NoError(t, err) defer os.RemoveAll(dir) //nolint[errcheck] @@ -103,17 +118,51 @@ func TestMBOXProviderTransferFromTo(t *testing.T) { rules, rulesClose := newTestRules(t) defer rulesClose() - setupEMLRules(rules) + setupMBOXRules(rules) testTransferFromTo(t, rules, source, target, 5*time.Second) checkMBOXFileStructure(t, dir, []string{ + "Archive.mbox", "Foo.mbox", "Inbox.mbox", }) } +func TestMBOXProviderGetMessageTargetsReturnsOnlyOneFolder(t *testing.T) { + provider := newTestMBOXProvider("") + + folderA := Mailbox{Name: "Folder A", IsExclusive: true} + folderB := Mailbox{Name: "Folder B", IsExclusive: true} + labelA := Mailbox{Name: "Label A", IsExclusive: false} + labelB := Mailbox{Name: "Label B", IsExclusive: false} + labelC := Mailbox{Name: "Label C", IsExclusive: false} + + rule1 := &Rule{TargetMailboxes: []Mailbox{folderA, labelA, labelB}} + rule2 := &Rule{TargetMailboxes: []Mailbox{folderB, labelC}} + rule3 := &Rule{TargetMailboxes: []Mailbox{folderB}} + + tests := []struct { + rules []*Rule + wantMailboxes []Mailbox + }{ + {[]*Rule{}, []Mailbox{}}, + {[]*Rule{rule1}, []Mailbox{folderA, labelA, labelB}}, + {[]*Rule{rule1, rule2}, []Mailbox{folderA, labelA, labelB, labelC}}, + {[]*Rule{rule1, rule3}, []Mailbox{folderA, labelA, labelB}}, + {[]*Rule{rule3, rule1}, []Mailbox{folderB, labelA, labelB}}, + } + for _, tc := range tests { + tc := tc + t.Run(fmt.Sprintf("%v", tc.rules), func(t *testing.T) { + mailboxes := provider.getMessageTargets(tc.rules, "", []byte("")) + r.Equal(t, tc.wantMailboxes, mailboxes) + }) + } +} + func setupMBOXRules(rules transferRules) { + _ = rules.setRule(Mailbox{Name: "All Mail"}, []Mailbox{{Name: "Archive"}}, 0, 0) _ = rules.setRule(Mailbox{Name: "Inbox"}, []Mailbox{{Name: "Inbox"}}, 0, 0) _ = rules.setRule(Mailbox{Name: "Foo"}, []Mailbox{{Name: "Foo"}}, 0, 0) } diff --git a/internal/transfer/provider_pmapi_source.go b/internal/transfer/provider_pmapi_source.go index 3bd870e7..f1957f78 100644 --- a/internal/transfer/provider_pmapi_source.go +++ b/internal/transfer/provider_pmapi_source.go @@ -123,7 +123,7 @@ func (p *PMAPIProvider) transferTo(rule *Rule, progress *Progress, ch chan<- Mes } msgID := fmt.Sprintf("%s_%s", rule.SourceMailbox.ID, pmapiMessage.ID) - progress.addMessage(msgID, rule) + progress.addMessage(msgID, []string{rule.SourceMailbox.Name}, rule.TargetMailboxNames()) msg, err := p.exportMessage(rule, progress, pmapiMessage.ID, msgID, skipEncryptedMessages) progress.messageExported(msgID, msg.Body, err) if err == nil { @@ -177,7 +177,7 @@ func (p *PMAPIProvider) exportMessage(rule *Rule, progress *Progress, pmapiMsgID ID: msgID, Unread: unread, Body: body, - Source: rule.SourceMailbox, + Sources: []Mailbox{rule.SourceMailbox}, Targets: rule.TargetMailboxes, }, nil } diff --git a/internal/transfer/provider_test.go b/internal/transfer/provider_test.go index 1b58a14d..62927425 100644 --- a/internal/transfer/provider_test.go +++ b/internal/transfer/provider_test.go @@ -43,7 +43,7 @@ hello `, subject)) } -func testTransferTo(t *testing.T, rules transferRules, provider SourceProvider, expectedMessageIDs []string) { +func testTransferTo(t *testing.T, rules transferRules, provider SourceProvider, expectedMessageIDs []string) []Message { progress := newProgress(log, nil) drainProgressUpdateChannel(&progress) @@ -53,13 +53,17 @@ func testTransferTo(t *testing.T, rules transferRules, provider SourceProvider, close(ch) }() + msgs := []Message{} gotMessageIDs := []string{} for msg := range ch { + msgs = append(msgs, msg) gotMessageIDs = append(gotMessageIDs, msg.ID) } r.ElementsMatch(t, expectedMessageIDs, gotMessageIDs) r.Empty(t, progress.GetFailedMessages()) + + return msgs } func testTransferFrom(t *testing.T, rules transferRules, provider TargetProvider, messages []Message) { @@ -69,7 +73,7 @@ func testTransferFrom(t *testing.T, rules transferRules, provider TargetProvider ch := make(chan Message) go func() { for _, message := range messages { - progress.addMessage(message.ID, nil) + progress.addMessage(message.ID, []string{}, []string{}) progress.messageExported(message.ID, []byte(""), nil) ch <- message } diff --git a/internal/transfer/report.go b/internal/transfer/report.go index a09d9b38..0e731e2d 100644 --- a/internal/transfer/report.go +++ b/internal/transfer/report.go @@ -114,7 +114,7 @@ type messageReport struct { SourceID string TargetID string BodyHash string - SourceMailbox string + SourceMailboxes []string TargetMailboxes []string Error string @@ -130,8 +130,8 @@ func newMessageReportFromMessageStatus(messageStatus *MessageStatus, includePriv SourceID: messageStatus.SourceID, TargetID: messageStatus.targetID, BodyHash: messageStatus.bodyHash, - SourceMailbox: messageStatus.rule.SourceMailbox.Name, - TargetMailboxes: messageStatus.rule.TargetMailboxNames(), + SourceMailboxes: messageStatus.sourceNames, + TargetMailboxes: messageStatus.targetNames, Error: messageStatus.GetErrorMessage(), } diff --git a/internal/transfer/testdata/mbox/All Mail.mbox b/internal/transfer/testdata/mbox/All Mail.mbox new file mode 100644 index 00000000..2e758e86 --- /dev/null +++ b/internal/transfer/testdata/mbox/All Mail.mbox @@ -0,0 +1,16 @@ +From - Mon May 4 16:40:31 2020 +From: Bridge Test +To: Bridge Test +Subject: Test 1 +X-Gmail-Labels: Foo,Bar + +hello + + +From - Mon May 4 16:40:31 2020 +From: Bridge Test +To: Bridge Test +Subject: Test 2 +X-Gmail-Labels: Foo + +hello diff --git a/internal/transfer/utils.go b/internal/transfer/utils.go index afa71c7d..157a399c 100644 --- a/internal/transfer/utils.go +++ b/internal/transfer/utils.go @@ -160,4 +160,13 @@ func sanitizeFileName(fileName string) string { } return r }, fileName) + +func appendIfNew(list []string, newItem string) []string { + for _, item := range list { + if item == newItem { + return list + } + } + return append(list, newItem) } +