From 5b7eabe21a08b264213e773e93aa64c524062290 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Wed, 4 Nov 2020 09:16:46 +0100 Subject: [PATCH] Skipped messages do not change total counts but shows as separate number --- Changelog.md | 4 +++ go.mod | 2 ++ go.sum | 6 ++++ internal/bridge/credits.go | 4 +-- internal/frontend/cli-ie/importexport.go | 14 ++++++-- .../qml/ImportExportUI/DialogImport.qml | 2 +- internal/frontend/qt-ie/frontend.go | 18 ++++++---- internal/frontend/qt-ie/import.go | 2 ++ internal/frontend/qt-ie/ui.go | 2 ++ internal/importexport/credits.go | 4 +-- internal/transfer/message.go | 6 +++- internal/transfer/progress.go | 34 ++++++++++++++----- internal/transfer/progress_counts.go | 28 +++++++++++++++ internal/transfer/progress_test.go | 19 +++++++---- internal/transfer/provider_eml_source.go | 12 ++----- internal/transfer/provider_mbox_source.go | 19 ++++------- internal/transfer/provider_mbox_test.go | 23 +++++++++++++ test/transfer_checks_test.go | 8 ++--- 18 files changed, 151 insertions(+), 56 deletions(-) create mode 100644 internal/transfer/progress_counts.go diff --git a/Changelog.md b/Changelog.md index 5cc6c3fe..01c98d59 100644 --- a/Changelog.md +++ b/Changelog.md @@ -4,6 +4,9 @@ Changelog [format](http://keepachangelog.com/en/1.0.0/) ## Unreleased +### Added +* GODT-799 Skipped messages do not change total counts but shows as separate number. + ### Changed * Updated go-mbox dependency back to upstream. @@ -11,6 +14,7 @@ Changelog [format](http://keepachangelog.com/en/1.0.0/) * GODT-847 Waiting for unilateral update during deleting the message. * GODT-849 Show in error counts in the end also lost messages. * GODT-835 Do not include conversation ID in references to show properly conversation threads in clients. +* GODT-799 Fix skipping unwanted folders importing from mbox files. ## [IE 1.2.0] Elbe diff --git a/go.mod b/go.mod index 385b1e1f..b9cf5264 100644 --- a/go.mod +++ b/go.mod @@ -65,6 +65,8 @@ require ( github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect github.com/stretchr/testify v1.6.1 github.com/therecipe/qt v0.0.0-20200701200531-7f61353ee73e + github.com/therecipe/qt/internal/binding/files/docs/5.12.0 v0.0.0-20200904063919-c0c124a5770d // indirect + github.com/therecipe/qt/internal/binding/files/docs/5.13.0 v0.0.0-20200904063919-c0c124a5770d // indirect github.com/twinj/uuid v1.0.0 // indirect github.com/urfave/cli v1.22.4 go.etcd.io/bbolt v1.3.5 diff --git a/go.sum b/go.sum index 74f0fb75..15ede6c9 100644 --- a/go.sum +++ b/go.sum @@ -166,6 +166,12 @@ github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/therecipe/qt v0.0.0-20200701200531-7f61353ee73e h1:G0DQ/TRQyrEZjtLlLwevFjaRiG8eeCMlq9WXQ2OO2bk= github.com/therecipe/qt v0.0.0-20200701200531-7f61353ee73e/go.mod h1:SUUR2j3aE1z6/g76SdD6NwACEpvCxb3fvG82eKbD6us= +github.com/therecipe/qt v0.0.0-20200904063919-c0c124a5770d h1:T+d8FnaLSvM/1BdlDXhW4d5dr2F07bAbB+LpgzMxx+o= +github.com/therecipe/qt/internal/binding/files/docs v0.0.0-20191019224306-1097424d656c h1:/VhcwU7WuFEVgDHZ9V8PIYAyYqQ6KNxFUjBMOf2aFZM= +github.com/therecipe/qt/internal/binding/files/docs/5.12.0 v0.0.0-20200904063919-c0c124a5770d h1:hAZyEG2swPRWjF0kqqdGERXUazYnRJdAk4a58f14z7Y= +github.com/therecipe/qt/internal/binding/files/docs/5.12.0 v0.0.0-20200904063919-c0c124a5770d/go.mod h1:7m8PDYDEtEVqfjoUQc2UrFqhG0CDmoVJjRlQxexndFc= +github.com/therecipe/qt/internal/binding/files/docs/5.13.0 v0.0.0-20200904063919-c0c124a5770d h1:AJRoBel/g9cDS+yE8BcN3E+TDD/xNAguG21aoR8DAIE= +github.com/therecipe/qt/internal/binding/files/docs/5.13.0 v0.0.0-20200904063919-c0c124a5770d/go.mod h1:mH55Ek7AZcdns5KPp99O0bg+78el64YCYWHiQKrOdt4= github.com/twinj/uuid v1.0.0 h1:fzz7COZnDrXGTAOHGuUGYd6sG+JMq+AoE7+Jlu0przk= github.com/twinj/uuid v1.0.0/go.mod h1:mMgcE1RHFUFqe5AfiwlINXisXfDGro23fWdPUfOMjRY= github.com/urfave/cli v1.22.4 h1:u7tSpNPPswAFymm8IehJhy4uJMlUuU/GmqSkvJ1InXA= diff --git a/internal/bridge/credits.go b/internal/bridge/credits.go index 1ba5112f..f6fb4c16 100644 --- a/internal/bridge/credits.go +++ b/internal/bridge/credits.go @@ -15,8 +15,8 @@ // You should have received a copy of the GNU General Public License // along with ProtonMail Bridge. If not, see . -// Code generated by ./credits.sh at Wed Nov 4 12:24:36 PM CET 2020. DO NOT EDIT. +// Code generated by ./credits.sh at Wed Nov 4 13:57:47 CET 2020. DO NOT EDIT. package bridge -const Credits = "github.com/0xAX/notificator;github.com/abiosoft/ishell;github.com/abiosoft/readline;github.com/allan-simon/go-singleinstance;github.com/antlr/antlr4;github.com/certifi/gocertifi;github.com/chzyer/logex;github.com/chzyer/test;github.com/cucumber/godog;github.com/docker/docker-credential-helpers;github.com/emersion/go-imap;github.com/emersion/go-imap-appendlimit;github.com/emersion/go-imap-idle;github.com/emersion/go-imap-move;github.com/emersion/go-imap-quota;github.com/emersion/go-imap-specialuse;github.com/emersion/go-imap-unselect;github.com/emersion/go-mbox;github.com/emersion/go-message;github.com/emersion/go-sasl;github.com/emersion/go-smtp;github.com/emersion/go-textwrapper;github.com/emersion/go-vcard;github.com/fatih/color;github.com/flynn-archive/go-shlex;github.com/getsentry/raven-go;github.com/golang/mock;github.com/google/go-cmp;github.com/google/uuid;github.com/gopherjs/gopherjs;github.com/go-resty/resty/v2;github.com/hashicorp/go-multierror;github.com/jameskeane/bcrypt;github.com/jaytaylor/html2text;github.com/kardianos/osext;github.com/keybase/go-keychain;github.com/logrusorgru/aurora;github.com/Masterminds/semver/v3;github.com/mattn/go-runewidth;github.com/miekg/dns;github.com/myesui/uuid;github.com/nsf/jsondiff;github.com/olekukonko/tablewriter;github.com/pkg/errors;github.com/ProtonMail/bcrypt;github.com/ProtonMail/crypto;github.com/ProtonMail/docker-credential-helpers;github.com/ProtonMail/go-appdir;github.com/ProtonMail/go-apple-mobileconfig;github.com/ProtonMail/go-autostart;github.com/ProtonMail/go-imap;github.com/ProtonMail/go-imap-id;github.com/ProtonMail/gopenpgp/v2;github.com/ProtonMail/go-smtp;github.com/ProtonMail/go-vcard;github.com/PuerkitoBio/goquery;github.com/sirupsen/logrus;github.com/skratchdot/open-golang;github.com/ssor/bom;github.com/stretchr/testify;github.com/therecipe/qt;github.com/twinj/uuid;github.com/urfave/cli;go.etcd.io/bbolt;golang.org/x/crypto;golang.org/x/net;golang.org/x/text;gopkg.in/stretchr/testify.v1;;Font Awesome 4.7.0;;Qt 5.13 by Qt group;" +const Credits = "github.com/0xAX/notificator;github.com/Masterminds/semver/v3;github.com/ProtonMail/bcrypt;github.com/ProtonMail/crypto;github.com/ProtonMail/docker-credential-helpers;github.com/ProtonMail/go-appdir;github.com/ProtonMail/go-apple-mobileconfig;github.com/ProtonMail/go-autostart;github.com/ProtonMail/go-imap;github.com/ProtonMail/go-imap-id;github.com/ProtonMail/go-smtp;github.com/ProtonMail/go-vcard;github.com/ProtonMail/gopenpgp/v2;github.com/PuerkitoBio/goquery;github.com/abiosoft/ishell;github.com/abiosoft/readline;github.com/allan-simon/go-singleinstance;github.com/antlr/antlr4;github.com/certifi/gocertifi;github.com/chzyer/logex;github.com/chzyer/test;github.com/cucumber/godog;github.com/docker/docker-credential-helpers;github.com/emersion/go-imap;github.com/emersion/go-imap-appendlimit;github.com/emersion/go-imap-idle;github.com/emersion/go-imap-move;github.com/emersion/go-imap-quota;github.com/emersion/go-imap-specialuse;github.com/emersion/go-imap-unselect;github.com/emersion/go-mbox;github.com/emersion/go-message;github.com/emersion/go-sasl;github.com/emersion/go-smtp;github.com/emersion/go-textwrapper;github.com/emersion/go-vcard;github.com/fatih/color;github.com/flynn-archive/go-shlex;github.com/getsentry/raven-go;github.com/go-resty/resty/v2;github.com/golang/mock;github.com/google/go-cmp;github.com/google/uuid;github.com/gopherjs/gopherjs;github.com/hashicorp/go-multierror;github.com/jameskeane/bcrypt;github.com/jaytaylor/html2text;github.com/kardianos/osext;github.com/keybase/go-keychain;github.com/logrusorgru/aurora;github.com/mattn/go-runewidth;github.com/miekg/dns;github.com/myesui/uuid;github.com/nsf/jsondiff;github.com/olekukonko/tablewriter;github.com/pkg/errors;github.com/sirupsen/logrus;github.com/skratchdot/open-golang;github.com/ssor/bom;github.com/stretchr/testify;github.com/therecipe/qt;github.com/twinj/uuid;github.com/urfave/cli;go.etcd.io/bbolt;golang.org/x/crypto;golang.org/x/net;golang.org/x/text;gopkg.in/stretchr/testify.v1;;Font Awesome 4.7.0;;Qt 5.13 by Qt group;" diff --git a/internal/frontend/cli-ie/importexport.go b/internal/frontend/cli-ie/importexport.go index ed63df34..ecf25d96 100644 --- a/internal/frontend/cli-ie/importexport.go +++ b/internal/frontend/cli-ie/importexport.go @@ -184,9 +184,17 @@ func (f *frontendCLI) setTransferRules(t *transfer.Transfer) bool { } func (f *frontendCLI) printTransferProgress(progress *transfer.Progress) { - failed, imported, exported, added, total := progress.GetCounts() - if total != 0 { - f.Println(fmt.Sprintf("Progress update: %d (%d / %d) / %d, failed: %d", imported, exported, added, total, failed)) + counts := progress.GetCounts() + if counts.Total != 0 { + f.Println(fmt.Sprintf( + "Progress update: %d (%d / %d) / %d, skipped: %d, failed: %d", + counts.Imported, + counts.Exported, + counts.Added, + counts.Total, + counts.Skipped, + counts.Failed, + )) } if progress.IsPaused() { diff --git a/internal/frontend/qml/ImportExportUI/DialogImport.qml b/internal/frontend/qml/ImportExportUI/DialogImport.qml index 170aed3a..bcf0d8b9 100644 --- a/internal/frontend/qml/ImportExportUI/DialogImport.qml +++ b/internal/frontend/qml/ImportExportUI/DialogImport.qml @@ -599,7 +599,7 @@ Dialog { } Text { - text: qsTr("Import summary:
Total number of emails: %1
Imported emails: %2
Errors: %3").arg(go.total).arg(finalReport.imported).arg(go.progressFails) + text: qsTr("Import summary:
Total number of emails: %1
Imported emails: %2
Filtered out emails: %3
Errors: %4").arg(go.total).arg(go.progressImported).arg(go.progressSkipped).arg(go.progressFails) anchors.horizontalCenter: parent.horizontalCenter textFormat: Text.RichText horizontalAlignment: Text.AlignHCenter diff --git a/internal/frontend/qt-ie/frontend.go b/internal/frontend/qt-ie/frontend.go index d131872c..d05f011d 100644 --- a/internal/frontend/qt-ie/frontend.go +++ b/internal/frontend/qt-ie/frontend.go @@ -337,12 +337,14 @@ func (f *FrontendQt) setProgressManager(progress *transfer.Progress) { if progress.IsStopped() { break } - failed, imported, _, _, total := progress.GetCounts() - f.Qml.SetTotal(int(total)) - f.Qml.SetProgressFails(int(failed)) + counts := progress.GetCounts() + f.Qml.SetTotal(int(counts.Total)) + f.Qml.SetProgressImported(int(counts.Imported)) + f.Qml.SetProgressSkipped(int(counts.Skipped)) + f.Qml.SetProgressFails(int(counts.Failed)) f.Qml.SetProgressDescription(progress.PauseReason()) - if total > 0 { - newProgress := float32(imported+failed) / float32(total) + if counts.Total > 0 { + newProgress := float32(counts.Imported+counts.Skipped+counts.Failed) / float32(counts.Total) if newProgress >= 0 && newProgress != f.Qml.Progress() { f.Qml.SetProgress(newProgress) f.Qml.ProgressChanged(newProgress) @@ -350,8 +352,10 @@ func (f *FrontendQt) setProgressManager(progress *transfer.Progress) { } } // Counts will add lost messages only once the progress is completeled. - failed, _, _, _, _ := progress.GetCounts() - f.Qml.SetProgressFails(int(failed)) + counts := progress.GetCounts() + f.Qml.SetProgressImported(int(counts.Imported)) + f.Qml.SetProgressSkipped(int(counts.Skipped)) + f.Qml.SetProgressFails(int(counts.Failed)) if err := progress.GetFatalError(); err != nil { f.Qml.SetProgressDescription(err.Error()) diff --git a/internal/frontend/qt-ie/import.go b/internal/frontend/qt-ie/import.go index c26bad32..2ad3e6e9 100644 --- a/internal/frontend/qt-ie/import.go +++ b/internal/frontend/qt-ie/import.go @@ -77,6 +77,8 @@ func (f *FrontendQt) StartImport(email string) { // TODO email not needed log.Trace("Starting import") f.Qml.SetProgressDescription("init") // TODO use const + f.Qml.SetProgressImported(0) + f.Qml.SetProgressSkipped(0) f.Qml.SetProgressFails(0) f.Qml.SetProgress(0.0) f.Qml.SetTotal(1) diff --git a/internal/frontend/qt-ie/ui.go b/internal/frontend/qt-ie/ui.go index 1ae5733a..1040d2ea 100644 --- a/internal/frontend/qt-ie/ui.go +++ b/internal/frontend/qt-ie/ui.go @@ -43,6 +43,8 @@ type GoQMLInterface struct { _ string `property:lastError` _ float32 `property:progress` _ string `property:progressDescription` + _ int `property:progressImported` + _ int `property:progressSkipped` _ int `property:progressFails` _ int `property:total` _ string `property:importLogFileName` diff --git a/internal/importexport/credits.go b/internal/importexport/credits.go index df66fe21..a7520e34 100644 --- a/internal/importexport/credits.go +++ b/internal/importexport/credits.go @@ -15,8 +15,8 @@ // You should have received a copy of the GNU General Public License // along with ProtonMail Bridge. If not, see . -// Code generated by ./credits.sh at Wed Nov 4 12:24:36 PM CET 2020. DO NOT EDIT. +// Code generated by ./credits.sh at Wed Nov 4 13:57:47 CET 2020. DO NOT EDIT. package importexport -const Credits = "github.com/0xAX/notificator;github.com/abiosoft/ishell;github.com/abiosoft/readline;github.com/allan-simon/go-singleinstance;github.com/antlr/antlr4;github.com/certifi/gocertifi;github.com/chzyer/logex;github.com/chzyer/test;github.com/cucumber/godog;github.com/docker/docker-credential-helpers;github.com/emersion/go-imap;github.com/emersion/go-imap-appendlimit;github.com/emersion/go-imap-idle;github.com/emersion/go-imap-move;github.com/emersion/go-imap-quota;github.com/emersion/go-imap-specialuse;github.com/emersion/go-imap-unselect;github.com/emersion/go-mbox;github.com/emersion/go-message;github.com/emersion/go-sasl;github.com/emersion/go-smtp;github.com/emersion/go-textwrapper;github.com/emersion/go-vcard;github.com/fatih/color;github.com/flynn-archive/go-shlex;github.com/getsentry/raven-go;github.com/golang/mock;github.com/google/go-cmp;github.com/google/uuid;github.com/gopherjs/gopherjs;github.com/go-resty/resty/v2;github.com/hashicorp/go-multierror;github.com/jameskeane/bcrypt;github.com/jaytaylor/html2text;github.com/kardianos/osext;github.com/keybase/go-keychain;github.com/logrusorgru/aurora;github.com/Masterminds/semver/v3;github.com/mattn/go-runewidth;github.com/miekg/dns;github.com/myesui/uuid;github.com/nsf/jsondiff;github.com/olekukonko/tablewriter;github.com/pkg/errors;github.com/ProtonMail/bcrypt;github.com/ProtonMail/crypto;github.com/ProtonMail/docker-credential-helpers;github.com/ProtonMail/go-appdir;github.com/ProtonMail/go-apple-mobileconfig;github.com/ProtonMail/go-autostart;github.com/ProtonMail/go-imap;github.com/ProtonMail/go-imap-id;github.com/ProtonMail/gopenpgp/v2;github.com/ProtonMail/go-smtp;github.com/ProtonMail/go-vcard;github.com/PuerkitoBio/goquery;github.com/sirupsen/logrus;github.com/skratchdot/open-golang;github.com/ssor/bom;github.com/stretchr/testify;github.com/therecipe/qt;github.com/twinj/uuid;github.com/urfave/cli;go.etcd.io/bbolt;golang.org/x/crypto;golang.org/x/net;golang.org/x/text;gopkg.in/stretchr/testify.v1;;Font Awesome 4.7.0;;Qt 5.13 by Qt group;" +const Credits = "github.com/0xAX/notificator;github.com/Masterminds/semver/v3;github.com/ProtonMail/bcrypt;github.com/ProtonMail/crypto;github.com/ProtonMail/docker-credential-helpers;github.com/ProtonMail/go-appdir;github.com/ProtonMail/go-apple-mobileconfig;github.com/ProtonMail/go-autostart;github.com/ProtonMail/go-imap;github.com/ProtonMail/go-imap-id;github.com/ProtonMail/go-smtp;github.com/ProtonMail/go-vcard;github.com/ProtonMail/gopenpgp/v2;github.com/PuerkitoBio/goquery;github.com/abiosoft/ishell;github.com/abiosoft/readline;github.com/allan-simon/go-singleinstance;github.com/antlr/antlr4;github.com/certifi/gocertifi;github.com/chzyer/logex;github.com/chzyer/test;github.com/cucumber/godog;github.com/docker/docker-credential-helpers;github.com/emersion/go-imap;github.com/emersion/go-imap-appendlimit;github.com/emersion/go-imap-idle;github.com/emersion/go-imap-move;github.com/emersion/go-imap-quota;github.com/emersion/go-imap-specialuse;github.com/emersion/go-imap-unselect;github.com/emersion/go-mbox;github.com/emersion/go-message;github.com/emersion/go-sasl;github.com/emersion/go-smtp;github.com/emersion/go-textwrapper;github.com/emersion/go-vcard;github.com/fatih/color;github.com/flynn-archive/go-shlex;github.com/getsentry/raven-go;github.com/go-resty/resty/v2;github.com/golang/mock;github.com/google/go-cmp;github.com/google/uuid;github.com/gopherjs/gopherjs;github.com/hashicorp/go-multierror;github.com/jameskeane/bcrypt;github.com/jaytaylor/html2text;github.com/kardianos/osext;github.com/keybase/go-keychain;github.com/logrusorgru/aurora;github.com/mattn/go-runewidth;github.com/miekg/dns;github.com/myesui/uuid;github.com/nsf/jsondiff;github.com/olekukonko/tablewriter;github.com/pkg/errors;github.com/sirupsen/logrus;github.com/skratchdot/open-golang;github.com/ssor/bom;github.com/stretchr/testify;github.com/therecipe/qt;github.com/twinj/uuid;github.com/urfave/cli;go.etcd.io/bbolt;golang.org/x/crypto;golang.org/x/net;golang.org/x/text;gopkg.in/stretchr/testify.v1;;Font Awesome 4.7.0;;Qt 5.13 by Qt group;" diff --git a/internal/transfer/message.go b/internal/transfer/message.go index dcec9288..f710fc52 100644 --- a/internal/transfer/message.go +++ b/internal/transfer/message.go @@ -58,6 +58,7 @@ type MessageStatus struct { targetID string // Message ID at the target (if any). bodyHash string // Hash of the message body. + skipped bool exported bool imported bool exportErr error @@ -96,7 +97,7 @@ func (status *MessageStatus) setDetailsFromHeader(header mail.Header) { } func (status *MessageStatus) hasError(includeMissing bool) bool { - return status.exportErr != nil || status.importErr != nil || (includeMissing && !status.imported) + return status.exportErr != nil || status.importErr != nil || (includeMissing && !status.skipped && !status.imported) } // GetErrorMessage returns error message. @@ -105,6 +106,9 @@ func (status *MessageStatus) GetErrorMessage() string { } func (status *MessageStatus) getErrorMessage(includeMissing bool) string { + if status.skipped { + return "" + } if status.exportErr != nil { return fmt.Sprintf("failed to export: %s", status.exportErr) } diff --git a/internal/transfer/progress.go b/internal/transfer/progress.go index 375eb37b..bc2dc6c9 100644 --- a/internal/transfer/progress.go +++ b/internal/transfer/progress.go @@ -140,6 +140,19 @@ func (p *Progress) addMessage(messageID string, sourceNames, targetNames []strin } } +// messageSkipped should be called once the message is skipped due to some +// filter such as time or folder and so on. +func (p *Progress) messageSkipped(messageID string) { + p.lock.Lock() + defer p.lock.Unlock() + defer p.update() + + p.log.WithField("id", messageID).Debug("Message skipped") + + p.messageStatuses[messageID].skipped = true + p.logMessage(messageID) +} + // messageExported should be called right before message is exported. func (p *Progress) messageExported(messageID string, body []byte, err error) { p.lock.Lock() @@ -330,35 +343,40 @@ func (p *Progress) GetFailedMessages() []*MessageStatus { } // GetCounts returns counts of exported and imported messages. -func (p *Progress) GetCounts() (failed, imported, exported, added, total uint) { +func (p *Progress) GetCounts() ProgressCounts { p.lock.Lock() defer p.lock.Unlock() + counts := ProgressCounts{} + // Return counts only once total is estimated or the process already // ended (for a case when it ended quickly to report it correctly). if p.updateCh != nil && !p.messageCounted { - return + return counts } // Include lost messages in the process only when transfer is done. includeMissing := p.updateCh == nil for _, mailboxCount := range p.messageCounts { - total += mailboxCount + counts.Total += mailboxCount } for _, status := range p.messageStatuses { - added++ + counts.Added++ + if status.skipped { + counts.Skipped++ + } if status.exported { - exported++ + counts.Exported++ } if status.imported { - imported++ + counts.Imported++ } if status.hasError(includeMissing) { - failed++ + counts.Failed++ } } - return + return counts } // GenerateBugReport generates similar file to import log except private information. diff --git a/internal/transfer/progress_counts.go b/internal/transfer/progress_counts.go new file mode 100644 index 00000000..2a3d1531 --- /dev/null +++ b/internal/transfer/progress_counts.go @@ -0,0 +1,28 @@ +// Copyright (c) 2020 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package transfer + +// ProgressCounts holds counts counted by Progress. +type ProgressCounts struct { + Failed, + Skipped, + Imported, + Exported, + Added, + Total uint +} diff --git a/internal/transfer/progress_test.go b/internal/transfer/progress_test.go index e005a1cb..c0ff22e1 100644 --- a/internal/transfer/progress_test.go +++ b/internal/transfer/progress_test.go @@ -39,8 +39,8 @@ func TestProgressUpdateCount(t *testing.T) { progress.finish() - _, _, _, _, total := progress.GetCounts() //nolint[dogsled] - r.Equal(t, uint(42), total) + counts := progress.GetCounts() + r.Equal(t, uint(42), counts.Total) } func TestProgressAddingMessages(t *testing.T) { @@ -66,13 +66,18 @@ func TestProgressAddingMessages(t *testing.T) { progress.messageExported("msg4", []byte(""), errors.New("failed export")) progress.messageImported("msg4", "", nil) + // msg5 is skipped. + progress.addMessage("msg5", []string{}, []string{}) + progress.messageSkipped("msg5") + progress.finish() - failed, imported, exported, added, _ := progress.GetCounts() - a.Equal(t, uint(4), added) - a.Equal(t, uint(2), exported) - a.Equal(t, uint(2), imported) - a.Equal(t, uint(3), failed) + counts := progress.GetCounts() + a.Equal(t, uint(5), counts.Added) + a.Equal(t, uint(2), counts.Exported) + a.Equal(t, uint(2), counts.Imported) + a.Equal(t, uint(1), counts.Skipped) + a.Equal(t, uint(3), counts.Failed) errorsMap := map[string]string{} for _, status := range progress.GetFailedMessages() { diff --git a/internal/transfer/provider_eml_source.go b/internal/transfer/provider_eml_source.go index 2576d49b..7211e174 100644 --- a/internal/transfer/provider_eml_source.go +++ b/internal/transfer/provider_eml_source.go @@ -82,8 +82,6 @@ func (p *EMLProvider) getFilePathsPerFolder(rules transferRules) (map[string][]s } func (p *EMLProvider) exportMessages(rule *Rule, filePaths []string, progress *Progress, ch chan<- Message) { - count := uint(len(filePaths)) - for _, filePath := range filePaths { if progress.shouldStop() { break @@ -91,6 +89,8 @@ func (p *EMLProvider) exportMessages(rule *Rule, filePaths []string, progress *P msg, err := p.exportMessage(rule, filePath) + progress.addMessage(filePath, msg.sourceNames(), msg.targetNames()) + // Read and check time in body only if the rule specifies it // to not waste energy. if err == nil && rule.HasTimeLimit() { @@ -99,17 +99,11 @@ func (p *EMLProvider) exportMessages(rule *Rule, filePaths []string, progress *P err = msgTimeErr } else if !rule.isTimeInRange(msgTime) { log.WithField("msg", filePath).Debug("Message skipped due to time") - - count-- - progress.updateCount(rule.SourceMailbox.Name, count) + progress.messageSkipped(filePath) continue } } - // addMessage is called after time check to not report message - // which should not be exported but any error from reading body - // or parsing time is reported as an error. - progress.addMessage(filePath, msg.sourceNames(), msg.targetNames()) progress.messageExported(filePath, msg.Body, err) if err == nil { ch <- msg diff --git a/internal/transfer/provider_mbox_source.go b/internal/transfer/provider_mbox_source.go index 5b11ef17..e0a64d11 100644 --- a/internal/transfer/provider_mbox_source.go +++ b/internal/transfer/provider_mbox_source.go @@ -114,7 +114,6 @@ func (p *MBOXProvider) transferTo(rules transferRules, progress *Progress, ch ch } index := 0 - count := 0 for { if progress.shouldStop() { break @@ -133,24 +132,18 @@ func (p *MBOXProvider) transferTo(rules transferRules, progress *Progress, ch ch msg, err := p.exportMessage(rules, folderName, id, msgReader) + progress.addMessage(id, msg.sourceNames(), msg.targetNames()) + if err == nil && len(msg.Targets) == 0 { - // Here should be called progress.messageSkipped(id) once we have - // this feature, and following progress.updateCount can be removed. + progress.messageSkipped(id) continue } - count++ - - // addMessage is called after time check to not report message - // which should not be exported but any error from reading body - // or parsing time is reported as an error. - progress.addMessage(id, msg.sourceNames(), msg.targetNames()) progress.messageExported(id, msg.Body, err) if err == nil { ch <- msg } } - progress.updateCount(filePath, uint(count)) } func (p *MBOXProvider) exportMessage(rules transferRules, folderName, id string, msgReader io.Reader) (Message, error) { @@ -177,7 +170,7 @@ func (p *MBOXProvider) getMessageRules(rules transferRules, folderName, id strin folderRule, err := rules.getRuleBySourceMailboxName(folderName) if err != nil { log.WithField("msg", id).WithField("source", folderName).Debug("Message source doesn't have a rule") - } else { + } else if folderRule.Active { msgRules = append(msgRules, folderRule) } @@ -191,7 +184,9 @@ func (p *MBOXProvider) getMessageRules(rules transferRules, folderName, id strin log.WithField("msg", id).WithField("source", label).Debug("Message source doesn't have a rule") continue } - msgRules = append(msgRules, rule) + if rule.Active { + msgRules = append(msgRules, rule) + } } } diff --git a/internal/transfer/provider_mbox_test.go b/internal/transfer/provider_mbox_test.go index acd54406..51f1028f 100644 --- a/internal/transfer/provider_mbox_test.go +++ b/internal/transfer/provider_mbox_test.go @@ -155,6 +155,29 @@ func TestMBOXProviderTransferFromTo(t *testing.T) { }) } +func TestMBOXProviderGetMessageRules(t *testing.T) { + provider := newTestMBOXProvider("") + + body := []byte(`Subject: Test +X-Gmail-Labels: foo,bar + +`) + rules := transferRules{ + rules: map[string]*Rule{ + "1": {Active: true, SourceMailbox: Mailbox{Name: "folder"}}, + "2": {Active: false, SourceMailbox: Mailbox{Name: "foo"}}, + "3": {Active: true, SourceMailbox: Mailbox{Name: "bar"}}, + "4": {Active: false, SourceMailbox: Mailbox{Name: "baz"}}, + "5": {Active: true, SourceMailbox: Mailbox{Name: "other"}}, + }, + } + + gotRules := provider.getMessageRules(rules, "folder", "id", body) + r.Equal(t, 2, len(gotRules)) + r.Equal(t, "folder", gotRules[0].SourceMailbox.Name) + r.Equal(t, "bar", gotRules[1].SourceMailbox.Name) +} + func TestMBOXProviderGetMessageTargetsReturnsOnlyOneFolder(t *testing.T) { provider := newTestMBOXProvider("") diff --git a/test/transfer_checks_test.go b/test/transfer_checks_test.go index 0ae0b745..4c8e55f9 100644 --- a/test/transfer_checks_test.go +++ b/test/transfer_checks_test.go @@ -65,15 +65,15 @@ func progressFinishedWith(wantResponse string) error { func transferExportedNumberOfMessages(wantCount int) error { progress := ctx.GetTransferProgress() - _, _, exported, _, _ := progress.GetCounts() //nolint[dogsled] - a.Equal(ctx.GetTestingT(), uint(wantCount), exported) + counts := progress.GetCounts() + a.Equal(ctx.GetTestingT(), uint(wantCount), counts.Exported) return ctx.GetTestingError() } func transferImportedNumberOfMessages(wantCount int) error { progress := ctx.GetTransferProgress() - _, imported, _, _, _ := progress.GetCounts() //nolint[dogsled] - a.Equal(ctx.GetTestingT(), uint(wantCount), imported) + counts := progress.GetCounts() + a.Equal(ctx.GetTestingT(), uint(wantCount), counts.Imported) return ctx.GetTestingError() }