Import/Export GUI

internal/transfer/mailbox.go

@@ -33,6 +33,11 @@ type Mailbox struct {
 	IsExclusive bool
 }
 
+// IsSystemFolder returns true when ID corresponds to PM system folder.
+func (m Mailbox) IsSystemFolder() bool {
+	return pmapi.IsSystemLabel(m.ID)
+}
+
 // Hash returns unique identifier to be used for matching.
 func (m Mailbox) Hash() string {
 	return fmt.Sprintf("%x", sha256.Sum256([]byte(m.Name)))

internal/transfer/progress.go

@@ -198,7 +198,7 @@ func (p *Progress) callWrap(callback func() error) {
 			break
 		}
 
-		p.Pause(err.Error())
+		p.Pause("paused due to " + err.Error())
 	}
 }
 
@@ -333,3 +333,11 @@ func (p *Progress) GenerateBugReport() []byte {
 	}
 	return bugReport.getData()
 }
+
+func (p *Progress) FileReport() (path string) {
+	if r := p.fileReport; r != nil {
+		path = r.path
+	}
+
+	return
+}

internal/transfer/provider_eml.go

@@ -39,9 +39,13 @@ func (p *EMLProvider) ID() string {
 // Mailboxes returns all available folder names from root of EML files.
 // In case the same folder name is used more than once (for example root/a/foo
 // and root/b/foo), it's treated as the same folder.
-func (p *EMLProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) {
+func (p *EMLProvider) Mailboxes(includeEmpty, includeAllMail bool) (mailboxes []Mailbox, err error) {
+	// Special case for exporting--we don't know the path before setup if finished.
+	if p.root == "" {
+		return
+	}
+
 	var folderNames []string
-	var err error
 	if includeEmpty {
 		folderNames, err = getFolderNames(p.root)
 	} else {
@@ -51,7 +55,6 @@ func (p *EMLProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, e
 		return nil, err
 	}
 
-	mailboxes := []Mailbox{}
 	for _, folderName := range folderNames {
 		mailboxes = append(mailboxes, Mailbox{
 			ID:          "",

internal/transfer/provider_imap_errors.go

@@ -0,0 +1,66 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+// imapError is base for all IMAP errors.
+type imapError struct {
+	Message string
+	Err     error
+}
+
+func (e imapError) Error() string {
+	return e.Message + ": " + e.Err.Error()
+}
+
+func (e imapError) Unwrap() error {
+	return e.Err
+}
+
+func (e imapError) Cause() error {
+	return e.Err
+}
+
+// ErrIMAPConnection is error representing connection issues.
+type ErrIMAPConnection struct {
+	imapError
+}
+
+func (e ErrIMAPConnection) Is(target error) bool {
+	_, ok := target.(*ErrIMAPConnection)
+	return ok
+}
+
+// ErrIMAPAuth is error representing authentication issues.
+type ErrIMAPAuth struct {
+	imapError
+}
+
+func (e ErrIMAPAuth) Is(target error) bool {
+	_, ok := target.(*ErrIMAPAuth)
+	return ok
+}
+
+// ErrIMAPAuthMethod is error representing wrong auth method.
+type ErrIMAPAuthMethod struct {
+	imapError
+}
+
+func (e ErrIMAPAuthMethod) Is(target error) bool {
+	_, ok := target.(*ErrIMAPAuthMethod)
+	return ok
+}

internal/transfer/provider_imap_utils.go

@@ -137,7 +137,7 @@ func (p *IMAPProvider) auth() error { //nolint[funlen]
 	log.Info("Connecting to server")
 
 	if _, err := net.DialTimeout("tcp", p.addr, imapDialTimeout); err != nil {
-		return errors.Wrap(err, "failed to dial server")
+		return ErrIMAPConnection{imapError{Err: err, Message: "failed to dial server"}}
 	}
 
 	var client *imapClient.Client
@@ -149,7 +149,7 @@ func (p *IMAPProvider) auth() error { //nolint[funlen]
 		client, err = imapClient.DialTLS(p.addr, nil)
 	}
 	if err != nil {
-		return errors.Wrap(err, "failed to connect to server")
+		return ErrIMAPConnection{imapError{Err: err, Message: "failed to connect to server"}}
 	}
 
 	client.ErrorLog = &imapErrorLogger{logrus.WithField("pkg", "imap-client")}
@@ -170,7 +170,7 @@ func (p *IMAPProvider) auth() error { //nolint[funlen]
 	capability, err := p.client.Capability()
 	log.WithField("capability", capability).WithError(err).Debug("Server capability")
 	if err != nil {
-		return errors.Wrap(err, "failed to get capabilities")
+		return ErrIMAPConnection{imapError{Err: err, Message: "failed to get capabilities"}}
 	}
 
 	// SASL AUTH PLAIN
@@ -178,7 +178,7 @@ func (p *IMAPProvider) auth() error { //nolint[funlen]
 		log.Debug("Trying plain auth")
 		authPlain := sasl.NewPlainClient("", p.username, p.password)
 		if err = p.client.Authenticate(authPlain); err != nil {
-			return errors.Wrap(err, "plain auth failed")
+			return ErrIMAPAuth{imapError{Err: err, Message: "plain auth failed"}}
 		}
 	}
 
@@ -186,12 +186,12 @@ func (p *IMAPProvider) auth() error { //nolint[funlen]
 	if ok, _ := p.client.Support("IMAP4rev1"); p.client.State() == imap.NotAuthenticatedState && ok {
 		log.Debug("Trying login")
 		if err = p.client.Login(p.username, p.password); err != nil {
-			return errors.Wrap(err, "login failed")
+			return ErrIMAPAuth{imapError{Err: err, Message: "login failed"}}
 		}
 	}
 
 	if p.client.State() == imap.NotAuthenticatedState {
-		return errors.New("unknown auth method")
+		return ErrIMAPAuthMethod{imapError{Err: err, Message: "unknown auth method"}}
 	}
 
 	log.Info("Logged in")

internal/transfer/provider_pmapi.go

@@ -38,20 +38,24 @@ type PMAPIProvider struct {
 
 // NewPMAPIProvider returns new PMAPIProvider.
 func NewPMAPIProvider(clientManager ClientManager, userID, addressID string) (*PMAPIProvider, error) {
-	keyRing, err := clientManager.GetClient(userID).KeyRingForAddressID(addressID)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to get key ring")
-	}
-
-	return &PMAPIProvider{
+	provider := &PMAPIProvider{
 		clientManager: clientManager,
 		userID:        userID,
 		addressID:     addressID,
-		keyRing:       keyRing,
 
 		importMsgReqMap:  map[string]*pmapi.ImportMsgReq{},
 		importMsgReqSize: 0,
-	}, nil
+	}
+
+	if addressID != "" {
+		keyRing, err := clientManager.GetClient(userID).KeyRingForAddressID(addressID)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to get key ring")
+		}
+		provider.keyRing = keyRing
+	}
+
+	return provider, nil
 }
 
 func (p *PMAPIProvider) client() pmapi.Client {
@@ -86,7 +90,14 @@ func (p *PMAPIProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox,
 		}
 	}
 
-	mailboxes := getSystemMailboxes(includeAllMail)
+	mailboxes := []Mailbox{}
+	for _, mailbox := range getSystemMailboxes(includeAllMail) {
+		if !includeEmpty && emptyLabelsMap[mailbox.ID] {
+			continue
+		}
+
+		mailboxes = append(mailboxes, mailbox)
+	}
 	for _, label := range sortedLabels {
 		if !includeEmpty && emptyLabelsMap[label.ID] {
 			continue

internal/transfer/provider_pmapi_source.go

@@ -86,14 +86,15 @@ func (p *PMAPIProvider) transferTo(rule *Rule, progress *Progress, ch chan<- Mes
 		progress.callWrap(func() error {
 			desc := false
 			pmapiMessages, count, err := p.listMessages(&pmapi.MessagesFilter{
-				LabelID:  rule.SourceMailbox.ID,
-				Begin:    rule.FromTime,
-				End:      rule.ToTime,
-				BeginID:  nextID,
-				PageSize: pmapiListPageSize,
-				Page:     0,
-				Sort:     "ID",
-				Desc:     &desc,
+				AddressID: p.addressID,
+				LabelID:   rule.SourceMailbox.ID,
+				Begin:     rule.FromTime,
+				End:       rule.ToTime,
+				BeginID:   nextID,
+				PageSize:  pmapiListPageSize,
+				Page:      0,
+				Sort:      "ID",
+				Desc:      &desc,
 			})
 			if err != nil {
 				return err

internal/transfer/rules.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"time"
 
@@ -42,6 +43,11 @@ type transferRules struct {
 	// E.g., every message will be imported into this mailbox.
 	globalMailbox *Mailbox
 
+	// globalFromTime and globalToTime is applied to every rule right
+	// before the transfer (propagateGlobalTime has to be called).
+	globalFromTime int64
+	globalToTime   int64
+
 	// skipEncryptedMessages determines whether message which cannot
 	// be decrypted should be exported or skipped.
 	skipEncryptedMessages bool
@@ -81,10 +87,18 @@ func (r *transferRules) setGlobalMailbox(mailbox *Mailbox) {
 }
 
 func (r *transferRules) setGlobalTimeLimit(fromTime, toTime int64) {
+	r.globalFromTime = fromTime
+	r.globalToTime = toTime
+}
+
+func (r *transferRules) propagateGlobalTime() {
+	if r.globalFromTime == 0 && r.globalToTime == 0 {
+		return
+	}
 	for _, rule := range r.rules {
 		if !rule.HasTimeLimit() {
-			rule.FromTime = fromTime
-			rule.ToTime = toTime
+			rule.FromTime = r.globalFromTime
+			rule.ToTime = r.globalToTime
 		}
 	}
 }
@@ -122,8 +136,9 @@ func (r *transferRules) setDefaultRules(sourceMailboxes []Mailbox, targetMailbox
 		}
 
 		targetMailboxes := sourceMailbox.findMatchingMailboxes(targetMailboxes)
-		if len(targetMailboxes) == 0 {
-			targetMailboxes = defaultCallback(sourceMailbox)
+
+		if !containsExclusive(targetMailboxes) {
+			targetMailboxes = append(targetMailboxes, defaultCallback(sourceMailbox)...)
 		}
 
 		active := true
@@ -147,10 +162,14 @@ func (r *transferRules) setDefaultRules(sourceMailboxes []Mailbox, targetMailbox
 		}
 	}
 
-	for _, rule := range r.rules {
-		if !rule.Active {
-			continue
-		}
+	// There is no point showing rule which has no action (i.e., source mailbox
+	// is not available).
+	// A good reason to keep all rules and only deactivate them would be for
+	// multiple imports from different sources with the same or similar enough
+	// mailbox setup to reuse configuration. That is very minor feature which
+	// can be implemented in more reasonable way by allowing users to save and
+	// load configurations.
+	for key, rule := range r.rules {
 		found := false
 		for _, sourceMailbox := range sourceMailboxes {
 			if sourceMailbox.Name == rule.SourceMailbox.Name {
@@ -158,7 +177,7 @@ func (r *transferRules) setDefaultRules(sourceMailboxes []Mailbox, targetMailbox
 			}
 		}
 		if !found {
-			rule.Active = false
+			delete(r.rules, key)
 		}
 	}
 
@@ -216,6 +235,7 @@ func (r *transferRules) getRules() []*Rule {
 	for _, rule := range r.rules {
 		rules = append(rules, rule)
 	}
+	sort.Sort(byRuleOrder(rules))
 	return rules
 }
 
@@ -288,3 +308,59 @@ func (r *Rule) TargetMailboxNames() (names []string) {
 	}
 	return
 }
+
+// byRuleOrder implements sort.Interface. Sort order:
+//  * System folders first (as defined in getSystemMailboxes).
+//  * Custom folders by name.
+//  * Custom labels by name.
+type byRuleOrder []*Rule
+
+func (a byRuleOrder) Len() int {
+	return len(a)
+}
+
+func (a byRuleOrder) Swap(i, j int) {
+	a[i], a[j] = a[j], a[i]
+}
+
+func (a byRuleOrder) Less(i, j int) bool {
+	if a[i].SourceMailbox.IsExclusive && !a[j].SourceMailbox.IsExclusive {
+		return true
+	}
+	if !a[i].SourceMailbox.IsExclusive && a[j].SourceMailbox.IsExclusive {
+		return false
+	}
+
+	iSystemIndex := -1
+	jSystemIndex := -1
+	for index, systemFolders := range getSystemMailboxes(true) {
+		if a[i].SourceMailbox.Name == systemFolders.Name {
+			iSystemIndex = index
+		}
+		if a[j].SourceMailbox.Name == systemFolders.Name {
+			jSystemIndex = index
+		}
+	}
+	if iSystemIndex != -1 && jSystemIndex == -1 {
+		return true
+	}
+	if iSystemIndex == -1 && jSystemIndex != -1 {
+		return false
+	}
+	if iSystemIndex != -1 && jSystemIndex != -1 {
+		return iSystemIndex < jSystemIndex
+	}
+
+	return a[i].SourceMailbox.Name < a[j].SourceMailbox.Name
+}
+
+// containsExclusive returns true if there is at least one exclusive mailbox.
+func containsExclusive(mailboxes []Mailbox) bool {
+	for _, m := range mailboxes {
+		if m.IsExclusive {
+			return true
+		}
+	}
+
+	return false
+}

internal/transfer/rules_test.go

@@ -86,6 +86,7 @@ func TestSetGlobalTimeLimit(t *testing.T) {
 	r.NoError(t, rules.setRule(mailboxB, []Mailbox{}, 0, 0))
 
 	rules.setGlobalTimeLimit(30, 40)
+	rules.propagateGlobalTime()
 
 	r.Equal(t, map[string]*Rule{
 		mailboxA.Hash(): {Active: true, SourceMailbox: mailboxA, TargetMailboxes: []Mailbox{}, FromTime: 10, ToTime: 20},
@@ -154,7 +155,6 @@ func TestSetDefaultRulesDeactivateMissing(t *testing.T) {
 
 	r.Equal(t, map[string]*Rule{
 		mailboxA.Hash(): {Active: true, SourceMailbox: mailboxA, TargetMailboxes: []Mailbox{mailboxB}, FromTime: 0, ToTime: 0},
-		mailboxB.Hash(): {Active: false, SourceMailbox: mailboxB, TargetMailboxes: []Mailbox{mailboxB}, FromTime: 0, ToTime: 0},
 	}, rules.rules)
 }
 
@@ -208,3 +208,40 @@ func generateTimeRule(from, to int64) Rule {
 		ToTime:          to,
 	}
 }
+
+func TestOrderRules(t *testing.T) {
+	wantMailboxOrder := []Mailbox{
+		{Name: "Inbox", IsExclusive: true},
+		{Name: "Drafts", IsExclusive: true},
+		{Name: "Sent", IsExclusive: true},
+		{Name: "Starred", IsExclusive: true},
+		{Name: "Archive", IsExclusive: true},
+		{Name: "Spam", IsExclusive: true},
+		{Name: "All Mail", IsExclusive: true},
+		{Name: "Folder A", IsExclusive: true},
+		{Name: "Folder B", IsExclusive: true},
+		{Name: "Folder C", IsExclusive: true},
+		{Name: "Label A", IsExclusive: false},
+		{Name: "Label B", IsExclusive: false},
+		{Name: "Label C", IsExclusive: false},
+	}
+	wantMailboxNames := []string{}
+
+	rules := map[string]*Rule{}
+	for _, mailbox := range wantMailboxOrder {
+		wantMailboxNames = append(wantMailboxNames, mailbox.Name)
+		rules[mailbox.Hash()] = &Rule{
+			SourceMailbox: mailbox,
+		}
+	}
+	transferRules := transferRules{
+		rules: rules,
+	}
+
+	gotMailboxNames := []string{}
+	for _, rule := range transferRules.getRules() {
+		gotMailboxNames = append(gotMailboxNames, rule.SourceMailbox.Name)
+	}
+
+	r.Equal(t, wantMailboxNames, gotMailboxNames)
+}

internal/transfer/transfer.go

@@ -31,12 +31,14 @@ var log = logrus.WithField("pkg", "transfer") //nolint[gochecknoglobals]
 // Transfer is facade on top of import rules, progress manager and source
 // and target providers. This is the main object which should be used.
 type Transfer struct {
-	panicHandler PanicHandler
-	id           string
-	dir          string
-	rules        transferRules
-	source       SourceProvider
-	target       TargetProvider
+	panicHandler    PanicHandler
+	id              string
+	dir             string
+	rules           transferRules
+	source          SourceProvider
+	target          TargetProvider
+	sourceMboxCache []Mailbox
+	targetMboxCache []Mailbox
 }
 
 // New creates Transfer for specific source and target. Usage:
@@ -127,23 +129,33 @@ func (t *Transfer) GetRules() []*Rule {
 }
 
 // SourceMailboxes returns mailboxes available at source side.
-func (t *Transfer) SourceMailboxes() ([]Mailbox, error) {
-	return t.source.Mailboxes(false, true)
+func (t *Transfer) SourceMailboxes() (m []Mailbox, err error) {
+	if t.sourceMboxCache == nil {
+		t.sourceMboxCache, err = t.source.Mailboxes(false, true)
+	}
+	return t.sourceMboxCache, err
 }
 
 // TargetMailboxes returns mailboxes available at target side.
-func (t *Transfer) TargetMailboxes() ([]Mailbox, error) {
-	return t.target.Mailboxes(true, false)
+func (t *Transfer) TargetMailboxes() (m []Mailbox, err error) {
+	if t.targetMboxCache == nil {
+		t.targetMboxCache, err = t.target.Mailboxes(true, false)
+	}
+	return t.targetMboxCache, err
 }
 
 // CreateTargetMailbox creates mailbox in target provider.
 func (t *Transfer) CreateTargetMailbox(mailbox Mailbox) (Mailbox, error) {
+	t.targetMboxCache = nil
+
 	return t.target.CreateMailbox(mailbox)
 }
 
 // ChangeTarget changes the target. It is safe to change target for export,
 // must not be changed for import. Do not set after you started transfer.
 func (t *Transfer) ChangeTarget(target TargetProvider) {
+	t.targetMboxCache = nil
+
 	t.target = target
 }
 
@@ -151,6 +163,7 @@ func (t *Transfer) ChangeTarget(target TargetProvider) {
 func (t *Transfer) Start() *Progress {
 	log.Debug("Transfer started")
 	t.rules.save()
+	t.rules.propagateGlobalTime()
 
 	log := log.WithField("id", t.id)
 	reportFile := newFileReport(t.dir, t.id)