Import/Export backend prep

internal/transfer/mailbox.go

@@ -0,0 +1,69 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"strings"
+)
+
+// Mailbox is universal data holder of mailbox details for every provider.
+type Mailbox struct {
+	ID          string
+	Name        string
+	Color       string
+	IsExclusive bool
+}
+
+// Hash returns unique identifier to be used for matching.
+func (m Mailbox) Hash() string {
+	return fmt.Sprintf("%x", sha256.Sum256([]byte(m.Name)))
+}
+
+// findMatchingMailboxes returns all matching mailboxes from `mailboxes`.
+// Only one exclusive mailbox is returned.
+func (m Mailbox) findMatchingMailboxes(mailboxes []Mailbox) []Mailbox {
+	nameVariants := []string{}
+	if strings.Contains(m.Name, "/") || strings.Contains(m.Name, "|") {
+		for _, slashPart := range strings.Split(m.Name, "/") {
+			for _, part := range strings.Split(slashPart, "|") {
+				nameVariants = append(nameVariants, strings.ToLower(part))
+			}
+		}
+	}
+	nameVariants = append(nameVariants, strings.ToLower(m.Name))
+
+	isExclusiveIncluded := false
+	matches := []Mailbox{}
+	for i := range nameVariants {
+		nameVariant := nameVariants[len(nameVariants)-1-i]
+		for _, mailbox := range mailboxes {
+			if mailbox.IsExclusive && isExclusiveIncluded {
+				continue
+			}
+			if strings.ToLower(mailbox.Name) == nameVariant {
+				matches = append(matches, mailbox)
+				if mailbox.IsExclusive {
+					isExclusiveIncluded = true
+				}
+			}
+		}
+	}
+	return matches
+}

internal/transfer/mailbox_test.go

@@ -0,0 +1,61 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"testing"
+
+	r "github.com/stretchr/testify/require"
+)
+
+func TestFindMatchingMailboxes(t *testing.T) {
+	mailboxes := []Mailbox{
+		{Name: "Inbox", IsExclusive: true},
+		{Name: "Sent", IsExclusive: true},
+		{Name: "Archive", IsExclusive: true},
+		{Name: "Foo", IsExclusive: false},
+		{Name: "hello/world", IsExclusive: true},
+		{Name: "Hello", IsExclusive: false},
+		{Name: "WORLD", IsExclusive: true},
+	}
+
+	tests := []struct {
+		name      string
+		wantNames []string
+	}{
+		{"inbox", []string{"Inbox"}},
+		{"foo", []string{"Foo"}},
+		{"hello", []string{"Hello"}},
+		{"world", []string{"WORLD"}},
+		{"hello/world", []string{"hello/world", "Hello"}},
+		{"hello|world", []string{"WORLD", "Hello"}},
+		{"nomailbox", []string{}},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			mailbox := Mailbox{Name: tc.name}
+			got := mailbox.findMatchingMailboxes(mailboxes)
+			gotNames := []string{}
+			for _, m := range got {
+				gotNames = append(gotNames, m.Name)
+			}
+			r.Equal(t, tc.wantNames, gotNames)
+		})
+	}
+}

internal/transfer/message.go

@@ -0,0 +1,100 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"fmt"
+	"mime"
+	"net/mail"
+	"time"
+)
+
+// Message is data holder passed between import and export.
+type Message struct {
+	ID      string
+	Unread  bool
+	Body    []byte
+	Source  Mailbox
+	Targets []Mailbox
+}
+
+// MessageStatus holds status for message used by progress manager.
+type MessageStatus struct {
+	eventTime time.Time // Time of adding message to the process.
+	rule      *Rule     // Rule with source and target mailboxes.
+	SourceID  string    // Message ID at the source.
+	targetID  string    // Message ID at the target (if any).
+	bodyHash  string    // Hash of the message body.
+
+	exported  bool
+	imported  bool
+	exportErr error
+	importErr error
+
+	// Info about message displayed to user.
+	// This is needed only for failed messages, but we cannot know in advance
+	// which message will fail. We could clear it once the message passed
+	// without any error. However, if we say one message takes about 100 bytes
+	// in average, it's about 100 MB per million of messages, which is fine.
+	Subject string
+	From    string
+	Time    time.Time
+}
+
+func (status *MessageStatus) setDetailsFromHeader(header mail.Header) {
+	dec := &mime.WordDecoder{}
+
+	status.Subject = header.Get("subject")
+	if subject, err := dec.Decode(status.Subject); err == nil {
+		status.Subject = subject
+	}
+
+	status.From = header.Get("from")
+	if from, err := dec.Decode(status.From); err == nil {
+		status.From = from
+	}
+
+	if msgTime, err := header.Date(); err == nil {
+		status.Time = msgTime
+	}
+}
+
+func (status *MessageStatus) hasError(includeMissing bool) bool {
+	return status.exportErr != nil || status.importErr != nil || (includeMissing && !status.imported)
+}
+
+// GetErrorMessage returns error message.
+func (status *MessageStatus) GetErrorMessage() string {
+	return status.getErrorMessage(true)
+}
+
+func (status *MessageStatus) getErrorMessage(includeMissing bool) string {
+	if status.exportErr != nil {
+		return fmt.Sprintf("failed to export: %s", status.exportErr)
+	}
+	if status.importErr != nil {
+		return fmt.Sprintf("failed to import: %s", status.importErr)
+	}
+	if includeMissing && !status.imported {
+		if !status.exported {
+			return "failed to import: lost before read"
+		}
+		return "failed to import: lost in the process"
+	}
+	return ""
+}

internal/transfer/mocks/mocks.go

@@ -0,0 +1,97 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/ProtonMail/proton-bridge/internal/transfer (interfaces: PanicHandler,ClientManager)
+
+// Package mocks is a generated GoMock package.
+package mocks
+
+import (
+	pmapi "github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	gomock "github.com/golang/mock/gomock"
+	reflect "reflect"
+)
+
+// MockPanicHandler is a mock of PanicHandler interface
+type MockPanicHandler struct {
+	ctrl     *gomock.Controller
+	recorder *MockPanicHandlerMockRecorder
+}
+
+// MockPanicHandlerMockRecorder is the mock recorder for MockPanicHandler
+type MockPanicHandlerMockRecorder struct {
+	mock *MockPanicHandler
+}
+
+// NewMockPanicHandler creates a new mock instance
+func NewMockPanicHandler(ctrl *gomock.Controller) *MockPanicHandler {
+	mock := &MockPanicHandler{ctrl: ctrl}
+	mock.recorder = &MockPanicHandlerMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockPanicHandler) EXPECT() *MockPanicHandlerMockRecorder {
+	return m.recorder
+}
+
+// HandlePanic mocks base method
+func (m *MockPanicHandler) HandlePanic() {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "HandlePanic")
+}
+
+// HandlePanic indicates an expected call of HandlePanic
+func (mr *MockPanicHandlerMockRecorder) HandlePanic() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HandlePanic", reflect.TypeOf((*MockPanicHandler)(nil).HandlePanic))
+}
+
+// MockClientManager is a mock of ClientManager interface
+type MockClientManager struct {
+	ctrl     *gomock.Controller
+	recorder *MockClientManagerMockRecorder
+}
+
+// MockClientManagerMockRecorder is the mock recorder for MockClientManager
+type MockClientManagerMockRecorder struct {
+	mock *MockClientManager
+}
+
+// NewMockClientManager creates a new mock instance
+func NewMockClientManager(ctrl *gomock.Controller) *MockClientManager {
+	mock := &MockClientManager{ctrl: ctrl}
+	mock.recorder = &MockClientManagerMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockClientManager) EXPECT() *MockClientManagerMockRecorder {
+	return m.recorder
+}
+
+// CheckConnection mocks base method
+func (m *MockClientManager) CheckConnection() error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "CheckConnection")
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// CheckConnection indicates an expected call of CheckConnection
+func (mr *MockClientManagerMockRecorder) CheckConnection() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckConnection", reflect.TypeOf((*MockClientManager)(nil).CheckConnection))
+}
+
+// GetClient mocks base method
+func (m *MockClientManager) GetClient(arg0 string) pmapi.Client {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetClient", arg0)
+	ret0, _ := ret[0].(pmapi.Client)
+	return ret0
+}
+
+// GetClient indicates an expected call of GetClient
+func (mr *MockClientManagerMockRecorder) GetClient(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetClient", reflect.TypeOf((*MockClientManager)(nil).GetClient), arg0)
+}

internal/transfer/progress.go

@@ -0,0 +1,331 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+// Progress maintains progress between import, export and user interface.
+// Import and export update progress about processing messages and progress
+// informs user interface, vice versa action (such as pause or resume) from
+// user interface is passed down to import and export.
+type Progress struct {
+	log  *logrus.Entry
+	lock sync.RWMutex
+
+	updateCh        chan struct{}
+	messageCounts   map[string]uint
+	messageStatuses map[string]*MessageStatus
+	pauseReason     string
+	isStopped       bool
+	fatalError      error
+	fileReport      *fileReport
+}
+
+func newProgress(log *logrus.Entry, fileReport *fileReport) Progress {
+	return Progress{
+		log: log,
+
+		updateCh:        make(chan struct{}),
+		messageCounts:   map[string]uint{},
+		messageStatuses: map[string]*MessageStatus{},
+		fileReport:      fileReport,
+	}
+}
+
+// update is helper to notify listener for updates.
+func (p *Progress) update() {
+	if p.updateCh == nil {
+		// If the progress was ended by fatal instead finish, we ignore error.
+		if p.fatalError != nil {
+			return
+		}
+		panic("update should not be called after finish was called")
+	}
+
+	// In case no one listens for an update, do not block the progress.
+	select {
+	case p.updateCh <- struct{}{}:
+	case <-time.After(100 * time.Millisecond):
+	}
+}
+
+// start should be called before anything starts.
+func (p *Progress) start() {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+}
+
+// finish should be called as the last call once everything is done.
+func (p *Progress) finish() {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	p.cleanUpdateCh()
+}
+
+// fatal should be called once there is error with no possible continuation.
+func (p *Progress) fatal(err error) {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	p.isStopped = true
+	p.fatalError = err
+	p.cleanUpdateCh()
+}
+
+func (p *Progress) cleanUpdateCh() {
+	if p.updateCh == nil {
+		// If the progress was ended by fatal instead finish, we ignore error.
+		if p.fatalError != nil {
+			return
+		}
+		panic("update should not be called after finish was called")
+	}
+
+	close(p.updateCh)
+	p.updateCh = nil
+}
+
+func (p *Progress) updateCount(mailbox string, count uint) {
+	p.lock.Lock()
+	defer p.update()
+	defer p.lock.Unlock()
+
+	log.WithField("mailbox", mailbox).WithField("count", count).Debug("Mailbox count updated")
+	p.messageCounts[mailbox] = count
+}
+
+// addMessage should be called as soon as there is ID of the message.
+func (p *Progress) addMessage(messageID string, rule *Rule) {
+	p.lock.Lock()
+	defer p.update()
+	defer p.lock.Unlock()
+
+	p.log.WithField("id", messageID).Trace("Message added")
+	p.messageStatuses[messageID] = &MessageStatus{
+		eventTime: time.Now(),
+		rule:      rule,
+		SourceID:  messageID,
+	}
+}
+
+// messageExported should be called right before message is exported.
+func (p *Progress) messageExported(messageID string, body []byte, err error) {
+	p.lock.Lock()
+	defer p.update()
+	defer p.lock.Unlock()
+
+	p.log.WithField("id", messageID).WithError(err).Debug("Message exported")
+	status := p.messageStatuses[messageID]
+	status.exported = true
+	status.exportErr = err
+
+	if len(body) > 0 {
+		status.bodyHash = fmt.Sprintf("%x", sha256.Sum256(body))
+
+		if header, err := getMessageHeader(body); err != nil {
+			p.log.WithField("id", messageID).WithError(err).Warning("Failed to parse headers for reporting")
+		} else {
+			status.setDetailsFromHeader(header)
+		}
+	}
+
+	// If export failed, no other step will be done with message and we can log it to the report file.
+	if err != nil {
+		p.logMessage(messageID)
+	}
+}
+
+// messageImported should be called right after message is imported.
+func (p *Progress) messageImported(messageID, importID string, err error) {
+	p.lock.Lock()
+	defer p.update()
+	defer p.lock.Unlock()
+
+	p.log.WithField("id", messageID).WithError(err).Debug("Message imported")
+	p.messageStatuses[messageID].targetID = importID
+	p.messageStatuses[messageID].imported = true
+	p.messageStatuses[messageID].importErr = err
+
+	// Import is the last step, now we can log the result to the report file.
+	p.logMessage(messageID)
+}
+
+// logMessage writes message status to log file.
+func (p *Progress) logMessage(messageID string) {
+	if p.fileReport == nil {
+		return
+	}
+	p.fileReport.writeMessageStatus(p.messageStatuses[messageID])
+}
+
+// callWrap calls the callback and in case of problem it pause the process.
+// Then it waits for user action to fix it and click on continue or abort.
+func (p *Progress) callWrap(callback func() error) {
+	for {
+		if p.shouldStop() {
+			break
+		}
+
+		err := callback()
+		if err == nil {
+			break
+		}
+
+		p.Pause(err.Error())
+	}
+}
+
+// shouldStop is utility for providers to automatically wait during pause
+// and returned value determines whether the process shouls be fully stopped.
+func (p *Progress) shouldStop() bool {
+	for p.IsPaused() {
+		time.Sleep(time.Second)
+	}
+	return p.IsStopped()
+}
+
+// GetUpdateChannel returns channel notifying any update from import or export.
+func (p *Progress) GetUpdateChannel() chan struct{} {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	return p.updateCh
+}
+
+// Pause pauses the progress.
+func (p *Progress) Pause(reason string) {
+	p.lock.Lock()
+	defer p.update()
+	defer p.lock.Unlock()
+
+	p.log.Info("Progress paused")
+	p.pauseReason = reason
+}
+
+// Resume resumes the progress.
+func (p *Progress) Resume() {
+	p.lock.Lock()
+	defer p.update()
+	defer p.lock.Unlock()
+
+	p.log.Info("Progress resumed")
+	p.pauseReason = ""
+}
+
+// IsPaused returns whether progress is paused.
+func (p *Progress) IsPaused() bool {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	return p.pauseReason != ""
+}
+
+// PauseReason returns pause reason.
+func (p *Progress) PauseReason() string {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	return p.pauseReason
+}
+
+// Stop stops the process.
+func (p *Progress) Stop() {
+	p.lock.Lock()
+	defer p.update()
+	defer p.lock.Unlock()
+
+	p.log.Info("Progress stopped")
+	p.isStopped = true
+	p.pauseReason = "" // Clear pause to run paused code and stop it.
+}
+
+// IsStopped returns whether progress is stopped.
+func (p *Progress) IsStopped() bool {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	return p.isStopped
+}
+
+// GetFatalError returns fatal error (progress failed and did not finish).
+func (p *Progress) GetFatalError() error {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	return p.fatalError
+}
+
+// GetFailedMessages returns statuses of failed messages.
+func (p *Progress) GetFailedMessages() []*MessageStatus {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	// Include lost messages in the process only when transfer is done.
+	includeMissing := p.updateCh == nil
+
+	statuses := []*MessageStatus{}
+	for _, status := range p.messageStatuses {
+		if status.hasError(includeMissing) {
+			statuses = append(statuses, status)
+		}
+	}
+	return statuses
+}
+
+// GetCounts returns counts of exported and imported messages.
+func (p *Progress) GetCounts() (failed, imported, exported, added, total uint) {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	// Include lost messages in the process only when transfer is done.
+	includeMissing := p.updateCh == nil
+
+	for _, mailboxCount := range p.messageCounts {
+		total += mailboxCount
+	}
+	for _, status := range p.messageStatuses {
+		added++
+		if status.exported {
+			exported++
+		}
+		if status.imported {
+			imported++
+		}
+		if status.hasError(includeMissing) {
+			failed++
+		}
+	}
+	return
+}
+
+// GenerateBugReport generates similar file to import log except private information.
+func (p *Progress) GenerateBugReport() []byte {
+	bugReport := bugReport{}
+	for _, status := range p.messageStatuses {
+		bugReport.writeMessageStatus(status)
+	}
+	return bugReport.getData()
+}

internal/transfer/progress_test.go

@@ -0,0 +1,120 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"testing"
+
+	"github.com/pkg/errors"
+	a "github.com/stretchr/testify/assert"
+	r "github.com/stretchr/testify/require"
+)
+
+func TestProgressUpdateCount(t *testing.T) {
+	progress := newProgress(log, nil)
+	drainProgressUpdateChannel(&progress)
+
+	progress.start()
+
+	progress.updateCount("inbox", 10)
+	progress.updateCount("archive", 20)
+	progress.updateCount("inbox", 12)
+	progress.updateCount("sent", 5)
+	progress.updateCount("foo", 4)
+	progress.updateCount("foo", 5)
+
+	progress.finish()
+
+	_, _, _, _, total := progress.GetCounts() //nolint[dogsled]
+	r.Equal(t, uint(42), total)
+}
+
+func TestProgressAddingMessages(t *testing.T) {
+	progress := newProgress(log, nil)
+	drainProgressUpdateChannel(&progress)
+
+	progress.start()
+
+	// msg1 has no problem.
+	progress.addMessage("msg1", nil)
+	progress.messageExported("msg1", []byte(""), nil)
+	progress.messageImported("msg1", "", nil)
+
+	// msg2 has an import problem.
+	progress.addMessage("msg2", nil)
+	progress.messageExported("msg2", []byte(""), nil)
+	progress.messageImported("msg2", "", errors.New("failed import"))
+
+	// msg3 has an export problem.
+	progress.addMessage("msg3", nil)
+	progress.messageExported("msg3", []byte(""), errors.New("failed export"))
+
+	// msg4 has an export problem and import is also called.
+	progress.addMessage("msg4", nil)
+	progress.messageExported("msg4", []byte(""), errors.New("failed export"))
+	progress.messageImported("msg4", "", nil)
+
+	progress.finish()
+
+	failed, imported, exported, added, _ := progress.GetCounts()
+	a.Equal(t, uint(4), added)
+	a.Equal(t, uint(4), exported)
+	a.Equal(t, uint(3), imported)
+	a.Equal(t, uint(3), failed)
+
+	errorsMap := map[string]string{}
+	for _, status := range progress.GetFailedMessages() {
+		errorsMap[status.SourceID] = status.GetErrorMessage()
+	}
+	a.Equal(t, map[string]string{
+		"msg2": "failed to import: failed import",
+		"msg3": "failed to export: failed export",
+		"msg4": "failed to export: failed export",
+	}, errorsMap)
+}
+
+func TestProgressFinish(t *testing.T) {
+	progress := newProgress(log, nil)
+	drainProgressUpdateChannel(&progress)
+
+	progress.start()
+	progress.finish()
+	r.Nil(t, progress.updateCh)
+
+	r.Panics(t, func() { progress.addMessage("msg", nil) })
+}
+
+func TestProgressFatalError(t *testing.T) {
+	progress := newProgress(log, nil)
+	drainProgressUpdateChannel(&progress)
+
+	progress.start()
+	progress.fatal(errors.New("fatal error"))
+	r.Nil(t, progress.updateCh)
+
+	r.NotPanics(t, func() { progress.addMessage("msg", nil) })
+}
+
+func drainProgressUpdateChannel(progress *Progress) {
+	// updateCh is not needed to drain under tests - timeout is implemented.
+	// But timeout takes time which would slow down tests.
+	go func() {
+		for range progress.updateCh {
+		}
+	}()
+}

internal/transfer/provider.go

@@ -0,0 +1,51 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+// Provider provides interface for common operation with provider.
+type Provider interface {
+	// ID is used for generating transfer ID by combining source and target ID.
+	ID() string
+
+	// Mailboxes returns all available mailboxes.
+	// Provider used as source returns only non-empty maibloxes.
+	// Provider used as target does not return all mail maiblox.
+	Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error)
+}
+
+// SourceProvider provides interface of provider with support of export.
+type SourceProvider interface {
+	Provider
+
+	// TransferTo exports messages based on rules to channel.
+	TransferTo(transferRules, *Progress, chan<- Message)
+}
+
+// TargetProvider provides interface of provider with support of import.
+type TargetProvider interface {
+	Provider
+
+	// DefaultMailboxes returns the default mailboxes for default rules if no other is found.
+	DefaultMailboxes(sourceMailbox Mailbox) (targetMailboxes []Mailbox)
+
+	// CreateMailbox creates new mailbox to be used as target in transfer rules.
+	CreateMailbox(Mailbox) (Mailbox, error)
+
+	// TransferFrom imports messages from channel.
+	TransferFrom(transferRules, *Progress, <-chan Message)
+}

internal/transfer/provider_eml.go

@@ -0,0 +1,65 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+// EMLProvider implements import and export to/from EML file structure.
+type EMLProvider struct {
+	root string
+}
+
+// NewEMLProvider creates EMLProvider.
+func NewEMLProvider(root string) *EMLProvider {
+	return &EMLProvider{
+		root: root,
+	}
+}
+
+// ID is used for generating transfer ID by combining source and target ID.
+// We want to keep the same rules for import from or export to local files
+// no matter exact path, therefore it returns constant. The same as EML.
+func (p *EMLProvider) ID() string {
+	return "local" //nolint[goconst]
+}
+
+// Mailboxes returns all available folder names from root of EML files.
+// In case the same folder name is used more than once (for example root/a/foo
+// and root/b/foo), it's treated as the same folder.
+func (p *EMLProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) {
+	var folderNames []string
+	var err error
+	if includeEmpty {
+		folderNames, err = getFolderNames(p.root)
+	} else {
+		folderNames, err = getFolderNamesWithFileSuffix(p.root, ".eml")
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	mailboxes := []Mailbox{}
+	for _, folderName := range folderNames {
+		mailboxes = append(mailboxes, Mailbox{
+			ID:          "",
+			Name:        folderName,
+			Color:       "",
+			IsExclusive: false,
+		})
+	}
+
+	return mailboxes, nil
+}

internal/transfer/provider_eml_source.go

@@ -0,0 +1,135 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+)
+
+// TransferTo exports messages based on rules to channel.
+func (p *EMLProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) {
+	log.Info("Started transfer from EML to channel")
+	defer log.Info("Finished transfer from EML to channel")
+
+	filePathsPerFolder, err := p.getFilePathsPerFolder(rules)
+	if err != nil {
+		progress.fatal(err)
+		return
+	}
+
+	// This list is not filtered by time but instead going throgh each file
+	// twice or keeping all in memory we will tell rough estimation which
+	// will be updated during processing each file.
+	for folderName, filePaths := range filePathsPerFolder {
+		if progress.shouldStop() {
+			break
+		}
+
+		progress.updateCount(folderName, uint(len(filePaths)))
+	}
+
+	for folderName, filePaths := range filePathsPerFolder {
+		// No error guaranteed by getFilePathsPerFolder.
+		rule, _ := rules.getRuleBySourceMailboxName(folderName)
+		log.WithField("rule", rule).Debug("Processing rule")
+		p.exportMessages(rule, filePaths, progress, ch)
+	}
+}
+
+func (p *EMLProvider) getFilePathsPerFolder(rules transferRules) (map[string][]string, error) {
+	filePaths, err := getFilePathsWithSuffix(p.root, ".eml")
+	if err != nil {
+		return nil, err
+	}
+
+	filePathsMap := map[string][]string{}
+	for _, filePath := range filePaths {
+		folder := filepath.Base(filepath.Dir(filepath.Join(p.root, filePath)))
+		_, err := rules.getRuleBySourceMailboxName(folder)
+		if err != nil {
+			log.WithField("msg", filePath).Trace("Message skipped due to folder name")
+			continue
+		}
+
+		filePathsMap[folder] = append(filePathsMap[folder], filePath)
+	}
+
+	return filePathsMap, nil
+}
+
+func (p *EMLProvider) exportMessages(rule *Rule, filePaths []string, progress *Progress, ch chan<- Message) {
+	count := uint(len(filePaths))
+
+	for _, filePath := range filePaths {
+		if progress.shouldStop() {
+			break
+		}
+
+		msg, err := p.exportMessage(rule, filePath)
+
+		// Read and check time in body only if the rule specifies it
+		// to not waste energy.
+		if err == nil && rule.HasTimeLimit() {
+			msgTime, msgTimeErr := getMessageTime(msg.Body)
+			if msgTimeErr != nil {
+				err = msgTimeErr
+			} else if !rule.isTimeInRange(msgTime) {
+				log.WithField("msg", filePath).Debug("Message skipped due to time")
+
+				count--
+				progress.updateCount(rule.SourceMailbox.Name, count)
+				continue
+			}
+		}
+
+		// addMessage is called after time check to not report message
+		// which should not be exported but any error from reading body
+		// or parsing time is reported as an error.
+		progress.addMessage(filePath, rule)
+		progress.messageExported(filePath, msg.Body, err)
+		if err == nil {
+			ch <- msg
+		}
+	}
+}
+
+func (p *EMLProvider) exportMessage(rule *Rule, filePath string) (Message, error) {
+	fullFilePath := filepath.Clean(filepath.Join(p.root, filePath))
+	file, err := os.Open(fullFilePath) //nolint[gosec]
+	if err != nil {
+		return Message{}, errors.Wrap(err, "failed to open message")
+	}
+	defer file.Close() //nolint[errcheck]
+
+	body, err := ioutil.ReadAll(file)
+	if err != nil {
+		return Message{}, errors.Wrap(err, "failed to read message")
+	}
+
+	return Message{
+		ID:      filePath,
+		Unread:  false,
+		Body:    body,
+		Source:  rule.SourceMailbox,
+		Targets: rule.TargetMailboxes,
+	}, nil
+}

internal/transfer/provider_eml_target.go

@@ -0,0 +1,89 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/hashicorp/go-multierror"
+)
+
+// DefaultMailboxes returns the default mailboxes for default rules if no other is found.
+func (p *EMLProvider) DefaultMailboxes(sourceMailbox Mailbox) []Mailbox {
+	return []Mailbox{{
+		Name: sourceMailbox.Name,
+	}}
+}
+
+// CreateMailbox does nothing. Folders are created dynamically during the import.
+func (p *EMLProvider) CreateMailbox(mailbox Mailbox) (Mailbox, error) {
+	return mailbox, nil
+}
+
+// TransferFrom imports messages from channel.
+func (p *EMLProvider) TransferFrom(rules transferRules, progress *Progress, ch <-chan Message) {
+	log.Info("Started transfer from channel to EML")
+	defer log.Info("Finished transfer from channel to EML")
+
+	err := p.createFolders(rules)
+	if err != nil {
+		progress.fatal(err)
+		return
+	}
+
+	for msg := range ch {
+		for progress.shouldStop() {
+			break
+		}
+
+		err := p.writeFile(msg)
+		progress.messageImported(msg.ID, "", err)
+	}
+}
+
+func (p *EMLProvider) createFolders(rules transferRules) error {
+	for rule := range rules.iterateActiveRules() {
+		for _, mailbox := range rule.TargetMailboxes {
+			path := filepath.Join(p.root, mailbox.Name)
+			if err := os.MkdirAll(path, os.ModePerm); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (p *EMLProvider) writeFile(msg Message) error {
+	fileName := filepath.Base(msg.ID)
+	if !strings.HasSuffix(fileName, ".eml") {
+		fileName += ".eml"
+	}
+
+	var err error
+	for _, mailbox := range msg.Targets {
+		path := filepath.Join(p.root, mailbox.Name, fileName)
+
+		if localErr := ioutil.WriteFile(path, msg.Body, 0600); localErr != nil {
+			err = multierror.Append(err, localErr)
+		}
+	}
+	return err
+}

internal/transfer/provider_eml_test.go

@@ -0,0 +1,126 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+	"time"
+
+	r "github.com/stretchr/testify/require"
+)
+
+func newTestEMLProvider(path string) *EMLProvider {
+	if path == "" {
+		path = "testdata/eml"
+	}
+	return NewEMLProvider(path)
+}
+
+func TestEMLProviderMailboxes(t *testing.T) {
+	provider := newTestEMLProvider("")
+
+	tests := []struct {
+		includeEmpty  bool
+		wantMailboxes []Mailbox
+	}{
+		{true, []Mailbox{
+			{Name: "Foo"},
+			{Name: "Inbox"},
+			{Name: "eml"},
+		}},
+		{false, []Mailbox{
+			{Name: "Foo"},
+			{Name: "Inbox"},
+		}},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(fmt.Sprintf("%v", tc.includeEmpty), func(t *testing.T) {
+			mailboxes, err := provider.Mailboxes(tc.includeEmpty, false)
+			r.NoError(t, err)
+			r.Equal(t, tc.wantMailboxes, mailboxes)
+		})
+	}
+}
+
+func TestEMLProviderTransferTo(t *testing.T) {
+	provider := newTestEMLProvider("")
+
+	rules, rulesClose := newTestRules(t)
+	defer rulesClose()
+	setupEMLRules(rules)
+
+	testTransferTo(t, rules, provider, []string{
+		"Foo/msg.eml",
+		"Inbox/msg.eml",
+	})
+}
+
+func TestEMLProviderTransferFrom(t *testing.T) {
+	dir, err := ioutil.TempDir("", "eml")
+	r.NoError(t, err)
+	defer os.RemoveAll(dir) //nolint[errcheck]
+
+	provider := newTestEMLProvider(dir)
+
+	rules, rulesClose := newTestRules(t)
+	defer rulesClose()
+	setupEMLRules(rules)
+
+	testTransferFrom(t, rules, provider, []Message{
+		{ID: "Foo/msg.eml", Body: getTestMsgBody("msg"), Targets: []Mailbox{{Name: "Foo"}}},
+	})
+
+	checkEMLFileStructure(t, dir, []string{
+		"Foo/msg.eml",
+	})
+}
+
+func TestEMLProviderTransferFromTo(t *testing.T) {
+	dir, err := ioutil.TempDir("", "eml")
+	r.NoError(t, err)
+	defer os.RemoveAll(dir) //nolint[errcheck]
+
+	source := newTestEMLProvider("")
+	target := newTestEMLProvider(dir)
+
+	rules, rulesClose := newTestRules(t)
+	defer rulesClose()
+	setupEMLRules(rules)
+
+	testTransferFromTo(t, rules, source, target, 5*time.Second)
+
+	checkEMLFileStructure(t, dir, []string{
+		"Foo/msg.eml",
+		"Inbox/msg.eml",
+	})
+}
+
+func setupEMLRules(rules transferRules) {
+	_ = rules.setRule(Mailbox{Name: "Inbox"}, []Mailbox{{Name: "Inbox"}}, 0, 0)
+	_ = rules.setRule(Mailbox{Name: "Foo"}, []Mailbox{{Name: "Foo"}}, 0, 0)
+}
+
+func checkEMLFileStructure(t *testing.T, root string, expectedFiles []string) {
+	files, err := getFilePathsWithSuffix(root, ".eml")
+	r.NoError(t, err)
+	r.Equal(t, expectedFiles, files)
+}

internal/transfer/provider_imap.go

@@ -0,0 +1,98 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"net"
+	"strings"
+
+	imapClient "github.com/emersion/go-imap/client"
+)
+
+// IMAPProvider implements export from IMAP server.
+type IMAPProvider struct {
+	username string
+	password string
+	addr     string
+
+	client *imapClient.Client
+}
+
+// NewIMAPProvider returns new IMAPProvider.
+func NewIMAPProvider(username, password, host, port string) (*IMAPProvider, error) {
+	p := &IMAPProvider{
+		username: username,
+		password: password,
+		addr:     net.JoinHostPort(host, port),
+	}
+
+	if err := p.auth(); err != nil {
+		return nil, err
+	}
+
+	return p, nil
+}
+
+// ID is used for generating transfer ID by combining source and target ID.
+// We want to keep the same rules for import from any IMAP server, therefore
+// it returns constant.
+func (p *IMAPProvider) ID() string {
+	return "imap"
+}
+
+// Mailboxes returns all available folder names from root of EML files.
+// In case the same folder name is used more than once (for example root/a/foo
+// and root/b/foo), it's treated as the same folder.
+func (p *IMAPProvider) Mailboxes(includEmpty, includeAllMail bool) ([]Mailbox, error) {
+	mailboxesInfo, err := p.list()
+	if err != nil {
+		return nil, err
+	}
+
+	mailboxes := []Mailbox{}
+	for _, mailbox := range mailboxesInfo {
+		hasNoSelect := false
+		for _, attrib := range mailbox.Attributes {
+			if strings.ToLower(attrib) == "\\noselect" {
+				hasNoSelect = true
+				break
+			}
+		}
+		if hasNoSelect || mailbox.Name == "[Gmail]" {
+			continue
+		}
+
+		if !includEmpty || true {
+			mailboxStatus, err := p.selectIn(mailbox.Name)
+			if err != nil {
+				return nil, err
+			}
+			if mailboxStatus.Messages == 0 {
+				continue
+			}
+		}
+
+		mailboxes = append(mailboxes, Mailbox{
+			ID:          "",
+			Name:        mailbox.Name,
+			Color:       "",
+			IsExclusive: false,
+		})
+	}
+	return mailboxes, nil
+}

internal/transfer/provider_imap_source.go

@@ -0,0 +1,210 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"fmt"
+	"io/ioutil"
+
+	"github.com/emersion/go-imap"
+)
+
+type imapMessageInfo struct {
+	id   string
+	uid  uint32
+	size uint32
+}
+
+const (
+	imapPageSize     = uint32(2000)             // Optimized on Gmail.
+	imapMaxFetchSize = uint32(50 * 1000 * 1000) // Size in octets. If 0, it will use one fetch per message.
+)
+
+// TransferTo exports messages based on rules to channel.
+func (p *IMAPProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) {
+	log.Info("Started transfer from IMAP to channel")
+	defer log.Info("Finished transfer from IMAP to channel")
+
+	imapMessageInfoMap := p.loadMessageInfoMap(rules, progress)
+
+	for rule := range rules.iterateActiveRules() {
+		log.WithField("rule", rule).Debug("Processing rule")
+		messagesInfo := imapMessageInfoMap[rule.SourceMailbox.Name]
+		p.transferTo(rule, messagesInfo, progress, ch)
+	}
+}
+
+func (p *IMAPProvider) loadMessageInfoMap(rules transferRules, progress *Progress) map[string]map[string]imapMessageInfo {
+	res := map[string]map[string]imapMessageInfo{}
+
+	for rule := range rules.iterateActiveRules() {
+		if progress.shouldStop() {
+			break
+		}
+
+		mailboxName := rule.SourceMailbox.Name
+		var mailbox *imap.MailboxStatus
+		progress.callWrap(func() error {
+			var err error
+			mailbox, err = p.selectIn(mailboxName)
+			return err
+		})
+		if mailbox.Messages == 0 {
+			continue
+		}
+
+		messagesInfo := p.loadMessagesInfo(rule, progress, mailbox.UidValidity)
+		res[rule.SourceMailbox.Name] = messagesInfo
+		progress.updateCount(rule.SourceMailbox.Name, uint(len(messagesInfo)))
+	}
+
+	return res
+}
+
+func (p *IMAPProvider) loadMessagesInfo(rule *Rule, progress *Progress, uidValidity uint32) map[string]imapMessageInfo {
+	messagesInfo := map[string]imapMessageInfo{}
+
+	pageStart := uint32(1)
+	pageEnd := imapPageSize
+	for {
+		if progress.shouldStop() {
+			break
+		}
+
+		seqSet := &imap.SeqSet{}
+		seqSet.AddRange(pageStart, pageEnd)
+
+		items := []imap.FetchItem{imap.FetchUid, imap.FetchRFC822Size}
+		if rule.HasTimeLimit() {
+			items = append(items, imap.FetchEnvelope)
+		}
+
+		pageMsgCount := uint32(0)
+		processMessageCallback := func(imapMessage *imap.Message) {
+			pageMsgCount++
+			if rule.HasTimeLimit() {
+				t := imapMessage.Envelope.Date.Unix()
+				if t != 0 && !rule.isTimeInRange(t) {
+					log.WithField("uid", imapMessage.Uid).Debug("Message skipped due to time")
+					return
+				}
+			}
+			id := fmt.Sprintf("%s_%d:%d", rule.SourceMailbox.Name, uidValidity, imapMessage.Uid)
+			messagesInfo[id] = imapMessageInfo{
+				id:   id,
+				uid:  imapMessage.Uid,
+				size: imapMessage.Size,
+			}
+			progress.addMessage(id, rule)
+		}
+
+		progress.callWrap(func() error {
+			return p.fetch(seqSet, items, processMessageCallback)
+		})
+
+		if pageMsgCount < imapPageSize {
+			break
+		}
+
+		pageStart = pageEnd
+		pageEnd += imapPageSize
+	}
+
+	return messagesInfo
+}
+
+func (p *IMAPProvider) transferTo(rule *Rule, messagesInfo map[string]imapMessageInfo, progress *Progress, ch chan<- Message) {
+	progress.callWrap(func() error {
+		_, err := p.selectIn(rule.SourceMailbox.Name)
+		return err
+	})
+
+	seqSet := &imap.SeqSet{}
+	seqSetSize := uint32(0)
+	uidToID := map[uint32]string{}
+
+	for _, messageInfo := range messagesInfo {
+		if progress.shouldStop() {
+			break
+		}
+
+		if seqSetSize != 0 && (seqSetSize+messageInfo.size) > imapMaxFetchSize {
+			log.WithField("mailbox", rule.SourceMailbox.Name).WithField("seq", seqSet).WithField("size", seqSetSize).Debug("Fetching messages")
+
+			p.exportMessages(rule, progress, ch, seqSet, uidToID)
+
+			seqSet = &imap.SeqSet{}
+			seqSetSize = 0
+			uidToID = map[uint32]string{}
+		}
+
+		seqSet.AddNum(messageInfo.uid)
+		seqSetSize += messageInfo.size
+		uidToID[messageInfo.uid] = messageInfo.id
+	}
+}
+
+func (p *IMAPProvider) exportMessages(rule *Rule, progress *Progress, ch chan<- Message, seqSet *imap.SeqSet, uidToID map[uint32]string) {
+	section := &imap.BodySectionName{}
+	items := []imap.FetchItem{imap.FetchUid, imap.FetchFlags, section.FetchItem()}
+
+	processMessageCallback := func(imapMessage *imap.Message) {
+		id, ok := uidToID[imapMessage.Uid]
+
+		// Sometimes, server sends not requested messages.
+		if !ok {
+			log.WithField("uid", imapMessage.Uid).Warning("Message skipped: not requested")
+			return
+		}
+
+		// Sometimes, server sends message twice, once with body and once without it.
+		bodyReader := imapMessage.GetBody(section)
+		if bodyReader == nil {
+			log.WithField("uid", imapMessage.Uid).Warning("Message skipped: no body")
+			return
+		}
+
+		body, err := ioutil.ReadAll(bodyReader)
+		progress.messageExported(id, body, err)
+		if err == nil {
+			msg := p.exportMessage(rule, id, imapMessage, body)
+			ch <- msg
+		}
+	}
+
+	progress.callWrap(func() error {
+		return p.uidFetch(seqSet, items, processMessageCallback)
+	})
+}
+
+func (p *IMAPProvider) exportMessage(rule *Rule, id string, imapMessage *imap.Message, body []byte) Message {
+	unread := true
+	for _, flag := range imapMessage.Flags {
+		if flag == imap.SeenFlag {
+			unread = false
+		}
+	}
+
+	return Message{
+		ID:      id,
+		Unread:  unread,
+		Body:    body,
+		Source:  rule.SourceMailbox,
+		Targets: rule.TargetMailboxes,
+	}
+}

internal/transfer/provider_imap_utils.go

@@ -0,0 +1,236 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"net"
+	"time"
+
+	imapID "github.com/ProtonMail/go-imap-id"
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/emersion/go-imap"
+	imapClient "github.com/emersion/go-imap/client"
+	sasl "github.com/emersion/go-sasl"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	imapDialTimeout      = 5 * time.Second
+	imapRetries          = 10
+	imapReconnectTimeout = 30 * time.Minute
+	imapReconnectSleep   = time.Minute
+)
+
+type imapErrorLogger struct {
+	log *logrus.Entry
+}
+
+func (l *imapErrorLogger) Printf(f string, v ...interface{}) {
+	l.log.Errorf(f, v...)
+}
+
+func (l *imapErrorLogger) Println(v ...interface{}) {
+	l.log.Errorln(v...)
+}
+
+type imapDebugLogger struct { //nolint[unused]
+	log *logrus.Entry
+}
+
+func (l *imapDebugLogger) Write(data []byte) (int, error) {
+	l.log.Trace(string(data))
+	return len(data), nil
+}
+
+func (p *IMAPProvider) ensureConnection(callback func() error) error {
+	var callErr error
+	for i := 1; i <= imapRetries; i++ {
+		callErr = callback()
+		if callErr == nil {
+			return nil
+		}
+
+		log.WithField("attempt", i).WithError(callErr).Warning("Call failed, trying reconnect")
+		err := p.tryReconnect()
+		if err != nil {
+			return err
+		}
+	}
+	return errors.Wrap(callErr, "too many retries")
+}
+
+func (p *IMAPProvider) tryReconnect() error {
+	start := time.Now()
+	var previousErr error
+	for {
+		if time.Since(start) > imapReconnectTimeout {
+			return previousErr
+		}
+
+		err := pmapi.CheckConnection()
+		if err != nil {
+			time.Sleep(imapReconnectSleep)
+			previousErr = err
+			continue
+		}
+
+		err = p.reauth()
+		if err != nil {
+			time.Sleep(imapReconnectSleep)
+			previousErr = err
+			continue
+		}
+
+		break
+	}
+	return nil
+}
+
+func (p *IMAPProvider) reauth() error {
+	if _, err := p.client.Capability(); err != nil {
+		state := p.client.State()
+		log.WithField("addr", p.addr).WithField("state", state).WithError(err).Debug("Reconnecting")
+		p.client = nil
+	}
+
+	return p.auth()
+}
+
+func (p *IMAPProvider) auth() error { //nolint[funlen]
+	log := log.WithField("addr", p.addr)
+
+	log.Info("Connecting to server")
+
+	if _, err := net.DialTimeout("tcp", p.addr, imapDialTimeout); err != nil {
+		return errors.Wrap(err, "failed to dial server")
+	}
+
+	client, err := imapClient.DialTLS(p.addr, nil)
+	if err != nil {
+		return errors.Wrap(err, "failed to connect to server")
+	}
+	client.ErrorLog = &imapErrorLogger{logrus.WithField("pkg", "imap-client")}
+	// Logrus have Writer helper but it fails for big messages because of
+	// bufio.MaxScanTokenSize limit.
+	// This spams a lot, uncomment once needed during development.
+	//client.SetDebug(&imapDebugLogger{logrus.WithField("pkg", "imap-client")})
+	p.client = client
+
+	log.Info("Connected")
+
+	if (p.client.State() & imap.AuthenticatedState) == imap.AuthenticatedState {
+		return nil
+	}
+
+	capability, err := p.client.Capability()
+	log.WithField("capability", capability).WithError(err).Debug("Server capability")
+	if err != nil {
+		return errors.Wrap(err, "failed to get capabilities")
+	}
+
+	// SASL AUTH PLAIN
+	if ok, _ := p.client.SupportAuth("PLAIN"); p.client.State() == imap.NotAuthenticatedState && ok {
+		log.Debug("Trying plain auth")
+		authPlain := sasl.NewPlainClient("", p.username, p.password)
+		if err = p.client.Authenticate(authPlain); err != nil {
+			return errors.Wrap(err, "plain auth failed")
+		}
+	}
+
+	// LOGIN: if the server reports the IMAP4rev1 capability then it is standards conformant and must support login.
+	if ok, _ := p.client.Support("IMAP4rev1"); p.client.State() == imap.NotAuthenticatedState && ok {
+		log.Debug("Trying login")
+		if err = p.client.Login(p.username, p.password); err != nil {
+			return errors.Wrap(err, "login failed")
+		}
+	}
+
+	if p.client.State() == imap.NotAuthenticatedState {
+		return errors.New("unknown auth method")
+	}
+
+	log.Info("Logged in")
+
+	idClient := imapID.NewClient(p.client)
+	if ok, err := idClient.SupportID(); err == nil && ok {
+		serverID, err := idClient.ID(imapID.ID{
+			imapID.FieldName:    "ImportExport",
+			imapID.FieldVersion: "beta",
+		})
+		log.WithField("ID", serverID).WithError(err).Debug("Server info")
+	}
+
+	return err
+}
+
+func (p *IMAPProvider) list() (mailboxes []*imap.MailboxInfo, err error) {
+	err = p.ensureConnection(func() error {
+		mailboxesCh := make(chan *imap.MailboxInfo)
+		doneCh := make(chan error)
+
+		go func() {
+			doneCh <- p.client.List("", "*", mailboxesCh)
+		}()
+
+		for mailbox := range mailboxesCh {
+			mailboxes = append(mailboxes, mailbox)
+		}
+
+		return <-doneCh
+	})
+	return
+}
+
+func (p *IMAPProvider) selectIn(mailboxName string) (mailbox *imap.MailboxStatus, err error) {
+	err = p.ensureConnection(func() error {
+		mailbox, err = p.client.Select(mailboxName, true)
+		return err
+	})
+	return
+}
+
+func (p *IMAPProvider) fetch(seqSet *imap.SeqSet, items []imap.FetchItem, processMessageCallback func(m *imap.Message)) error {
+	return p.fetchHelper(false, seqSet, items, processMessageCallback)
+}
+
+func (p *IMAPProvider) uidFetch(seqSet *imap.SeqSet, items []imap.FetchItem, processMessageCallback func(m *imap.Message)) error {
+	return p.fetchHelper(true, seqSet, items, processMessageCallback)
+}
+
+func (p *IMAPProvider) fetchHelper(uid bool, seqSet *imap.SeqSet, items []imap.FetchItem, processMessageCallback func(m *imap.Message)) error {
+	return p.ensureConnection(func() error {
+		messagesCh := make(chan *imap.Message)
+		doneCh := make(chan error)
+
+		go func() {
+			if uid {
+				doneCh <- p.client.UidFetch(seqSet, items, messagesCh)
+			} else {
+				doneCh <- p.client.Fetch(seqSet, items, messagesCh)
+			}
+		}()
+
+		for message := range messagesCh {
+			processMessageCallback(message)
+		}
+
+		err := <-doneCh
+		return err
+	})
+}

internal/transfer/provider_local.go

@@ -0,0 +1,68 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+// LocalProvider implements import from local EML and MBOX file structure.
+type LocalProvider struct {
+	root         string
+	emlProvider  *EMLProvider
+	mboxProvider *MBOXProvider
+}
+
+func NewLocalProvider(root string) *LocalProvider {
+	return &LocalProvider{
+		root:         root,
+		emlProvider:  NewEMLProvider(root),
+		mboxProvider: NewMBOXProvider(root),
+	}
+}
+
+// ID is used for generating transfer ID by combining source and target ID.
+// We want to keep the same rules for import from or export to local files
+// no matter exact path, therefore it returns constant.
+// The same as EML and MBOX.
+func (p *LocalProvider) ID() string {
+	return "local" //nolint[goconst]
+}
+
+// Mailboxes returns all available folder names from root of EML and MBOX files.
+func (p *LocalProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) {
+	mailboxes, err := p.emlProvider.Mailboxes(includeEmpty, includeAllMail)
+	if err != nil {
+		return nil, err
+	}
+
+	mboxMailboxes, err := p.mboxProvider.Mailboxes(includeEmpty, includeAllMail)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, mboxMailbox := range mboxMailboxes {
+		found := false
+		for _, mailboxes := range mailboxes {
+			if mboxMailbox.Name == mailboxes.Name {
+				found = true
+				break
+			}
+		}
+		if !found {
+			mailboxes = append(mailboxes, mboxMailbox)
+		}
+	}
+	return mailboxes, nil
+}

internal/transfer/provider_local_source.go

@@ -0,0 +1,42 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"sync"
+)
+
+// TransferTo exports messages based on rules to channel.
+func (p *LocalProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) {
+	log.Info("Started transfer from EML and MBOX to channel")
+	defer log.Info("Finished transfer from EML and MBOX to channel")
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+
+	go func() {
+		defer wg.Done()
+		p.emlProvider.TransferTo(rules, progress, ch)
+	}()
+	go func() {
+		defer wg.Done()
+		p.mboxProvider.TransferTo(rules, progress, ch)
+	}()
+
+	wg.Wait()
+}

internal/transfer/provider_local_test.go

@@ -0,0 +1,77 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"fmt"
+	"testing"
+
+	r "github.com/stretchr/testify/require"
+)
+
+func newTestLocalProvider(path string) *LocalProvider {
+	if path == "" {
+		path = "testdata/emlmbox"
+	}
+	return NewLocalProvider(path)
+}
+
+func TestLocalProviderMailboxes(t *testing.T) {
+	provider := newTestLocalProvider("")
+
+	tests := []struct {
+		includeEmpty  bool
+		wantMailboxes []Mailbox
+	}{
+		{true, []Mailbox{
+			{Name: "Foo"},
+			{Name: "emlmbox"},
+			{Name: "Inbox"},
+		}},
+		{false, []Mailbox{
+			{Name: "Foo"},
+			{Name: "Inbox"},
+		}},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(fmt.Sprintf("%v", tc.includeEmpty), func(t *testing.T) {
+			mailboxes, err := provider.Mailboxes(tc.includeEmpty, false)
+			r.NoError(t, err)
+			r.Equal(t, tc.wantMailboxes, mailboxes)
+		})
+	}
+}
+
+func TestLocalProviderTransferTo(t *testing.T) {
+	provider := newTestLocalProvider("")
+
+	rules, rulesClose := newTestRules(t)
+	defer rulesClose()
+	setupEMLMBOXRules(rules)
+
+	testTransferTo(t, rules, provider, []string{
+		"Foo/msg.eml",
+		"Inbox.mbox:1",
+	})
+}
+
+func setupEMLMBOXRules(rules transferRules) {
+	_ = rules.setRule(Mailbox{Name: "Inbox"}, []Mailbox{{Name: "Inbox"}}, 0, 0)
+	_ = rules.setRule(Mailbox{Name: "Foo"}, []Mailbox{{Name: "Foo"}}, 0, 0)
+}

internal/transfer/provider_mbox.go

@@ -0,0 +1,66 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"path/filepath"
+	"strings"
+)
+
+// MBOXProvider implements import and export to/from MBOX structure.
+type MBOXProvider struct {
+	root string
+}
+
+func NewMBOXProvider(root string) *MBOXProvider {
+	return &MBOXProvider{
+		root: root,
+	}
+}
+
+// ID is used for generating transfer ID by combining source and target ID.
+// We want to keep the same rules for import from or export to local files
+// no matter exact path, therefore it returns constant. The same as EML.
+func (p *MBOXProvider) ID() string {
+	return "local" //nolint[goconst]
+}
+
+// Mailboxes returns all available folder names from root of EML files.
+// In case the same folder name is used more than once (for example root/a/foo
+// and root/b/foo), it's treated as the same folder.
+func (p *MBOXProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) {
+	filePaths, err := getFilePathsWithSuffix(p.root, "mbox")
+	if err != nil {
+		return nil, err
+	}
+
+	mailboxes := []Mailbox{}
+	for _, filePath := range filePaths {
+		fileName := filepath.Base(filePath)
+		mailboxName := strings.TrimSuffix(fileName, ".mbox")
+
+		mailboxes = append(mailboxes, Mailbox{
+			ID:          "",
+			Name:        mailboxName,
+			Color:       "",
+			IsExclusive: false,
+		})
+	}
+
+	return mailboxes, nil
+}

internal/transfer/provider_mbox_source.go

@@ -0,0 +1,183 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/emersion/go-mbox"
+	"github.com/pkg/errors"
+)
+
+// TransferTo exports messages based on rules to channel.
+func (p *MBOXProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) {
+	log.Info("Started transfer from MBOX to channel")
+	defer log.Info("Finished transfer from MBOX to channel")
+
+	filePathsPerFolder, err := p.getFilePathsPerFolder(rules)
+	if err != nil {
+		progress.fatal(err)
+		return
+	}
+
+	for folderName, filePaths := range filePathsPerFolder {
+		// No error guaranteed by getFilePathsPerFolder.
+		rule, _ := rules.getRuleBySourceMailboxName(folderName)
+		for _, filePath := range filePaths {
+			if progress.shouldStop() {
+				break
+			}
+			p.updateCount(rule, progress, filePath)
+		}
+	}
+
+	for folderName, filePaths := range filePathsPerFolder {
+		// No error guaranteed by getFilePathsPerFolder.
+		rule, _ := rules.getRuleBySourceMailboxName(folderName)
+		log.WithField("rule", rule).Debug("Processing rule")
+		for _, filePath := range filePaths {
+			if progress.shouldStop() {
+				break
+			}
+			p.transferTo(rule, progress, ch, filePath)
+		}
+	}
+}
+
+func (p *MBOXProvider) getFilePathsPerFolder(rules transferRules) (map[string][]string, error) {
+	filePaths, err := getFilePathsWithSuffix(p.root, ".mbox")
+	if err != nil {
+		return nil, err
+	}
+
+	filePathsMap := map[string][]string{}
+	for _, filePath := range filePaths {
+		fileName := filepath.Base(filePath)
+		folder := strings.TrimSuffix(fileName, ".mbox")
+		_, err := rules.getRuleBySourceMailboxName(folder)
+		if err != nil {
+			log.WithField("msg", filePath).Trace("Mailbox skipped due to folder name")
+			continue
+		}
+
+		filePathsMap[folder] = append(filePathsMap[folder], filePath)
+	}
+	return filePathsMap, nil
+}
+
+func (p *MBOXProvider) updateCount(rule *Rule, progress *Progress, filePath string) {
+	mboxReader := p.openMbox(progress, filePath)
+	if mboxReader == nil {
+		return
+	}
+
+	count := 0
+	for {
+		_, err := mboxReader.NextMessage()
+		if err != nil {
+			break
+		}
+		count++
+	}
+	progress.updateCount(rule.SourceMailbox.Name, uint(count))
+}
+
+func (p *MBOXProvider) transferTo(rule *Rule, progress *Progress, ch chan<- Message, filePath string) {
+	mboxReader := p.openMbox(progress, filePath)
+	if mboxReader == nil {
+		return
+	}
+
+	index := 0
+	count := 0
+	for {
+		if progress.shouldStop() {
+			break
+		}
+
+		index++
+		id := fmt.Sprintf("%s:%d", filePath, index)
+
+		msgReader, err := mboxReader.NextMessage()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			progress.fatal(err)
+			break
+		}
+
+		msg, err := p.exportMessage(rule, id, msgReader)
+
+		// Read and check time in body only if the rule specifies it
+		// to not waste energy.
+		if err == nil && rule.HasTimeLimit() {
+			msgTime, msgTimeErr := getMessageTime(msg.Body)
+			if msgTimeErr != nil {
+				err = msgTimeErr
+			} else if !rule.isTimeInRange(msgTime) {
+				log.WithField("msg", id).Debug("Message skipped due to time")
+				continue
+			}
+		}
+
+		// Counting only messages filtered by time to update count to correct total.
+		count++
+
+		// addMessage is called after time check to not report message
+		// which should not be exported but any error from reading body
+		// or parsing time is reported as an error.
+		progress.addMessage(id, rule)
+		progress.messageExported(id, msg.Body, err)
+		if err == nil {
+			ch <- msg
+		}
+	}
+	progress.updateCount(rule.SourceMailbox.Name, uint(count))
+}
+
+func (p *MBOXProvider) exportMessage(rule *Rule, id string, msgReader io.Reader) (Message, error) {
+	body, err := ioutil.ReadAll(msgReader)
+	if err != nil {
+		return Message{}, errors.Wrap(err, "failed to read message")
+	}
+
+	return Message{
+		ID:      id,
+		Unread:  false,
+		Body:    body,
+		Source:  rule.SourceMailbox,
+		Targets: rule.TargetMailboxes,
+	}, nil
+}
+
+func (p *MBOXProvider) openMbox(progress *Progress, mboxPath string) *mbox.Reader {
+	mboxPath = filepath.Join(p.root, mboxPath)
+	mboxFile, err := os.Open(mboxPath) //nolint[gosec]
+	if os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		progress.fatal(err)
+		return nil
+	}
+	return mbox.NewReader(mboxFile)
+}

internal/transfer/provider_mbox_target.go

@@ -0,0 +1,97 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/emersion/go-mbox"
+	"github.com/hashicorp/go-multierror"
+)
+
+// DefaultMailboxes returns the default mailboxes for default rules if no other is found.
+func (p *MBOXProvider) DefaultMailboxes(sourceMailbox Mailbox) []Mailbox {
+	return []Mailbox{{
+		Name: sourceMailbox.Name,
+	}}
+}
+
+// CreateMailbox does nothing. Files are created dynamically during the import.
+func (p *MBOXProvider) CreateMailbox(mailbox Mailbox) (Mailbox, error) {
+	return mailbox, nil
+}
+
+// TransferFrom imports messages from channel.
+func (p *MBOXProvider) TransferFrom(rules transferRules, progress *Progress, ch <-chan Message) {
+	log.Info("Started transfer from channel to MBOX")
+	defer log.Info("Finished transfer from channel to MBOX")
+
+	for msg := range ch {
+		for progress.shouldStop() {
+			break
+		}
+
+		err := p.writeMessage(msg)
+		progress.messageImported(msg.ID, "", err)
+	}
+}
+
+func (p *MBOXProvider) writeMessage(msg Message) error {
+	var multiErr error
+	for _, mailbox := range msg.Targets {
+		mboxName := filepath.Base(mailbox.Name)
+		if !strings.HasSuffix(mboxName, ".mbox") {
+			mboxName += ".mbox"
+		}
+
+		mboxPath := filepath.Join(p.root, mboxName)
+		mboxFile, err := os.OpenFile(mboxPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
+		if err != nil {
+			multiErr = multierror.Append(multiErr, err)
+			continue
+		}
+
+		msgFrom := ""
+		msgTime := time.Now()
+		if header, err := getMessageHeader(msg.Body); err == nil {
+			if date, err := header.Date(); err == nil {
+				msgTime = date
+			}
+			if addresses, err := header.AddressList("from"); err == nil && len(addresses) > 0 {
+				msgFrom = addresses[0].Address
+			}
+		}
+
+		mboxWriter := mbox.NewWriter(mboxFile)
+		messageWriter, err := mboxWriter.CreateMessage(msgFrom, msgTime)
+		if err != nil {
+			multiErr = multierror.Append(multiErr, err)
+			continue
+		}
+
+		_, err = messageWriter.Write(msg.Body)
+		if err != nil {
+			multiErr = multierror.Append(multiErr, err)
+			continue
+		}
+	}
+	return multiErr
+}

internal/transfer/provider_mbox_test.go

@@ -0,0 +1,125 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+	"time"
+
+	r "github.com/stretchr/testify/require"
+)
+
+func newTestMBOXProvider(path string) *MBOXProvider {
+	if path == "" {
+		path = "testdata/mbox"
+	}
+	return NewMBOXProvider(path)
+}
+
+func TestMBOXProviderMailboxes(t *testing.T) {
+	provider := newTestMBOXProvider("")
+
+	tests := []struct {
+		includeEmpty  bool
+		wantMailboxes []Mailbox
+	}{
+		{true, []Mailbox{
+			{Name: "Foo"},
+			{Name: "Inbox"},
+		}},
+		{false, []Mailbox{
+			{Name: "Foo"},
+			{Name: "Inbox"},
+		}},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(fmt.Sprintf("%v", tc.includeEmpty), func(t *testing.T) {
+			mailboxes, err := provider.Mailboxes(tc.includeEmpty, false)
+			r.NoError(t, err)
+			r.Equal(t, tc.wantMailboxes, mailboxes)
+		})
+	}
+}
+
+func TestMBOXProviderTransferTo(t *testing.T) {
+	provider := newTestMBOXProvider("")
+
+	rules, rulesClose := newTestRules(t)
+	defer rulesClose()
+	setupMBOXRules(rules)
+
+	testTransferTo(t, rules, provider, []string{
+		"Foo.mbox:1",
+		"Inbox.mbox:1",
+	})
+}
+
+func TestMBOXProviderTransferFrom(t *testing.T) {
+	dir, err := ioutil.TempDir("", "eml")
+	r.NoError(t, err)
+	defer os.RemoveAll(dir) //nolint[errcheck]
+
+	provider := newTestMBOXProvider(dir)
+
+	rules, rulesClose := newTestRules(t)
+	defer rulesClose()
+	setupMBOXRules(rules)
+
+	testTransferFrom(t, rules, provider, []Message{
+		{ID: "Foo.mbox:1", Body: getTestMsgBody("msg"), Targets: []Mailbox{{Name: "Foo"}}},
+	})
+
+	checkMBOXFileStructure(t, dir, []string{
+		"Foo.mbox",
+	})
+}
+
+func TestMBOXProviderTransferFromTo(t *testing.T) {
+	dir, err := ioutil.TempDir("", "eml")
+	r.NoError(t, err)
+	defer os.RemoveAll(dir) //nolint[errcheck]
+
+	source := newTestMBOXProvider("")
+	target := newTestMBOXProvider(dir)
+
+	rules, rulesClose := newTestRules(t)
+	defer rulesClose()
+	setupEMLRules(rules)
+
+	testTransferFromTo(t, rules, source, target, 5*time.Second)
+
+	checkMBOXFileStructure(t, dir, []string{
+		"Foo.mbox",
+		"Inbox.mbox",
+	})
+}
+
+func setupMBOXRules(rules transferRules) {
+	_ = rules.setRule(Mailbox{Name: "Inbox"}, []Mailbox{{Name: "Inbox"}}, 0, 0)
+	_ = rules.setRule(Mailbox{Name: "Foo"}, []Mailbox{{Name: "Foo"}}, 0, 0)
+}
+
+func checkMBOXFileStructure(t *testing.T, root string, expectedFiles []string) {
+	files, err := getFilePathsWithSuffix(root, ".mbox")
+	r.NoError(t, err)
+	r.Equal(t, expectedFiles, files)
+}

internal/transfer/provider_pmapi.go

@@ -0,0 +1,140 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"sort"
+
+	"github.com/ProtonMail/gopenpgp/crypto"
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/pkg/errors"
+)
+
+// PMAPIProvider implements import and export to/from ProtonMail server.
+type PMAPIProvider struct {
+	clientManager ClientManager
+	userID        string
+	addressID     string
+	keyRing       *crypto.KeyRing
+}
+
+// NewPMAPIProvider returns new PMAPIProvider.
+func NewPMAPIProvider(clientManager ClientManager, userID, addressID string) (*PMAPIProvider, error) {
+	keyRing, err := clientManager.GetClient(userID).KeyRingForAddressID(addressID)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get key ring")
+	}
+
+	return &PMAPIProvider{
+		clientManager: clientManager,
+		userID:        userID,
+		addressID:     addressID,
+		keyRing:       keyRing,
+	}, nil
+}
+
+func (p *PMAPIProvider) client() pmapi.Client {
+	return p.clientManager.GetClient(p.userID)
+}
+
+// ID returns identifier of current setup of PMAPI provider.
+// Identification is unique per user.
+func (p *PMAPIProvider) ID() string {
+	return p.userID
+}
+
+// Mailboxes returns all available labels in ProtonMail account.
+func (p *PMAPIProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) {
+	labels, err := p.client().ListLabels()
+	if err != nil {
+		return nil, err
+	}
+	sortedLabels := byFoldersLabels(labels)
+	sort.Sort(sortedLabels)
+
+	emptyLabelsMap := map[string]bool{}
+	if !includeEmpty {
+		messagesCounts, err := p.client().CountMessages(p.addressID)
+		if err != nil {
+			return nil, err
+		}
+		for _, messagesCount := range messagesCounts {
+			if messagesCount.Total == 0 {
+				emptyLabelsMap[messagesCount.LabelID] = true
+			}
+		}
+	}
+
+	mailboxes := getSystemMailboxes(includeAllMail)
+	for _, label := range sortedLabels {
+		if !includeEmpty && emptyLabelsMap[label.ID] {
+			continue
+		}
+
+		mailboxes = append(mailboxes, Mailbox{
+			ID:          label.ID,
+			Name:        label.Name,
+			Color:       label.Color,
+			IsExclusive: label.Exclusive == 1,
+		})
+	}
+	return mailboxes, nil
+}
+
+func getSystemMailboxes(includeAllMail bool) []Mailbox {
+	mailboxes := []Mailbox{
+		{ID: pmapi.InboxLabel, Name: "Inbox", IsExclusive: true},
+		{ID: pmapi.DraftLabel, Name: "Drafts", IsExclusive: true},
+		{ID: pmapi.SentLabel, Name: "Sent", IsExclusive: true},
+		{ID: pmapi.StarredLabel, Name: "Starred", IsExclusive: true},
+		{ID: pmapi.ArchiveLabel, Name: "Archive", IsExclusive: true},
+		{ID: pmapi.SpamLabel, Name: "Spam", IsExclusive: true},
+		{ID: pmapi.TrashLabel, Name: "Trash", IsExclusive: true},
+	}
+
+	if includeAllMail {
+		mailboxes = append(mailboxes, Mailbox{
+			ID:          pmapi.AllMailLabel,
+			Name:        "All Mail",
+			IsExclusive: true,
+		})
+	}
+
+	return mailboxes
+}
+
+type byFoldersLabels []*pmapi.Label
+
+func (l byFoldersLabels) Len() int {
+	return len(l)
+}
+
+func (l byFoldersLabels) Swap(i, j int) {
+	l[i], l[j] = l[j], l[i]
+}
+
+// Less sorts first folders, then labels, by user order.
+func (l byFoldersLabels) Less(i, j int) bool {
+	if l[i].Exclusive == 1 && l[j].Exclusive == 0 {
+		return true
+	}
+	if l[i].Exclusive == 0 && l[j].Exclusive == 1 {
+		return false
+	}
+	return l[i].Order < l[j].Order
+}

internal/transfer/provider_pmapi_source.go

@@ -0,0 +1,161 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"fmt"
+
+	pkgMessage "github.com/ProtonMail/proton-bridge/pkg/message"
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/pkg/errors"
+)
+
+const pmapiListPageSize = 150
+
+// TransferTo exports messages based on rules to channel.
+func (p *PMAPIProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) {
+	log.Info("Started transfer from PMAPI to channel")
+	defer log.Info("Finished transfer from PMAPI to channel")
+
+	go p.loadCounts(rules, progress)
+
+	for rule := range rules.iterateActiveRules() {
+		p.transferTo(rule, progress, ch, rules.skipEncryptedMessages)
+	}
+}
+
+func (p *PMAPIProvider) loadCounts(rules transferRules, progress *Progress) {
+	for rule := range rules.iterateActiveRules() {
+		if progress.shouldStop() {
+			break
+		}
+
+		rule := rule
+		progress.callWrap(func() error {
+			_, total, err := p.listMessages(&pmapi.MessagesFilter{
+				LabelID: rule.SourceMailbox.ID,
+				Begin:   rule.FromTime,
+				End:     rule.ToTime,
+				Limit:   0,
+			})
+			if err != nil {
+				log.WithError(err).Warning("Problem to load counts")
+				return err
+			}
+			progress.updateCount(rule.SourceMailbox.Name, uint(total))
+			return nil
+		})
+	}
+}
+
+func (p *PMAPIProvider) transferTo(rule *Rule, progress *Progress, ch chan<- Message, skipEncryptedMessages bool) {
+	nextID := ""
+	for {
+		if progress.shouldStop() {
+			break
+		}
+
+		isLastPage := true
+
+		progress.callWrap(func() error {
+			desc := false
+			pmapiMessages, count, err := p.listMessages(&pmapi.MessagesFilter{
+				LabelID:  rule.SourceMailbox.ID,
+				Begin:    rule.FromTime,
+				End:      rule.ToTime,
+				BeginID:  nextID,
+				PageSize: pmapiListPageSize,
+				Page:     0,
+				Sort:     "ID",
+				Desc:     &desc,
+			})
+			if err != nil {
+				return err
+			}
+			log.WithField("label", rule.SourceMailbox.ID).WithField("next", nextID).WithField("count", count).Debug("Listing messages")
+
+			isLastPage = len(pmapiMessages) < pmapiListPageSize
+
+			// The first ID is the last one from the last page (= do not export twice the same one).
+			if nextID != "" {
+				pmapiMessages = pmapiMessages[1:]
+			}
+
+			for _, pmapiMessage := range pmapiMessages {
+				if progress.shouldStop() {
+					break
+				}
+
+				msgID := fmt.Sprintf("%s_%s", rule.SourceMailbox.ID, pmapiMessage.ID)
+				progress.addMessage(msgID, rule)
+				msg, err := p.exportMessage(rule, progress, pmapiMessage.ID, msgID, skipEncryptedMessages)
+				progress.messageExported(msgID, msg.Body, err)
+				if err == nil {
+					ch <- msg
+				}
+			}
+
+			if !isLastPage {
+				nextID = pmapiMessages[len(pmapiMessages)-1].ID
+			}
+
+			return nil
+		})
+
+		if isLastPage {
+			break
+		}
+	}
+}
+
+func (p *PMAPIProvider) exportMessage(rule *Rule, progress *Progress, pmapiMsgID, msgID string, skipEncryptedMessages bool) (Message, error) {
+	var msg *pmapi.Message
+	progress.callWrap(func() error {
+		var err error
+		msg, err = p.getMessage(pmapiMsgID)
+		return err
+	})
+
+	msgBuilder := pkgMessage.NewBuilder(p.client(), msg)
+	msgBuilder.EncryptedToHTML = false
+	_, body, err := msgBuilder.BuildMessage()
+	if err != nil {
+		return Message{
+			Body: body, // Keep body to show details about the message to user.
+		}, errors.Wrap(err, "failed to build message")
+	}
+
+	if !msgBuilder.SuccessfullyDecrypted() && skipEncryptedMessages {
+		return Message{
+			Body: body, // Keep body to show details about the message to user.
+		}, errors.New("skipping encrypted message")
+	}
+
+	unread := false
+	if msg.Unread == 1 {
+		unread = true
+	}
+
+	return Message{
+		ID:      msgID,
+		Unread:  unread,
+		Body:    body,
+		Source:  rule.SourceMailbox,
+		Targets: rule.TargetMailboxes,
+	}, nil
+}

internal/transfer/provider_pmapi_target.go

@@ -0,0 +1,220 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+
+	pkgMessage "github.com/ProtonMail/proton-bridge/pkg/message"
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/pkg/errors"
+)
+
+// DefaultMailboxes returns the default mailboxes for default rules if no other is found.
+func (p *PMAPIProvider) DefaultMailboxes(_ Mailbox) []Mailbox {
+	return []Mailbox{{
+		ID:          pmapi.ArchiveLabel,
+		Name:        "Archive",
+		IsExclusive: true,
+	}}
+}
+
+// CreateMailbox creates label in ProtonMail account.
+func (p *PMAPIProvider) CreateMailbox(mailbox Mailbox) (Mailbox, error) {
+	if mailbox.ID != "" {
+		return Mailbox{}, errors.New("mailbox is already created")
+	}
+
+	exclusive := 0
+	if mailbox.IsExclusive {
+		exclusive = 1
+	}
+
+	label, err := p.client().CreateLabel(&pmapi.Label{
+		Name:      mailbox.Name,
+		Color:     mailbox.Color,
+		Exclusive: exclusive,
+		Type:      pmapi.LabelTypeMailbox,
+	})
+	if err != nil {
+		return Mailbox{}, errors.Wrap(err, fmt.Sprintf("failed to create mailbox %s", mailbox.Name))
+	}
+	mailbox.ID = label.ID
+	return mailbox, nil
+}
+
+// TransferFrom imports messages from channel.
+func (p *PMAPIProvider) TransferFrom(rules transferRules, progress *Progress, ch <-chan Message) {
+	log.Info("Started transfer from channel to PMAPI")
+	defer log.Info("Finished transfer from channel to PMAPI")
+
+	for msg := range ch {
+		for progress.shouldStop() {
+			break
+		}
+
+		var importedID string
+		var err error
+		if p.isMessageDraft(msg) {
+			importedID, err = p.importDraft(msg, rules.globalMailbox)
+		} else {
+			importedID, err = p.importMessage(msg, rules.globalMailbox)
+		}
+		progress.messageImported(msg.ID, importedID, err)
+	}
+}
+
+func (p *PMAPIProvider) isMessageDraft(msg Message) bool {
+	for _, target := range msg.Targets {
+		if target.ID == pmapi.DraftLabel {
+			return true
+		}
+	}
+	return false
+}
+
+func (p *PMAPIProvider) importDraft(msg Message, globalMailbox *Mailbox) (string, error) {
+	message, attachmentReaders, err := p.parseMessage(msg)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to parse message")
+	}
+
+	if err := message.Encrypt(p.keyRing, nil); err != nil {
+		return "", errors.Wrap(err, "failed to encrypt draft")
+	}
+
+	if globalMailbox != nil {
+		message.LabelIDs = append(message.LabelIDs, globalMailbox.ID)
+	}
+
+	attachments := message.Attachments
+	message.Attachments = nil
+
+	draft, err := p.createDraft(message, "", pmapi.DraftActionReply)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to create draft")
+	}
+
+	for idx, attachment := range attachments {
+		attachment.MessageID = draft.ID
+		attachmentBody, _ := ioutil.ReadAll(attachmentReaders[idx])
+
+		r := bytes.NewReader(attachmentBody)
+		sigReader, err := attachment.DetachedSign(p.keyRing, r)
+		if err != nil {
+			return "", errors.Wrap(err, "failed to sign attachment")
+		}
+
+		r = bytes.NewReader(attachmentBody)
+		encReader, err := attachment.Encrypt(p.keyRing, r)
+		if err != nil {
+			return "", errors.Wrap(err, "failed to encrypt attachment")
+		}
+
+		_, err = p.createAttachment(attachment, encReader, sigReader)
+		if err != nil {
+			return "", errors.Wrap(err, "failed to create attachment")
+		}
+	}
+
+	return draft.ID, nil
+}
+
+func (p *PMAPIProvider) importMessage(msg Message, globalMailbox *Mailbox) (string, error) {
+	message, attachmentReaders, err := p.parseMessage(msg)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to parse message")
+	}
+
+	body, err := p.encryptMessage(message, attachmentReaders)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to encrypt message")
+	}
+
+	unread := 0
+	if msg.Unread {
+		unread = 1
+	}
+
+	labelIDs := []string{}
+	for _, target := range msg.Targets {
+		// Frontend should not set All Mail to Rules, but to be sure...
+		if target.ID != pmapi.AllMailLabel {
+			labelIDs = append(labelIDs, target.ID)
+		}
+	}
+	if globalMailbox != nil {
+		labelIDs = append(labelIDs, globalMailbox.ID)
+	}
+
+	importMsgReq := &pmapi.ImportMsgReq{
+		AddressID: p.addressID,
+		Body:      body,
+		Unread:    unread,
+		Time:      message.Time,
+		Flags:     computeMessageFlags(labelIDs),
+		LabelIDs:  labelIDs,
+	}
+
+	results, err := p.importRequest([]*pmapi.ImportMsgReq{importMsgReq})
+	if err != nil {
+		return "", errors.Wrap(err, "failed to import messages")
+	}
+	if len(results) == 0 {
+		return "", errors.New("import ended with no result")
+	}
+	if results[0].Error != nil {
+		return "", errors.Wrap(results[0].Error, "failed to import message")
+	}
+	return results[0].MessageID, nil
+}
+
+func (p *PMAPIProvider) parseMessage(msg Message) (*pmapi.Message, []io.Reader, error) {
+	message, _, _, attachmentReaders, err := pkgMessage.Parse(bytes.NewBuffer(msg.Body), "", "")
+	return message, attachmentReaders, err
+}
+
+func (p *PMAPIProvider) encryptMessage(msg *pmapi.Message, attachmentReaders []io.Reader) ([]byte, error) {
+	if msg.MIMEType == pmapi.ContentTypeMultipartEncrypted {
+		return []byte(msg.Body), nil
+	}
+	return pkgMessage.BuildEncrypted(msg, attachmentReaders, p.keyRing)
+}
+
+func computeMessageFlags(labels []string) (flag int64) {
+	for _, labelID := range labels {
+		switch labelID {
+		case pmapi.SentLabel:
+			flag = (flag | pmapi.FlagSent)
+		case pmapi.ArchiveLabel, pmapi.InboxLabel:
+			flag = (flag | pmapi.FlagReceived)
+		case pmapi.DraftLabel:
+			log.Error("Found draft target in non-draft import")
+		}
+	}
+
+	// NOTE: if the labels are custom only
+	if flag == 0 {
+		flag = pmapi.FlagReceived
+	}
+
+	return flag
+}

internal/transfer/provider_pmapi_test.go

@@ -0,0 +1,201 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	gomock "github.com/golang/mock/gomock"
+	r "github.com/stretchr/testify/require"
+)
+
+func TestPMAPIProviderMailboxes(t *testing.T) {
+	m := initMocks(t)
+	defer m.ctrl.Finish()
+
+	setupPMAPIClientExpectationForExport(&m)
+	provider, err := NewPMAPIProvider(m.clientManager, "user", "addressID")
+	r.NoError(t, err)
+
+	tests := []struct {
+		includeEmpty   bool
+		includeAllMail bool
+		wantMailboxes  []Mailbox
+	}{
+		{true, false, []Mailbox{
+			{ID: "folder1", Name: "One", Color: "red", IsExclusive: true},
+			{ID: "folder2", Name: "Two", Color: "orange", IsExclusive: true},
+			{ID: "label2", Name: "Bar", Color: "green", IsExclusive: false},
+			{ID: "label1", Name: "Foo", Color: "blue", IsExclusive: false},
+		}},
+		{false, true, []Mailbox{
+			{ID: pmapi.AllMailLabel, Name: "All Mail", IsExclusive: true},
+			{ID: "folder1", Name: "One", Color: "red", IsExclusive: true},
+			{ID: "folder2", Name: "Two", Color: "orange", IsExclusive: true},
+			{ID: "label1", Name: "Foo", Color: "blue", IsExclusive: false},
+		}},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(fmt.Sprintf("%v-%v", tc.includeEmpty, tc.includeAllMail), func(t *testing.T) {
+			mailboxes, err := provider.Mailboxes(tc.includeEmpty, tc.includeAllMail)
+			r.NoError(t, err)
+			r.Equal(t, []Mailbox{
+				{ID: pmapi.InboxLabel, Name: "Inbox", IsExclusive: true},
+				{ID: pmapi.DraftLabel, Name: "Drafts", IsExclusive: true},
+				{ID: pmapi.SentLabel, Name: "Sent", IsExclusive: true},
+				{ID: pmapi.StarredLabel, Name: "Starred", IsExclusive: true},
+				{ID: pmapi.ArchiveLabel, Name: "Archive", IsExclusive: true},
+				{ID: pmapi.SpamLabel, Name: "Spam", IsExclusive: true},
+				{ID: pmapi.TrashLabel, Name: "Trash", IsExclusive: true},
+			}, mailboxes[:7])
+			r.Equal(t, tc.wantMailboxes, mailboxes[7:])
+		})
+	}
+}
+
+func TestPMAPIProviderTransferTo(t *testing.T) {
+	m := initMocks(t)
+	defer m.ctrl.Finish()
+
+	setupPMAPIClientExpectationForExport(&m)
+	provider, err := NewPMAPIProvider(m.clientManager, "user", "addressID")
+	r.NoError(t, err)
+
+	rules, rulesClose := newTestRules(t)
+	defer rulesClose()
+	setupPMAPIRules(rules)
+
+	testTransferTo(t, rules, provider, []string{
+		"0_msg1",
+		"0_msg2",
+	})
+}
+
+func TestPMAPIProviderTransferFrom(t *testing.T) {
+	m := initMocks(t)
+	defer m.ctrl.Finish()
+
+	setupPMAPIClientExpectationForImport(&m)
+	provider, err := NewPMAPIProvider(m.clientManager, "user", "addressID")
+	r.NoError(t, err)
+
+	rules, rulesClose := newTestRules(t)
+	defer rulesClose()
+	setupPMAPIRules(rules)
+
+	testTransferFrom(t, rules, provider, []Message{
+		{ID: "msg1", Body: getTestMsgBody("msg1"), Targets: []Mailbox{{ID: pmapi.InboxLabel}}},
+		{ID: "msg2", Body: getTestMsgBody("msg2"), Targets: []Mailbox{{ID: pmapi.InboxLabel}}},
+	})
+}
+
+func TestPMAPIProviderTransferFromDraft(t *testing.T) {
+	m := initMocks(t)
+	defer m.ctrl.Finish()
+
+	setupPMAPIClientExpectationForImportDraft(&m)
+	provider, err := NewPMAPIProvider(m.clientManager, "user", "addressID")
+	r.NoError(t, err)
+
+	rules, rulesClose := newTestRules(t)
+	defer rulesClose()
+	setupPMAPIRules(rules)
+
+	testTransferFrom(t, rules, provider, []Message{
+		{ID: "draft1", Body: getTestMsgBody("draft1"), Targets: []Mailbox{{ID: pmapi.DraftLabel}}},
+	})
+}
+
+func TestPMAPIProviderTransferFromTo(t *testing.T) {
+	m := initMocks(t)
+	defer m.ctrl.Finish()
+
+	setupPMAPIClientExpectationForExport(&m)
+	setupPMAPIClientExpectationForImport(&m)
+
+	source, err := NewPMAPIProvider(m.clientManager, "user", "addressID")
+	r.NoError(t, err)
+	target, err := NewPMAPIProvider(m.clientManager, "user", "addressID")
+	r.NoError(t, err)
+
+	rules, rulesClose := newTestRules(t)
+	defer rulesClose()
+	setupPMAPIRules(rules)
+
+	testTransferFromTo(t, rules, source, target, 5*time.Second)
+}
+
+func setupPMAPIRules(rules transferRules) {
+	_ = rules.setRule(Mailbox{ID: pmapi.InboxLabel}, []Mailbox{{ID: pmapi.InboxLabel}}, 0, 0)
+}
+
+func setupPMAPIClientExpectationForExport(m *mocks) {
+	m.pmapiClient.EXPECT().KeyRingForAddressID(gomock.Any()).Return(m.keyring, nil).AnyTimes()
+	m.pmapiClient.EXPECT().ListLabels().Return([]*pmapi.Label{
+		{ID: "label1", Name: "Foo", Color: "blue", Exclusive: 0, Order: 2},
+		{ID: "label2", Name: "Bar", Color: "green", Exclusive: 0, Order: 1},
+		{ID: "folder1", Name: "One", Color: "red", Exclusive: 1, Order: 1},
+		{ID: "folder2", Name: "Two", Color: "orange", Exclusive: 1, Order: 2},
+	}, nil).AnyTimes()
+	m.pmapiClient.EXPECT().CountMessages(gomock.Any()).Return([]*pmapi.MessagesCount{
+		{LabelID: "label1", Total: 10},
+		{LabelID: "label2", Total: 0},
+		{LabelID: "folder1", Total: 20},
+	}, nil).AnyTimes()
+	m.pmapiClient.EXPECT().ListMessages(gomock.Any()).Return([]*pmapi.Message{
+		{ID: "msg1"},
+		{ID: "msg2"},
+	}, 2, nil).AnyTimes()
+	m.pmapiClient.EXPECT().GetMessage(gomock.Any()).DoAndReturn(func(msgID string) (*pmapi.Message, error) {
+		return &pmapi.Message{
+			ID:       msgID,
+			Body:     string(getTestMsgBody(msgID)),
+			MIMEType: pmapi.ContentTypeMultipartMixed,
+		}, nil
+	}).AnyTimes()
+}
+
+func setupPMAPIClientExpectationForImport(m *mocks) {
+	m.pmapiClient.EXPECT().KeyRingForAddressID(gomock.Any()).Return(m.keyring, nil).AnyTimes()
+	m.pmapiClient.EXPECT().Import(gomock.Any()).DoAndReturn(func(requests []*pmapi.ImportMsgReq) ([]*pmapi.ImportMsgRes, error) {
+		r.Equal(m.t, 1, len(requests))
+
+		request := requests[0]
+		for _, msgID := range []string{"msg1", "msg2"} {
+			if bytes.Contains(request.Body, []byte(msgID)) {
+				return []*pmapi.ImportMsgRes{{MessageID: msgID, Error: nil}}, nil
+			}
+		}
+		r.Fail(m.t, "No message found")
+		return nil, nil
+	}).Times(2)
+}
+
+func setupPMAPIClientExpectationForImportDraft(m *mocks) {
+	m.pmapiClient.EXPECT().KeyRingForAddressID(gomock.Any()).Return(m.keyring, nil).AnyTimes()
+	m.pmapiClient.EXPECT().CreateDraft(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(msg *pmapi.Message, parentID string, action int) (*pmapi.Message, error) {
+		r.Equal(m.t, msg.Subject, "draft1")
+		msg.ID = "draft1"
+		return msg, nil
+	})
+}

internal/transfer/provider_pmapi_utils.go

@@ -0,0 +1,109 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"io"
+	"time"
+
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/pkg/errors"
+)
+
+const (
+	pmapiRetries          = 10
+	pmapiReconnectTimeout = 30 * time.Minute
+	pmapiReconnectSleep   = time.Minute
+)
+
+func (p *PMAPIProvider) ensureConnection(callback func() error) error {
+	var callErr error
+	for i := 1; i <= pmapiRetries; i++ {
+		callErr = callback()
+		if callErr == nil {
+			return nil
+		}
+
+		log.WithField("attempt", i).WithError(callErr).Warning("Call failed, trying reconnect")
+		err := p.tryReconnect()
+		if err != nil {
+			return err
+		}
+	}
+	return errors.Wrap(callErr, "too many retries")
+}
+
+func (p *PMAPIProvider) tryReconnect() error {
+	start := time.Now()
+	var previousErr error
+	for {
+		if time.Since(start) > pmapiReconnectTimeout {
+			return previousErr
+		}
+
+		err := p.clientManager.CheckConnection()
+		if err != nil {
+			time.Sleep(pmapiReconnectSleep)
+			previousErr = err
+			continue
+		}
+
+		break
+	}
+	return nil
+}
+
+func (p *PMAPIProvider) listMessages(filter *pmapi.MessagesFilter) (messages []*pmapi.Message, count int, err error) {
+	err = p.ensureConnection(func() error {
+		messages, count, err = p.client().ListMessages(filter)
+		return err
+	})
+	return
+}
+
+func (p *PMAPIProvider) getMessage(msgID string) (message *pmapi.Message, err error) {
+	err = p.ensureConnection(func() error {
+		message, err = p.client().GetMessage(msgID)
+		return err
+	})
+	return
+}
+
+func (p *PMAPIProvider) importRequest(req []*pmapi.ImportMsgReq) (res []*pmapi.ImportMsgRes, err error) {
+	err = p.ensureConnection(func() error {
+		res, err = p.client().Import(req)
+		return err
+	})
+	return
+}
+
+func (p *PMAPIProvider) createDraft(message *pmapi.Message, parent string, action int) (draft *pmapi.Message, err error) {
+	err = p.ensureConnection(func() error {
+		draft, err = p.client().CreateDraft(message, parent, action)
+		return err
+	})
+	return
+}
+
+func (p *PMAPIProvider) createAttachment(att *pmapi.Attachment, r io.Reader, sig io.Reader) (created *pmapi.Attachment, err error) {
+	err = p.ensureConnection(func() error {
+		created, err = p.client().CreateAttachment(att, r, sig)
+		return err
+	})
+	return
+}

internal/transfer/provider_test.go

@@ -0,0 +1,111 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	a "github.com/stretchr/testify/assert"
+	r "github.com/stretchr/testify/require"
+)
+
+func getTestMsgBody(subject string) []byte {
+	return []byte(fmt.Sprintf(`Subject: %s
+From: Bridge Test <bridgetest@pm.test>
+To: Bridge Test <bridgetest@protonmail.com>
+Content-Type: multipart/mixed; boundary=c672b8d1ef56ed28ab87c3622c5114069bdd3ad7b8f9737498d0c01ecef0967a
+
+--c672b8d1ef56ed28ab87c3622c5114069bdd3ad7b8f9737498d0c01ecef0967a
+Content-Disposition: inline
+Content-Transfer-Encoding: 7bit
+Content-Type: text/plain; charset=utf-8
+
+hello
+
+--c672b8d1ef56ed28ab87c3622c5114069bdd3ad7b8f9737498d0c01ecef0967a--
+`, subject))
+}
+
+func testTransferTo(t *testing.T, rules transferRules, provider SourceProvider, expectedMessageIDs []string) {
+	progress := newProgress(log, nil)
+	drainProgressUpdateChannel(&progress)
+
+	ch := make(chan Message)
+	go func() {
+		provider.TransferTo(rules, &progress, ch)
+		close(ch)
+	}()
+
+	gotMessageIDs := []string{}
+	for msg := range ch {
+		gotMessageIDs = append(gotMessageIDs, msg.ID)
+	}
+	r.ElementsMatch(t, expectedMessageIDs, gotMessageIDs)
+
+	r.Empty(t, progress.GetFailedMessages())
+}
+
+func testTransferFrom(t *testing.T, rules transferRules, provider TargetProvider, messages []Message) {
+	progress := newProgress(log, nil)
+	drainProgressUpdateChannel(&progress)
+
+	ch := make(chan Message)
+	go func() {
+		for _, message := range messages {
+			progress.addMessage(message.ID, nil)
+			progress.messageExported(message.ID, []byte(""), nil)
+			ch <- message
+		}
+		close(ch)
+	}()
+
+	go func() {
+		provider.TransferFrom(rules, &progress, ch)
+		progress.finish()
+	}()
+
+	maxWait := time.Duration(len(messages)) * time.Second
+	a.Eventually(t, func() bool {
+		return progress.updateCh == nil
+	}, maxWait, 10*time.Millisecond, "Waiting for imported messages timed out")
+
+	r.Empty(t, progress.GetFailedMessages())
+}
+
+func testTransferFromTo(t *testing.T, rules transferRules, source SourceProvider, target TargetProvider, maxWait time.Duration) {
+	progress := newProgress(log, nil)
+	drainProgressUpdateChannel(&progress)
+
+	ch := make(chan Message)
+	go func() {
+		source.TransferTo(rules, &progress, ch)
+		close(ch)
+	}()
+	go func() {
+		target.TransferFrom(rules, &progress, ch)
+		progress.finish()
+	}()
+
+	a.Eventually(t, func() bool {
+		return progress.updateCh == nil
+	}, maxWait, 10*time.Millisecond, "Waiting for export and import timed out")
+
+	r.Empty(t, progress.GetFailedMessages())
+}

internal/transfer/report.go

@@ -0,0 +1,145 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+// fileReport is struct which can write and read message details.
+// File report includes private information.
+type fileReport struct {
+	path string
+}
+
+func openLastFileReport(reportsPath, importID string) (*fileReport, error) { //nolint[deadcode]
+	allLogFileNames, err := getFilePathsWithSuffix(reportsPath, ".log")
+	if err != nil {
+		return nil, err
+	}
+
+	reportFileNames := []string{}
+	for _, fileName := range allLogFileNames {
+		if strings.HasPrefix(fileName, fmt.Sprintf("import_%s_", importID)) {
+			reportFileNames = append(reportFileNames, fileName)
+		}
+	}
+	if len(reportFileNames) == 0 {
+		return nil, errors.New("no report found")
+	}
+
+	sort.Strings(reportFileNames)
+	reportFileName := reportFileNames[len(reportFileNames)-1]
+	path := filepath.Join(reportsPath, reportFileName)
+	return &fileReport{
+		path: path,
+	}, nil
+}
+
+func newFileReport(reportsPath, importID string) *fileReport {
+	fileName := fmt.Sprintf("import_%s_%d.log", importID, time.Now().Unix())
+	path := filepath.Join(reportsPath, fileName)
+
+	return &fileReport{
+		path: path,
+	}
+}
+
+func (r *fileReport) writeMessageStatus(messageStatus *MessageStatus) {
+	f, err := os.OpenFile(r.path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
+	if err != nil {
+		log.WithError(err).Error("Failed to open report file")
+	}
+	defer f.Close() //nolint[errcheck]
+
+	messageReport := newMessageReportFromMessageStatus(messageStatus, true)
+	data, err := json.Marshal(messageReport)
+	if err != nil {
+		log.WithError(err).Error("Failed to marshall message details")
+	}
+	data = append(data, '\n')
+
+	if _, err = f.Write(data); err != nil {
+		log.WithError(err).Error("Failed to write to report file")
+	}
+}
+
+// bugReport is struct which can create report for bug reporting.
+// Bug report does NOT include private information.
+type bugReport struct {
+	data bytes.Buffer
+}
+
+func (r *bugReport) writeMessageStatus(messageStatus *MessageStatus) {
+	messageReport := newMessageReportFromMessageStatus(messageStatus, false)
+	data, err := json.Marshal(messageReport)
+	if err != nil {
+		log.WithError(err).Error("Failed to marshall message details")
+	}
+	_, _ = r.data.Write(data)
+	_, _ = r.data.Write([]byte("\n"))
+}
+
+func (r *bugReport) getData() []byte {
+	return r.data.Bytes()
+}
+
+// messageReport is struct which holds data used by `fileReport` and `bugReport`.
+type messageReport struct {
+	EventTime       int64
+	SourceID        string
+	TargetID        string
+	BodyHash        string
+	SourceMailbox   string
+	TargetMailboxes []string
+	Error           string
+
+	// Private information for user.
+	Subject string
+	From    string
+	Time    string
+}
+
+func newMessageReportFromMessageStatus(messageStatus *MessageStatus, includePrivateInfo bool) messageReport {
+	md := messageReport{
+		EventTime:       messageStatus.eventTime.Unix(),
+		SourceID:        messageStatus.SourceID,
+		TargetID:        messageStatus.targetID,
+		BodyHash:        messageStatus.bodyHash,
+		SourceMailbox:   messageStatus.rule.SourceMailbox.Name,
+		TargetMailboxes: messageStatus.rule.TargetMailboxNames(),
+		Error:           messageStatus.GetErrorMessage(),
+	}
+
+	if includePrivateInfo {
+		md.Subject = messageStatus.Subject
+		md.From = messageStatus.From
+		md.Time = messageStatus.Time.Format(time.RFC1123Z)
+	}
+
+	return md
+}

internal/transfer/rules.go

@@ -0,0 +1,290 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/pkg/errors"
+)
+
+// transferRules maintains import rules, e.g. to which target mailbox should be
+// source mailbox imported or what time spans.
+type transferRules struct {
+	filePath string
+
+	// rules is map with key as hash of source mailbox to its rule.
+	// Every source mailbox should have rule, at least disabled one.
+	rules map[string]*Rule
+
+	// globalMailbox is applied to every message in the import phase.
+	// E.g., every message will be imported into this mailbox.
+	globalMailbox *Mailbox
+
+	// skipEncryptedMessages determines whether message which cannot
+	// be decrypted should be exported or skipped.
+	skipEncryptedMessages bool
+}
+
+// loadRules loads rules from `rulesPath` based on `ruleID`.
+func loadRules(rulesPath, ruleID string) transferRules {
+	fileName := fmt.Sprintf("rules_%s.json", ruleID)
+	filePath := filepath.Join(rulesPath, fileName)
+
+	var rules map[string]*Rule
+	f, err := os.Open(filePath) //nolint[gosec]
+	if err != nil {
+		log.WithError(err).Debug("Problem to read rules")
+	} else {
+		defer f.Close() //nolint[errcheck]
+		if err := json.NewDecoder(f).Decode(&rules); err != nil {
+			log.WithError(err).Warn("Problem to umarshal rules")
+		}
+	}
+	if rules == nil {
+		rules = map[string]*Rule{}
+	}
+
+	return transferRules{
+		filePath: filePath,
+		rules:    rules,
+	}
+}
+
+func (r *transferRules) setSkipEncryptedMessages(skip bool) {
+	r.skipEncryptedMessages = skip
+}
+
+func (r *transferRules) setGlobalMailbox(mailbox *Mailbox) {
+	r.globalMailbox = mailbox
+}
+
+func (r *transferRules) setGlobalTimeLimit(fromTime, toTime int64) {
+	for _, rule := range r.rules {
+		if !rule.HasTimeLimit() {
+			rule.FromTime = fromTime
+			rule.ToTime = toTime
+		}
+	}
+}
+
+func (r *transferRules) getRuleBySourceMailboxName(name string) (*Rule, error) {
+	for _, rule := range r.rules {
+		if rule.SourceMailbox.Name == name {
+			return rule, nil
+		}
+	}
+	return nil, fmt.Errorf("no rule for mailbox %s", name)
+}
+
+func (r *transferRules) iterateActiveRules() chan *Rule {
+	ch := make(chan *Rule)
+	go func() {
+		for _, rule := range r.rules {
+			if rule.Active {
+				ch <- rule
+			}
+		}
+		close(ch)
+	}()
+	return ch
+}
+
+// setDefaultRules iterates `sourceMailboxes` and sets missing rules with
+// matching mailboxes from `targetMailboxes`. In case no matching mailbox
+// is found, `defaultCallback` with a source mailbox as a parameter is used.
+func (r *transferRules) setDefaultRules(sourceMailboxes []Mailbox, targetMailboxes []Mailbox, defaultCallback func(Mailbox) []Mailbox) {
+	for _, sourceMailbox := range sourceMailboxes {
+		h := sourceMailbox.Hash()
+		if _, ok := r.rules[h]; ok {
+			continue
+		}
+
+		targetMailboxes := sourceMailbox.findMatchingMailboxes(targetMailboxes)
+		if len(targetMailboxes) == 0 {
+			targetMailboxes = defaultCallback(sourceMailbox)
+		}
+
+		active := true
+		if len(targetMailboxes) == 0 {
+			active = false
+		}
+
+		// For both import to or export from ProtonMail, spam and draft
+		// mailboxes are by default deactivated.
+		for _, mailbox := range append([]Mailbox{sourceMailbox}, targetMailboxes...) {
+			if mailbox.ID == pmapi.SpamLabel || mailbox.ID == pmapi.DraftLabel || mailbox.ID == pmapi.TrashLabel {
+				active = false
+				break
+			}
+		}
+
+		r.rules[h] = &Rule{
+			Active:          active,
+			SourceMailbox:   sourceMailbox,
+			TargetMailboxes: targetMailboxes,
+		}
+	}
+
+	for _, rule := range r.rules {
+		if !rule.Active {
+			continue
+		}
+		found := false
+		for _, sourceMailbox := range sourceMailboxes {
+			if sourceMailbox.Name == rule.SourceMailbox.Name {
+				found = true
+			}
+		}
+		if !found {
+			rule.Active = false
+		}
+	}
+
+	r.save()
+}
+
+// setRule sets messages from `sourceMailbox` between `fromData` and `toDate`
+// (if used) to be imported to all `targetMailboxes`.
+func (r *transferRules) setRule(sourceMailbox Mailbox, targetMailboxes []Mailbox, fromTime, toTime int64) error {
+	numberOfExclusiveMailboxes := 0
+	for _, mailbox := range targetMailboxes {
+		if mailbox.IsExclusive {
+			numberOfExclusiveMailboxes++
+		}
+	}
+	if numberOfExclusiveMailboxes > 1 {
+		return errors.New("rule can have only one exclusive target mailbox")
+	}
+
+	h := sourceMailbox.Hash()
+	r.rules[h] = &Rule{
+		Active:          true,
+		SourceMailbox:   sourceMailbox,
+		TargetMailboxes: targetMailboxes,
+		FromTime:        fromTime,
+		ToTime:          toTime,
+	}
+	r.save()
+	return nil
+}
+
+// unsetRule unsets messages from `sourceMailbox` to be exported.
+func (r *transferRules) unsetRule(sourceMailbox Mailbox) {
+	h := sourceMailbox.Hash()
+	if rule, ok := r.rules[h]; ok {
+		rule.Active = false
+	} else {
+		r.rules[h] = &Rule{
+			Active:        false,
+			SourceMailbox: sourceMailbox,
+		}
+	}
+	r.save()
+}
+
+// getRule returns rule for `sourceMailbox` or nil if it does not exist.
+func (r *transferRules) getRule(sourceMailbox Mailbox) *Rule {
+	h := sourceMailbox.Hash()
+	return r.rules[h]
+}
+
+// getRules returns all set rules.
+func (r *transferRules) getRules() []*Rule {
+	rules := []*Rule{}
+	for _, rule := range r.rules {
+		rules = append(rules, rule)
+	}
+	return rules
+}
+
+// reset wipes our all rules.
+func (r *transferRules) reset() {
+	r.rules = map[string]*Rule{}
+	r.save()
+}
+
+// save saves rules to file.
+func (r *transferRules) save() {
+	f, err := os.Create(r.filePath)
+	if err != nil {
+		log.WithError(err).Warn("Problem to write rules")
+		return
+	}
+	defer f.Close() //nolint[errcheck]
+
+	if err := json.NewEncoder(f).Encode(r.rules); err != nil {
+		log.WithError(err).Warn("Problem to marshal rules")
+	}
+}
+
+// Rule is data holder of rule for one source mailbox used by `transferRules`.
+type Rule struct {
+	Active          bool      `json:"active"`
+	SourceMailbox   Mailbox   `json:"source"`
+	TargetMailboxes []Mailbox `json:"targets"`
+	FromTime        int64     `json:"from"`
+	ToTime          int64     `json:"to"`
+}
+
+// String returns textual representation for log purposes.
+func (r *Rule) String() string {
+	return fmt.Sprintf(
+		"%s -> %s (%d - %d)",
+		r.SourceMailbox.Name,
+		strings.Join(r.TargetMailboxNames(), ", "),
+		r.FromTime,
+		r.ToTime,
+	)
+}
+
+func (r *Rule) isTimeInRange(t int64) bool {
+	if !r.HasTimeLimit() {
+		return true
+	}
+	return r.FromTime <= t && t <= r.ToTime
+}
+
+// HasTimeLimit returns whether rule defines time limit.
+func (r *Rule) HasTimeLimit() bool {
+	return r.FromTime != 0 || r.ToTime != 0
+}
+
+// FromDate returns time struct based on `FromTime`.
+func (r *Rule) FromDate() time.Time {
+	return time.Unix(r.FromTime, 0)
+}
+
+// ToDate returns time struct based on `ToTime`.
+func (r *Rule) ToDate() time.Time {
+	return time.Unix(r.ToTime, 0)
+}
+
+// TargetMailboxNames returns array of target mailbox names.
+func (r *Rule) TargetMailboxNames() (names []string) {
+	for _, mailbox := range r.TargetMailboxes {
+		names = append(names, mailbox.Name)
+	}
+	return
+}

internal/transfer/rules_test.go

@@ -0,0 +1,210 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	r "github.com/stretchr/testify/require"
+)
+
+func newTestRules(t *testing.T) (transferRules, func()) {
+	path, err := ioutil.TempDir("", "rules")
+	r.NoError(t, err)
+
+	ruleID := "rule"
+	rules := loadRules(path, ruleID)
+	return rules, func() {
+		_ = os.RemoveAll(path)
+	}
+}
+
+func TestLoadRules(t *testing.T) {
+	path, err := ioutil.TempDir("", "rules")
+	r.NoError(t, err)
+	defer os.RemoveAll(path) //nolint[errcheck]
+
+	ruleID := "rule"
+	rules := loadRules(path, ruleID)
+
+	mailboxA := Mailbox{ID: "1", Name: "One", Color: "orange", IsExclusive: true}
+	mailboxB := Mailbox{ID: "2", Name: "Two", Color: "", IsExclusive: true}
+	mailboxC := Mailbox{ID: "3", Name: "Three", Color: "", IsExclusive: false}
+
+	r.NoError(t, rules.setRule(mailboxA, []Mailbox{mailboxB, mailboxC}, 0, 0))
+	r.NoError(t, rules.setRule(mailboxB, []Mailbox{mailboxB}, 10, 20))
+	r.NoError(t, rules.setRule(mailboxC, []Mailbox{}, 0, 30))
+
+	rules2 := loadRules(path, ruleID)
+	r.Equal(t, map[string]*Rule{
+		mailboxA.Hash(): {Active: true, SourceMailbox: mailboxA, TargetMailboxes: []Mailbox{mailboxB, mailboxC}, FromTime: 0, ToTime: 0},
+		mailboxB.Hash(): {Active: true, SourceMailbox: mailboxB, TargetMailboxes: []Mailbox{mailboxB}, FromTime: 10, ToTime: 20},
+		mailboxC.Hash(): {Active: true, SourceMailbox: mailboxC, TargetMailboxes: []Mailbox{}, FromTime: 0, ToTime: 30},
+	}, rules2.rules)
+
+	rules2.unsetRule(mailboxA)
+	rules2.unsetRule(mailboxC)
+
+	rules3 := loadRules(path, ruleID)
+	r.Equal(t, map[string]*Rule{
+		mailboxA.Hash(): {Active: false, SourceMailbox: mailboxA, TargetMailboxes: []Mailbox{mailboxB, mailboxC}, FromTime: 0, ToTime: 0},
+		mailboxB.Hash(): {Active: true, SourceMailbox: mailboxB, TargetMailboxes: []Mailbox{mailboxB}, FromTime: 10, ToTime: 20},
+		mailboxC.Hash(): {Active: false, SourceMailbox: mailboxC, TargetMailboxes: []Mailbox{}, FromTime: 0, ToTime: 30},
+	}, rules3.rules)
+}
+
+func TestSetGlobalTimeLimit(t *testing.T) {
+	path, err := ioutil.TempDir("", "rules")
+	r.NoError(t, err)
+	defer os.RemoveAll(path) //nolint[errcheck]
+
+	rules := loadRules(path, "rule")
+
+	mailboxA := Mailbox{Name: "One"}
+	mailboxB := Mailbox{Name: "Two"}
+
+	r.NoError(t, rules.setRule(mailboxA, []Mailbox{}, 10, 20))
+	r.NoError(t, rules.setRule(mailboxB, []Mailbox{}, 0, 0))
+
+	rules.setGlobalTimeLimit(30, 40)
+
+	r.Equal(t, map[string]*Rule{
+		mailboxA.Hash(): {Active: true, SourceMailbox: mailboxA, TargetMailboxes: []Mailbox{}, FromTime: 10, ToTime: 20},
+		mailboxB.Hash(): {Active: true, SourceMailbox: mailboxB, TargetMailboxes: []Mailbox{}, FromTime: 30, ToTime: 40},
+	}, rules.rules)
+}
+
+func TestSetDefaultRules(t *testing.T) {
+	path, err := ioutil.TempDir("", "rules")
+	r.NoError(t, err)
+	defer os.RemoveAll(path) //nolint[errcheck]
+
+	rules := loadRules(path, "rule")
+
+	mailbox1 := Mailbox{Name: "One"}                       // Set manually, default will not override it.
+	mailbox2 := Mailbox{Name: "Two"}                       // Matched by `targetMailboxes`.
+	mailbox3 := Mailbox{Name: "Three"}                     // Matched by `defaultCallback`, not included in `targetMailboxes`.
+	mailbox4 := Mailbox{Name: "Four"}                      // Matched by nothing, will not be active.
+	mailbox5 := Mailbox{Name: "Spam", ID: pmapi.SpamLabel} // Spam is inactive by default (ID found in source).
+	mailbox6a := Mailbox{Name: "Draft"}                    // Draft is inactive by default (ID found in target, mailbox6b).
+	mailbox6b := Mailbox{Name: "Draft", ID: pmapi.DraftLabel}
+
+	sourceMailboxes := []Mailbox{mailbox1, mailbox2, mailbox3, mailbox4, mailbox5, mailbox6a}
+	targetMailboxes := []Mailbox{mailbox1, mailbox2, mailbox6b}
+
+	r.NoError(t, rules.setRule(mailbox1, []Mailbox{mailbox3}, 0, 0))
+
+	defaultCallback := func(mailbox Mailbox) []Mailbox {
+		if mailbox.Name == "Three" {
+			return []Mailbox{mailbox3}
+		}
+		return []Mailbox{}
+	}
+
+	rules.setDefaultRules(sourceMailboxes, targetMailboxes, defaultCallback)
+
+	r.Equal(t, map[string]*Rule{
+		mailbox1.Hash():  {Active: true, SourceMailbox: mailbox1, TargetMailboxes: []Mailbox{mailbox3}},
+		mailbox2.Hash():  {Active: true, SourceMailbox: mailbox2, TargetMailboxes: []Mailbox{mailbox2}},
+		mailbox3.Hash():  {Active: true, SourceMailbox: mailbox3, TargetMailboxes: []Mailbox{mailbox3}},
+		mailbox4.Hash():  {Active: false, SourceMailbox: mailbox4, TargetMailboxes: []Mailbox{}},
+		mailbox5.Hash():  {Active: false, SourceMailbox: mailbox5, TargetMailboxes: []Mailbox{}},
+		mailbox6a.Hash(): {Active: false, SourceMailbox: mailbox6a, TargetMailboxes: []Mailbox{mailbox6b}},
+	}, rules.rules)
+}
+
+func TestSetDefaultRulesDeactivateMissing(t *testing.T) {
+	path, err := ioutil.TempDir("", "rules")
+	r.NoError(t, err)
+	defer os.RemoveAll(path) //nolint[errcheck]
+
+	rules := loadRules(path, "rule")
+
+	mailboxA := Mailbox{ID: "1", Name: "One", Color: "", IsExclusive: true}
+	mailboxB := Mailbox{ID: "2", Name: "Two", Color: "", IsExclusive: true}
+
+	r.NoError(t, rules.setRule(mailboxA, []Mailbox{mailboxB}, 0, 0))
+	r.NoError(t, rules.setRule(mailboxB, []Mailbox{mailboxB}, 0, 0))
+
+	sourceMailboxes := []Mailbox{mailboxA}
+	targetMailboxes := []Mailbox{mailboxA, mailboxB}
+	defaultCallback := func(mailbox Mailbox) (mailboxes []Mailbox) {
+		return
+	}
+	rules.setDefaultRules(sourceMailboxes, targetMailboxes, defaultCallback)
+
+	r.Equal(t, map[string]*Rule{
+		mailboxA.Hash(): {Active: true, SourceMailbox: mailboxA, TargetMailboxes: []Mailbox{mailboxB}, FromTime: 0, ToTime: 0},
+		mailboxB.Hash(): {Active: false, SourceMailbox: mailboxB, TargetMailboxes: []Mailbox{mailboxB}, FromTime: 0, ToTime: 0},
+	}, rules.rules)
+}
+
+func TestIsTimeInRange(t *testing.T) {
+	tests := []struct {
+		rule Rule
+		time int64
+		want bool
+	}{
+		{generateTimeRule(0, 0), 0, true},
+		{generateTimeRule(0, 0), 10, true},
+		{generateTimeRule(0, 15), 10, true},
+		{generateTimeRule(5, 15), 10, true},
+		{generateTimeRule(0, 5), 10, false},
+		{generateTimeRule(5, 7), 10, false},
+		{generateTimeRule(15, 30), 10, false},
+		{generateTimeRule(15, 0), 10, false},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(fmt.Sprintf("%v / %d", tc.rule, tc.time), func(t *testing.T) {
+			got := tc.rule.isTimeInRange(tc.time)
+			r.Equal(t, tc.want, got)
+		})
+	}
+}
+
+func TestHasTimeLimit(t *testing.T) {
+	tests := []struct {
+		rule Rule
+		want bool
+	}{
+		{generateTimeRule(0, 0), false},
+		{generateTimeRule(0, 1), true},
+		{generateTimeRule(1, 2), true},
+		{generateTimeRule(1, 0), true},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(fmt.Sprintf("%v", tc.rule), func(t *testing.T) {
+			r.Equal(t, tc.want, tc.rule.HasTimeLimit())
+		})
+	}
+}
+
+func generateTimeRule(from, to int64) Rule {
+	return Rule{
+		SourceMailbox:   Mailbox{},
+		TargetMailboxes: []Mailbox{},
+		FromTime:        from,
+		ToTime:          to,
+	}
+}

internal/transfer/testdata/eml/Foo/msg.eml

@@ -0,0 +1,4 @@
+From: Bridge Test <bridgetest@pm.test>
+To: Bridge Test <bridgetest@protonmail.com>
+
+hello

internal/transfer/testdata/eml/Inbox/msg.eml

@@ -0,0 +1,4 @@
+From: Bridge Test <bridgetest@pm.test>
+To: Bridge Test <bridgetest@protonmail.com>
+
+hello

internal/transfer/testdata/emlmbox/Foo/msg.eml

@@ -0,0 +1,4 @@
+From: Bridge Test <bridgetest@pm.test>
+To: Bridge Test <bridgetest@protonmail.com>
+
+hello

internal/transfer/testdata/emlmbox/Inbox.mbox

@@ -0,0 +1,5 @@
+From - Mon May  4 16:40:31 2020
+From: Bridge Test <bridgetest@pm.test>
+To: Bridge Test <bridgetest@protonmail.com>
+
+hello

internal/transfer/testdata/keyring_userKey

@@ -0,0 +1,62 @@
+-----BEGIN PGP PRIVATE KEY BLOCK-----
+Version: OpenPGP.js v4.4.5
+Comment: testpassphrase
+
+xcLYBFzGzhEBCADBxfqTFMqfQzT77A5tuuhPFwPq8dfC2evs8u1OvTqFbztY
+5FOuSxzduyeDqQ1Fx6dKEOKgcYE8t1Uh4VSS7z6bTdY8j9yrL81kCVB46sE1
+OzStzyx/5l7OdH/pM4F+aKslnLvqlw0UeJr+UNizVtOCEUaNfVjPK3cc1ocx
+v+36K4RnnyfEtjUW9gDZbhgaF02G5ILHmWmbgM7I+77gCd2wI0EdY9s/JZQ+
+VmkMFqoMdY9PyBchoOIPUkkGQi1SaF4IEzMaAUSbnCYkHHY/SbfDTcR46VGq
+cXlkB1rq5xskaUQ9r+giCC/K4pc7bBkI1lQ7ADVuWvdrWnWapK0FO6CfABEB
+AAEAB/0YPhPJ0phA/EWviN+16bmGVOZNaVapjt2zMMybWmrtEQv3OeWgO3nP
+4cohRi/zaCBCphcm+dxbLhftW7AFi/9PVcR09436MB+oTCQFugpUWw+4TmA5
+BidxTpDxf4X2vH3rquQLBufWL6U7JlPeKAGL1xZ2aCq0DIeOk5D+xTjZizV2
+GIyQRVCLWb+LfDmvvcp3Y94X60KXdBAMuS1ZMKcY3Sl8VAXNB4KQsC/kByzf
+6FCB097XZRYV7lvJJQ7+6Wisb3yVi8sEQx2sFm5fAp+0qi3a6zRTEp49r6Hr
+gyWViH5zOOpA7DcNwx1Bwhi7GG0tak6EUnnKUNLfOupglcphBADmpXCgT4nc
+uSBYTiZSVcB/ICCkTxVsHL1WcXtPK2Ikzussx2n9kb0rapvuC0YLipX9lUkQ
+fyeC3jQJeCyN79AkDGkOfWaESueT2hM0Po+RwDgMibKn6yJ1zebz4Lc2J3C9
+oVFcAnql+9KyGsAPn03fyQzDnvhNnJvHJi4Hx8AWoQQA1xLoXeVBjRi0IjjU
+E6Mqaq5RLEog4kXRp86VSSEGHBwyIYnDiM//gjseo/CXuVyHwL7UXitp8s1B
+D1uE3APrhqUS66fD5pkF+z+RcSqiIv7I76NJ24Cdg38L6seGSjOHrq7/dEeG
+K6WqfQUCEjta3yNSg7pXb2wn2WZqKIK+rz8EALZRuMXeql/FtO3Cjb0sv7oT
+9dLP4cn1bskGRJ+Vok9lfCERbfXGccoAk3V+qSfpHgKxsebkRbUhf+trOGnw
+tW+kBWo/5hYGQuN+A9JogSJViT+nuZyE+x1/rKswDFmlMSdf2GIDARWIV0gc
+b1yOEwUmNBSthPcnFXvBr4BG3XTtNPTNLSJhcm9uMjEtM0BzYWRlbWJlLm9y
+ZyIgPGFyb24yMS0zQHNhZGVtYmUub3JnPsLAdQQQAQgAHwUCXMbOEQYLCQcI
+AwIEFQgKAgMWAgECGQECGwMCHgEACgkQZ/B4v2b2xB6XUgf/dHGRHimyMR78
+QYbEm2cuaEvOtq4a+J6Zv3P4VOWAbvkGWS9LDKSvVi60vq4oYOmF54HgPzur
+nA4OtZDf0HKwQK45VZ7CYD693o70jkKPrAAJG3yTsbesfiS7RbFyGKzKJ7EL
+nsUIJkfgm/SlKmXU/u8MOBO5Wg7/TcsS33sRWHl90j+9jbhqdl92R+vY/CwC
+ieFkQA7/TDv1u+NAalH+Lpkd8AIuEcki+TAogZ7oi/SnofwnoB7BxRm+mIkp
+ZZhIDSCaPOzLG8CSZ81d3HVHhqbf8dh0DFKFoUYyKdbOqIkNWWASf+c/ZEme
+IWcekY8hqwf/raZ56tGM/bRwYPcotMfC1wRcxs4RAQgAsMb5/ELWmrfPy3ba
+5qif+RXhGSbjitATNgHpoPUHrfTC7cn4JWHqehoXLAQpFAoKd+O/ZNpZozK9
+ilpqGUx05yMw06jNQEhYIbgIF4wzPpz02Lp6YeMwdF5LF+Rw83PHdHrA/wRV
+/QjL04+kZnN+G5HmzMlhFY+oZSpL+Gp1bTXgtAVDkhCnMB5tP2VwULMGyJ+X
+vRYxwTK2CrLjIVZv5n1VYY+caCowU6j/XFqvlCJj+G5oV+UhFOWffaMRXhOh
+a64RrhqT1Np7wCLvLMP2wpys9xlMcLQJLqDNxqOTp504V7dm67ncC0fKUsT4
+m4oTktnxKPd6MU+4VYveaLCquwARAQABAAf4u9s7gpGErs1USxmDO9TlyGZK
+aBlri8nMf3s+hOJCOo3cRaRHJBfdY6pu/baG6H6JTsWzeY4MHwr6N+dhVIEh
+FPMa9EZAjagyc4GugxWGiMVTfU+2AEfdrdynhQKMgXSctnnNCdkRuX0nwqb3
+nlupm1hsz2ze4+Wg0BKSLS0FQdoUbITdJUR69OHr4dNJVHWYI0JSBx4SdhV3
+y9163dDvmc+lW9AEaD53vyZWfzCHZxsR/gI32VmT0z5gn1t8w9AOdXo2lA1H
+bf7wh4/qCyujGu64ToZtiEny/GCyM6PofLtiZuJNLw3s/y+B2tKv22aTJ760
++Gib1xB9WcWjKyrxBADoeCyq+nHGrl0CwOkmjanlFymgo7mnBOXuiFOvGrKk
+M1meMU1TI4TEBWkVnDVMcSejgjAf/bX1dtouba1tMAMu7DlaV/0EwbSADRel
+RSqEbIzIOys+y9TY/BMI/uCKNyEKHvu1KUXADb+CBpdBpCfMBWDANFlo9xLz
+Ajcmu2dyawQAwquwC0VXQcvzfs+Hd5au5XvHdm1KidOiAdu6PH1SrOgenIN4
+lkEjHrJD9jmloO2/GVcxDBB2pmf0B4HEg7DuY9LXBrksP5eSbbRc5+UH1HUv
+u82AqQnfNKTd/jae+lLwaOS++ohtwMkkD6W0LdWnHPjyyXg4Oi9zPID3asRu
+3PED/3CYyjl6S8GTMY4FNH7Yxu9+NV2xpKE92Hf8K/hnYlmSSVKDCEeOJtLt
+BkkcSqY6liCNSMmJdVyAF2GrR+zmDac7UQRssf57oOWtSsGozt0aqJXuspMT
+6aB+P1UhZ8Ly9rWZNiJ0jwyfnQNOLCYDaqjFmiSpqrNnJ2Q1Xge3+k80P9DC
+wF8EGAEIAAkFAlzGzhECGwwACgkQZ/B4v2b2xB5wlwgAjZA1zdv5irFjyWVo
+4/itONtyO1NbdpyYpcct7vD0oV+a4wahQP0J3Kk1GhZ5tvAoZF/jakQQOM5o
+GjUYpXAGnr09Mv9EiQ2pDwXc2yq0WfXnGxNrpzOqdtV+IqY9NYkl55Tme7x+
+WRvrkPSUeUsyEGvxwR1stdv8eg9jUmxdl8Io3PYoFJJlrM/6aXeC1r3KOj7q
+XAnR0XHJ+QBSNKCWLlQv5hui9BKfcLiVKFK/dNhs82nRyhPr4sWFw6MTqdAK
+4zkn7l0jmy6Evi1AiiGPiHPnxeNErnofOIEh4REQj00deZADHrixTLtx2FuR
+uaSC3IcBmBsj1fNb4eYXElILjQ==
+=fMOl
+-----END PGP PRIVATE KEY BLOCK-----

internal/transfer/testdata/mbox/Foo.mbox

@@ -0,0 +1,5 @@
+From - Mon May  4 16:40:31 2020
+From: Bridge Test <bridgetest@pm.test>
+To: Bridge Test <bridgetest@protonmail.com>
+
+hello

internal/transfer/testdata/mbox/Inbox.mbox

@@ -0,0 +1,5 @@
+From - Mon May  4 16:40:31 2020
+From: Bridge Test <bridgetest@pm.test>
+To: Bridge Test <bridgetest@protonmail.com>
+
+hello

internal/transfer/transfer.go

@@ -0,0 +1,177 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+// Package transfer provides tools to export messages from one provider and
+// import them to another provider. Provider can be EML, MBOX, IMAP or PMAPI.
+package transfer
+
+import (
+	"crypto/sha256"
+	"fmt"
+
+	"github.com/sirupsen/logrus"
+)
+
+var log = logrus.WithField("pkg", "transfer") //nolint[gochecknoglobals]
+
+// Transfer is facade on top of import rules, progress manager and source
+// and target providers. This is the main object which should be used.
+type Transfer struct {
+	panicHandler PanicHandler
+	id           string
+	dir          string
+	rules        transferRules
+	source       SourceProvider
+	target       TargetProvider
+}
+
+// New creates Transfer for specific source and target. Usage:
+//
+//   source := transfer.NewEMLProvider(...)
+//   target := transfer.NewPMAPIProvider(...)
+//   transfer.New(source, target, ...)
+func New(panicHandler PanicHandler, transferDir string, source SourceProvider, target TargetProvider) (*Transfer, error) {
+	transferID := fmt.Sprintf("%x", sha256.Sum256([]byte(source.ID()+"-"+target.ID())))
+	rules := loadRules(transferDir, transferID)
+	transfer := &Transfer{
+		panicHandler: panicHandler,
+		id:           transferID,
+		dir:          transferDir,
+		rules:        rules,
+		source:       source,
+		target:       target,
+	}
+	if err := transfer.setDefaultRules(); err != nil {
+		return nil, err
+	}
+	return transfer, nil
+}
+
+// SetDefaultRules sets missing rules for source mailboxes with matching
+// target mailboxes. In case no matching mailbox is found, `defaultCallback`
+// with a source mailbox as a parameter is used.
+func (t *Transfer) setDefaultRules() error {
+	sourceMailboxes, err := t.SourceMailboxes()
+	if err != nil {
+		return err
+	}
+
+	targetMailboxes, err := t.TargetMailboxes()
+	if err != nil {
+		return err
+	}
+
+	defaultCallback := func(sourceMailbox Mailbox) []Mailbox {
+		return t.target.DefaultMailboxes(sourceMailbox)
+	}
+
+	t.rules.setDefaultRules(sourceMailboxes, targetMailboxes, defaultCallback)
+	return nil
+}
+
+// SetSkipEncryptedMessages sets whether message which cannot be decrypted
+// should be exported or skipped.
+func (t *Transfer) SetSkipEncryptedMessages(skip bool) {
+	t.rules.setSkipEncryptedMessages(skip)
+}
+
+// SetGlobalMailbox sets mailbox that is applied to every message in
+// the import phase.
+func (t *Transfer) SetGlobalMailbox(mailbox *Mailbox) {
+	t.rules.setGlobalMailbox(mailbox)
+}
+
+// SetGlobalTimeLimit sets time limit that is applied to rules without any
+// specified time limit.
+func (t *Transfer) SetGlobalTimeLimit(fromTime, toTime int64) {
+	t.rules.setGlobalTimeLimit(fromTime, toTime)
+}
+
+// SetRule sets sourceMailbox for transfer.
+func (t *Transfer) SetRule(sourceMailbox Mailbox, targetMailboxes []Mailbox, fromTime, toTime int64) error {
+	return t.rules.setRule(sourceMailbox, targetMailboxes, fromTime, toTime)
+}
+
+// UnsetRule unsets sourceMailbox from transfer.
+func (t *Transfer) UnsetRule(sourceMailbox Mailbox) {
+	t.rules.unsetRule(sourceMailbox)
+}
+
+// ResetRules unsets all rules.
+func (t *Transfer) ResetRules() {
+	t.rules.reset()
+}
+
+// GetRule returns rule for given mailbox.
+func (t *Transfer) GetRule(sourceMailbox Mailbox) *Rule {
+	return t.rules.getRule(sourceMailbox)
+}
+
+// GetRules returns all set transfer rules.
+func (t *Transfer) GetRules() []*Rule {
+	return t.rules.getRules()
+}
+
+// SourceMailboxes returns mailboxes available at source side.
+func (t *Transfer) SourceMailboxes() ([]Mailbox, error) {
+	return t.source.Mailboxes(false, true)
+}
+
+// TargetMailboxes returns mailboxes available at target side.
+func (t *Transfer) TargetMailboxes() ([]Mailbox, error) {
+	return t.target.Mailboxes(true, false)
+}
+
+// CreateTargetMailbox creates mailbox in target provider.
+func (t *Transfer) CreateTargetMailbox(mailbox Mailbox) (Mailbox, error) {
+	return t.target.CreateMailbox(mailbox)
+}
+
+// ChangeTarget allows to change target. Ideally should not be used.
+// Useful for situration after user changes mind where to export files and similar.
+func (t *Transfer) ChangeTarget(target TargetProvider) {
+	t.target = target
+}
+
+// Start starts the transfer from source to target.
+func (t *Transfer) Start() *Progress {
+	log.Debug("Transfer started")
+	t.rules.save()
+
+	log := log.WithField("id", t.id)
+	reportFile := newFileReport(t.dir, t.id)
+	progress := newProgress(log, reportFile)
+
+	ch := make(chan Message)
+
+	go func() {
+		defer t.panicHandler.HandlePanic()
+
+		progress.start()
+		t.source.TransferTo(t.rules, &progress, ch)
+		close(ch)
+	}()
+
+	go func() {
+		defer t.panicHandler.HandlePanic()
+
+		t.target.TransferFrom(t.rules, &progress, ch)
+		progress.finish()
+	}()
+
+	return &progress
+}

internal/transfer/transfer_test.go

@@ -0,0 +1,73 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"bytes"
+	"io/ioutil"
+	"testing"
+
+	"github.com/ProtonMail/gopenpgp/crypto"
+	transfermocks "github.com/ProtonMail/proton-bridge/internal/transfer/mocks"
+	pmapimocks "github.com/ProtonMail/proton-bridge/pkg/pmapi/mocks"
+	gomock "github.com/golang/mock/gomock"
+)
+
+type mocks struct {
+	t *testing.T
+
+	ctrl          *gomock.Controller
+	panicHandler  *transfermocks.MockPanicHandler
+	clientManager *transfermocks.MockClientManager
+	pmapiClient   *pmapimocks.MockClient
+
+	keyring *crypto.KeyRing
+}
+
+func initMocks(t *testing.T) mocks {
+	mockCtrl := gomock.NewController(t)
+
+	m := mocks{
+		t: t,
+
+		ctrl:          mockCtrl,
+		panicHandler:  transfermocks.NewMockPanicHandler(mockCtrl),
+		clientManager: transfermocks.NewMockClientManager(mockCtrl),
+		pmapiClient:   pmapimocks.NewMockClient(mockCtrl),
+		keyring:       newTestKeyring(),
+	}
+
+	m.clientManager.EXPECT().GetClient("user").Return(m.pmapiClient).AnyTimes()
+
+	return m
+}
+
+func newTestKeyring() *crypto.KeyRing {
+	data, err := ioutil.ReadFile("testdata/keyring_userKey")
+	if err != nil {
+		panic(err)
+	}
+	userKey, err := crypto.ReadArmoredKeyRing(bytes.NewReader(data))
+	if err != nil {
+		panic(err)
+	}
+	if err := userKey.Unlock([]byte("testpassphrase")); err != nil {
+		panic(err)
+	}
+	return userKey
+}

internal/transfer/types.go

@@ -0,0 +1,31 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+)
+
+type PanicHandler interface {
+	HandlePanic()
+}
+
+type ClientManager interface {
+	GetClient(userID string) pmapi.Client
+	CheckConnection() error
+}

internal/transfer/utils.go

@@ -0,0 +1,141 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"bufio"
+	"bytes"
+	"io/ioutil"
+	"net/mail"
+	"net/textproto"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+// getFolderNames collects all folder names under `root`.
+// Folder names will be without a path.
+func getFolderNames(root string) ([]string, error) {
+	return getFolderNamesWithFileSuffix(root, "")
+}
+
+// getFolderNamesWithFileSuffix collects all folder names under `root`, which
+// contains some file with a give `fileSuffix`. Names will be without a path.
+func getFolderNamesWithFileSuffix(root, fileSuffix string) ([]string, error) {
+	folders := []string{}
+
+	files, err := ioutil.ReadDir(root)
+	if err != nil {
+		return nil, err
+	}
+
+	hasFileWithSuffix := fileSuffix == ""
+	for _, file := range files {
+		if file.IsDir() {
+			subfolders, err := getFolderNamesWithFileSuffix(filepath.Join(root, file.Name()), fileSuffix)
+			if err != nil {
+				return nil, err
+			}
+			for _, subfolder := range subfolders {
+				match := false
+				for _, folder := range folders {
+					if folder == subfolder {
+						match = true
+						break
+					}
+				}
+				if !match {
+					folders = append(folders, subfolder)
+				}
+			}
+		} else if fileSuffix == "" || strings.HasSuffix(file.Name(), fileSuffix) {
+			hasFileWithSuffix = true
+		}
+	}
+
+	if hasFileWithSuffix {
+		folders = append(folders, filepath.Base(root))
+	}
+
+	sort.Strings(folders)
+	return folders, nil
+}
+
+// getFilePathsWithSuffix collects all file names with `suffix` under `root`.
+// File names will be with relative path based to `root`.
+func getFilePathsWithSuffix(root, suffix string) ([]string, error) {
+	fileNames, err := getFilePathsWithSuffixInner("", root, suffix)
+	if err != nil {
+		return nil, err
+	}
+	sort.Strings(fileNames)
+	return fileNames, err
+}
+
+func getFilePathsWithSuffixInner(prefix, root, suffix string) ([]string, error) {
+	fileNames := []string{}
+
+	files, err := ioutil.ReadDir(root)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, file := range files {
+		if !file.IsDir() {
+			if strings.HasSuffix(file.Name(), suffix) {
+				fileNames = append(fileNames, filepath.Join(prefix, file.Name()))
+			}
+		} else {
+			subfolderFileNames, err := getFilePathsWithSuffixInner(
+				filepath.Join(prefix, file.Name()),
+				filepath.Join(root, file.Name()),
+				suffix,
+			)
+			if err != nil {
+				return nil, err
+			}
+			fileNames = append(fileNames, subfolderFileNames...)
+		}
+	}
+
+	return fileNames, nil
+}
+
+// getMessageTime returns time of the message specified in the message header.
+func getMessageTime(body []byte) (int64, error) {
+	mailHeader, err := getMessageHeader(body)
+	if err != nil {
+		return 0, err
+	}
+	if t, err := mailHeader.Date(); err == nil && !t.IsZero() {
+		return t.Unix(), nil
+	}
+	return 0, nil
+}
+
+// getMessageHeader returns headers of the message body.
+func getMessageHeader(body []byte) (mail.Header, error) {
+	tpr := textproto.NewReader(bufio.NewReader(bytes.NewBuffer(body)))
+	header, err := tpr.ReadMIMEHeader()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read headers")
+	}
+	return mail.Header(header), nil
+}

internal/transfer/utils_test.go

@@ -0,0 +1,190 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	r "github.com/stretchr/testify/require"
+)
+
+func TestGetFolderNames(t *testing.T) {
+	root, clean := createTestingFolderStructure(t)
+	defer clean()
+
+	tests := []struct {
+		suffix    string
+		wantNames []string
+	}{
+		{
+			"",
+			[]string{
+				"bar",
+				"baz",
+				filepath.Base(root),
+				"foo",
+				"qwerty",
+				"test",
+			},
+		},
+		{
+			".eml",
+			[]string{
+				"bar",
+				"baz",
+				filepath.Base(root),
+				"foo",
+			},
+		},
+		{
+			".txt",
+			[]string{
+				filepath.Base(root),
+			},
+		},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.suffix, func(t *testing.T) {
+			names, err := getFolderNamesWithFileSuffix(root, tc.suffix)
+			r.NoError(t, err)
+			r.Equal(t, tc.wantNames, names)
+		})
+	}
+}
+
+func TestGetFilePathsWithSuffix(t *testing.T) {
+	root, clean := createTestingFolderStructure(t)
+	defer clean()
+
+	tests := []struct {
+		suffix    string
+		wantPaths []string
+	}{
+		{
+			".eml",
+			[]string{
+				"foo/bar/baz/msg1.eml",
+				"foo/bar/baz/msg2.eml",
+				"foo/bar/baz/msg3.eml",
+				"foo/bar/msg4.eml",
+				"foo/bar/msg5.eml",
+				"foo/baz/msg6.eml",
+				"foo/msg7.eml",
+				"msg10.eml",
+				"test/foo/msg8.eml",
+				"test/foo/msg9.eml",
+			},
+		},
+		{
+			".txt",
+			[]string{
+				"info.txt",
+			},
+		},
+		{
+			".hello",
+			[]string{},
+		},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.suffix, func(t *testing.T) {
+			paths, err := getFilePathsWithSuffix(root, tc.suffix)
+			r.NoError(t, err)
+			r.Equal(t, tc.wantPaths, paths)
+		})
+	}
+}
+
+func createTestingFolderStructure(t *testing.T) (string, func()) {
+	root, err := ioutil.TempDir("", "folderstructure")
+	r.NoError(t, err)
+
+	for _, path := range []string{
+		"foo/bar/baz",
+		"foo/baz",
+		"test/foo",
+		"qwerty",
+	} {
+		err = os.MkdirAll(filepath.Join(root, path), os.ModePerm)
+		r.NoError(t, err)
+	}
+
+	for _, path := range []string{
+		"foo/bar/baz/msg1.eml",
+		"foo/bar/baz/msg2.eml",
+		"foo/bar/baz/msg3.eml",
+		"foo/bar/msg4.eml",
+		"foo/bar/msg5.eml",
+		"foo/baz/msg6.eml",
+		"foo/msg7.eml",
+		"test/foo/msg8.eml",
+		"test/foo/msg9.eml",
+		"msg10.eml",
+		"info.txt",
+	} {
+		f, err := os.Create(filepath.Join(root, path))
+		r.NoError(t, err)
+		err = f.Close()
+		r.NoError(t, err)
+	}
+
+	return root, func() {
+		_ = os.RemoveAll(root)
+	}
+}
+
+func TestGetMessageTime(t *testing.T) {
+	tests := []struct {
+		body     string
+		wantTime int64
+		wantErr  string
+	}{
+		{"", 0, "failed to read headers: EOF"},
+		{"Subject: hello\n\n", 0, ""},
+		{"Date: Thu, 23 Apr 2020 04:52:44 +0000\n\n", 1587617564, ""},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.body, func(t *testing.T) {
+			time, err := getMessageTime([]byte(tc.body))
+			if tc.wantErr == "" {
+				r.NoError(t, err)
+			} else {
+				r.EqualError(t, err, tc.wantErr)
+			}
+			r.Equal(t, tc.wantTime, time)
+		})
+	}
+}
+
+func TestGetMessageHeader(t *testing.T) {
+	body := `Subject: Hello
+From: user@example.com
+
+Body
+`
+	header, err := getMessageHeader([]byte(body))
+	r.NoError(t, err)
+	r.Equal(t, header.Get("subject"), "Hello")
+	r.Equal(t, header.Get("from"), "user@example.com")
+}