Import/Export backend prep

.golangci.yml

@@ -21,6 +21,12 @@ issues:
         - gochecknoinits
         - gosec
 
+linters-settings:
+  godox:
+    keywords:
+      - TODO
+      - FIXME
+
 linters:
   # setting disable-all will make only explicitly enabled linters run
   disable-all: true

Makefile

@@ -156,12 +156,17 @@ test: gofiles
 		./internal/frontend/cli/... \
 		./internal/imap/... \
 		./internal/metrics/... \
+		./internal/importexport/... \
 		./internal/preferences/... \
 		./internal/smtp/... \
 		./internal/store/... \
+		./internal/transfer/... \
 		./internal/users/... \
 		./pkg/...
 
+test-ie:
+	go test ./internal/transfer/...
+
 bench:
 	go test -run '^$$' -bench=. -memprofile bench_mem.pprof -cpuprofile bench_cpu.pprof ./internal/store
 	go tool pprof -png -output bench_mem.png bench_mem.pprof
@@ -172,6 +177,7 @@ coverage: test
 
 mocks:
 	mockgen --package mocks github.com/ProtonMail/proton-bridge/internal/users Configer,PanicHandler,ClientManager,CredentialsStorer,StoreMaker > internal/users/mocks/mocks.go
+	mockgen --package mocks github.com/ProtonMail/proton-bridge/internal/transfer PanicHandler,ClientManager > internal/transfer/mocks/mocks.go
 	mockgen --package mocks github.com/ProtonMail/proton-bridge/internal/store PanicHandler,ClientManager,BridgeUser > internal/store/mocks/mocks.go
 	mockgen --package mocks github.com/ProtonMail/proton-bridge/pkg/listener Listener > internal/store/mocks/utils_mocks.go
 	mockgen --package mocks github.com/ProtonMail/proton-bridge/pkg/pmapi Client > pkg/pmapi/mocks/mocks.go
@@ -195,11 +201,15 @@ doc:
 .PHONY: gofiles
 # Following files are for the whole app so it makes sense to have them in bridge package.
 # (Options like cmd or internal were considered and bridge package is the best place for them.)
-gofiles: ./internal/bridge/credits.go ./internal/bridge/release_notes.go
+gofiles: ./internal/bridge/credits.go ./internal/bridge/release_notes.go ./internal/importexport/credits.go ./internal/importexport/release_notes.go
 ./internal/bridge/credits.go: ./utils/credits.sh go.mod
-	cd ./utils/ && ./credits.sh
+	cd ./utils/ && ./credits.sh bridge
-./internal/bridge/release_notes.go: ./utils/release-notes.sh ./release-notes/notes.txt ./release-notes/bugs.txt
+./internal/bridge/release_notes.go: ./utils/release-notes.sh ./release-notes/notes-bridge.txt ./release-notes/bugs-bridge.txt
-	cd ./utils/ && ./release-notes.sh
+	cd ./utils/ && ./release-notes.sh bridge
+./internal/importexport/credits.go: ./utils/credits.sh go.mod
+	cd ./utils/ && ./credits.sh importexport
+./internal/importexport/release_notes.go: ./utils/release-notes.sh ./release-notes/notes-importexport.txt ./release-notes/bugs-importexport.txt
+	cd ./utils/ && ./release-notes.sh importexport
 
 
 ## Run and debug
@@ -219,6 +229,9 @@ run-nogui: clean-vendor gofiles
 run-nogui-cli: clean-vendor gofiles
 	PROTONMAIL_ENV=dev go run ${BUILD_FLAGS_NOGUI} cmd/Desktop-Bridge/main.go ${RUN_FLAGS} -c
 
+run-ie:
+	PROTONMAIL_ENV=dev go run ${BUILD_FLAGS_NOGUI} cmd/Import-Export/main.go ${RUN_FLAGS} -c
+
 run-debug:
 	PROTONMAIL_ENV=dev dlv debug --build-flags "${BUILD_FLAGS_NOGUI}" cmd/Desktop-Bridge/main.go -- ${RUN_FLAGS}
 

cmd/Desktop-Bridge/main.go

@@ -147,11 +147,11 @@ func (ph *panicHandler) HandlePanic() {
 	config.HandlePanic(ph.cfg, fmt.Sprintf("Recover: %v", r))
 	frontend.HandlePanic()
 
-	*ph.err = cli.NewExitError("Panic and restart", 666)
+	*ph.err = cli.NewExitError("Panic and restart", 255)
 	numberOfCrashes++
 	log.Error("Restarting after panic")
 	restartApp()
-	os.Exit(666)
+	os.Exit(255)
 }
 
 // run initializes and starts everything in a precise order.

cmd/Import-Export/main.go

@@ -0,0 +1,296 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"runtime/pprof"
+	"strconv"
+	"strings"
+
+	"github.com/ProtonMail/proton-bridge/internal/events"
+	"github.com/ProtonMail/proton-bridge/internal/frontend"
+	"github.com/ProtonMail/proton-bridge/internal/importexport"
+	"github.com/ProtonMail/proton-bridge/internal/users/credentials"
+	"github.com/ProtonMail/proton-bridge/pkg/args"
+	"github.com/ProtonMail/proton-bridge/pkg/config"
+	"github.com/ProtonMail/proton-bridge/pkg/constants"
+	"github.com/ProtonMail/proton-bridge/pkg/listener"
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/ProtonMail/proton-bridge/pkg/updates"
+	"github.com/getsentry/raven-go"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var (
+	log = logrus.WithField("pkg", "main") //nolint[gochecknoglobals]
+
+	// How many crashes in a row.
+	numberOfCrashes = 0 //nolint[gochecknoglobals]
+
+	// After how many crashes import/export gives up starting.
+	maxAllowedCrashes = 10 //nolint[gochecknoglobals]
+)
+
+func main() {
+	constants.AppShortName = "importExport" //TODO
+
+	if err := raven.SetDSN(constants.DSNSentry); err != nil {
+		log.WithError(err).Errorln("Can not setup sentry DSN")
+	}
+	raven.SetRelease(constants.Revision)
+
+	args.FilterProcessSerialNumberFromArgs()
+	filterRestartNumberFromArgs()
+
+	app := cli.NewApp()
+	app.Name = "Protonmail Import/Export"
+	app.Version = constants.BuildVersion
+	app.Flags = []cli.Flag{
+		cli.StringFlag{
+			Name:  "log-level, l",
+			Usage: "Set the log level (one of panic, fatal, error, warn, info, debug, debug-client, debug-server)"},
+		cli.BoolFlag{
+			Name:  "cli, c",
+			Usage: "Use command line interface"},
+		cli.StringFlag{
+			Name:  "version-json, g",
+			Usage: "Generate json version file"},
+		cli.BoolFlag{
+			Name:  "mem-prof, m",
+			Usage: "Generate memory profile"},
+		cli.BoolFlag{
+			Name:  "cpu-prof, p",
+			Usage: "Generate CPU profile"},
+	}
+	app.Usage = "ProtonMail Import/Export"
+	app.Action = run
+
+	// Always log the basic info about current import/export.
+	logrus.SetLevel(logrus.InfoLevel)
+	log.WithField("version", constants.Version).
+		WithField("revision", constants.Revision).
+		WithField("runtime", runtime.GOOS).
+		WithField("build", constants.BuildTime).
+		WithField("args", os.Args).
+		WithField("appLong", app.Name).
+		WithField("appShort", constants.AppShortName).
+		Info("Run app")
+	if err := app.Run(os.Args); err != nil {
+		log.Error("Program exited with error: ", err)
+	}
+}
+
+type panicHandler struct {
+	cfg *config.Config
+	err *error // Pointer to error of cli action.
+}
+
+func (ph *panicHandler) HandlePanic() {
+	r := recover()
+	if r == nil {
+		return
+	}
+
+	config.HandlePanic(ph.cfg, fmt.Sprintf("Recover: %v", r))
+	frontend.HandlePanic()
+
+	*ph.err = cli.NewExitError("Panic and restart", 255)
+	numberOfCrashes++
+	log.Error("Restarting after panic")
+	restartApp()
+	os.Exit(255)
+}
+
+// run initializes and starts everything in a precise order.
+//
+// IMPORTANT: ***Read the comments before CHANGING the order ***
+func run(context *cli.Context) (contextError error) { // nolint[funlen]
+	// We need to have config instance to setup a logs, panic handler, etc ...
+	cfg := config.New(constants.AppShortName, constants.Version, constants.Revision, "")
+
+	// We want to know about any problem. Our PanicHandler calls sentry which is
+	// not dependent on anything else. If that fails, it tries to create crash
+	// report which will not be possible if no folder can be created. That's the
+	// only problem we will not be notified about in any way.
+	panicHandler := &panicHandler{cfg, &contextError}
+	defer panicHandler.HandlePanic()
+
+	// First we need config and create necessary folder; it's dependency for everything.
+	if err := cfg.CreateDirs(); err != nil {
+		log.Fatal("Cannot create necessary folders: ", err)
+	}
+
+	// Setup of logs should be as soon as possible to ensure we record every wanted report in the log.
+	logLevel := context.GlobalString("log-level")
+	_, _ = config.SetupLog(cfg, logLevel)
+
+	// Doesn't make sense to continue when Import/Export was invoked with wrong arguments.
+	// We should tell that to the user before we do anything else.
+	if context.Args().First() != "" {
+		_ = cli.ShowAppHelp(context)
+		return cli.NewExitError("Unknown argument", 4)
+	}
+
+	// It's safe to get version JSON file even when other instance is running.
+	// (thus we put it before check of presence of other Import/Export instance).
+	updates := updates.New(
+		constants.AppShortName,
+		constants.Version,
+		constants.Revision,
+		constants.BuildTime,
+		importexport.ReleaseNotes,
+		importexport.ReleaseFixedBugs,
+		cfg.GetUpdateDir(),
+	)
+
+	if dir := context.GlobalString("version-json"); dir != "" {
+		generateVersionFiles(updates, dir)
+		return nil
+	}
+
+	// In case user wants to do CPU or memory profiles...
+	if doCPUProfile := context.GlobalBool("cpu-prof"); doCPUProfile {
+		f, err := os.Create("cpu.pprof")
+		if err != nil {
+			log.Fatal("Could not create CPU profile: ", err)
+		}
+		if err := pprof.StartCPUProfile(f); err != nil {
+			log.Fatal("Could not start CPU profile: ", err)
+		}
+		defer pprof.StopCPUProfile()
+	}
+
+	if doMemoryProfile := context.GlobalBool("mem-prof"); doMemoryProfile {
+		defer makeMemoryProfile()
+	}
+
+	// Now we initialize all Import/Export parts.
+	log.Debug("Initializing import/export...")
+	eventListener := listener.New()
+	events.SetupEvents(eventListener)
+
+	credentialsStore, credentialsError := credentials.NewStore("import-export")
+	if credentialsError != nil {
+		log.Error("Could not get credentials store: ", credentialsError)
+	}
+
+	cm := pmapi.NewClientManager(cfg.GetAPIConfig())
+
+	// Different build types have different roundtrippers (e.g. we want to enable
+	// TLS fingerprint checks in production builds). GetRoundTripper has a different
+	// implementation depending on whether build flag pmapi_prod is used or not.
+	cm.SetRoundTripper(cfg.GetRoundTripper(cm, eventListener))
+
+	importexportInstance := importexport.New(cfg, panicHandler, eventListener, cm, credentialsStore)
+
+	// Decide about frontend mode before initializing rest of import/export.
+	var frontendMode string
+	switch {
+	case context.GlobalBool("cli"):
+		frontendMode = "cli"
+	default:
+		frontendMode = "qt"
+	}
+	log.WithField("mode", frontendMode).Debug("Determined frontend mode to use")
+
+	frontend := frontend.NewImportExport(constants.Version, constants.BuildVersion, frontendMode, panicHandler, cfg, eventListener, updates, importexportInstance)
+
+	// Last part is to start everything.
+	log.Debug("Starting frontend...")
+	if err := frontend.Loop(credentialsError); err != nil {
+		log.Error("Frontend failed with error: ", err)
+		return cli.NewExitError("Frontend error", 2)
+	}
+
+	if frontend.IsAppRestarting() {
+		restartApp()
+	}
+
+	return nil
+}
+
+// generateVersionFiles writes a JSON file with details about current build.
+// Those files are used for upgrading the app.
+func generateVersionFiles(updates *updates.Updates, dir string) {
+	log.Info("Generating version files")
+	for _, goos := range []string{"windows", "darwin", "linux"} {
+		log.Debug("Generating JSON for ", goos)
+		if err := updates.CreateJSONAndSign(dir, goos); err != nil {
+			log.Error(err)
+		}
+	}
+}
+
+func makeMemoryProfile() {
+	name := "./mem.pprof"
+	f, err := os.Create(name)
+	if err != nil {
+		log.Error("Could not create memory profile: ", err)
+	}
+	if abs, err := filepath.Abs(name); err == nil {
+		name = abs
+	}
+	log.Info("Writing memory profile to ", name)
+	runtime.GC() // get up-to-date statistics
+	if err := pprof.WriteHeapProfile(f); err != nil {
+		log.Error("Could not write memory profile: ", err)
+	}
+	_ = f.Close()
+}
+
+// filterRestartNumberFromArgs removes flag with a number how many restart we already did.
+// See restartApp how that number is used.
+func filterRestartNumberFromArgs() {
+	tmp := os.Args[:0]
+	for i, arg := range os.Args {
+		if !strings.HasPrefix(arg, "--restart_") {
+			tmp = append(tmp, arg)
+			continue
+		}
+		var err error
+		numberOfCrashes, err = strconv.Atoi(os.Args[i][10:])
+		if err != nil {
+			numberOfCrashes = maxAllowedCrashes
+		}
+	}
+	os.Args = tmp
+}
+
+// restartApp starts a new instance in background.
+func restartApp() {
+	if numberOfCrashes >= maxAllowedCrashes {
+		log.Error("Too many crashes")
+		return
+	}
+	if exeFile, err := os.Executable(); err == nil {
+		arguments := append(os.Args[1:], fmt.Sprintf("--restart_%d", numberOfCrashes))
+		cmd := exec.Command(exeFile, arguments...) //nolint[gosec]
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		cmd.Stdin = os.Stdin
+		if err := cmd.Start(); err != nil {
+			log.Error("Restart failed: ", err)
+		}
+	}
+}

doc/importexport.md

@@ -0,0 +1,135 @@
+# Import/Export
+
+## Main blocks
+
+This is basic overview of the main import/export blocks.
+
+```mermaid
+graph LR
+    S[ProtonMail server]
+    U[User]
+
+    subgraph "Import/Export app"
+        Users
+        Frontend["Qt / CLI"]
+        ImportExport
+        Transfer
+
+        Frontend --> ImportExport
+        Frontend --> Transfer
+        ImportExport --> Users
+        ImportExport --> Transfer
+    end
+
+    EML --> Transfer
+    MBOX --> Transfer
+    IMAP --> Transfer
+    S --> Transfer
+
+    Transfer --> EML
+    Transfer --> MBOX
+    Transfer --> S
+
+    U --> Frontend
+```
+
+## Code structure
+
+More detailed graph of main types used in Import/Export app and connection between them.
+
+```mermaid
+graph TD
+    PM[ProtonMail Server]
+    EML[EML]
+    MBOX[MBOX]
+    IMAP[IMAP]
+
+    subgraph "Import/Export app"
+        subgraph PkgUsers
+            subgraph PkgCredentials
+                CredStore[Store]
+                Creds[Credentials]
+
+                CredStore --> Creds
+            end
+
+            US[Users]
+            U[User]
+
+            US --> U
+        end
+
+        subgraph PkgFrontend
+            CLI
+            Qt
+        end
+
+        subgraph PkgImportExport
+            IE[ImportExport]
+        end
+
+        subgraph PkgTransfer
+            Transfer
+            Rules
+            Progress
+
+            Provider
+            LocalProvider
+            EMLProvider
+            MBOXProvider
+            IMAPProvider
+            PMAPIProvider
+
+            Mailbox
+            Message
+
+            Transfer --> |source|Provider
+            Transfer --> |target|Provider
+            Transfer --> Rules
+            Transfer --> Progress
+
+            Provider --> LocalProvider
+            Provider --> EMLProvider
+            Provider --> MBOXProvider
+            Provider --> IMAPProvider
+            Provider --> PMAPIProvider
+
+            LocalProvider --> EMLProvider
+            LocalProvider --> MBOXProvider
+
+            Provider --> Mailbox
+            Provider --> Message
+
+        end
+
+        subgraph PMAPI
+            APIM[ClientManager]
+            APIC[Client]
+
+            APIM --> APIC
+        end
+    end
+
+    CLI --> IE
+    CLI --> Transfer
+    CLI --> Progress
+    Qt --> IE
+    Qt --> Transfer
+    Qt --> Progress
+
+    U --> CredStore
+    U --> Creds
+
+    US --> APIM
+    U --> APIM
+
+    PMAPIProvider --> APIM
+    EMLProvider --> EML
+    MBOXProvider --> MBOX
+    IMAPProvider --> IMAP
+
+    IE --> US
+    IE --> Transfer
+
+    APIC --> PM
+```

doc/index.md

@@ -2,8 +2,13 @@
 
 Documentation pages in order to read for a novice:
 
-* [Development cycle](development.md)
+## Bridge
+
 * [Bridge code](bridge.md)
 * [Internal Bridge database](database.md)
 * [Communication between Bridge, Client and Server](communication.md)
 * [Encryption](encryption.md)
+
+## Import/Export
+
+* [Import/Export code](importexport.md)

go.mod

@@ -34,6 +34,7 @@ require (
 	github.com/emersion/go-imap-quota v0.0.0-20200423100218-dcfd1b7d2b41
 	github.com/emersion/go-imap-specialuse v0.0.0-20200722111535-598ff00e4075
 	github.com/emersion/go-imap-unselect v0.0.0-20171113212723-b985794e5f26
+	github.com/emersion/go-mbox v1.0.0
 	github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21
 	github.com/emersion/go-textwrapper v0.0.0-20160606182133-d0e65e56babe
 	github.com/emersion/go-vcard v0.0.0-20190105225839-8856043f13c5 // indirect

go.sum

@@ -61,6 +61,8 @@ github.com/emersion/go-imap-specialuse v0.0.0-20200722111535-598ff00e4075 h1:z8T
 github.com/emersion/go-imap-specialuse v0.0.0-20200722111535-598ff00e4075/go.mod h1:/nybxhI8kXom8Tw6BrHMl42usALvka6meORflnnYwe4=
 github.com/emersion/go-imap-unselect v0.0.0-20171113212723-b985794e5f26 h1:FiSb8+XBQQSkcX3ubr+1tAtlRJBYaFmRZqOAweZ9Wy8=
 github.com/emersion/go-imap-unselect v0.0.0-20171113212723-b985794e5f26/go.mod h1:+gnnZx3Mg3MnCzZrv0eZdp5puxXQUgGT/6N6L7ShKfM=
+github.com/emersion/go-mbox v1.0.0 h1:HN6aKbyqmgIfK9fS/gen+NRr2wXLSxZXWfdAIAnzQPc=
+github.com/emersion/go-mbox v1.0.0/go.mod h1:Yp9IVuuOYLEuMv4yjgDHvhb5mHOcYH6x92Oas3QqEZI=
 github.com/emersion/go-message v0.11.1 h1:0C/S4JIXDTSfXB1vpqdimAYyK4+79fgEAMQ0dSL+Kac=
 github.com/emersion/go-message v0.11.1/go.mod h1:C4jnca5HOTo4bGN9YdqNQM9sITuT3Y0K6bSUw9RklvY=
 github.com/emersion/go-sasl v0.0.0-20191210011802-430746ea8b9b h1:uhWtEWBHgop1rqEk2klKaxPAkVDCXexai6hSuRQ7Nvs=

internal/bridge/bridge.go

@@ -60,7 +60,7 @@ func New(
 	}
 
 	storeFactory := newStoreFactory(config, panicHandler, clientManager, eventListener)
-	u := users.New(config, panicHandler, eventListener, clientManager, credStorer, storeFactory)
+	u := users.New(config, panicHandler, eventListener, clientManager, credStorer, storeFactory, true)
 	b := &Bridge{
 		Users: u,
 

internal/frontend/autoconfig/applemail.go

@@ -43,7 +43,7 @@ func (c *appleMail) Name() string {
 	return "Apple Mail"
 }
 
-func (c *appleMail) Configure(imapPort, smtpPort int, imapSSL, smtpSSL bool, user types.BridgeUser, addressIndex int) error { //nolint[funlen]
+func (c *appleMail) Configure(imapPort, smtpPort int, imapSSL, smtpSSL bool, user types.User, addressIndex int) error { //nolint[funlen]
 	var addresses string
 	var displayName string
 

internal/frontend/autoconfig/autoconfig.go

@@ -23,7 +23,7 @@ import "github.com/ProtonMail/proton-bridge/internal/frontend/types"
 
 type AutoConfig interface {
 	Name() string
-	Configure(imapPort int, smtpPort int, imapSSl, smtpSSL bool, user types.BridgeUser, addressIndex int) error
+	Configure(imapPort int, smtpPort int, imapSSl, smtpSSL bool, user types.User, addressIndex int) error
 }
 
 var available []AutoConfig //nolint[gochecknoglobals]

internal/frontend/cli-ie/account_utils.go

@@ -0,0 +1,100 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package cli
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/ProtonMail/proton-bridge/internal/frontend/types"
+	"github.com/abiosoft/ishell"
+)
+
+// completeUsernames is a helper to complete usernames as the user types.
+func (f *frontendCLI) completeUsernames(args []string) (usernames []string) {
+	if len(args) > 1 {
+		return
+	}
+	arg := ""
+	if len(args) == 1 {
+		arg = args[0]
+	}
+	for _, user := range f.ie.GetUsers() {
+		if strings.HasPrefix(strings.ToLower(user.Username()), strings.ToLower(arg)) {
+			usernames = append(usernames, user.Username())
+		}
+	}
+	return
+}
+
+// noAccountWrapper is a decorator for functions which need any account to be properly functional.
+func (f *frontendCLI) noAccountWrapper(callback func(*ishell.Context)) func(*ishell.Context) {
+	return func(c *ishell.Context) {
+		users := f.ie.GetUsers()
+		if len(users) == 0 {
+			f.Println("No active accounts. Please add account to continue.")
+		} else {
+			callback(c)
+		}
+	}
+}
+
+func (f *frontendCLI) askUserByIndexOrName(c *ishell.Context) types.User {
+	user := f.getUserByIndexOrName("")
+	if user != nil {
+		return user
+	}
+
+	numberOfAccounts := len(f.ie.GetUsers())
+	indexRange := fmt.Sprintf("number between 0 and %d", numberOfAccounts-1)
+	if len(c.Args) == 0 {
+		f.Printf("Please choose %s or username.\n", indexRange)
+		return nil
+	}
+	arg := c.Args[0]
+	user = f.getUserByIndexOrName(arg)
+	if user == nil {
+		f.Printf("Wrong input '%s'. Choose %s or username.\n", bold(arg), indexRange)
+		return nil
+	}
+	return user
+}
+
+func (f *frontendCLI) getUserByIndexOrName(arg string) types.User {
+	users := f.ie.GetUsers()
+	numberOfAccounts := len(users)
+	if numberOfAccounts == 0 {
+		return nil
+	}
+	if numberOfAccounts == 1 {
+		return users[0]
+	}
+	if index, err := strconv.Atoi(arg); err == nil {
+		if index < 0 || index >= numberOfAccounts {
+			return nil
+		}
+		return users[index]
+	}
+	for _, user := range users {
+		if user.Username() == arg {
+			return user
+		}
+	}
+	return nil
+}

internal/frontend/cli-ie/accounts.go

@@ -0,0 +1,153 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package cli
+
+import (
+	"strings"
+
+	"github.com/abiosoft/ishell"
+)
+
+func (f *frontendCLI) listAccounts(c *ishell.Context) {
+	spacing := "%-2d: %-20s (%-15s, %-15s)\n"
+	f.Printf(bold(strings.Replace(spacing, "d", "s", -1)), "#", "account", "status", "address mode")
+	for idx, user := range f.ie.GetUsers() {
+		connected := "disconnected"
+		if user.IsConnected() {
+			connected = "connected"
+		}
+		mode := "split"
+		if user.IsCombinedAddressMode() {
+			mode = "combined"
+		}
+		f.Printf(spacing, idx, user.Username(), connected, mode)
+	}
+	f.Println()
+}
+
+func (f *frontendCLI) loginAccount(c *ishell.Context) { // nolint[funlen]
+	f.ShowPrompt(false)
+	defer f.ShowPrompt(true)
+
+	loginName := ""
+	if len(c.Args) > 0 {
+		user := f.getUserByIndexOrName(c.Args[0])
+		if user != nil {
+			loginName = user.GetPrimaryAddress()
+		}
+	}
+
+	if loginName == "" {
+		loginName = f.readStringInAttempts("Username", c.ReadLine, isNotEmpty)
+		if loginName == "" {
+			return
+		}
+	} else {
+		f.Println("Username:", loginName)
+	}
+
+	password := f.readStringInAttempts("Password", c.ReadPassword, isNotEmpty)
+	if password == "" {
+		return
+	}
+
+	f.Println("Authenticating ... ")
+	client, auth, err := f.ie.Login(loginName, password)
+	if err != nil {
+		f.processAPIError(err)
+		return
+	}
+
+	if auth.HasTwoFactor() {
+		twoFactor := f.readStringInAttempts("Two factor code", c.ReadLine, isNotEmpty)
+		if twoFactor == "" {
+			return
+		}
+
+		_, err = client.Auth2FA(twoFactor, auth)
+		if err != nil {
+			f.processAPIError(err)
+			return
+		}
+	}
+
+	mailboxPassword := password
+	if auth.HasMailboxPassword() {
+		mailboxPassword = f.readStringInAttempts("Mailbox password", c.ReadPassword, isNotEmpty)
+	}
+	if mailboxPassword == "" {
+		return
+	}
+
+	f.Println("Adding account ...")
+	user, err := f.ie.FinishLogin(client, auth, mailboxPassword)
+	if err != nil {
+		log.WithField("username", loginName).WithError(err).Error("Login was unsuccessful")
+		f.Println("Adding account was unsuccessful:", err)
+		return
+	}
+
+	f.Printf("Account %s was added successfully.\n", bold(user.Username()))
+}
+
+func (f *frontendCLI) logoutAccount(c *ishell.Context) {
+	f.ShowPrompt(false)
+	defer f.ShowPrompt(true)
+
+	user := f.askUserByIndexOrName(c)
+	if user == nil {
+		return
+	}
+	if f.yesNoQuestion("Are you sure you want to logout account " + bold(user.Username())) {
+		if err := user.Logout(); err != nil {
+			f.printAndLogError("Logging out failed: ", err)
+		}
+	}
+}
+
+func (f *frontendCLI) deleteAccount(c *ishell.Context) {
+	f.ShowPrompt(false)
+	defer f.ShowPrompt(true)
+
+	user := f.askUserByIndexOrName(c)
+	if user == nil {
+		return
+	}
+	if f.yesNoQuestion("Are you sure you want to " + bold("remove account "+user.Username())) {
+		clearCache := f.yesNoQuestion("Do you want to remove cache for this account")
+		if err := f.ie.DeleteUser(user.ID(), clearCache); err != nil {
+			f.printAndLogError("Cannot delete account: ", err)
+			return
+		}
+	}
+}
+
+func (f *frontendCLI) deleteAccounts(c *ishell.Context) {
+	f.ShowPrompt(false)
+	defer f.ShowPrompt(true)
+
+	if !f.yesNoQuestion("Do you really want remove all accounts") {
+		return
+	}
+	for _, user := range f.ie.GetUsers() {
+		if err := f.ie.DeleteUser(user.ID(), false); err != nil {
+			f.printAndLogError("Cannot delete account ", user.Username(), ": ", err)
+		}
+	}
+	c.Println("Keychain cleared")
+}

internal/frontend/cli-ie/frontend.go

@@ -0,0 +1,233 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+// Package cli provides CLI interface of the Bridge.
+package cli
+
+import (
+	"github.com/ProtonMail/proton-bridge/internal/events"
+	"github.com/ProtonMail/proton-bridge/internal/frontend/types"
+	"github.com/ProtonMail/proton-bridge/pkg/config"
+	"github.com/ProtonMail/proton-bridge/pkg/listener"
+
+	"github.com/abiosoft/ishell"
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	log = logrus.WithField("pkg", "frontend/cli-ie") //nolint[gochecknoglobals]
+)
+
+type frontendCLI struct {
+	*ishell.Shell
+
+	config        *config.Config
+	eventListener listener.Listener
+	updates       types.Updater
+	ie            types.ImportExporter
+
+	appRestart bool
+}
+
+// New returns a new CLI frontend configured with the given options.
+func New( //nolint[funlen]
+	panicHandler types.PanicHandler,
+	config *config.Config,
+	eventListener listener.Listener,
+	updates types.Updater,
+	ie types.ImportExporter,
+) *frontendCLI { //nolint[golint]
+	fe := &frontendCLI{
+		Shell: ishell.New(),
+
+		config:        config,
+		eventListener: eventListener,
+		updates:       updates,
+		ie:            ie,
+
+		appRestart: false,
+	}
+
+	// Clear commands.
+	clearCmd := &ishell.Cmd{Name: "clear",
+		Help:    "remove stored accounts and preferences. (alias: cl)",
+		Aliases: []string{"cl"},
+	}
+	clearCmd.AddCmd(&ishell.Cmd{Name: "accounts",
+		Help:    "remove all accounts from keychain. (aliases: k, keychain)",
+		Aliases: []string{"a", "k", "keychain"},
+		Func:    fe.deleteAccounts,
+	})
+	fe.AddCmd(clearCmd)
+
+	// Check commands.
+	checkCmd := &ishell.Cmd{Name: "check", Help: "check internet connection or new version."}
+	checkCmd.AddCmd(&ishell.Cmd{Name: "updates",
+		Help:    "check for Import/Export updates. (aliases: u, v, version)",
+		Aliases: []string{"u", "version", "v"},
+		Func:    fe.checkUpdates,
+	})
+	checkCmd.AddCmd(&ishell.Cmd{Name: "internet",
+		Help:    "check internet connection. (aliases: i, conn, connection)",
+		Aliases: []string{"i", "con", "connection"},
+		Func:    fe.checkInternetConnection,
+	})
+	fe.AddCmd(checkCmd)
+
+	// Print info commands.
+	fe.AddCmd(&ishell.Cmd{Name: "log-dir",
+		Help:    "print path to directory with logs. (aliases: log, logs)",
+		Aliases: []string{"log", "logs"},
+		Func:    fe.printLogDir,
+	})
+	fe.AddCmd(&ishell.Cmd{Name: "manual",
+		Help:    "print URL with instructions. (alias: man)",
+		Aliases: []string{"man"},
+		Func:    fe.printManual,
+	})
+	fe.AddCmd(&ishell.Cmd{Name: "release-notes",
+		Help:    "print release notes. (aliases: notes, fixed-bugs, bugs, ver, version)",
+		Aliases: []string{"notes", "fixed-bugs", "bugs", "ver", "version"},
+		Func:    fe.printLocalReleaseNotes,
+	})
+	fe.AddCmd(&ishell.Cmd{Name: "credits",
+		Help: "print used resources.",
+		Func: fe.printCredits,
+	})
+
+	// Account commands.
+	fe.AddCmd(&ishell.Cmd{Name: "list",
+		Help:    "print the list of accounts. (aliases: l, ls)",
+		Func:    fe.noAccountWrapper(fe.listAccounts),
+		Aliases: []string{"l", "ls"},
+	})
+	fe.AddCmd(&ishell.Cmd{Name: "login",
+		Help:      "login procedure to add or connect account. Optionally use index or account as parameter. (aliases: a, add, con, connect)",
+		Func:      fe.loginAccount,
+		Aliases:   []string{"add", "a", "con", "connect"},
+		Completer: fe.completeUsernames,
+	})
+	fe.AddCmd(&ishell.Cmd{Name: "logout",
+		Help:      "disconnect the account. Use index or account name as parameter. (aliases: d, disconnect)",
+		Func:      fe.noAccountWrapper(fe.logoutAccount),
+		Aliases:   []string{"d", "disconnect"},
+		Completer: fe.completeUsernames,
+	})
+	fe.AddCmd(&ishell.Cmd{Name: "delete",
+		Help:      "remove the account from keychain. Use index or account name as parameter. (aliases: del, rm, remove)",
+		Func:      fe.noAccountWrapper(fe.deleteAccount),
+		Aliases:   []string{"del", "rm", "remove"},
+		Completer: fe.completeUsernames,
+	})
+
+	// Import/Export commands.
+	importCmd := &ishell.Cmd{Name: "import",
+		Help:    "import messages. (alias: imp)",
+		Aliases: []string{"imp"},
+	}
+	importCmd.AddCmd(&ishell.Cmd{Name: "local",
+		Help:    "import local messages. (aliases: loc)",
+		Func:    fe.noAccountWrapper(fe.importLocalMessages),
+		Aliases: []string{"loc"},
+	})
+	importCmd.AddCmd(&ishell.Cmd{Name: "remote",
+		Help:    "import remote messages. (aliases: rem)",
+		Func:    fe.noAccountWrapper(fe.importRemoteMessages),
+		Aliases: []string{"rem"},
+	})
+	fe.AddCmd(importCmd)
+
+	exportCmd := &ishell.Cmd{Name: "export",
+		Help:    "export messages. (alias: exp)",
+		Aliases: []string{"exp"},
+	}
+	exportCmd.AddCmd(&ishell.Cmd{Name: "eml",
+		Help: "export messages to eml files.",
+		Func: fe.noAccountWrapper(fe.exportMessagesToEML),
+	})
+	exportCmd.AddCmd(&ishell.Cmd{Name: "mbox",
+		Help: "export messages to mbox files.",
+		Func: fe.noAccountWrapper(fe.exportMessagesToMBOX),
+	})
+	fe.AddCmd(exportCmd)
+
+	// System commands.
+	fe.AddCmd(&ishell.Cmd{Name: "restart",
+		Help: "restart the import/export.",
+		Func: fe.restart,
+	})
+
+	go func() {
+		defer panicHandler.HandlePanic()
+		fe.watchEvents()
+	}()
+	fe.eventListener.RetryEmit(events.TLSCertIssue)
+	fe.eventListener.RetryEmit(events.ErrorEvent)
+	return fe
+}
+
+func (f *frontendCLI) watchEvents() {
+	errorCh := f.getEventChannel(events.ErrorEvent)
+	internetOffCh := f.getEventChannel(events.InternetOffEvent)
+	internetOnCh := f.getEventChannel(events.InternetOnEvent)
+	addressChangedLogoutCh := f.getEventChannel(events.AddressChangedLogoutEvent)
+	logoutCh := f.getEventChannel(events.LogoutEvent)
+	certIssue := f.getEventChannel(events.TLSCertIssue)
+	for {
+		select {
+		case errorDetails := <-errorCh:
+			f.Println("Import/Export failed:", errorDetails)
+		case <-internetOffCh:
+			f.notifyInternetOff()
+		case <-internetOnCh:
+			f.notifyInternetOn()
+		case address := <-addressChangedLogoutCh:
+			f.notifyLogout(address)
+		case userID := <-logoutCh:
+			user, err := f.ie.GetUser(userID)
+			if err != nil {
+				return
+			}
+			f.notifyLogout(user.Username())
+		case <-certIssue:
+			f.notifyCertIssue()
+		}
+	}
+}
+
+func (f *frontendCLI) getEventChannel(event string) <-chan string {
+	ch := make(chan string)
+	f.eventListener.Add(event, ch)
+	return ch
+}
+
+// IsAppRestarting returns whether the app is currently set to restart.
+func (f *frontendCLI) IsAppRestarting() bool {
+	return f.appRestart
+}
+
+// Loop starts the frontend loop with an interactive shell.
+func (f *frontendCLI) Loop(credentialsError error) error {
+	if credentialsError != nil {
+		f.notifyCredentialsError()
+		return credentialsError
+	}
+
+	f.Print(`Welcome to ProtonMail Import/Export interactive shell`)
+	f.Run()
+	return nil
+}

internal/frontend/cli-ie/importexport.go

@@ -0,0 +1,222 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package cli
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/ProtonMail/proton-bridge/internal/frontend/types"
+	"github.com/ProtonMail/proton-bridge/internal/transfer"
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/abiosoft/ishell"
+)
+
+func (f *frontendCLI) importLocalMessages(c *ishell.Context) {
+	f.ShowPrompt(false)
+	defer f.ShowPrompt(true)
+
+	user, path := f.getUserAndPath(c, false)
+	if user == nil || path == "" {
+		return
+	}
+
+	t, err := f.ie.GetLocalImporter(user.GetPrimaryAddress(), path)
+	f.transfer(t, err, false, true)
+}
+
+func (f *frontendCLI) importRemoteMessages(c *ishell.Context) {
+	f.ShowPrompt(false)
+	defer f.ShowPrompt(true)
+
+	user := f.askUserByIndexOrName(c)
+	if user == nil {
+		return
+	}
+
+	username := f.readStringInAttempts("IMAP username", c.ReadLine, isNotEmpty)
+	if username == "" {
+		return
+	}
+	password := f.readStringInAttempts("IMAP password", c.ReadPassword, isNotEmpty)
+	if password == "" {
+		return
+	}
+	host := f.readStringInAttempts("IMAP host", c.ReadLine, isNotEmpty)
+	if host == "" {
+		return
+	}
+	port := f.readStringInAttempts("IMAP port", c.ReadLine, isNotEmpty)
+	if port == "" {
+		return
+	}
+
+	t, err := f.ie.GetRemoteImporter(user.GetPrimaryAddress(), username, password, host, port)
+	f.transfer(t, err, false, true)
+}
+
+func (f *frontendCLI) exportMessagesToEML(c *ishell.Context) {
+	f.ShowPrompt(false)
+	defer f.ShowPrompt(true)
+
+	user, path := f.getUserAndPath(c, true)
+	if user == nil || path == "" {
+		return
+	}
+
+	t, err := f.ie.GetEMLExporter(user.GetPrimaryAddress(), path)
+	f.transfer(t, err, true, false)
+}
+
+func (f *frontendCLI) exportMessagesToMBOX(c *ishell.Context) {
+	f.ShowPrompt(false)
+	defer f.ShowPrompt(true)
+
+	user, path := f.getUserAndPath(c, true)
+	if user == nil || path == "" {
+		return
+	}
+
+	t, err := f.ie.GetMBOXExporter(user.GetPrimaryAddress(), path)
+	f.transfer(t, err, true, false)
+}
+
+func (f *frontendCLI) getUserAndPath(c *ishell.Context, createPath bool) (types.User, string) {
+	user := f.askUserByIndexOrName(c)
+	if user == nil {
+		return nil, ""
+	}
+
+	path := f.readStringInAttempts("Path of EML and MBOX files", c.ReadLine, isNotEmpty)
+	if path == "" {
+		return nil, ""
+	}
+
+	if createPath {
+		_ = os.Mkdir(path, os.ModePerm)
+	}
+
+	return user, path
+}
+
+func (f *frontendCLI) transfer(t *transfer.Transfer, err error, askSkipEncrypted bool, askGlobalMailbox bool) {
+	if err != nil {
+		f.printAndLogError("Failed to init transferrer: ", err)
+		return
+	}
+
+	if askSkipEncrypted {
+		skipEncryptedMessages := f.yesNoQuestion("Skip encrypted messages")
+		t.SetSkipEncryptedMessages(skipEncryptedMessages)
+	}
+
+	if !f.setTransferRules(t) {
+		return
+	}
+
+	if askGlobalMailbox {
+		if err := f.setTransferGlobalMailbox(t); err != nil {
+			f.printAndLogError("Failed to create global mailbox: ", err)
+			return
+		}
+	}
+
+	progress := t.Start()
+	for range progress.GetUpdateChannel() {
+		f.printTransferProgress(progress)
+	}
+	f.printTransferResult(progress)
+}
+
+func (f *frontendCLI) setTransferGlobalMailbox(t *transfer.Transfer) error {
+	labelName := fmt.Sprintf("Imported %s", time.Now().Format("Jan-02-2006 15:04"))
+
+	useGlobalLabel := f.yesNoQuestion("Use global label " + labelName)
+	if !useGlobalLabel {
+		return nil
+	}
+
+	globalMailbox, err := t.CreateTargetMailbox(transfer.Mailbox{
+		Name:        labelName,
+		Color:       pmapi.LabelColors[0],
+		IsExclusive: false,
+	})
+	if err != nil {
+		return err
+	}
+
+	t.SetGlobalMailbox(&globalMailbox)
+	return nil
+}
+
+func (f *frontendCLI) setTransferRules(t *transfer.Transfer) bool {
+	f.Println("Rules:")
+	for _, rule := range t.GetRules() {
+		if !rule.Active {
+			continue
+		}
+		targets := strings.Join(rule.TargetMailboxNames(), ", ")
+		if rule.HasTimeLimit() {
+			f.Printf(" %-30s → %s (%s - %s)\n", rule.SourceMailbox.Name, targets, rule.FromDate(), rule.ToDate())
+		} else {
+			f.Printf(" %-30s → %s\n", rule.SourceMailbox.Name, targets)
+		}
+	}
+
+	return f.yesNoQuestion("Proceed")
+}
+
+func (f *frontendCLI) printTransferProgress(progress *transfer.Progress) {
+	failed, imported, exported, added, total := progress.GetCounts()
+	f.Println(fmt.Sprintf("Progress update: %d (%d / %d) / %d, failed: %d", imported, exported, added, total, failed))
+
+	if progress.IsPaused() {
+		f.Printf("Transfer is paused bacause %s", progress.PauseReason())
+		if !f.yesNoQuestion("Continue (y) or stop (n)") {
+			progress.Stop()
+		}
+	}
+}
+
+func (f *frontendCLI) printTransferResult(progress *transfer.Progress) {
+	err := progress.GetFatalError()
+	if err != nil {
+		f.Println("Transfer failed: " + err.Error())
+		return
+	}
+
+	statuses := progress.GetFailedMessages()
+	if len(statuses) == 0 {
+		f.Println("Transfer finished!")
+		return
+	}
+
+	f.Println("Transfer finished with errors:")
+	for _, messageStatus := range statuses {
+		f.Printf(
+			" %-17s | %-30s | %-30s\n  %s: %s\n",
+			messageStatus.Time.Format("Jan 02 2006 15:04"),
+			messageStatus.From,
+			messageStatus.Subject,
+			messageStatus.SourceID,
+			messageStatus.GetErrorMessage(),
+		)
+	}
+}

internal/frontend/cli-ie/system.go

@@ -0,0 +1,50 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package cli
+
+import (
+	"github.com/abiosoft/ishell"
+)
+
+var (
+	currentPort = "" //nolint[gochecknoglobals]
+)
+
+func (f *frontendCLI) restart(c *ishell.Context) {
+	if f.yesNoQuestion("Are you sure you want to restart the Import/Export") {
+		f.Println("Restarting Import/Export...")
+		f.appRestart = true
+		f.Stop()
+	}
+}
+
+func (f *frontendCLI) checkInternetConnection(c *ishell.Context) {
+	if f.ie.CheckConnection() == nil {
+		f.Println("Internet connection is available.")
+	} else {
+		f.Println("Can not contact the server, please check you internet connection.")
+	}
+}
+
+func (f *frontendCLI) printLogDir(c *ishell.Context) {
+	f.Println("Log files are stored in\n\n ", f.config.GetLogDir())
+}
+
+func (f *frontendCLI) printManual(c *ishell.Context) {
+	f.Println("More instructions about the Import/Export can be found at\n\n  https://protonmail.com/support/categories/import-export/")
+}

internal/frontend/cli-ie/updates.go

@@ -0,0 +1,65 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package cli
+
+import (
+	"strings"
+
+	"github.com/ProtonMail/proton-bridge/internal/bridge"
+	"github.com/ProtonMail/proton-bridge/pkg/updates"
+	"github.com/abiosoft/ishell"
+)
+
+func (f *frontendCLI) checkUpdates(c *ishell.Context) {
+	isUpToDate, latestVersionInfo, err := f.updates.CheckIsUpToDate()
+	if err != nil {
+		f.printAndLogError("Cannot retrieve version info: ", err)
+		f.checkInternetConnection(c)
+		return
+	}
+	if isUpToDate {
+		f.Println("Your version is up to date.")
+	} else {
+		f.notifyNeedUpgrade()
+		f.Println("")
+		f.printReleaseNotes(latestVersionInfo)
+	}
+}
+
+func (f *frontendCLI) printLocalReleaseNotes(c *ishell.Context) {
+	localVersion := f.updates.GetLocalVersion()
+	f.printReleaseNotes(localVersion)
+}
+
+func (f *frontendCLI) printReleaseNotes(versionInfo updates.VersionInfo) {
+	f.Println(bold("ProtonMail Bridge "+versionInfo.Version), "\n")
+	if versionInfo.ReleaseNotes != "" {
+		f.Println(bold("Release Notes"))
+		f.Println(versionInfo.ReleaseNotes)
+	}
+	if versionInfo.ReleaseFixedBugs != "" {
+		f.Println(bold("Fixed bugs"))
+		f.Println(versionInfo.ReleaseFixedBugs)
+	}
+}
+
+func (f *frontendCLI) printCredits(c *ishell.Context) {
+	for _, pkg := range strings.Split(bridge.Credits, ";") {
+		f.Println(pkg)
+	}
+}

internal/frontend/cli-ie/utils.go

@@ -0,0 +1,123 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package cli
+
+import (
+	"strings"
+
+	pmapi "github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/fatih/color"
+)
+
+const (
+	maxInputRepeat = 2
+)
+
+var (
+	bold = color.New(color.Bold).SprintFunc() //nolint[gochecknoglobals]
+)
+
+func isNotEmpty(val string) bool {
+	return val != ""
+}
+
+func (f *frontendCLI) yesNoQuestion(question string) bool {
+	f.Print(question, "? yes/"+bold("no")+": ")
+	yes := "yes"
+	answer := strings.ToLower(f.ReadLine())
+	for i := 0; i < len(answer); i++ {
+		if i >= len(yes) || answer[i] != yes[i] {
+			return false // Everything else is false.
+		}
+	}
+	return len(answer) > 0 // Empty is false.
+}
+
+func (f *frontendCLI) readStringInAttempts(title string, readFunc func() string, isOK func(string) bool) (value string) {
+	f.Printf("%s: ", title)
+	value = readFunc()
+	title = strings.ToLower(string(title[0])) + title[1:]
+	for i := 0; !isOK(value); i++ {
+		if i >= maxInputRepeat {
+			f.Println("Too many attempts")
+			return ""
+		}
+		f.Printf("Please fill %s: ", title)
+		value = readFunc()
+	}
+	return
+}
+
+func (f *frontendCLI) printAndLogError(args ...interface{}) {
+	log.Error(args...)
+	f.Println(args...)
+}
+
+func (f *frontendCLI) processAPIError(err error) {
+	log.Warn("API error: ", err)
+	switch err {
+	case pmapi.ErrAPINotReachable:
+		f.notifyInternetOff()
+	case pmapi.ErrUpgradeApplication:
+		f.notifyNeedUpgrade()
+	default:
+		f.Println("Server error:", err.Error())
+	}
+}
+
+func (f *frontendCLI) notifyInternetOff() {
+	f.Println("Internet connection is not available.")
+}
+
+func (f *frontendCLI) notifyInternetOn() {
+	f.Println("Internet connection is available again.")
+}
+
+func (f *frontendCLI) notifyLogout(address string) {
+	f.Printf("Account %s is disconnected. Login to continue using this account with email client.", address)
+}
+
+func (f *frontendCLI) notifyNeedUpgrade() {
+	f.Println("Please download and install the newest version of application from", f.updates.GetDownloadLink())
+}
+
+func (f *frontendCLI) notifyCredentialsError() {
+	// Print in 80-column width.
+	f.Println("ProtonMail Bridge is not able to detect a supported password manager")
+	f.Println("(pass, gnome-keyring). Please install and set up a supported password manager")
+	f.Println("and restart the application.")
+}
+
+func (f *frontendCLI) notifyCertIssue() {
+	// Print in 80-column width.
+	f.Println(`Connection security error: Your network connection to Proton services may
+be insecure.
+
+Description:
+ProtonMail Bridge was not able to establish a secure connection to Proton
+servers due to a TLS certificate error. This means your connection may
+potentially be insecure and susceptible to monitoring by third parties.
+
+Recommendation:
+* If you trust your network operator, you can continue to use ProtonMail
+  as usual.
+* If you don't trust your network operator, reconnect to ProtonMail over a VPN
+  (such as ProtonVPN) which encrypts your Internet connection, or use
+  a different network to access ProtonMail.
+`)
+}

internal/frontend/cli/account_utils.go

@@ -55,7 +55,7 @@ func (f *frontendCLI) noAccountWrapper(callback func(*ishell.Context)) func(*ish
 	}
 }
 
-func (f *frontendCLI) askUserByIndexOrName(c *ishell.Context) types.BridgeUser {
+func (f *frontendCLI) askUserByIndexOrName(c *ishell.Context) types.User {
 	user := f.getUserByIndexOrName("")
 	if user != nil {
 		return user
@@ -76,7 +76,7 @@ func (f *frontendCLI) askUserByIndexOrName(c *ishell.Context) types.BridgeUser {
 	return user
 }
 
-func (f *frontendCLI) getUserByIndexOrName(arg string) types.BridgeUser {
+func (f *frontendCLI) getUserByIndexOrName(arg string) types.User {
 	users := f.bridge.GetUsers()
 	numberOfAccounts := len(users)
 	if numberOfAccounts == 0 {

internal/frontend/cli/accounts.go

@@ -63,7 +63,7 @@ func (f *frontendCLI) showAccountInfo(c *ishell.Context) {
 	}
 }
 
-func (f *frontendCLI) showAccountAddressInfo(user types.BridgeUser, address string) {
+func (f *frontendCLI) showAccountAddressInfo(user types.User, address string) {
 	smtpSecurity := "STARTTLS"
 	if f.preferences.GetBool(preferences.SMTPSSLKey) {
 		smtpSecurity = "SSL"

internal/frontend/cli/system.go

@@ -43,7 +43,7 @@ func (f *frontendCLI) checkInternetConnection(c *ishell.Context) {
 	if f.bridge.CheckConnection() == nil {
 		f.Println("Internet connection is available.")
 	} else {
-		f.Println("Can not contact server please check you internet connection.")
+		f.Println("Can not contact the server, please check you internet connection.")
 	}
 }
 

internal/frontend/cli/updates.go

@@ -26,7 +26,7 @@ import (
 )
 
 func (f *frontendCLI) checkUpdates(c *ishell.Context) {
-	isUpToDate, latestVersionInfo, err := f.updates.CheckIsBridgeUpToDate()
+	isUpToDate, latestVersionInfo, err := f.updates.CheckIsUpToDate()
 	if err != nil {
 		f.printAndLogError("Cannot retrieve version info: ", err)
 		f.checkInternetConnection(c)
@@ -47,7 +47,7 @@ func (f *frontendCLI) printLocalReleaseNotes(c *ishell.Context) {
 }
 
 func (f *frontendCLI) printReleaseNotes(versionInfo updates.VersionInfo) {
-	f.Println(bold("ProtonMail Bridge "+versionInfo.Version), "\n")
+	f.Println(bold("ProtonMail Import/Export "+versionInfo.Version), "\n")
 	if versionInfo.ReleaseNotes != "" {
 		f.Println(bold("Release Notes"))
 		f.Println(versionInfo.ReleaseNotes)

internal/frontend/frontend.go

@@ -22,8 +22,10 @@ import (
 	"github.com/0xAX/notificator"
 	"github.com/ProtonMail/proton-bridge/internal/bridge"
 	"github.com/ProtonMail/proton-bridge/internal/frontend/cli"
+	cliie "github.com/ProtonMail/proton-bridge/internal/frontend/cli-ie"
 	"github.com/ProtonMail/proton-bridge/internal/frontend/qt"
 	"github.com/ProtonMail/proton-bridge/internal/frontend/types"
+	"github.com/ProtonMail/proton-bridge/internal/importexport"
 	"github.com/ProtonMail/proton-bridge/pkg/config"
 	"github.com/ProtonMail/proton-bridge/pkg/listener"
 	"github.com/sirupsen/logrus"
@@ -86,3 +88,37 @@ func new(
 		return qt.New(version, buildVersion, showWindowOnStart, panicHandler, config, preferences, eventListener, updates, bridge, noEncConfirmator)
 	}
 }
+
+// NewImportExport returns initialized frontend based on `frontendType`, which can be `cli` or `qt`.
+func NewImportExport(
+	version,
+	buildVersion,
+	frontendType string,
+	panicHandler types.PanicHandler,
+	config *config.Config,
+	eventListener listener.Listener,
+	updates types.Updater,
+	ie *importexport.ImportExport,
+) Frontend {
+	ieWrap := types.NewImportExportWrap(ie)
+	return newImportExport(version, buildVersion, frontendType, panicHandler, config, eventListener, updates, ieWrap)
+}
+
+func newImportExport(
+	version,
+	buildVersion,
+	frontendType string,
+	panicHandler types.PanicHandler,
+	config *config.Config,
+	eventListener listener.Listener,
+	updates types.Updater,
+	ie types.ImportExporter,
+) Frontend {
+	switch frontendType {
+	case "cli":
+		return cliie.New(panicHandler, config, eventListener, updates, ie)
+	default:
+		return cliie.New(panicHandler, config, eventListener, updates, ie)
+		//return qt.New(panicHandler, config, eventListener, updates, ie)
+	}
+}

internal/frontend/qt/frontend.go

@@ -410,7 +410,7 @@ func (s *FrontendQt) isNewVersionAvailable(showMessage bool) {
 	go func() {
 		defer s.panicHandler.HandlePanic()
 		defer s.Qml.ProcessFinished()
-		isUpToDate, latestVersionInfo, err := s.updates.CheckIsBridgeUpToDate()
+		isUpToDate, latestVersionInfo, err := s.updates.CheckIsUpToDate()
 		if err != nil {
 			log.Warn("Can not retrieve version info: ", err)
 			s.checkInternet()

internal/frontend/types/types.go

@@ -20,6 +20,8 @@ package types
 
 import (
 	"github.com/ProtonMail/proton-bridge/internal/bridge"
+	"github.com/ProtonMail/proton-bridge/internal/importexport"
+	"github.com/ProtonMail/proton-bridge/internal/transfer"
 	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
 	"github.com/ProtonMail/proton-bridge/pkg/updates"
 )
@@ -31,7 +33,7 @@ type PanicHandler interface {
 
 // Updater is an interface for handling Bridge upgrades.
 type Updater interface {
-	CheckIsBridgeUpToDate() (isUpToDate bool, latestVersion updates.VersionInfo, err error)
+	CheckIsUpToDate() (isUpToDate bool, latestVersion updates.VersionInfo, err error)
 	GetDownloadLink() string
 	GetLocalVersion() updates.VersionInfo
 	StartUpgrade(currentStatus chan<- updates.Progress)
@@ -41,24 +43,19 @@ type NoEncConfirmator interface {
 	ConfirmNoEncryption(string, bool)
 }
 
-// Bridger is an interface of bridge needed by frontend.
+// UserManager is an interface of users needed by frontend.
-type Bridger interface {
+type UserManager interface {
-	GetCurrentClient() string
-	SetCurrentOS(os string)
 	Login(username, password string) (pmapi.Client, *pmapi.Auth, error)
-	FinishLogin(client pmapi.Client, auth *pmapi.Auth, mailboxPassword string) (BridgeUser, error)
+	FinishLogin(client pmapi.Client, auth *pmapi.Auth, mailboxPassword string) (User, error)
-	GetUsers() []BridgeUser
+	GetUsers() []User
-	GetUser(query string) (BridgeUser, error)
+	GetUser(query string) (User, error)
 	DeleteUser(userID string, clearCache bool) error
-	ReportBug(osType, osVersion, description, accountName, address, emailClient string) error
 	ClearData() error
-	AllowProxy()
-	DisallowProxy()
 	CheckConnection() error
 }
 
-// BridgeUser is an interface of user needed by frontend.
+// User is an interface of user needed by frontend.
-type BridgeUser interface {
+type User interface {
 	ID() string
 	Username() string
 	IsConnected() bool
@@ -70,6 +67,17 @@ type BridgeUser interface {
 	Logout() error
 }
 
+// Bridger is an interface of bridge needed by frontend.
+type Bridger interface {
+	UserManager
+
+	GetCurrentClient() string
+	SetCurrentOS(os string)
+	ReportBug(osType, osVersion, description, accountName, address, emailClient string) error
+	AllowProxy()
+	DisallowProxy()
+}
+
 type bridgeWrap struct {
 	*bridge.Bridge
 }
@@ -81,17 +89,55 @@ func NewBridgeWrap(bridge *bridge.Bridge) *bridgeWrap { //nolint[golint]
 	return &bridgeWrap{Bridge: bridge}
 }
 
-func (b *bridgeWrap) FinishLogin(client pmapi.Client, auth *pmapi.Auth, mailboxPassword string) (BridgeUser, error) {
+func (b *bridgeWrap) FinishLogin(client pmapi.Client, auth *pmapi.Auth, mailboxPassword string) (User, error) {
 	return b.Bridge.FinishLogin(client, auth, mailboxPassword)
 }
 
-func (b *bridgeWrap) GetUsers() (users []BridgeUser) {
+func (b *bridgeWrap) GetUsers() (users []User) {
 	for _, user := range b.Bridge.GetUsers() {
 		users = append(users, user)
 	}
 	return
 }
 
-func (b *bridgeWrap) GetUser(query string) (BridgeUser, error) {
+func (b *bridgeWrap) GetUser(query string) (User, error) {
 	return b.Bridge.GetUser(query)
 }
+
+// ImportExporter is an interface of import/export needed by frontend.
+type ImportExporter interface {
+	UserManager
+
+	GetLocalImporter(string, string) (*transfer.Transfer, error)
+	GetRemoteImporter(string, string, string, string, string) (*transfer.Transfer, error)
+	GetEMLExporter(string, string) (*transfer.Transfer, error)
+	GetMBOXExporter(string, string) (*transfer.Transfer, error)
+}
+
+type importExportWrap struct {
+	*importexport.ImportExport
+}
+
+// NewImportExportWrap wraps import/export struct into local importExportWrap
+// to implement local interface.
+// The problem is that Import/Export returns the importexport package's User
+// type. Every method which returns User therefore has to be overridden to
+// fulfill the interface.
+func NewImportExportWrap(ie *importexport.ImportExport) *importExportWrap { //nolint[golint]
+	return &importExportWrap{ImportExport: ie}
+}
+
+func (b *importExportWrap) FinishLogin(client pmapi.Client, auth *pmapi.Auth, mailboxPassword string) (User, error) {
+	return b.ImportExport.FinishLogin(client, auth, mailboxPassword)
+}
+
+func (b *importExportWrap) GetUsers() (users []User) {
+	for _, user := range b.ImportExport.GetUsers() {
+		users = append(users, user)
+	}
+	return
+}
+
+func (b *importExportWrap) GetUser(query string) (User, error) {
+	return b.ImportExport.GetUser(query)
+}

internal/imap/mailbox_message.go

@@ -19,17 +19,14 @@ package imap
 
 import (
 	"bytes"
-	"encoding/base64"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"mime/multipart"
 	"net/mail"
 	"net/textproto"
 	"regexp"
 	"sort"
 	"strings"
-	"text/template"
 	"time"
 
 	"github.com/ProtonMail/gopenpgp/v2/crypto"
@@ -39,7 +36,6 @@ import (
 	"github.com/ProtonMail/proton-bridge/pkg/parallel"
 	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
 	"github.com/emersion/go-imap"
-	"github.com/emersion/go-textwrapper"
 	"github.com/hashicorp/go-multierror"
 	enmime "github.com/jhillyerd/enmime"
 	"github.com/pkg/errors"
@@ -185,74 +181,8 @@ func (im *imapMailbox) CreateMessage(flags []string, date time.Time, body imap.L
 }
 
 func (im *imapMailbox) importMessage(m *pmapi.Message, readers []io.Reader, kr *crypto.KeyRing) (err error) { // nolint[funlen]
-	b := &bytes.Buffer{}
+	body, err := message.BuildEncrypted(m, readers, kr)
-
+	if err != nil {
-	// Overwrite content for main header for import.
-	// Even if message has just simple body we should upload as multipart/mixed.
-	// Each part has encrypted body and header reflects the original header.
-	mainHeader := message.GetHeader(m)
-	mainHeader.Set("Content-Type", "multipart/mixed; boundary="+message.GetBoundary(m))
-	mainHeader.Del("Content-Disposition")
-	mainHeader.Del("Content-Transfer-Encoding")
-	if err = writeHeader(b, mainHeader); err != nil {
-		return
-	}
-	mw := multipart.NewWriter(b)
-	if err = mw.SetBoundary(message.GetBoundary(m)); err != nil {
-		return
-	}
-
-	// Write the body part.
-	bodyHeader := make(textproto.MIMEHeader)
-	bodyHeader.Set("Content-Type", m.MIMEType+"; charset=utf-8")
-	bodyHeader.Set("Content-Disposition", "inline")
-	bodyHeader.Set("Content-Transfer-Encoding", "7bit")
-
-	var p io.Writer
-	if p, err = mw.CreatePart(bodyHeader); err != nil {
-		return
-	}
-	// First, encrypt the message body.
-	if err = m.Encrypt(kr, kr); err != nil {
-		return err
-	}
-	if _, err := io.WriteString(p, m.Body); err != nil {
-		return err
-	}
-
-	// Write the attachments parts.
-	for i := 0; i < len(m.Attachments); i++ {
-		att := m.Attachments[i]
-		r := readers[i]
-		h := message.GetAttachmentHeader(att)
-		if p, err = mw.CreatePart(h); err != nil {
-			return
-		}
-		// Create line wrapper writer.
-		ww := textwrapper.NewRFC822(p)
-
-		// Create base64 writer.
-		bw := base64.NewEncoder(base64.StdEncoding, ww)
-
-		data, err := ioutil.ReadAll(r)
-		if err != nil {
-			return err
-		}
-
-		// Create encrypted writer.
-		pgpMessage, err := kr.Encrypt(crypto.NewPlainMessage(data), nil)
-		if err != nil {
-			return err
-		}
-		if _, err := bw.Write(pgpMessage.GetBinary()); err != nil {
-			return err
-		}
-		if err := bw.Close(); err != nil {
-			return err
-		}
-	}
-
-	if err := mw.Close(); err != nil {
 		return err
 	}
 
@@ -263,7 +193,7 @@ func (im *imapMailbox) importMessage(m *pmapi.Message, readers []io.Reader, kr *
 		}
 	}
 
-	return im.storeMailbox.ImportMessage(m, b.Bytes(), labels)
+	return im.storeMailbox.ImportMessage(m, body, labels)
 }
 
 func (im *imapMailbox) getMessage(storeMessage storeMessageProvider, items []imap.FetchItem) (msg *imap.Message, err error) {
@@ -489,55 +419,6 @@ func (im *imapMailbox) fetchMessage(m *pmapi.Message) (err error) {
 	return
 }
 
-const customMessageTemplate = `
-<html>
-	<head></head>
-	<body style="font-family: Arial,'Helvetica Neue',Helvetica,sans-serif; font-size: 14px;">
-		<div style="color:#555; background-color:#cf9696; padding:20px; border-radius: 4px;">
-			<strong>Decryption error</strong><br/>
-			Decryption of this message's encrypted content failed.
-			<pre>{{.Error}}</pre>
-		</div>
-
-		{{if .AttachBody}}
-		<div style="color:#333; background-color:#f4f4f4;  border: 1px solid #acb0bf; border-radius: 2px; padding:1rem; margin:1rem 0; font-family:monospace; font-size: 1em;">
-			<pre>{{.Body}}</pre>
-		</div>
-		{{- end}}
-	</body>
-</html>
-`
-
-type customMessageData struct {
-	Error      string
-	AttachBody bool
-	Body       string
-}
-
-func (im *imapMailbox) makeCustomMessage(m *pmapi.Message, decodeError error, attachBody bool) (err error) {
-	t := template.Must(template.New("customMessage").Parse(customMessageTemplate))
-
-	b := new(bytes.Buffer)
-
-	if err = t.Execute(b, customMessageData{
-		Error:      decodeError.Error(),
-		AttachBody: attachBody,
-		Body:       m.Body,
-	}); err != nil {
-		return
-	}
-
-	m.MIMEType = pmapi.ContentTypeHTML
-	m.Body = b.String()
-
-	// NOTE: we need to set header in custom message header, so we check that is non-nil.
-	if m.Header == nil {
-		m.Header = make(mail.Header)
-	}
-
-	return
-}
-
 func (im *imapMailbox) writeMessageBody(w io.Writer, m *pmapi.Message) (err error) {
 	im.log.Trace("Writing message body")
 
@@ -555,7 +436,7 @@ func (im *imapMailbox) writeMessageBody(w io.Writer, m *pmapi.Message) (err erro
 
 	err = message.WriteBody(w, kr, m)
 	if err != nil {
-		if customMessageErr := im.makeCustomMessage(m, err, true); customMessageErr != nil {
+		if customMessageErr := message.CustomMessage(m, err, true); customMessageErr != nil {
 			im.log.WithError(customMessageErr).Warn("Failed to make custom message")
 		}
 		_, _ = io.WriteString(w, m.Body)
@@ -692,7 +573,7 @@ func (im *imapMailbox) buildMessage(m *pmapi.Message) (structure *message.BodySt
 
 	if errDecrypt != nil && errDecrypt != openpgperrors.ErrSignatureExpired {
 		errNoCache.add(errDecrypt)
-		if customMessageErr := im.makeCustomMessage(m, errDecrypt, true); customMessageErr != nil {
+		if customMessageErr := message.CustomMessage(m, errDecrypt, true); customMessageErr != nil {
 			im.log.WithError(customMessageErr).Warn("Failed to make custom message")
 		}
 	}
@@ -708,7 +589,7 @@ func (im *imapMailbox) buildMessage(m *pmapi.Message) (structure *message.BodySt
 		return nil, nil, err
 	} else if err != nil {
 		errNoCache.add(err)
-		if customMessageErr := im.makeCustomMessage(m, err, true); customMessageErr != nil {
+		if customMessageErr := message.CustomMessage(m, err, true); customMessageErr != nil {
 			im.log.WithError(customMessageErr).Warn("Failed to make custom message")
 		}
 		structure, msgBody, err = im.buildMessageInner(m, kr)

internal/importexport/credits.go

@@ -0,0 +1,22 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+// Code generated by ./credits.sh at Thu Jun 4 15:54:31 CEST 2020. DO NOT EDIT.
+
+package importexport
+
+const Credits = "github.com/0xAX/notificator;github.com/ProtonMail/bcrypt;github.com/ProtonMail/crypto;github.com/ProtonMail/docker-credential-helpers;github.com/ProtonMail/go-appdir;github.com/ProtonMail/go-apple-mobileconfig;github.com/ProtonMail/go-autostart;github.com/ProtonMail/go-imap-id;github.com/ProtonMail/go-smtp;github.com/ProtonMail/go-vcard;github.com/ProtonMail/gopenpgp;github.com/abiosoft/ishell;github.com/abiosoft/readline;github.com/allan-simon/go-singleinstance;github.com/andybalholm/cascadia;github.com/certifi/gocertifi;github.com/chzyer/logex;github.com/chzyer/test;github.com/cucumber/godog;github.com/docker/docker-credential-helpers;github.com/emersion/go-imap;github.com/emersion/go-imap-appendlimit;github.com/emersion/go-imap-idle;github.com/emersion/go-imap-move;github.com/emersion/go-imap-quota;github.com/emersion/go-imap-specialuse;github.com/emersion/go-imap-unselect;github.com/emersion/go-mbox;github.com/emersion/go-sasl;github.com/emersion/go-smtp;github.com/emersion/go-textwrapper;github.com/emersion/go-vcard;github.com/fatih/color;github.com/flynn-archive/go-shlex;github.com/getsentry/raven-go;github.com/go-resty/resty/v2;github.com/golang/mock;github.com/google/go-cmp;github.com/gopherjs/gopherjs;github.com/hashicorp/go-multierror;github.com/jameskeane/bcrypt;github.com/jaytaylor/html2text;github.com/jhillyerd/enmime;github.com/kardianos/osext;github.com/keybase/go-keychain;github.com/logrusorgru/aurora;github.com/miekg/dns;github.com/myesui/uuid;github.com/nsf/jsondiff;github.com/pkg/errors;github.com/sirupsen/logrus;github.com/skratchdot/open-golang;github.com/stretchr/testify;github.com/therecipe/qt;github.com/twinj/uuid;github.com/urfave/cli;go.etcd.io/bbolt;golang.org/x/crypto;golang.org/x/net;golang.org/x/text;gopkg.in/stretchr/testify.v1;;Font Awesome 4.7.0;;Qt 5.13 by Qt group;"

internal/importexport/importexport.go

@@ -0,0 +1,113 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+// Package importexport provides core functionality of Import/Export app.
+package importexport
+
+import (
+	"github.com/ProtonMail/proton-bridge/internal/transfer"
+	"github.com/ProtonMail/proton-bridge/internal/users"
+
+	"github.com/ProtonMail/proton-bridge/pkg/listener"
+	logrus "github.com/sirupsen/logrus"
+)
+
+var (
+	log = logrus.WithField("pkg", "importexport") //nolint[gochecknoglobals]
+)
+
+type ImportExport struct {
+	*users.Users
+
+	config        Configer
+	panicHandler  users.PanicHandler
+	clientManager users.ClientManager
+}
+
+func New(
+	config Configer,
+	panicHandler users.PanicHandler,
+	eventListener listener.Listener,
+	clientManager users.ClientManager,
+	credStorer users.CredentialsStorer,
+) *ImportExport {
+	u := users.New(config, panicHandler, eventListener, clientManager, credStorer, &storeFactory{}, false)
+	return &ImportExport{
+		Users: u,
+
+		config:        config,
+		panicHandler:  panicHandler,
+		clientManager: clientManager,
+	}
+}
+
+// GetLocalImporter returns transferrer from local EML or MBOX structure to ProtonMail account.
+func (ie *ImportExport) GetLocalImporter(address, path string) (*transfer.Transfer, error) {
+	source := transfer.NewLocalProvider(path)
+	target, err := ie.getPMAPIProvider(address)
+	if err != nil {
+		return nil, err
+	}
+	return transfer.New(ie.panicHandler, ie.config.GetTransferDir(), source, target)
+}
+
+// GetRemoteImporter returns transferrer from remote IMAP to ProtonMail account.
+func (ie *ImportExport) GetRemoteImporter(address, username, password, host, port string) (*transfer.Transfer, error) {
+	source, err := transfer.NewIMAPProvider(username, password, host, port)
+	if err != nil {
+		return nil, err
+	}
+	target, err := ie.getPMAPIProvider(address)
+	if err != nil {
+		return nil, err
+	}
+	return transfer.New(ie.panicHandler, ie.config.GetTransferDir(), source, target)
+}
+
+// GetEMLExporter returns transferrer from ProtonMail account to local EML structure.
+func (ie *ImportExport) GetEMLExporter(address, path string) (*transfer.Transfer, error) {
+	source, err := ie.getPMAPIProvider(address)
+	if err != nil {
+		return nil, err
+	}
+	target := transfer.NewEMLProvider(path)
+	return transfer.New(ie.panicHandler, ie.config.GetTransferDir(), source, target)
+}
+
+// GetMBOXExporter returns transferrer from ProtonMail account to local MBOX structure.
+func (ie *ImportExport) GetMBOXExporter(address, path string) (*transfer.Transfer, error) {
+	source, err := ie.getPMAPIProvider(address)
+	if err != nil {
+		return nil, err
+	}
+	target := transfer.NewMBOXProvider(path)
+	return transfer.New(ie.panicHandler, ie.config.GetTransferDir(), source, target)
+}
+
+func (ie *ImportExport) getPMAPIProvider(address string) (*transfer.PMAPIProvider, error) {
+	user, err := ie.Users.GetUser(address)
+	if err != nil {
+		return nil, err
+	}
+
+	addressID, err := user.GetAddressID(address)
+	if err != nil {
+		return nil, err
+	}
+
+	return transfer.NewPMAPIProvider(ie.clientManager, user.ID(), addressID)
+}

internal/importexport/release_notes.go

@@ -0,0 +1,26 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+// Code generated by ./release-notes.sh at 'Thu Jun  4 15:54:31 CEST 2020'. DO NOT EDIT.
+
+package importexport
+
+const ReleaseNotes = `
+`
+
+const ReleaseFixedBugs = `
+`

internal/importexport/store_factory.go

@@ -0,0 +1,35 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package importexport
+
+import (
+	"github.com/ProtonMail/proton-bridge/internal/store"
+)
+
+// storeFactory implements dummy factory creating no store (not needed by Import/Export).
+type storeFactory struct{}
+
+// New does nothing.
+func (f *storeFactory) New(user store.BridgeUser) (*store.Store, error) {
+	return nil, nil
+}
+
+// Remove does nothing.
+func (f *storeFactory) Remove(userID string) error {
+	return nil
+}

internal/importexport/types.go

@@ -0,0 +1,26 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package importexport
+
+import "github.com/ProtonMail/proton-bridge/internal/users"
+
+type Configer interface {
+	users.Configer
+
+	GetTransferDir() string
+}

internal/transfer/mailbox.go

@@ -0,0 +1,69 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"strings"
+)
+
+// Mailbox is universal data holder of mailbox details for every provider.
+type Mailbox struct {
+	ID          string
+	Name        string
+	Color       string
+	IsExclusive bool
+}
+
+// Hash returns unique identifier to be used for matching.
+func (m Mailbox) Hash() string {
+	return fmt.Sprintf("%x", sha256.Sum256([]byte(m.Name)))
+}
+
+// findMatchingMailboxes returns all matching mailboxes from `mailboxes`.
+// Only one exclusive mailbox is returned.
+func (m Mailbox) findMatchingMailboxes(mailboxes []Mailbox) []Mailbox {
+	nameVariants := []string{}
+	if strings.Contains(m.Name, "/") || strings.Contains(m.Name, "|") {
+		for _, slashPart := range strings.Split(m.Name, "/") {
+			for _, part := range strings.Split(slashPart, "|") {
+				nameVariants = append(nameVariants, strings.ToLower(part))
+			}
+		}
+	}
+	nameVariants = append(nameVariants, strings.ToLower(m.Name))
+
+	isExclusiveIncluded := false
+	matches := []Mailbox{}
+	for i := range nameVariants {
+		nameVariant := nameVariants[len(nameVariants)-1-i]
+		for _, mailbox := range mailboxes {
+			if mailbox.IsExclusive && isExclusiveIncluded {
+				continue
+			}
+			if strings.ToLower(mailbox.Name) == nameVariant {
+				matches = append(matches, mailbox)
+				if mailbox.IsExclusive {
+					isExclusiveIncluded = true
+				}
+			}
+		}
+	}
+	return matches
+}

internal/transfer/mailbox_test.go

@@ -0,0 +1,61 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"testing"
+
+	r "github.com/stretchr/testify/require"
+)
+
+func TestFindMatchingMailboxes(t *testing.T) {
+	mailboxes := []Mailbox{
+		{Name: "Inbox", IsExclusive: true},
+		{Name: "Sent", IsExclusive: true},
+		{Name: "Archive", IsExclusive: true},
+		{Name: "Foo", IsExclusive: false},
+		{Name: "hello/world", IsExclusive: true},
+		{Name: "Hello", IsExclusive: false},
+		{Name: "WORLD", IsExclusive: true},
+	}
+
+	tests := []struct {
+		name      string
+		wantNames []string
+	}{
+		{"inbox", []string{"Inbox"}},
+		{"foo", []string{"Foo"}},
+		{"hello", []string{"Hello"}},
+		{"world", []string{"WORLD"}},
+		{"hello/world", []string{"hello/world", "Hello"}},
+		{"hello|world", []string{"WORLD", "Hello"}},
+		{"nomailbox", []string{}},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			mailbox := Mailbox{Name: tc.name}
+			got := mailbox.findMatchingMailboxes(mailboxes)
+			gotNames := []string{}
+			for _, m := range got {
+				gotNames = append(gotNames, m.Name)
+			}
+			r.Equal(t, tc.wantNames, gotNames)
+		})
+	}
+}

internal/transfer/message.go

@@ -0,0 +1,100 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"fmt"
+	"mime"
+	"net/mail"
+	"time"
+)
+
+// Message is data holder passed between import and export.
+type Message struct {
+	ID      string
+	Unread  bool
+	Body    []byte
+	Source  Mailbox
+	Targets []Mailbox
+}
+
+// MessageStatus holds status for message used by progress manager.
+type MessageStatus struct {
+	eventTime time.Time // Time of adding message to the process.
+	rule      *Rule     // Rule with source and target mailboxes.
+	SourceID  string    // Message ID at the source.
+	targetID  string    // Message ID at the target (if any).
+	bodyHash  string    // Hash of the message body.
+
+	exported  bool
+	imported  bool
+	exportErr error
+	importErr error
+
+	// Info about message displayed to user.
+	// This is needed only for failed messages, but we cannot know in advance
+	// which message will fail. We could clear it once the message passed
+	// without any error. However, if we say one message takes about 100 bytes
+	// in average, it's about 100 MB per million of messages, which is fine.
+	Subject string
+	From    string
+	Time    time.Time
+}
+
+func (status *MessageStatus) setDetailsFromHeader(header mail.Header) {
+	dec := &mime.WordDecoder{}
+
+	status.Subject = header.Get("subject")
+	if subject, err := dec.Decode(status.Subject); err == nil {
+		status.Subject = subject
+	}
+
+	status.From = header.Get("from")
+	if from, err := dec.Decode(status.From); err == nil {
+		status.From = from
+	}
+
+	if msgTime, err := header.Date(); err == nil {
+		status.Time = msgTime
+	}
+}
+
+func (status *MessageStatus) hasError(includeMissing bool) bool {
+	return status.exportErr != nil || status.importErr != nil || (includeMissing && !status.imported)
+}
+
+// GetErrorMessage returns error message.
+func (status *MessageStatus) GetErrorMessage() string {
+	return status.getErrorMessage(true)
+}
+
+func (status *MessageStatus) getErrorMessage(includeMissing bool) string {
+	if status.exportErr != nil {
+		return fmt.Sprintf("failed to export: %s", status.exportErr)
+	}
+	if status.importErr != nil {
+		return fmt.Sprintf("failed to import: %s", status.importErr)
+	}
+	if includeMissing && !status.imported {
+		if !status.exported {
+			return "failed to import: lost before read"
+		}
+		return "failed to import: lost in the process"
+	}
+	return ""
+}

internal/transfer/mocks/mocks.go

@@ -0,0 +1,97 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/ProtonMail/proton-bridge/internal/transfer (interfaces: PanicHandler,ClientManager)
+
+// Package mocks is a generated GoMock package.
+package mocks
+
+import (
+	pmapi "github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	gomock "github.com/golang/mock/gomock"
+	reflect "reflect"
+)
+
+// MockPanicHandler is a mock of PanicHandler interface
+type MockPanicHandler struct {
+	ctrl     *gomock.Controller
+	recorder *MockPanicHandlerMockRecorder
+}
+
+// MockPanicHandlerMockRecorder is the mock recorder for MockPanicHandler
+type MockPanicHandlerMockRecorder struct {
+	mock *MockPanicHandler
+}
+
+// NewMockPanicHandler creates a new mock instance
+func NewMockPanicHandler(ctrl *gomock.Controller) *MockPanicHandler {
+	mock := &MockPanicHandler{ctrl: ctrl}
+	mock.recorder = &MockPanicHandlerMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockPanicHandler) EXPECT() *MockPanicHandlerMockRecorder {
+	return m.recorder
+}
+
+// HandlePanic mocks base method
+func (m *MockPanicHandler) HandlePanic() {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "HandlePanic")
+}
+
+// HandlePanic indicates an expected call of HandlePanic
+func (mr *MockPanicHandlerMockRecorder) HandlePanic() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HandlePanic", reflect.TypeOf((*MockPanicHandler)(nil).HandlePanic))
+}
+
+// MockClientManager is a mock of ClientManager interface
+type MockClientManager struct {
+	ctrl     *gomock.Controller
+	recorder *MockClientManagerMockRecorder
+}
+
+// MockClientManagerMockRecorder is the mock recorder for MockClientManager
+type MockClientManagerMockRecorder struct {
+	mock *MockClientManager
+}
+
+// NewMockClientManager creates a new mock instance
+func NewMockClientManager(ctrl *gomock.Controller) *MockClientManager {
+	mock := &MockClientManager{ctrl: ctrl}
+	mock.recorder = &MockClientManagerMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockClientManager) EXPECT() *MockClientManagerMockRecorder {
+	return m.recorder
+}
+
+// CheckConnection mocks base method
+func (m *MockClientManager) CheckConnection() error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "CheckConnection")
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// CheckConnection indicates an expected call of CheckConnection
+func (mr *MockClientManagerMockRecorder) CheckConnection() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckConnection", reflect.TypeOf((*MockClientManager)(nil).CheckConnection))
+}
+
+// GetClient mocks base method
+func (m *MockClientManager) GetClient(arg0 string) pmapi.Client {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetClient", arg0)
+	ret0, _ := ret[0].(pmapi.Client)
+	return ret0
+}
+
+// GetClient indicates an expected call of GetClient
+func (mr *MockClientManagerMockRecorder) GetClient(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetClient", reflect.TypeOf((*MockClientManager)(nil).GetClient), arg0)
+}

internal/transfer/progress.go

@@ -0,0 +1,331 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+// Progress maintains progress between import, export and user interface.
+// Import and export update progress about processing messages and progress
+// informs user interface, vice versa action (such as pause or resume) from
+// user interface is passed down to import and export.
+type Progress struct {
+	log  *logrus.Entry
+	lock sync.RWMutex
+
+	updateCh        chan struct{}
+	messageCounts   map[string]uint
+	messageStatuses map[string]*MessageStatus
+	pauseReason     string
+	isStopped       bool
+	fatalError      error
+	fileReport      *fileReport
+}
+
+func newProgress(log *logrus.Entry, fileReport *fileReport) Progress {
+	return Progress{
+		log: log,
+
+		updateCh:        make(chan struct{}),
+		messageCounts:   map[string]uint{},
+		messageStatuses: map[string]*MessageStatus{},
+		fileReport:      fileReport,
+	}
+}
+
+// update is helper to notify listener for updates.
+func (p *Progress) update() {
+	if p.updateCh == nil {
+		// If the progress was ended by fatal instead finish, we ignore error.
+		if p.fatalError != nil {
+			return
+		}
+		panic("update should not be called after finish was called")
+	}
+
+	// In case no one listens for an update, do not block the progress.
+	select {
+	case p.updateCh <- struct{}{}:
+	case <-time.After(100 * time.Millisecond):
+	}
+}
+
+// start should be called before anything starts.
+func (p *Progress) start() {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+}
+
+// finish should be called as the last call once everything is done.
+func (p *Progress) finish() {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	p.cleanUpdateCh()
+}
+
+// fatal should be called once there is error with no possible continuation.
+func (p *Progress) fatal(err error) {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	p.isStopped = true
+	p.fatalError = err
+	p.cleanUpdateCh()
+}
+
+func (p *Progress) cleanUpdateCh() {
+	if p.updateCh == nil {
+		// If the progress was ended by fatal instead finish, we ignore error.
+		if p.fatalError != nil {
+			return
+		}
+		panic("update should not be called after finish was called")
+	}
+
+	close(p.updateCh)
+	p.updateCh = nil
+}
+
+func (p *Progress) updateCount(mailbox string, count uint) {
+	p.lock.Lock()
+	defer p.update()
+	defer p.lock.Unlock()
+
+	log.WithField("mailbox", mailbox).WithField("count", count).Debug("Mailbox count updated")
+	p.messageCounts[mailbox] = count
+}
+
+// addMessage should be called as soon as there is ID of the message.
+func (p *Progress) addMessage(messageID string, rule *Rule) {
+	p.lock.Lock()
+	defer p.update()
+	defer p.lock.Unlock()
+
+	p.log.WithField("id", messageID).Trace("Message added")
+	p.messageStatuses[messageID] = &MessageStatus{
+		eventTime: time.Now(),
+		rule:      rule,
+		SourceID:  messageID,
+	}
+}
+
+// messageExported should be called right before message is exported.
+func (p *Progress) messageExported(messageID string, body []byte, err error) {
+	p.lock.Lock()
+	defer p.update()
+	defer p.lock.Unlock()
+
+	p.log.WithField("id", messageID).WithError(err).Debug("Message exported")
+	status := p.messageStatuses[messageID]
+	status.exported = true
+	status.exportErr = err
+
+	if len(body) > 0 {
+		status.bodyHash = fmt.Sprintf("%x", sha256.Sum256(body))
+
+		if header, err := getMessageHeader(body); err != nil {
+			p.log.WithField("id", messageID).WithError(err).Warning("Failed to parse headers for reporting")
+		} else {
+			status.setDetailsFromHeader(header)
+		}
+	}
+
+	// If export failed, no other step will be done with message and we can log it to the report file.
+	if err != nil {
+		p.logMessage(messageID)
+	}
+}
+
+// messageImported should be called right after message is imported.
+func (p *Progress) messageImported(messageID, importID string, err error) {
+	p.lock.Lock()
+	defer p.update()
+	defer p.lock.Unlock()
+
+	p.log.WithField("id", messageID).WithError(err).Debug("Message imported")
+	p.messageStatuses[messageID].targetID = importID
+	p.messageStatuses[messageID].imported = true
+	p.messageStatuses[messageID].importErr = err
+
+	// Import is the last step, now we can log the result to the report file.
+	p.logMessage(messageID)
+}
+
+// logMessage writes message status to log file.
+func (p *Progress) logMessage(messageID string) {
+	if p.fileReport == nil {
+		return
+	}
+	p.fileReport.writeMessageStatus(p.messageStatuses[messageID])
+}
+
+// callWrap calls the callback and in case of problem it pause the process.
+// Then it waits for user action to fix it and click on continue or abort.
+func (p *Progress) callWrap(callback func() error) {
+	for {
+		if p.shouldStop() {
+			break
+		}
+
+		err := callback()
+		if err == nil {
+			break
+		}
+
+		p.Pause(err.Error())
+	}
+}
+
+// shouldStop is utility for providers to automatically wait during pause
+// and returned value determines whether the process shouls be fully stopped.
+func (p *Progress) shouldStop() bool {
+	for p.IsPaused() {
+		time.Sleep(time.Second)
+	}
+	return p.IsStopped()
+}
+
+// GetUpdateChannel returns channel notifying any update from import or export.
+func (p *Progress) GetUpdateChannel() chan struct{} {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	return p.updateCh
+}
+
+// Pause pauses the progress.
+func (p *Progress) Pause(reason string) {
+	p.lock.Lock()
+	defer p.update()
+	defer p.lock.Unlock()
+
+	p.log.Info("Progress paused")
+	p.pauseReason = reason
+}
+
+// Resume resumes the progress.
+func (p *Progress) Resume() {
+	p.lock.Lock()
+	defer p.update()
+	defer p.lock.Unlock()
+
+	p.log.Info("Progress resumed")
+	p.pauseReason = ""
+}
+
+// IsPaused returns whether progress is paused.
+func (p *Progress) IsPaused() bool {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	return p.pauseReason != ""
+}
+
+// PauseReason returns pause reason.
+func (p *Progress) PauseReason() string {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	return p.pauseReason
+}
+
+// Stop stops the process.
+func (p *Progress) Stop() {
+	p.lock.Lock()
+	defer p.update()
+	defer p.lock.Unlock()
+
+	p.log.Info("Progress stopped")
+	p.isStopped = true
+	p.pauseReason = "" // Clear pause to run paused code and stop it.
+}
+
+// IsStopped returns whether progress is stopped.
+func (p *Progress) IsStopped() bool {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	return p.isStopped
+}
+
+// GetFatalError returns fatal error (progress failed and did not finish).
+func (p *Progress) GetFatalError() error {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	return p.fatalError
+}
+
+// GetFailedMessages returns statuses of failed messages.
+func (p *Progress) GetFailedMessages() []*MessageStatus {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	// Include lost messages in the process only when transfer is done.
+	includeMissing := p.updateCh == nil
+
+	statuses := []*MessageStatus{}
+	for _, status := range p.messageStatuses {
+		if status.hasError(includeMissing) {
+			statuses = append(statuses, status)
+		}
+	}
+	return statuses
+}
+
+// GetCounts returns counts of exported and imported messages.
+func (p *Progress) GetCounts() (failed, imported, exported, added, total uint) {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	// Include lost messages in the process only when transfer is done.
+	includeMissing := p.updateCh == nil
+
+	for _, mailboxCount := range p.messageCounts {
+		total += mailboxCount
+	}
+	for _, status := range p.messageStatuses {
+		added++
+		if status.exported {
+			exported++
+		}
+		if status.imported {
+			imported++
+		}
+		if status.hasError(includeMissing) {
+			failed++
+		}
+	}
+	return
+}
+
+// GenerateBugReport generates similar file to import log except private information.
+func (p *Progress) GenerateBugReport() []byte {
+	bugReport := bugReport{}
+	for _, status := range p.messageStatuses {
+		bugReport.writeMessageStatus(status)
+	}
+	return bugReport.getData()
+}

internal/transfer/progress_test.go

@@ -0,0 +1,120 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"testing"
+
+	"github.com/pkg/errors"
+	a "github.com/stretchr/testify/assert"
+	r "github.com/stretchr/testify/require"
+)
+
+func TestProgressUpdateCount(t *testing.T) {
+	progress := newProgress(log, nil)
+	drainProgressUpdateChannel(&progress)
+
+	progress.start()
+
+	progress.updateCount("inbox", 10)
+	progress.updateCount("archive", 20)
+	progress.updateCount("inbox", 12)
+	progress.updateCount("sent", 5)
+	progress.updateCount("foo", 4)
+	progress.updateCount("foo", 5)
+
+	progress.finish()
+
+	_, _, _, _, total := progress.GetCounts() //nolint[dogsled]
+	r.Equal(t, uint(42), total)
+}
+
+func TestProgressAddingMessages(t *testing.T) {
+	progress := newProgress(log, nil)
+	drainProgressUpdateChannel(&progress)
+
+	progress.start()
+
+	// msg1 has no problem.
+	progress.addMessage("msg1", nil)
+	progress.messageExported("msg1", []byte(""), nil)
+	progress.messageImported("msg1", "", nil)
+
+	// msg2 has an import problem.
+	progress.addMessage("msg2", nil)
+	progress.messageExported("msg2", []byte(""), nil)
+	progress.messageImported("msg2", "", errors.New("failed import"))
+
+	// msg3 has an export problem.
+	progress.addMessage("msg3", nil)
+	progress.messageExported("msg3", []byte(""), errors.New("failed export"))
+
+	// msg4 has an export problem and import is also called.
+	progress.addMessage("msg4", nil)
+	progress.messageExported("msg4", []byte(""), errors.New("failed export"))
+	progress.messageImported("msg4", "", nil)
+
+	progress.finish()
+
+	failed, imported, exported, added, _ := progress.GetCounts()
+	a.Equal(t, uint(4), added)
+	a.Equal(t, uint(4), exported)
+	a.Equal(t, uint(3), imported)
+	a.Equal(t, uint(3), failed)
+
+	errorsMap := map[string]string{}
+	for _, status := range progress.GetFailedMessages() {
+		errorsMap[status.SourceID] = status.GetErrorMessage()
+	}
+	a.Equal(t, map[string]string{
+		"msg2": "failed to import: failed import",
+		"msg3": "failed to export: failed export",
+		"msg4": "failed to export: failed export",
+	}, errorsMap)
+}
+
+func TestProgressFinish(t *testing.T) {
+	progress := newProgress(log, nil)
+	drainProgressUpdateChannel(&progress)
+
+	progress.start()
+	progress.finish()
+	r.Nil(t, progress.updateCh)
+
+	r.Panics(t, func() { progress.addMessage("msg", nil) })
+}
+
+func TestProgressFatalError(t *testing.T) {
+	progress := newProgress(log, nil)
+	drainProgressUpdateChannel(&progress)
+
+	progress.start()
+	progress.fatal(errors.New("fatal error"))
+	r.Nil(t, progress.updateCh)
+
+	r.NotPanics(t, func() { progress.addMessage("msg", nil) })
+}
+
+func drainProgressUpdateChannel(progress *Progress) {
+	// updateCh is not needed to drain under tests - timeout is implemented.
+	// But timeout takes time which would slow down tests.
+	go func() {
+		for range progress.updateCh {
+		}
+	}()
+}

internal/transfer/provider.go

@@ -0,0 +1,51 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+// Provider provides interface for common operation with provider.
+type Provider interface {
+	// ID is used for generating transfer ID by combining source and target ID.
+	ID() string
+
+	// Mailboxes returns all available mailboxes.
+	// Provider used as source returns only non-empty maibloxes.
+	// Provider used as target does not return all mail maiblox.
+	Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error)
+}
+
+// SourceProvider provides interface of provider with support of export.
+type SourceProvider interface {
+	Provider
+
+	// TransferTo exports messages based on rules to channel.
+	TransferTo(transferRules, *Progress, chan<- Message)
+}
+
+// TargetProvider provides interface of provider with support of import.
+type TargetProvider interface {
+	Provider
+
+	// DefaultMailboxes returns the default mailboxes for default rules if no other is found.
+	DefaultMailboxes(sourceMailbox Mailbox) (targetMailboxes []Mailbox)
+
+	// CreateMailbox creates new mailbox to be used as target in transfer rules.
+	CreateMailbox(Mailbox) (Mailbox, error)
+
+	// TransferFrom imports messages from channel.
+	TransferFrom(transferRules, *Progress, <-chan Message)
+}

internal/transfer/provider_eml.go

@@ -0,0 +1,65 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+// EMLProvider implements import and export to/from EML file structure.
+type EMLProvider struct {
+	root string
+}
+
+// NewEMLProvider creates EMLProvider.
+func NewEMLProvider(root string) *EMLProvider {
+	return &EMLProvider{
+		root: root,
+	}
+}
+
+// ID is used for generating transfer ID by combining source and target ID.
+// We want to keep the same rules for import from or export to local files
+// no matter exact path, therefore it returns constant. The same as EML.
+func (p *EMLProvider) ID() string {
+	return "local" //nolint[goconst]
+}
+
+// Mailboxes returns all available folder names from root of EML files.
+// In case the same folder name is used more than once (for example root/a/foo
+// and root/b/foo), it's treated as the same folder.
+func (p *EMLProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) {
+	var folderNames []string
+	var err error
+	if includeEmpty {
+		folderNames, err = getFolderNames(p.root)
+	} else {
+		folderNames, err = getFolderNamesWithFileSuffix(p.root, ".eml")
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	mailboxes := []Mailbox{}
+	for _, folderName := range folderNames {
+		mailboxes = append(mailboxes, Mailbox{
+			ID:          "",
+			Name:        folderName,
+			Color:       "",
+			IsExclusive: false,
+		})
+	}
+
+	return mailboxes, nil
+}

internal/transfer/provider_eml_source.go

@@ -0,0 +1,135 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+)
+
+// TransferTo exports messages based on rules to channel.
+func (p *EMLProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) {
+	log.Info("Started transfer from EML to channel")
+	defer log.Info("Finished transfer from EML to channel")
+
+	filePathsPerFolder, err := p.getFilePathsPerFolder(rules)
+	if err != nil {
+		progress.fatal(err)
+		return
+	}
+
+	// This list is not filtered by time but instead going throgh each file
+	// twice or keeping all in memory we will tell rough estimation which
+	// will be updated during processing each file.
+	for folderName, filePaths := range filePathsPerFolder {
+		if progress.shouldStop() {
+			break
+		}
+
+		progress.updateCount(folderName, uint(len(filePaths)))
+	}
+
+	for folderName, filePaths := range filePathsPerFolder {
+		// No error guaranteed by getFilePathsPerFolder.
+		rule, _ := rules.getRuleBySourceMailboxName(folderName)
+		log.WithField("rule", rule).Debug("Processing rule")
+		p.exportMessages(rule, filePaths, progress, ch)
+	}
+}
+
+func (p *EMLProvider) getFilePathsPerFolder(rules transferRules) (map[string][]string, error) {
+	filePaths, err := getFilePathsWithSuffix(p.root, ".eml")
+	if err != nil {
+		return nil, err
+	}
+
+	filePathsMap := map[string][]string{}
+	for _, filePath := range filePaths {
+		folder := filepath.Base(filepath.Dir(filepath.Join(p.root, filePath)))
+		_, err := rules.getRuleBySourceMailboxName(folder)
+		if err != nil {
+			log.WithField("msg", filePath).Trace("Message skipped due to folder name")
+			continue
+		}
+
+		filePathsMap[folder] = append(filePathsMap[folder], filePath)
+	}
+
+	return filePathsMap, nil
+}
+
+func (p *EMLProvider) exportMessages(rule *Rule, filePaths []string, progress *Progress, ch chan<- Message) {
+	count := uint(len(filePaths))
+
+	for _, filePath := range filePaths {
+		if progress.shouldStop() {
+			break
+		}
+
+		msg, err := p.exportMessage(rule, filePath)
+
+		// Read and check time in body only if the rule specifies it
+		// to not waste energy.
+		if err == nil && rule.HasTimeLimit() {
+			msgTime, msgTimeErr := getMessageTime(msg.Body)
+			if msgTimeErr != nil {
+				err = msgTimeErr
+			} else if !rule.isTimeInRange(msgTime) {
+				log.WithField("msg", filePath).Debug("Message skipped due to time")
+
+				count--
+				progress.updateCount(rule.SourceMailbox.Name, count)
+				continue
+			}
+		}
+
+		// addMessage is called after time check to not report message
+		// which should not be exported but any error from reading body
+		// or parsing time is reported as an error.
+		progress.addMessage(filePath, rule)
+		progress.messageExported(filePath, msg.Body, err)
+		if err == nil {
+			ch <- msg
+		}
+	}
+}
+
+func (p *EMLProvider) exportMessage(rule *Rule, filePath string) (Message, error) {
+	fullFilePath := filepath.Clean(filepath.Join(p.root, filePath))
+	file, err := os.Open(fullFilePath) //nolint[gosec]
+	if err != nil {
+		return Message{}, errors.Wrap(err, "failed to open message")
+	}
+	defer file.Close() //nolint[errcheck]
+
+	body, err := ioutil.ReadAll(file)
+	if err != nil {
+		return Message{}, errors.Wrap(err, "failed to read message")
+	}
+
+	return Message{
+		ID:      filePath,
+		Unread:  false,
+		Body:    body,
+		Source:  rule.SourceMailbox,
+		Targets: rule.TargetMailboxes,
+	}, nil
+}

internal/transfer/provider_eml_target.go

@@ -0,0 +1,89 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/hashicorp/go-multierror"
+)
+
+// DefaultMailboxes returns the default mailboxes for default rules if no other is found.
+func (p *EMLProvider) DefaultMailboxes(sourceMailbox Mailbox) []Mailbox {
+	return []Mailbox{{
+		Name: sourceMailbox.Name,
+	}}
+}
+
+// CreateMailbox does nothing. Folders are created dynamically during the import.
+func (p *EMLProvider) CreateMailbox(mailbox Mailbox) (Mailbox, error) {
+	return mailbox, nil
+}
+
+// TransferFrom imports messages from channel.
+func (p *EMLProvider) TransferFrom(rules transferRules, progress *Progress, ch <-chan Message) {
+	log.Info("Started transfer from channel to EML")
+	defer log.Info("Finished transfer from channel to EML")
+
+	err := p.createFolders(rules)
+	if err != nil {
+		progress.fatal(err)
+		return
+	}
+
+	for msg := range ch {
+		for progress.shouldStop() {
+			break
+		}
+
+		err := p.writeFile(msg)
+		progress.messageImported(msg.ID, "", err)
+	}
+}
+
+func (p *EMLProvider) createFolders(rules transferRules) error {
+	for rule := range rules.iterateActiveRules() {
+		for _, mailbox := range rule.TargetMailboxes {
+			path := filepath.Join(p.root, mailbox.Name)
+			if err := os.MkdirAll(path, os.ModePerm); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (p *EMLProvider) writeFile(msg Message) error {
+	fileName := filepath.Base(msg.ID)
+	if !strings.HasSuffix(fileName, ".eml") {
+		fileName += ".eml"
+	}
+
+	var err error
+	for _, mailbox := range msg.Targets {
+		path := filepath.Join(p.root, mailbox.Name, fileName)
+
+		if localErr := ioutil.WriteFile(path, msg.Body, 0600); localErr != nil {
+			err = multierror.Append(err, localErr)
+		}
+	}
+	return err
+}

internal/transfer/provider_eml_test.go

@@ -0,0 +1,126 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+	"time"
+
+	r "github.com/stretchr/testify/require"
+)
+
+func newTestEMLProvider(path string) *EMLProvider {
+	if path == "" {
+		path = "testdata/eml"
+	}
+	return NewEMLProvider(path)
+}
+
+func TestEMLProviderMailboxes(t *testing.T) {
+	provider := newTestEMLProvider("")
+
+	tests := []struct {
+		includeEmpty  bool
+		wantMailboxes []Mailbox
+	}{
+		{true, []Mailbox{
+			{Name: "Foo"},
+			{Name: "Inbox"},
+			{Name: "eml"},
+		}},
+		{false, []Mailbox{
+			{Name: "Foo"},
+			{Name: "Inbox"},
+		}},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(fmt.Sprintf("%v", tc.includeEmpty), func(t *testing.T) {
+			mailboxes, err := provider.Mailboxes(tc.includeEmpty, false)
+			r.NoError(t, err)
+			r.Equal(t, tc.wantMailboxes, mailboxes)
+		})
+	}
+}
+
+func TestEMLProviderTransferTo(t *testing.T) {
+	provider := newTestEMLProvider("")
+
+	rules, rulesClose := newTestRules(t)
+	defer rulesClose()
+	setupEMLRules(rules)
+
+	testTransferTo(t, rules, provider, []string{
+		"Foo/msg.eml",
+		"Inbox/msg.eml",
+	})
+}
+
+func TestEMLProviderTransferFrom(t *testing.T) {
+	dir, err := ioutil.TempDir("", "eml")
+	r.NoError(t, err)
+	defer os.RemoveAll(dir) //nolint[errcheck]
+
+	provider := newTestEMLProvider(dir)
+
+	rules, rulesClose := newTestRules(t)
+	defer rulesClose()
+	setupEMLRules(rules)
+
+	testTransferFrom(t, rules, provider, []Message{
+		{ID: "Foo/msg.eml", Body: getTestMsgBody("msg"), Targets: []Mailbox{{Name: "Foo"}}},
+	})
+
+	checkEMLFileStructure(t, dir, []string{
+		"Foo/msg.eml",
+	})
+}
+
+func TestEMLProviderTransferFromTo(t *testing.T) {
+	dir, err := ioutil.TempDir("", "eml")
+	r.NoError(t, err)
+	defer os.RemoveAll(dir) //nolint[errcheck]
+
+	source := newTestEMLProvider("")
+	target := newTestEMLProvider(dir)
+
+	rules, rulesClose := newTestRules(t)
+	defer rulesClose()
+	setupEMLRules(rules)
+
+	testTransferFromTo(t, rules, source, target, 5*time.Second)
+
+	checkEMLFileStructure(t, dir, []string{
+		"Foo/msg.eml",
+		"Inbox/msg.eml",
+	})
+}
+
+func setupEMLRules(rules transferRules) {
+	_ = rules.setRule(Mailbox{Name: "Inbox"}, []Mailbox{{Name: "Inbox"}}, 0, 0)
+	_ = rules.setRule(Mailbox{Name: "Foo"}, []Mailbox{{Name: "Foo"}}, 0, 0)
+}
+
+func checkEMLFileStructure(t *testing.T, root string, expectedFiles []string) {
+	files, err := getFilePathsWithSuffix(root, ".eml")
+	r.NoError(t, err)
+	r.Equal(t, expectedFiles, files)
+}

internal/transfer/provider_imap.go

@@ -0,0 +1,98 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"net"
+	"strings"
+
+	imapClient "github.com/emersion/go-imap/client"
+)
+
+// IMAPProvider implements export from IMAP server.
+type IMAPProvider struct {
+	username string
+	password string
+	addr     string
+
+	client *imapClient.Client
+}
+
+// NewIMAPProvider returns new IMAPProvider.
+func NewIMAPProvider(username, password, host, port string) (*IMAPProvider, error) {
+	p := &IMAPProvider{
+		username: username,
+		password: password,
+		addr:     net.JoinHostPort(host, port),
+	}
+
+	if err := p.auth(); err != nil {
+		return nil, err
+	}
+
+	return p, nil
+}
+
+// ID is used for generating transfer ID by combining source and target ID.
+// We want to keep the same rules for import from any IMAP server, therefore
+// it returns constant.
+func (p *IMAPProvider) ID() string {
+	return "imap"
+}
+
+// Mailboxes returns all available folder names from root of EML files.
+// In case the same folder name is used more than once (for example root/a/foo
+// and root/b/foo), it's treated as the same folder.
+func (p *IMAPProvider) Mailboxes(includEmpty, includeAllMail bool) ([]Mailbox, error) {
+	mailboxesInfo, err := p.list()
+	if err != nil {
+		return nil, err
+	}
+
+	mailboxes := []Mailbox{}
+	for _, mailbox := range mailboxesInfo {
+		hasNoSelect := false
+		for _, attrib := range mailbox.Attributes {
+			if strings.ToLower(attrib) == "\\noselect" {
+				hasNoSelect = true
+				break
+			}
+		}
+		if hasNoSelect || mailbox.Name == "[Gmail]" {
+			continue
+		}
+
+		if !includEmpty || true {
+			mailboxStatus, err := p.selectIn(mailbox.Name)
+			if err != nil {
+				return nil, err
+			}
+			if mailboxStatus.Messages == 0 {
+				continue
+			}
+		}
+
+		mailboxes = append(mailboxes, Mailbox{
+			ID:          "",
+			Name:        mailbox.Name,
+			Color:       "",
+			IsExclusive: false,
+		})
+	}
+	return mailboxes, nil
+}

internal/transfer/provider_imap_source.go

@@ -0,0 +1,210 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"fmt"
+	"io/ioutil"
+
+	"github.com/emersion/go-imap"
+)
+
+type imapMessageInfo struct {
+	id   string
+	uid  uint32
+	size uint32
+}
+
+const (
+	imapPageSize     = uint32(2000)             // Optimized on Gmail.
+	imapMaxFetchSize = uint32(50 * 1000 * 1000) // Size in octets. If 0, it will use one fetch per message.
+)
+
+// TransferTo exports messages based on rules to channel.
+func (p *IMAPProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) {
+	log.Info("Started transfer from IMAP to channel")
+	defer log.Info("Finished transfer from IMAP to channel")
+
+	imapMessageInfoMap := p.loadMessageInfoMap(rules, progress)
+
+	for rule := range rules.iterateActiveRules() {
+		log.WithField("rule", rule).Debug("Processing rule")
+		messagesInfo := imapMessageInfoMap[rule.SourceMailbox.Name]
+		p.transferTo(rule, messagesInfo, progress, ch)
+	}
+}
+
+func (p *IMAPProvider) loadMessageInfoMap(rules transferRules, progress *Progress) map[string]map[string]imapMessageInfo {
+	res := map[string]map[string]imapMessageInfo{}
+
+	for rule := range rules.iterateActiveRules() {
+		if progress.shouldStop() {
+			break
+		}
+
+		mailboxName := rule.SourceMailbox.Name
+		var mailbox *imap.MailboxStatus
+		progress.callWrap(func() error {
+			var err error
+			mailbox, err = p.selectIn(mailboxName)
+			return err
+		})
+		if mailbox.Messages == 0 {
+			continue
+		}
+
+		messagesInfo := p.loadMessagesInfo(rule, progress, mailbox.UidValidity)
+		res[rule.SourceMailbox.Name] = messagesInfo
+		progress.updateCount(rule.SourceMailbox.Name, uint(len(messagesInfo)))
+	}
+
+	return res
+}
+
+func (p *IMAPProvider) loadMessagesInfo(rule *Rule, progress *Progress, uidValidity uint32) map[string]imapMessageInfo {
+	messagesInfo := map[string]imapMessageInfo{}
+
+	pageStart := uint32(1)
+	pageEnd := imapPageSize
+	for {
+		if progress.shouldStop() {
+			break
+		}
+
+		seqSet := &imap.SeqSet{}
+		seqSet.AddRange(pageStart, pageEnd)
+
+		items := []imap.FetchItem{imap.FetchUid, imap.FetchRFC822Size}
+		if rule.HasTimeLimit() {
+			items = append(items, imap.FetchEnvelope)
+		}
+
+		pageMsgCount := uint32(0)
+		processMessageCallback := func(imapMessage *imap.Message) {
+			pageMsgCount++
+			if rule.HasTimeLimit() {
+				t := imapMessage.Envelope.Date.Unix()
+				if t != 0 && !rule.isTimeInRange(t) {
+					log.WithField("uid", imapMessage.Uid).Debug("Message skipped due to time")
+					return
+				}
+			}
+			id := fmt.Sprintf("%s_%d:%d", rule.SourceMailbox.Name, uidValidity, imapMessage.Uid)
+			messagesInfo[id] = imapMessageInfo{
+				id:   id,
+				uid:  imapMessage.Uid,
+				size: imapMessage.Size,
+			}
+			progress.addMessage(id, rule)
+		}
+
+		progress.callWrap(func() error {
+			return p.fetch(seqSet, items, processMessageCallback)
+		})
+
+		if pageMsgCount < imapPageSize {
+			break
+		}
+
+		pageStart = pageEnd
+		pageEnd += imapPageSize
+	}
+
+	return messagesInfo
+}
+
+func (p *IMAPProvider) transferTo(rule *Rule, messagesInfo map[string]imapMessageInfo, progress *Progress, ch chan<- Message) {
+	progress.callWrap(func() error {
+		_, err := p.selectIn(rule.SourceMailbox.Name)
+		return err
+	})
+
+	seqSet := &imap.SeqSet{}
+	seqSetSize := uint32(0)
+	uidToID := map[uint32]string{}
+
+	for _, messageInfo := range messagesInfo {
+		if progress.shouldStop() {
+			break
+		}
+
+		if seqSetSize != 0 && (seqSetSize+messageInfo.size) > imapMaxFetchSize {
+			log.WithField("mailbox", rule.SourceMailbox.Name).WithField("seq", seqSet).WithField("size", seqSetSize).Debug("Fetching messages")
+
+			p.exportMessages(rule, progress, ch, seqSet, uidToID)
+
+			seqSet = &imap.SeqSet{}
+			seqSetSize = 0
+			uidToID = map[uint32]string{}
+		}
+
+		seqSet.AddNum(messageInfo.uid)
+		seqSetSize += messageInfo.size
+		uidToID[messageInfo.uid] = messageInfo.id
+	}
+}
+
+func (p *IMAPProvider) exportMessages(rule *Rule, progress *Progress, ch chan<- Message, seqSet *imap.SeqSet, uidToID map[uint32]string) {
+	section := &imap.BodySectionName{}
+	items := []imap.FetchItem{imap.FetchUid, imap.FetchFlags, section.FetchItem()}
+
+	processMessageCallback := func(imapMessage *imap.Message) {
+		id, ok := uidToID[imapMessage.Uid]
+
+		// Sometimes, server sends not requested messages.
+		if !ok {
+			log.WithField("uid", imapMessage.Uid).Warning("Message skipped: not requested")
+			return
+		}
+
+		// Sometimes, server sends message twice, once with body and once without it.
+		bodyReader := imapMessage.GetBody(section)
+		if bodyReader == nil {
+			log.WithField("uid", imapMessage.Uid).Warning("Message skipped: no body")
+			return
+		}
+
+		body, err := ioutil.ReadAll(bodyReader)
+		progress.messageExported(id, body, err)
+		if err == nil {
+			msg := p.exportMessage(rule, id, imapMessage, body)
+			ch <- msg
+		}
+	}
+
+	progress.callWrap(func() error {
+		return p.uidFetch(seqSet, items, processMessageCallback)
+	})
+}
+
+func (p *IMAPProvider) exportMessage(rule *Rule, id string, imapMessage *imap.Message, body []byte) Message {
+	unread := true
+	for _, flag := range imapMessage.Flags {
+		if flag == imap.SeenFlag {
+			unread = false
+		}
+	}
+
+	return Message{
+		ID:      id,
+		Unread:  unread,
+		Body:    body,
+		Source:  rule.SourceMailbox,
+		Targets: rule.TargetMailboxes,
+	}
+}

internal/transfer/provider_imap_utils.go

@@ -0,0 +1,236 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"net"
+	"time"
+
+	imapID "github.com/ProtonMail/go-imap-id"
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/emersion/go-imap"
+	imapClient "github.com/emersion/go-imap/client"
+	sasl "github.com/emersion/go-sasl"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	imapDialTimeout      = 5 * time.Second
+	imapRetries          = 10
+	imapReconnectTimeout = 30 * time.Minute
+	imapReconnectSleep   = time.Minute
+)
+
+type imapErrorLogger struct {
+	log *logrus.Entry
+}
+
+func (l *imapErrorLogger) Printf(f string, v ...interface{}) {
+	l.log.Errorf(f, v...)
+}
+
+func (l *imapErrorLogger) Println(v ...interface{}) {
+	l.log.Errorln(v...)
+}
+
+type imapDebugLogger struct { //nolint[unused]
+	log *logrus.Entry
+}
+
+func (l *imapDebugLogger) Write(data []byte) (int, error) {
+	l.log.Trace(string(data))
+	return len(data), nil
+}
+
+func (p *IMAPProvider) ensureConnection(callback func() error) error {
+	var callErr error
+	for i := 1; i <= imapRetries; i++ {
+		callErr = callback()
+		if callErr == nil {
+			return nil
+		}
+
+		log.WithField("attempt", i).WithError(callErr).Warning("Call failed, trying reconnect")
+		err := p.tryReconnect()
+		if err != nil {
+			return err
+		}
+	}
+	return errors.Wrap(callErr, "too many retries")
+}
+
+func (p *IMAPProvider) tryReconnect() error {
+	start := time.Now()
+	var previousErr error
+	for {
+		if time.Since(start) > imapReconnectTimeout {
+			return previousErr
+		}
+
+		err := pmapi.CheckConnection()
+		if err != nil {
+			time.Sleep(imapReconnectSleep)
+			previousErr = err
+			continue
+		}
+
+		err = p.reauth()
+		if err != nil {
+			time.Sleep(imapReconnectSleep)
+			previousErr = err
+			continue
+		}
+
+		break
+	}
+	return nil
+}
+
+func (p *IMAPProvider) reauth() error {
+	if _, err := p.client.Capability(); err != nil {
+		state := p.client.State()
+		log.WithField("addr", p.addr).WithField("state", state).WithError(err).Debug("Reconnecting")
+		p.client = nil
+	}
+
+	return p.auth()
+}
+
+func (p *IMAPProvider) auth() error { //nolint[funlen]
+	log := log.WithField("addr", p.addr)
+
+	log.Info("Connecting to server")
+
+	if _, err := net.DialTimeout("tcp", p.addr, imapDialTimeout); err != nil {
+		return errors.Wrap(err, "failed to dial server")
+	}
+
+	client, err := imapClient.DialTLS(p.addr, nil)
+	if err != nil {
+		return errors.Wrap(err, "failed to connect to server")
+	}
+	client.ErrorLog = &imapErrorLogger{logrus.WithField("pkg", "imap-client")}
+	// Logrus have Writer helper but it fails for big messages because of
+	// bufio.MaxScanTokenSize limit.
+	// This spams a lot, uncomment once needed during development.
+	//client.SetDebug(&imapDebugLogger{logrus.WithField("pkg", "imap-client")})
+	p.client = client
+
+	log.Info("Connected")
+
+	if (p.client.State() & imap.AuthenticatedState) == imap.AuthenticatedState {
+		return nil
+	}
+
+	capability, err := p.client.Capability()
+	log.WithField("capability", capability).WithError(err).Debug("Server capability")
+	if err != nil {
+		return errors.Wrap(err, "failed to get capabilities")
+	}
+
+	// SASL AUTH PLAIN
+	if ok, _ := p.client.SupportAuth("PLAIN"); p.client.State() == imap.NotAuthenticatedState && ok {
+		log.Debug("Trying plain auth")
+		authPlain := sasl.NewPlainClient("", p.username, p.password)
+		if err = p.client.Authenticate(authPlain); err != nil {
+			return errors.Wrap(err, "plain auth failed")
+		}
+	}
+
+	// LOGIN: if the server reports the IMAP4rev1 capability then it is standards conformant and must support login.
+	if ok, _ := p.client.Support("IMAP4rev1"); p.client.State() == imap.NotAuthenticatedState && ok {
+		log.Debug("Trying login")
+		if err = p.client.Login(p.username, p.password); err != nil {
+			return errors.Wrap(err, "login failed")
+		}
+	}
+
+	if p.client.State() == imap.NotAuthenticatedState {
+		return errors.New("unknown auth method")
+	}
+
+	log.Info("Logged in")
+
+	idClient := imapID.NewClient(p.client)
+	if ok, err := idClient.SupportID(); err == nil && ok {
+		serverID, err := idClient.ID(imapID.ID{
+			imapID.FieldName:    "ImportExport",
+			imapID.FieldVersion: "beta",
+		})
+		log.WithField("ID", serverID).WithError(err).Debug("Server info")
+	}
+
+	return err
+}
+
+func (p *IMAPProvider) list() (mailboxes []*imap.MailboxInfo, err error) {
+	err = p.ensureConnection(func() error {
+		mailboxesCh := make(chan *imap.MailboxInfo)
+		doneCh := make(chan error)
+
+		go func() {
+			doneCh <- p.client.List("", "*", mailboxesCh)
+		}()
+
+		for mailbox := range mailboxesCh {
+			mailboxes = append(mailboxes, mailbox)
+		}
+
+		return <-doneCh
+	})
+	return
+}
+
+func (p *IMAPProvider) selectIn(mailboxName string) (mailbox *imap.MailboxStatus, err error) {
+	err = p.ensureConnection(func() error {
+		mailbox, err = p.client.Select(mailboxName, true)
+		return err
+	})
+	return
+}
+
+func (p *IMAPProvider) fetch(seqSet *imap.SeqSet, items []imap.FetchItem, processMessageCallback func(m *imap.Message)) error {
+	return p.fetchHelper(false, seqSet, items, processMessageCallback)
+}
+
+func (p *IMAPProvider) uidFetch(seqSet *imap.SeqSet, items []imap.FetchItem, processMessageCallback func(m *imap.Message)) error {
+	return p.fetchHelper(true, seqSet, items, processMessageCallback)
+}
+
+func (p *IMAPProvider) fetchHelper(uid bool, seqSet *imap.SeqSet, items []imap.FetchItem, processMessageCallback func(m *imap.Message)) error {
+	return p.ensureConnection(func() error {
+		messagesCh := make(chan *imap.Message)
+		doneCh := make(chan error)
+
+		go func() {
+			if uid {
+				doneCh <- p.client.UidFetch(seqSet, items, messagesCh)
+			} else {
+				doneCh <- p.client.Fetch(seqSet, items, messagesCh)
+			}
+		}()
+
+		for message := range messagesCh {
+			processMessageCallback(message)
+		}
+
+		err := <-doneCh
+		return err
+	})
+}

internal/transfer/provider_local.go

@@ -0,0 +1,68 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+// LocalProvider implements import from local EML and MBOX file structure.
+type LocalProvider struct {
+	root         string
+	emlProvider  *EMLProvider
+	mboxProvider *MBOXProvider
+}
+
+func NewLocalProvider(root string) *LocalProvider {
+	return &LocalProvider{
+		root:         root,
+		emlProvider:  NewEMLProvider(root),
+		mboxProvider: NewMBOXProvider(root),
+	}
+}
+
+// ID is used for generating transfer ID by combining source and target ID.
+// We want to keep the same rules for import from or export to local files
+// no matter exact path, therefore it returns constant.
+// The same as EML and MBOX.
+func (p *LocalProvider) ID() string {
+	return "local" //nolint[goconst]
+}
+
+// Mailboxes returns all available folder names from root of EML and MBOX files.
+func (p *LocalProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) {
+	mailboxes, err := p.emlProvider.Mailboxes(includeEmpty, includeAllMail)
+	if err != nil {
+		return nil, err
+	}
+
+	mboxMailboxes, err := p.mboxProvider.Mailboxes(includeEmpty, includeAllMail)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, mboxMailbox := range mboxMailboxes {
+		found := false
+		for _, mailboxes := range mailboxes {
+			if mboxMailbox.Name == mailboxes.Name {
+				found = true
+				break
+			}
+		}
+		if !found {
+			mailboxes = append(mailboxes, mboxMailbox)
+		}
+	}
+	return mailboxes, nil
+}

internal/transfer/provider_local_source.go

@@ -0,0 +1,42 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"sync"
+)
+
+// TransferTo exports messages based on rules to channel.
+func (p *LocalProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) {
+	log.Info("Started transfer from EML and MBOX to channel")
+	defer log.Info("Finished transfer from EML and MBOX to channel")
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+
+	go func() {
+		defer wg.Done()
+		p.emlProvider.TransferTo(rules, progress, ch)
+	}()
+	go func() {
+		defer wg.Done()
+		p.mboxProvider.TransferTo(rules, progress, ch)
+	}()
+
+	wg.Wait()
+}

internal/transfer/provider_local_test.go

@@ -0,0 +1,77 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"fmt"
+	"testing"
+
+	r "github.com/stretchr/testify/require"
+)
+
+func newTestLocalProvider(path string) *LocalProvider {
+	if path == "" {
+		path = "testdata/emlmbox"
+	}
+	return NewLocalProvider(path)
+}
+
+func TestLocalProviderMailboxes(t *testing.T) {
+	provider := newTestLocalProvider("")
+
+	tests := []struct {
+		includeEmpty  bool
+		wantMailboxes []Mailbox
+	}{
+		{true, []Mailbox{
+			{Name: "Foo"},
+			{Name: "emlmbox"},
+			{Name: "Inbox"},
+		}},
+		{false, []Mailbox{
+			{Name: "Foo"},
+			{Name: "Inbox"},
+		}},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(fmt.Sprintf("%v", tc.includeEmpty), func(t *testing.T) {
+			mailboxes, err := provider.Mailboxes(tc.includeEmpty, false)
+			r.NoError(t, err)
+			r.Equal(t, tc.wantMailboxes, mailboxes)
+		})
+	}
+}
+
+func TestLocalProviderTransferTo(t *testing.T) {
+	provider := newTestLocalProvider("")
+
+	rules, rulesClose := newTestRules(t)
+	defer rulesClose()
+	setupEMLMBOXRules(rules)
+
+	testTransferTo(t, rules, provider, []string{
+		"Foo/msg.eml",
+		"Inbox.mbox:1",
+	})
+}
+
+func setupEMLMBOXRules(rules transferRules) {
+	_ = rules.setRule(Mailbox{Name: "Inbox"}, []Mailbox{{Name: "Inbox"}}, 0, 0)
+	_ = rules.setRule(Mailbox{Name: "Foo"}, []Mailbox{{Name: "Foo"}}, 0, 0)
+}

internal/transfer/provider_mbox.go

@@ -0,0 +1,66 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"path/filepath"
+	"strings"
+)
+
+// MBOXProvider implements import and export to/from MBOX structure.
+type MBOXProvider struct {
+	root string
+}
+
+func NewMBOXProvider(root string) *MBOXProvider {
+	return &MBOXProvider{
+		root: root,
+	}
+}
+
+// ID is used for generating transfer ID by combining source and target ID.
+// We want to keep the same rules for import from or export to local files
+// no matter exact path, therefore it returns constant. The same as EML.
+func (p *MBOXProvider) ID() string {
+	return "local" //nolint[goconst]
+}
+
+// Mailboxes returns all available folder names from root of EML files.
+// In case the same folder name is used more than once (for example root/a/foo
+// and root/b/foo), it's treated as the same folder.
+func (p *MBOXProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) {
+	filePaths, err := getFilePathsWithSuffix(p.root, "mbox")
+	if err != nil {
+		return nil, err
+	}
+
+	mailboxes := []Mailbox{}
+	for _, filePath := range filePaths {
+		fileName := filepath.Base(filePath)
+		mailboxName := strings.TrimSuffix(fileName, ".mbox")
+
+		mailboxes = append(mailboxes, Mailbox{
+			ID:          "",
+			Name:        mailboxName,
+			Color:       "",
+			IsExclusive: false,
+		})
+	}
+
+	return mailboxes, nil
+}

internal/transfer/provider_mbox_source.go

@@ -0,0 +1,183 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/emersion/go-mbox"
+	"github.com/pkg/errors"
+)
+
+// TransferTo exports messages based on rules to channel.
+func (p *MBOXProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) {
+	log.Info("Started transfer from MBOX to channel")
+	defer log.Info("Finished transfer from MBOX to channel")
+
+	filePathsPerFolder, err := p.getFilePathsPerFolder(rules)
+	if err != nil {
+		progress.fatal(err)
+		return
+	}
+
+	for folderName, filePaths := range filePathsPerFolder {
+		// No error guaranteed by getFilePathsPerFolder.
+		rule, _ := rules.getRuleBySourceMailboxName(folderName)
+		for _, filePath := range filePaths {
+			if progress.shouldStop() {
+				break
+			}
+			p.updateCount(rule, progress, filePath)
+		}
+	}
+
+	for folderName, filePaths := range filePathsPerFolder {
+		// No error guaranteed by getFilePathsPerFolder.
+		rule, _ := rules.getRuleBySourceMailboxName(folderName)
+		log.WithField("rule", rule).Debug("Processing rule")
+		for _, filePath := range filePaths {
+			if progress.shouldStop() {
+				break
+			}
+			p.transferTo(rule, progress, ch, filePath)
+		}
+	}
+}
+
+func (p *MBOXProvider) getFilePathsPerFolder(rules transferRules) (map[string][]string, error) {
+	filePaths, err := getFilePathsWithSuffix(p.root, ".mbox")
+	if err != nil {
+		return nil, err
+	}
+
+	filePathsMap := map[string][]string{}
+	for _, filePath := range filePaths {
+		fileName := filepath.Base(filePath)
+		folder := strings.TrimSuffix(fileName, ".mbox")
+		_, err := rules.getRuleBySourceMailboxName(folder)
+		if err != nil {
+			log.WithField("msg", filePath).Trace("Mailbox skipped due to folder name")
+			continue
+		}
+
+		filePathsMap[folder] = append(filePathsMap[folder], filePath)
+	}
+	return filePathsMap, nil
+}
+
+func (p *MBOXProvider) updateCount(rule *Rule, progress *Progress, filePath string) {
+	mboxReader := p.openMbox(progress, filePath)
+	if mboxReader == nil {
+		return
+	}
+
+	count := 0
+	for {
+		_, err := mboxReader.NextMessage()
+		if err != nil {
+			break
+		}
+		count++
+	}
+	progress.updateCount(rule.SourceMailbox.Name, uint(count))
+}
+
+func (p *MBOXProvider) transferTo(rule *Rule, progress *Progress, ch chan<- Message, filePath string) {
+	mboxReader := p.openMbox(progress, filePath)
+	if mboxReader == nil {
+		return
+	}
+
+	index := 0
+	count := 0
+	for {
+		if progress.shouldStop() {
+			break
+		}
+
+		index++
+		id := fmt.Sprintf("%s:%d", filePath, index)
+
+		msgReader, err := mboxReader.NextMessage()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			progress.fatal(err)
+			break
+		}
+
+		msg, err := p.exportMessage(rule, id, msgReader)
+
+		// Read and check time in body only if the rule specifies it
+		// to not waste energy.
+		if err == nil && rule.HasTimeLimit() {
+			msgTime, msgTimeErr := getMessageTime(msg.Body)
+			if msgTimeErr != nil {
+				err = msgTimeErr
+			} else if !rule.isTimeInRange(msgTime) {
+				log.WithField("msg", id).Debug("Message skipped due to time")
+				continue
+			}
+		}
+
+		// Counting only messages filtered by time to update count to correct total.
+		count++
+
+		// addMessage is called after time check to not report message
+		// which should not be exported but any error from reading body
+		// or parsing time is reported as an error.
+		progress.addMessage(id, rule)
+		progress.messageExported(id, msg.Body, err)
+		if err == nil {
+			ch <- msg
+		}
+	}
+	progress.updateCount(rule.SourceMailbox.Name, uint(count))
+}
+
+func (p *MBOXProvider) exportMessage(rule *Rule, id string, msgReader io.Reader) (Message, error) {
+	body, err := ioutil.ReadAll(msgReader)
+	if err != nil {
+		return Message{}, errors.Wrap(err, "failed to read message")
+	}
+
+	return Message{
+		ID:      id,
+		Unread:  false,
+		Body:    body,
+		Source:  rule.SourceMailbox,
+		Targets: rule.TargetMailboxes,
+	}, nil
+}
+
+func (p *MBOXProvider) openMbox(progress *Progress, mboxPath string) *mbox.Reader {
+	mboxPath = filepath.Join(p.root, mboxPath)
+	mboxFile, err := os.Open(mboxPath) //nolint[gosec]
+	if os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		progress.fatal(err)
+		return nil
+	}
+	return mbox.NewReader(mboxFile)
+}

internal/transfer/provider_mbox_target.go

@@ -0,0 +1,97 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/emersion/go-mbox"
+	"github.com/hashicorp/go-multierror"
+)
+
+// DefaultMailboxes returns the default mailboxes for default rules if no other is found.
+func (p *MBOXProvider) DefaultMailboxes(sourceMailbox Mailbox) []Mailbox {
+	return []Mailbox{{
+		Name: sourceMailbox.Name,
+	}}
+}
+
+// CreateMailbox does nothing. Files are created dynamically during the import.
+func (p *MBOXProvider) CreateMailbox(mailbox Mailbox) (Mailbox, error) {
+	return mailbox, nil
+}
+
+// TransferFrom imports messages from channel.
+func (p *MBOXProvider) TransferFrom(rules transferRules, progress *Progress, ch <-chan Message) {
+	log.Info("Started transfer from channel to MBOX")
+	defer log.Info("Finished transfer from channel to MBOX")
+
+	for msg := range ch {
+		for progress.shouldStop() {
+			break
+		}
+
+		err := p.writeMessage(msg)
+		progress.messageImported(msg.ID, "", err)
+	}
+}
+
+func (p *MBOXProvider) writeMessage(msg Message) error {
+	var multiErr error
+	for _, mailbox := range msg.Targets {
+		mboxName := filepath.Base(mailbox.Name)
+		if !strings.HasSuffix(mboxName, ".mbox") {
+			mboxName += ".mbox"
+		}
+
+		mboxPath := filepath.Join(p.root, mboxName)
+		mboxFile, err := os.OpenFile(mboxPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
+		if err != nil {
+			multiErr = multierror.Append(multiErr, err)
+			continue
+		}
+
+		msgFrom := ""
+		msgTime := time.Now()
+		if header, err := getMessageHeader(msg.Body); err == nil {
+			if date, err := header.Date(); err == nil {
+				msgTime = date
+			}
+			if addresses, err := header.AddressList("from"); err == nil && len(addresses) > 0 {
+				msgFrom = addresses[0].Address
+			}
+		}
+
+		mboxWriter := mbox.NewWriter(mboxFile)
+		messageWriter, err := mboxWriter.CreateMessage(msgFrom, msgTime)
+		if err != nil {
+			multiErr = multierror.Append(multiErr, err)
+			continue
+		}
+
+		_, err = messageWriter.Write(msg.Body)
+		if err != nil {
+			multiErr = multierror.Append(multiErr, err)
+			continue
+		}
+	}
+	return multiErr
+}

internal/transfer/provider_mbox_test.go

@@ -0,0 +1,125 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+	"time"
+
+	r "github.com/stretchr/testify/require"
+)
+
+func newTestMBOXProvider(path string) *MBOXProvider {
+	if path == "" {
+		path = "testdata/mbox"
+	}
+	return NewMBOXProvider(path)
+}
+
+func TestMBOXProviderMailboxes(t *testing.T) {
+	provider := newTestMBOXProvider("")
+
+	tests := []struct {
+		includeEmpty  bool
+		wantMailboxes []Mailbox
+	}{
+		{true, []Mailbox{
+			{Name: "Foo"},
+			{Name: "Inbox"},
+		}},
+		{false, []Mailbox{
+			{Name: "Foo"},
+			{Name: "Inbox"},
+		}},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(fmt.Sprintf("%v", tc.includeEmpty), func(t *testing.T) {
+			mailboxes, err := provider.Mailboxes(tc.includeEmpty, false)
+			r.NoError(t, err)
+			r.Equal(t, tc.wantMailboxes, mailboxes)
+		})
+	}
+}
+
+func TestMBOXProviderTransferTo(t *testing.T) {
+	provider := newTestMBOXProvider("")
+
+	rules, rulesClose := newTestRules(t)
+	defer rulesClose()
+	setupMBOXRules(rules)
+
+	testTransferTo(t, rules, provider, []string{
+		"Foo.mbox:1",
+		"Inbox.mbox:1",
+	})
+}
+
+func TestMBOXProviderTransferFrom(t *testing.T) {
+	dir, err := ioutil.TempDir("", "eml")
+	r.NoError(t, err)
+	defer os.RemoveAll(dir) //nolint[errcheck]
+
+	provider := newTestMBOXProvider(dir)
+
+	rules, rulesClose := newTestRules(t)
+	defer rulesClose()
+	setupMBOXRules(rules)
+
+	testTransferFrom(t, rules, provider, []Message{
+		{ID: "Foo.mbox:1", Body: getTestMsgBody("msg"), Targets: []Mailbox{{Name: "Foo"}}},
+	})
+
+	checkMBOXFileStructure(t, dir, []string{
+		"Foo.mbox",
+	})
+}
+
+func TestMBOXProviderTransferFromTo(t *testing.T) {
+	dir, err := ioutil.TempDir("", "eml")
+	r.NoError(t, err)
+	defer os.RemoveAll(dir) //nolint[errcheck]
+
+	source := newTestMBOXProvider("")
+	target := newTestMBOXProvider(dir)
+
+	rules, rulesClose := newTestRules(t)
+	defer rulesClose()
+	setupEMLRules(rules)
+
+	testTransferFromTo(t, rules, source, target, 5*time.Second)
+
+	checkMBOXFileStructure(t, dir, []string{
+		"Foo.mbox",
+		"Inbox.mbox",
+	})
+}
+
+func setupMBOXRules(rules transferRules) {
+	_ = rules.setRule(Mailbox{Name: "Inbox"}, []Mailbox{{Name: "Inbox"}}, 0, 0)
+	_ = rules.setRule(Mailbox{Name: "Foo"}, []Mailbox{{Name: "Foo"}}, 0, 0)
+}
+
+func checkMBOXFileStructure(t *testing.T, root string, expectedFiles []string) {
+	files, err := getFilePathsWithSuffix(root, ".mbox")
+	r.NoError(t, err)
+	r.Equal(t, expectedFiles, files)
+}

internal/transfer/provider_pmapi.go

@@ -0,0 +1,140 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"sort"
+
+	"github.com/ProtonMail/gopenpgp/crypto"
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/pkg/errors"
+)
+
+// PMAPIProvider implements import and export to/from ProtonMail server.
+type PMAPIProvider struct {
+	clientManager ClientManager
+	userID        string
+	addressID     string
+	keyRing       *crypto.KeyRing
+}
+
+// NewPMAPIProvider returns new PMAPIProvider.
+func NewPMAPIProvider(clientManager ClientManager, userID, addressID string) (*PMAPIProvider, error) {
+	keyRing, err := clientManager.GetClient(userID).KeyRingForAddressID(addressID)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get key ring")
+	}
+
+	return &PMAPIProvider{
+		clientManager: clientManager,
+		userID:        userID,
+		addressID:     addressID,
+		keyRing:       keyRing,
+	}, nil
+}
+
+func (p *PMAPIProvider) client() pmapi.Client {
+	return p.clientManager.GetClient(p.userID)
+}
+
+// ID returns identifier of current setup of PMAPI provider.
+// Identification is unique per user.
+func (p *PMAPIProvider) ID() string {
+	return p.userID
+}
+
+// Mailboxes returns all available labels in ProtonMail account.
+func (p *PMAPIProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) {
+	labels, err := p.client().ListLabels()
+	if err != nil {
+		return nil, err
+	}
+	sortedLabels := byFoldersLabels(labels)
+	sort.Sort(sortedLabels)
+
+	emptyLabelsMap := map[string]bool{}
+	if !includeEmpty {
+		messagesCounts, err := p.client().CountMessages(p.addressID)
+		if err != nil {
+			return nil, err
+		}
+		for _, messagesCount := range messagesCounts {
+			if messagesCount.Total == 0 {
+				emptyLabelsMap[messagesCount.LabelID] = true
+			}
+		}
+	}
+
+	mailboxes := getSystemMailboxes(includeAllMail)
+	for _, label := range sortedLabels {
+		if !includeEmpty && emptyLabelsMap[label.ID] {
+			continue
+		}
+
+		mailboxes = append(mailboxes, Mailbox{
+			ID:          label.ID,
+			Name:        label.Name,
+			Color:       label.Color,
+			IsExclusive: label.Exclusive == 1,
+		})
+	}
+	return mailboxes, nil
+}
+
+func getSystemMailboxes(includeAllMail bool) []Mailbox {
+	mailboxes := []Mailbox{
+		{ID: pmapi.InboxLabel, Name: "Inbox", IsExclusive: true},
+		{ID: pmapi.DraftLabel, Name: "Drafts", IsExclusive: true},
+		{ID: pmapi.SentLabel, Name: "Sent", IsExclusive: true},
+		{ID: pmapi.StarredLabel, Name: "Starred", IsExclusive: true},
+		{ID: pmapi.ArchiveLabel, Name: "Archive", IsExclusive: true},
+		{ID: pmapi.SpamLabel, Name: "Spam", IsExclusive: true},
+		{ID: pmapi.TrashLabel, Name: "Trash", IsExclusive: true},
+	}
+
+	if includeAllMail {
+		mailboxes = append(mailboxes, Mailbox{
+			ID:          pmapi.AllMailLabel,
+			Name:        "All Mail",
+			IsExclusive: true,
+		})
+	}
+
+	return mailboxes
+}
+
+type byFoldersLabels []*pmapi.Label
+
+func (l byFoldersLabels) Len() int {
+	return len(l)
+}
+
+func (l byFoldersLabels) Swap(i, j int) {
+	l[i], l[j] = l[j], l[i]
+}
+
+// Less sorts first folders, then labels, by user order.
+func (l byFoldersLabels) Less(i, j int) bool {
+	if l[i].Exclusive == 1 && l[j].Exclusive == 0 {
+		return true
+	}
+	if l[i].Exclusive == 0 && l[j].Exclusive == 1 {
+		return false
+	}
+	return l[i].Order < l[j].Order
+}

internal/transfer/provider_pmapi_source.go

@@ -0,0 +1,161 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"fmt"
+
+	pkgMessage "github.com/ProtonMail/proton-bridge/pkg/message"
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/pkg/errors"
+)
+
+const pmapiListPageSize = 150
+
+// TransferTo exports messages based on rules to channel.
+func (p *PMAPIProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) {
+	log.Info("Started transfer from PMAPI to channel")
+	defer log.Info("Finished transfer from PMAPI to channel")
+
+	go p.loadCounts(rules, progress)
+
+	for rule := range rules.iterateActiveRules() {
+		p.transferTo(rule, progress, ch, rules.skipEncryptedMessages)
+	}
+}
+
+func (p *PMAPIProvider) loadCounts(rules transferRules, progress *Progress) {
+	for rule := range rules.iterateActiveRules() {
+		if progress.shouldStop() {
+			break
+		}
+
+		rule := rule
+		progress.callWrap(func() error {
+			_, total, err := p.listMessages(&pmapi.MessagesFilter{
+				LabelID: rule.SourceMailbox.ID,
+				Begin:   rule.FromTime,
+				End:     rule.ToTime,
+				Limit:   0,
+			})
+			if err != nil {
+				log.WithError(err).Warning("Problem to load counts")
+				return err
+			}
+			progress.updateCount(rule.SourceMailbox.Name, uint(total))
+			return nil
+		})
+	}
+}
+
+func (p *PMAPIProvider) transferTo(rule *Rule, progress *Progress, ch chan<- Message, skipEncryptedMessages bool) {
+	nextID := ""
+	for {
+		if progress.shouldStop() {
+			break
+		}
+
+		isLastPage := true
+
+		progress.callWrap(func() error {
+			desc := false
+			pmapiMessages, count, err := p.listMessages(&pmapi.MessagesFilter{
+				LabelID:  rule.SourceMailbox.ID,
+				Begin:    rule.FromTime,
+				End:      rule.ToTime,
+				BeginID:  nextID,
+				PageSize: pmapiListPageSize,
+				Page:     0,
+				Sort:     "ID",
+				Desc:     &desc,
+			})
+			if err != nil {
+				return err
+			}
+			log.WithField("label", rule.SourceMailbox.ID).WithField("next", nextID).WithField("count", count).Debug("Listing messages")
+
+			isLastPage = len(pmapiMessages) < pmapiListPageSize
+
+			// The first ID is the last one from the last page (= do not export twice the same one).
+			if nextID != "" {
+				pmapiMessages = pmapiMessages[1:]
+			}
+
+			for _, pmapiMessage := range pmapiMessages {
+				if progress.shouldStop() {
+					break
+				}
+
+				msgID := fmt.Sprintf("%s_%s", rule.SourceMailbox.ID, pmapiMessage.ID)
+				progress.addMessage(msgID, rule)
+				msg, err := p.exportMessage(rule, progress, pmapiMessage.ID, msgID, skipEncryptedMessages)
+				progress.messageExported(msgID, msg.Body, err)
+				if err == nil {
+					ch <- msg
+				}
+			}
+
+			if !isLastPage {
+				nextID = pmapiMessages[len(pmapiMessages)-1].ID
+			}
+
+			return nil
+		})
+
+		if isLastPage {
+			break
+		}
+	}
+}
+
+func (p *PMAPIProvider) exportMessage(rule *Rule, progress *Progress, pmapiMsgID, msgID string, skipEncryptedMessages bool) (Message, error) {
+	var msg *pmapi.Message
+	progress.callWrap(func() error {
+		var err error
+		msg, err = p.getMessage(pmapiMsgID)
+		return err
+	})
+
+	msgBuilder := pkgMessage.NewBuilder(p.client(), msg)
+	msgBuilder.EncryptedToHTML = false
+	_, body, err := msgBuilder.BuildMessage()
+	if err != nil {
+		return Message{
+			Body: body, // Keep body to show details about the message to user.
+		}, errors.Wrap(err, "failed to build message")
+	}
+
+	if !msgBuilder.SuccessfullyDecrypted() && skipEncryptedMessages {
+		return Message{
+			Body: body, // Keep body to show details about the message to user.
+		}, errors.New("skipping encrypted message")
+	}
+
+	unread := false
+	if msg.Unread == 1 {
+		unread = true
+	}
+
+	return Message{
+		ID:      msgID,
+		Unread:  unread,
+		Body:    body,
+		Source:  rule.SourceMailbox,
+		Targets: rule.TargetMailboxes,
+	}, nil
+}

internal/transfer/provider_pmapi_target.go

@@ -0,0 +1,220 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+
+	pkgMessage "github.com/ProtonMail/proton-bridge/pkg/message"
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/pkg/errors"
+)
+
+// DefaultMailboxes returns the default mailboxes for default rules if no other is found.
+func (p *PMAPIProvider) DefaultMailboxes(_ Mailbox) []Mailbox {
+	return []Mailbox{{
+		ID:          pmapi.ArchiveLabel,
+		Name:        "Archive",
+		IsExclusive: true,
+	}}
+}
+
+// CreateMailbox creates label in ProtonMail account.
+func (p *PMAPIProvider) CreateMailbox(mailbox Mailbox) (Mailbox, error) {
+	if mailbox.ID != "" {
+		return Mailbox{}, errors.New("mailbox is already created")
+	}
+
+	exclusive := 0
+	if mailbox.IsExclusive {
+		exclusive = 1
+	}
+
+	label, err := p.client().CreateLabel(&pmapi.Label{
+		Name:      mailbox.Name,
+		Color:     mailbox.Color,
+		Exclusive: exclusive,
+		Type:      pmapi.LabelTypeMailbox,
+	})
+	if err != nil {
+		return Mailbox{}, errors.Wrap(err, fmt.Sprintf("failed to create mailbox %s", mailbox.Name))
+	}
+	mailbox.ID = label.ID
+	return mailbox, nil
+}
+
+// TransferFrom imports messages from channel.
+func (p *PMAPIProvider) TransferFrom(rules transferRules, progress *Progress, ch <-chan Message) {
+	log.Info("Started transfer from channel to PMAPI")
+	defer log.Info("Finished transfer from channel to PMAPI")
+
+	for msg := range ch {
+		for progress.shouldStop() {
+			break
+		}
+
+		var importedID string
+		var err error
+		if p.isMessageDraft(msg) {
+			importedID, err = p.importDraft(msg, rules.globalMailbox)
+		} else {
+			importedID, err = p.importMessage(msg, rules.globalMailbox)
+		}
+		progress.messageImported(msg.ID, importedID, err)
+	}
+}
+
+func (p *PMAPIProvider) isMessageDraft(msg Message) bool {
+	for _, target := range msg.Targets {
+		if target.ID == pmapi.DraftLabel {
+			return true
+		}
+	}
+	return false
+}
+
+func (p *PMAPIProvider) importDraft(msg Message, globalMailbox *Mailbox) (string, error) {
+	message, attachmentReaders, err := p.parseMessage(msg)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to parse message")
+	}
+
+	if err := message.Encrypt(p.keyRing, nil); err != nil {
+		return "", errors.Wrap(err, "failed to encrypt draft")
+	}
+
+	if globalMailbox != nil {
+		message.LabelIDs = append(message.LabelIDs, globalMailbox.ID)
+	}
+
+	attachments := message.Attachments
+	message.Attachments = nil
+
+	draft, err := p.createDraft(message, "", pmapi.DraftActionReply)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to create draft")
+	}
+
+	for idx, attachment := range attachments {
+		attachment.MessageID = draft.ID
+		attachmentBody, _ := ioutil.ReadAll(attachmentReaders[idx])
+
+		r := bytes.NewReader(attachmentBody)
+		sigReader, err := attachment.DetachedSign(p.keyRing, r)
+		if err != nil {
+			return "", errors.Wrap(err, "failed to sign attachment")
+		}
+
+		r = bytes.NewReader(attachmentBody)
+		encReader, err := attachment.Encrypt(p.keyRing, r)
+		if err != nil {
+			return "", errors.Wrap(err, "failed to encrypt attachment")
+		}
+
+		_, err = p.createAttachment(attachment, encReader, sigReader)
+		if err != nil {
+			return "", errors.Wrap(err, "failed to create attachment")
+		}
+	}
+
+	return draft.ID, nil
+}
+
+func (p *PMAPIProvider) importMessage(msg Message, globalMailbox *Mailbox) (string, error) {
+	message, attachmentReaders, err := p.parseMessage(msg)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to parse message")
+	}
+
+	body, err := p.encryptMessage(message, attachmentReaders)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to encrypt message")
+	}
+
+	unread := 0
+	if msg.Unread {
+		unread = 1
+	}
+
+	labelIDs := []string{}
+	for _, target := range msg.Targets {
+		// Frontend should not set All Mail to Rules, but to be sure...
+		if target.ID != pmapi.AllMailLabel {
+			labelIDs = append(labelIDs, target.ID)
+		}
+	}
+	if globalMailbox != nil {
+		labelIDs = append(labelIDs, globalMailbox.ID)
+	}
+
+	importMsgReq := &pmapi.ImportMsgReq{
+		AddressID: p.addressID,
+		Body:      body,
+		Unread:    unread,
+		Time:      message.Time,
+		Flags:     computeMessageFlags(labelIDs),
+		LabelIDs:  labelIDs,
+	}
+
+	results, err := p.importRequest([]*pmapi.ImportMsgReq{importMsgReq})
+	if err != nil {
+		return "", errors.Wrap(err, "failed to import messages")
+	}
+	if len(results) == 0 {
+		return "", errors.New("import ended with no result")
+	}
+	if results[0].Error != nil {
+		return "", errors.Wrap(results[0].Error, "failed to import message")
+	}
+	return results[0].MessageID, nil
+}
+
+func (p *PMAPIProvider) parseMessage(msg Message) (*pmapi.Message, []io.Reader, error) {
+	message, _, _, attachmentReaders, err := pkgMessage.Parse(bytes.NewBuffer(msg.Body), "", "")
+	return message, attachmentReaders, err
+}
+
+func (p *PMAPIProvider) encryptMessage(msg *pmapi.Message, attachmentReaders []io.Reader) ([]byte, error) {
+	if msg.MIMEType == pmapi.ContentTypeMultipartEncrypted {
+		return []byte(msg.Body), nil
+	}
+	return pkgMessage.BuildEncrypted(msg, attachmentReaders, p.keyRing)
+}
+
+func computeMessageFlags(labels []string) (flag int64) {
+	for _, labelID := range labels {
+		switch labelID {
+		case pmapi.SentLabel:
+			flag = (flag | pmapi.FlagSent)
+		case pmapi.ArchiveLabel, pmapi.InboxLabel:
+			flag = (flag | pmapi.FlagReceived)
+		case pmapi.DraftLabel:
+			log.Error("Found draft target in non-draft import")
+		}
+	}
+
+	// NOTE: if the labels are custom only
+	if flag == 0 {
+		flag = pmapi.FlagReceived
+	}
+
+	return flag
+}

internal/transfer/provider_pmapi_test.go

@@ -0,0 +1,201 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	gomock "github.com/golang/mock/gomock"
+	r "github.com/stretchr/testify/require"
+)
+
+func TestPMAPIProviderMailboxes(t *testing.T) {
+	m := initMocks(t)
+	defer m.ctrl.Finish()
+
+	setupPMAPIClientExpectationForExport(&m)
+	provider, err := NewPMAPIProvider(m.clientManager, "user", "addressID")
+	r.NoError(t, err)
+
+	tests := []struct {
+		includeEmpty   bool
+		includeAllMail bool
+		wantMailboxes  []Mailbox
+	}{
+		{true, false, []Mailbox{
+			{ID: "folder1", Name: "One", Color: "red", IsExclusive: true},
+			{ID: "folder2", Name: "Two", Color: "orange", IsExclusive: true},
+			{ID: "label2", Name: "Bar", Color: "green", IsExclusive: false},
+			{ID: "label1", Name: "Foo", Color: "blue", IsExclusive: false},
+		}},
+		{false, true, []Mailbox{
+			{ID: pmapi.AllMailLabel, Name: "All Mail", IsExclusive: true},
+			{ID: "folder1", Name: "One", Color: "red", IsExclusive: true},
+			{ID: "folder2", Name: "Two", Color: "orange", IsExclusive: true},
+			{ID: "label1", Name: "Foo", Color: "blue", IsExclusive: false},
+		}},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(fmt.Sprintf("%v-%v", tc.includeEmpty, tc.includeAllMail), func(t *testing.T) {
+			mailboxes, err := provider.Mailboxes(tc.includeEmpty, tc.includeAllMail)
+			r.NoError(t, err)
+			r.Equal(t, []Mailbox{
+				{ID: pmapi.InboxLabel, Name: "Inbox", IsExclusive: true},
+				{ID: pmapi.DraftLabel, Name: "Drafts", IsExclusive: true},
+				{ID: pmapi.SentLabel, Name: "Sent", IsExclusive: true},
+				{ID: pmapi.StarredLabel, Name: "Starred", IsExclusive: true},
+				{ID: pmapi.ArchiveLabel, Name: "Archive", IsExclusive: true},
+				{ID: pmapi.SpamLabel, Name: "Spam", IsExclusive: true},
+				{ID: pmapi.TrashLabel, Name: "Trash", IsExclusive: true},
+			}, mailboxes[:7])
+			r.Equal(t, tc.wantMailboxes, mailboxes[7:])
+		})
+	}
+}
+
+func TestPMAPIProviderTransferTo(t *testing.T) {
+	m := initMocks(t)
+	defer m.ctrl.Finish()
+
+	setupPMAPIClientExpectationForExport(&m)
+	provider, err := NewPMAPIProvider(m.clientManager, "user", "addressID")
+	r.NoError(t, err)
+
+	rules, rulesClose := newTestRules(t)
+	defer rulesClose()
+	setupPMAPIRules(rules)
+
+	testTransferTo(t, rules, provider, []string{
+		"0_msg1",
+		"0_msg2",
+	})
+}
+
+func TestPMAPIProviderTransferFrom(t *testing.T) {
+	m := initMocks(t)
+	defer m.ctrl.Finish()
+
+	setupPMAPIClientExpectationForImport(&m)
+	provider, err := NewPMAPIProvider(m.clientManager, "user", "addressID")
+	r.NoError(t, err)
+
+	rules, rulesClose := newTestRules(t)
+	defer rulesClose()
+	setupPMAPIRules(rules)
+
+	testTransferFrom(t, rules, provider, []Message{
+		{ID: "msg1", Body: getTestMsgBody("msg1"), Targets: []Mailbox{{ID: pmapi.InboxLabel}}},
+		{ID: "msg2", Body: getTestMsgBody("msg2"), Targets: []Mailbox{{ID: pmapi.InboxLabel}}},
+	})
+}
+
+func TestPMAPIProviderTransferFromDraft(t *testing.T) {
+	m := initMocks(t)
+	defer m.ctrl.Finish()
+
+	setupPMAPIClientExpectationForImportDraft(&m)
+	provider, err := NewPMAPIProvider(m.clientManager, "user", "addressID")
+	r.NoError(t, err)
+
+	rules, rulesClose := newTestRules(t)
+	defer rulesClose()
+	setupPMAPIRules(rules)
+
+	testTransferFrom(t, rules, provider, []Message{
+		{ID: "draft1", Body: getTestMsgBody("draft1"), Targets: []Mailbox{{ID: pmapi.DraftLabel}}},
+	})
+}
+
+func TestPMAPIProviderTransferFromTo(t *testing.T) {
+	m := initMocks(t)
+	defer m.ctrl.Finish()
+
+	setupPMAPIClientExpectationForExport(&m)
+	setupPMAPIClientExpectationForImport(&m)
+
+	source, err := NewPMAPIProvider(m.clientManager, "user", "addressID")
+	r.NoError(t, err)
+	target, err := NewPMAPIProvider(m.clientManager, "user", "addressID")
+	r.NoError(t, err)
+
+	rules, rulesClose := newTestRules(t)
+	defer rulesClose()
+	setupPMAPIRules(rules)
+
+	testTransferFromTo(t, rules, source, target, 5*time.Second)
+}
+
+func setupPMAPIRules(rules transferRules) {
+	_ = rules.setRule(Mailbox{ID: pmapi.InboxLabel}, []Mailbox{{ID: pmapi.InboxLabel}}, 0, 0)
+}
+
+func setupPMAPIClientExpectationForExport(m *mocks) {
+	m.pmapiClient.EXPECT().KeyRingForAddressID(gomock.Any()).Return(m.keyring, nil).AnyTimes()
+	m.pmapiClient.EXPECT().ListLabels().Return([]*pmapi.Label{
+		{ID: "label1", Name: "Foo", Color: "blue", Exclusive: 0, Order: 2},
+		{ID: "label2", Name: "Bar", Color: "green", Exclusive: 0, Order: 1},
+		{ID: "folder1", Name: "One", Color: "red", Exclusive: 1, Order: 1},
+		{ID: "folder2", Name: "Two", Color: "orange", Exclusive: 1, Order: 2},
+	}, nil).AnyTimes()
+	m.pmapiClient.EXPECT().CountMessages(gomock.Any()).Return([]*pmapi.MessagesCount{
+		{LabelID: "label1", Total: 10},
+		{LabelID: "label2", Total: 0},
+		{LabelID: "folder1", Total: 20},
+	}, nil).AnyTimes()
+	m.pmapiClient.EXPECT().ListMessages(gomock.Any()).Return([]*pmapi.Message{
+		{ID: "msg1"},
+		{ID: "msg2"},
+	}, 2, nil).AnyTimes()
+	m.pmapiClient.EXPECT().GetMessage(gomock.Any()).DoAndReturn(func(msgID string) (*pmapi.Message, error) {
+		return &pmapi.Message{
+			ID:       msgID,
+			Body:     string(getTestMsgBody(msgID)),
+			MIMEType: pmapi.ContentTypeMultipartMixed,
+		}, nil
+	}).AnyTimes()
+}
+
+func setupPMAPIClientExpectationForImport(m *mocks) {
+	m.pmapiClient.EXPECT().KeyRingForAddressID(gomock.Any()).Return(m.keyring, nil).AnyTimes()
+	m.pmapiClient.EXPECT().Import(gomock.Any()).DoAndReturn(func(requests []*pmapi.ImportMsgReq) ([]*pmapi.ImportMsgRes, error) {
+		r.Equal(m.t, 1, len(requests))
+
+		request := requests[0]
+		for _, msgID := range []string{"msg1", "msg2"} {
+			if bytes.Contains(request.Body, []byte(msgID)) {
+				return []*pmapi.ImportMsgRes{{MessageID: msgID, Error: nil}}, nil
+			}
+		}
+		r.Fail(m.t, "No message found")
+		return nil, nil
+	}).Times(2)
+}
+
+func setupPMAPIClientExpectationForImportDraft(m *mocks) {
+	m.pmapiClient.EXPECT().KeyRingForAddressID(gomock.Any()).Return(m.keyring, nil).AnyTimes()
+	m.pmapiClient.EXPECT().CreateDraft(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(msg *pmapi.Message, parentID string, action int) (*pmapi.Message, error) {
+		r.Equal(m.t, msg.Subject, "draft1")
+		msg.ID = "draft1"
+		return msg, nil
+	})
+}

internal/transfer/provider_pmapi_utils.go

@@ -0,0 +1,109 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"io"
+	"time"
+
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/pkg/errors"
+)
+
+const (
+	pmapiRetries          = 10
+	pmapiReconnectTimeout = 30 * time.Minute
+	pmapiReconnectSleep   = time.Minute
+)
+
+func (p *PMAPIProvider) ensureConnection(callback func() error) error {
+	var callErr error
+	for i := 1; i <= pmapiRetries; i++ {
+		callErr = callback()
+		if callErr == nil {
+			return nil
+		}
+
+		log.WithField("attempt", i).WithError(callErr).Warning("Call failed, trying reconnect")
+		err := p.tryReconnect()
+		if err != nil {
+			return err
+		}
+	}
+	return errors.Wrap(callErr, "too many retries")
+}
+
+func (p *PMAPIProvider) tryReconnect() error {
+	start := time.Now()
+	var previousErr error
+	for {
+		if time.Since(start) > pmapiReconnectTimeout {
+			return previousErr
+		}
+
+		err := p.clientManager.CheckConnection()
+		if err != nil {
+			time.Sleep(pmapiReconnectSleep)
+			previousErr = err
+			continue
+		}
+
+		break
+	}
+	return nil
+}
+
+func (p *PMAPIProvider) listMessages(filter *pmapi.MessagesFilter) (messages []*pmapi.Message, count int, err error) {
+	err = p.ensureConnection(func() error {
+		messages, count, err = p.client().ListMessages(filter)
+		return err
+	})
+	return
+}
+
+func (p *PMAPIProvider) getMessage(msgID string) (message *pmapi.Message, err error) {
+	err = p.ensureConnection(func() error {
+		message, err = p.client().GetMessage(msgID)
+		return err
+	})
+	return
+}
+
+func (p *PMAPIProvider) importRequest(req []*pmapi.ImportMsgReq) (res []*pmapi.ImportMsgRes, err error) {
+	err = p.ensureConnection(func() error {
+		res, err = p.client().Import(req)
+		return err
+	})
+	return
+}
+
+func (p *PMAPIProvider) createDraft(message *pmapi.Message, parent string, action int) (draft *pmapi.Message, err error) {
+	err = p.ensureConnection(func() error {
+		draft, err = p.client().CreateDraft(message, parent, action)
+		return err
+	})
+	return
+}
+
+func (p *PMAPIProvider) createAttachment(att *pmapi.Attachment, r io.Reader, sig io.Reader) (created *pmapi.Attachment, err error) {
+	err = p.ensureConnection(func() error {
+		created, err = p.client().CreateAttachment(att, r, sig)
+		return err
+	})
+	return
+}

internal/transfer/provider_test.go

@@ -0,0 +1,111 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	a "github.com/stretchr/testify/assert"
+	r "github.com/stretchr/testify/require"
+)
+
+func getTestMsgBody(subject string) []byte {
+	return []byte(fmt.Sprintf(`Subject: %s
+From: Bridge Test <bridgetest@pm.test>
+To: Bridge Test <bridgetest@protonmail.com>
+Content-Type: multipart/mixed; boundary=c672b8d1ef56ed28ab87c3622c5114069bdd3ad7b8f9737498d0c01ecef0967a
+
+--c672b8d1ef56ed28ab87c3622c5114069bdd3ad7b8f9737498d0c01ecef0967a
+Content-Disposition: inline
+Content-Transfer-Encoding: 7bit
+Content-Type: text/plain; charset=utf-8
+
+hello
+
+--c672b8d1ef56ed28ab87c3622c5114069bdd3ad7b8f9737498d0c01ecef0967a--
+`, subject))
+}
+
+func testTransferTo(t *testing.T, rules transferRules, provider SourceProvider, expectedMessageIDs []string) {
+	progress := newProgress(log, nil)
+	drainProgressUpdateChannel(&progress)
+
+	ch := make(chan Message)
+	go func() {
+		provider.TransferTo(rules, &progress, ch)
+		close(ch)
+	}()
+
+	gotMessageIDs := []string{}
+	for msg := range ch {
+		gotMessageIDs = append(gotMessageIDs, msg.ID)
+	}
+	r.ElementsMatch(t, expectedMessageIDs, gotMessageIDs)
+
+	r.Empty(t, progress.GetFailedMessages())
+}
+
+func testTransferFrom(t *testing.T, rules transferRules, provider TargetProvider, messages []Message) {
+	progress := newProgress(log, nil)
+	drainProgressUpdateChannel(&progress)
+
+	ch := make(chan Message)
+	go func() {
+		for _, message := range messages {
+			progress.addMessage(message.ID, nil)
+			progress.messageExported(message.ID, []byte(""), nil)
+			ch <- message
+		}
+		close(ch)
+	}()
+
+	go func() {
+		provider.TransferFrom(rules, &progress, ch)
+		progress.finish()
+	}()
+
+	maxWait := time.Duration(len(messages)) * time.Second
+	a.Eventually(t, func() bool {
+		return progress.updateCh == nil
+	}, maxWait, 10*time.Millisecond, "Waiting for imported messages timed out")
+
+	r.Empty(t, progress.GetFailedMessages())
+}
+
+func testTransferFromTo(t *testing.T, rules transferRules, source SourceProvider, target TargetProvider, maxWait time.Duration) {
+	progress := newProgress(log, nil)
+	drainProgressUpdateChannel(&progress)
+
+	ch := make(chan Message)
+	go func() {
+		source.TransferTo(rules, &progress, ch)
+		close(ch)
+	}()
+	go func() {
+		target.TransferFrom(rules, &progress, ch)
+		progress.finish()
+	}()
+
+	a.Eventually(t, func() bool {
+		return progress.updateCh == nil
+	}, maxWait, 10*time.Millisecond, "Waiting for export and import timed out")
+
+	r.Empty(t, progress.GetFailedMessages())
+}

internal/transfer/report.go

@@ -0,0 +1,145 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+// fileReport is struct which can write and read message details.
+// File report includes private information.
+type fileReport struct {
+	path string
+}
+
+func openLastFileReport(reportsPath, importID string) (*fileReport, error) { //nolint[deadcode]
+	allLogFileNames, err := getFilePathsWithSuffix(reportsPath, ".log")
+	if err != nil {
+		return nil, err
+	}
+
+	reportFileNames := []string{}
+	for _, fileName := range allLogFileNames {
+		if strings.HasPrefix(fileName, fmt.Sprintf("import_%s_", importID)) {
+			reportFileNames = append(reportFileNames, fileName)
+		}
+	}
+	if len(reportFileNames) == 0 {
+		return nil, errors.New("no report found")
+	}
+
+	sort.Strings(reportFileNames)
+	reportFileName := reportFileNames[len(reportFileNames)-1]
+	path := filepath.Join(reportsPath, reportFileName)
+	return &fileReport{
+		path: path,
+	}, nil
+}
+
+func newFileReport(reportsPath, importID string) *fileReport {
+	fileName := fmt.Sprintf("import_%s_%d.log", importID, time.Now().Unix())
+	path := filepath.Join(reportsPath, fileName)
+
+	return &fileReport{
+		path: path,
+	}
+}
+
+func (r *fileReport) writeMessageStatus(messageStatus *MessageStatus) {
+	f, err := os.OpenFile(r.path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
+	if err != nil {
+		log.WithError(err).Error("Failed to open report file")
+	}
+	defer f.Close() //nolint[errcheck]
+
+	messageReport := newMessageReportFromMessageStatus(messageStatus, true)
+	data, err := json.Marshal(messageReport)
+	if err != nil {
+		log.WithError(err).Error("Failed to marshall message details")
+	}
+	data = append(data, '\n')
+
+	if _, err = f.Write(data); err != nil {
+		log.WithError(err).Error("Failed to write to report file")
+	}
+}
+
+// bugReport is struct which can create report for bug reporting.
+// Bug report does NOT include private information.
+type bugReport struct {
+	data bytes.Buffer
+}
+
+func (r *bugReport) writeMessageStatus(messageStatus *MessageStatus) {
+	messageReport := newMessageReportFromMessageStatus(messageStatus, false)
+	data, err := json.Marshal(messageReport)
+	if err != nil {
+		log.WithError(err).Error("Failed to marshall message details")
+	}
+	_, _ = r.data.Write(data)
+	_, _ = r.data.Write([]byte("\n"))
+}
+
+func (r *bugReport) getData() []byte {
+	return r.data.Bytes()
+}
+
+// messageReport is struct which holds data used by `fileReport` and `bugReport`.
+type messageReport struct {
+	EventTime       int64
+	SourceID        string
+	TargetID        string
+	BodyHash        string
+	SourceMailbox   string
+	TargetMailboxes []string
+	Error           string
+
+	// Private information for user.
+	Subject string
+	From    string
+	Time    string
+}
+
+func newMessageReportFromMessageStatus(messageStatus *MessageStatus, includePrivateInfo bool) messageReport {
+	md := messageReport{
+		EventTime:       messageStatus.eventTime.Unix(),
+		SourceID:        messageStatus.SourceID,
+		TargetID:        messageStatus.targetID,
+		BodyHash:        messageStatus.bodyHash,
+		SourceMailbox:   messageStatus.rule.SourceMailbox.Name,
+		TargetMailboxes: messageStatus.rule.TargetMailboxNames(),
+		Error:           messageStatus.GetErrorMessage(),
+	}
+
+	if includePrivateInfo {
+		md.Subject = messageStatus.Subject
+		md.From = messageStatus.From
+		md.Time = messageStatus.Time.Format(time.RFC1123Z)
+	}
+
+	return md
+}

internal/transfer/rules.go

@@ -0,0 +1,290 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/pkg/errors"
+)
+
+// transferRules maintains import rules, e.g. to which target mailbox should be
+// source mailbox imported or what time spans.
+type transferRules struct {
+	filePath string
+
+	// rules is map with key as hash of source mailbox to its rule.
+	// Every source mailbox should have rule, at least disabled one.
+	rules map[string]*Rule
+
+	// globalMailbox is applied to every message in the import phase.
+	// E.g., every message will be imported into this mailbox.
+	globalMailbox *Mailbox
+
+	// skipEncryptedMessages determines whether message which cannot
+	// be decrypted should be exported or skipped.
+	skipEncryptedMessages bool
+}
+
+// loadRules loads rules from `rulesPath` based on `ruleID`.
+func loadRules(rulesPath, ruleID string) transferRules {
+	fileName := fmt.Sprintf("rules_%s.json", ruleID)
+	filePath := filepath.Join(rulesPath, fileName)
+
+	var rules map[string]*Rule
+	f, err := os.Open(filePath) //nolint[gosec]
+	if err != nil {
+		log.WithError(err).Debug("Problem to read rules")
+	} else {
+		defer f.Close() //nolint[errcheck]
+		if err := json.NewDecoder(f).Decode(&rules); err != nil {
+			log.WithError(err).Warn("Problem to umarshal rules")
+		}
+	}
+	if rules == nil {
+		rules = map[string]*Rule{}
+	}
+
+	return transferRules{
+		filePath: filePath,
+		rules:    rules,
+	}
+}
+
+func (r *transferRules) setSkipEncryptedMessages(skip bool) {
+	r.skipEncryptedMessages = skip
+}
+
+func (r *transferRules) setGlobalMailbox(mailbox *Mailbox) {
+	r.globalMailbox = mailbox
+}
+
+func (r *transferRules) setGlobalTimeLimit(fromTime, toTime int64) {
+	for _, rule := range r.rules {
+		if !rule.HasTimeLimit() {
+			rule.FromTime = fromTime
+			rule.ToTime = toTime
+		}
+	}
+}
+
+func (r *transferRules) getRuleBySourceMailboxName(name string) (*Rule, error) {
+	for _, rule := range r.rules {
+		if rule.SourceMailbox.Name == name {
+			return rule, nil
+		}
+	}
+	return nil, fmt.Errorf("no rule for mailbox %s", name)
+}
+
+func (r *transferRules) iterateActiveRules() chan *Rule {
+	ch := make(chan *Rule)
+	go func() {
+		for _, rule := range r.rules {
+			if rule.Active {
+				ch <- rule
+			}
+		}
+		close(ch)
+	}()
+	return ch
+}
+
+// setDefaultRules iterates `sourceMailboxes` and sets missing rules with
+// matching mailboxes from `targetMailboxes`. In case no matching mailbox
+// is found, `defaultCallback` with a source mailbox as a parameter is used.
+func (r *transferRules) setDefaultRules(sourceMailboxes []Mailbox, targetMailboxes []Mailbox, defaultCallback func(Mailbox) []Mailbox) {
+	for _, sourceMailbox := range sourceMailboxes {
+		h := sourceMailbox.Hash()
+		if _, ok := r.rules[h]; ok {
+			continue
+		}
+
+		targetMailboxes := sourceMailbox.findMatchingMailboxes(targetMailboxes)
+		if len(targetMailboxes) == 0 {
+			targetMailboxes = defaultCallback(sourceMailbox)
+		}
+
+		active := true
+		if len(targetMailboxes) == 0 {
+			active = false
+		}
+
+		// For both import to or export from ProtonMail, spam and draft
+		// mailboxes are by default deactivated.
+		for _, mailbox := range append([]Mailbox{sourceMailbox}, targetMailboxes...) {
+			if mailbox.ID == pmapi.SpamLabel || mailbox.ID == pmapi.DraftLabel || mailbox.ID == pmapi.TrashLabel {
+				active = false
+				break
+			}
+		}
+
+		r.rules[h] = &Rule{
+			Active:          active,
+			SourceMailbox:   sourceMailbox,
+			TargetMailboxes: targetMailboxes,
+		}
+	}
+
+	for _, rule := range r.rules {
+		if !rule.Active {
+			continue
+		}
+		found := false
+		for _, sourceMailbox := range sourceMailboxes {
+			if sourceMailbox.Name == rule.SourceMailbox.Name {
+				found = true
+			}
+		}
+		if !found {
+			rule.Active = false
+		}
+	}
+
+	r.save()
+}
+
+// setRule sets messages from `sourceMailbox` between `fromData` and `toDate`
+// (if used) to be imported to all `targetMailboxes`.
+func (r *transferRules) setRule(sourceMailbox Mailbox, targetMailboxes []Mailbox, fromTime, toTime int64) error {
+	numberOfExclusiveMailboxes := 0
+	for _, mailbox := range targetMailboxes {
+		if mailbox.IsExclusive {
+			numberOfExclusiveMailboxes++
+		}
+	}
+	if numberOfExclusiveMailboxes > 1 {
+		return errors.New("rule can have only one exclusive target mailbox")
+	}
+
+	h := sourceMailbox.Hash()
+	r.rules[h] = &Rule{
+		Active:          true,
+		SourceMailbox:   sourceMailbox,
+		TargetMailboxes: targetMailboxes,
+		FromTime:        fromTime,
+		ToTime:          toTime,
+	}
+	r.save()
+	return nil
+}
+
+// unsetRule unsets messages from `sourceMailbox` to be exported.
+func (r *transferRules) unsetRule(sourceMailbox Mailbox) {
+	h := sourceMailbox.Hash()
+	if rule, ok := r.rules[h]; ok {
+		rule.Active = false
+	} else {
+		r.rules[h] = &Rule{
+			Active:        false,
+			SourceMailbox: sourceMailbox,
+		}
+	}
+	r.save()
+}
+
+// getRule returns rule for `sourceMailbox` or nil if it does not exist.
+func (r *transferRules) getRule(sourceMailbox Mailbox) *Rule {
+	h := sourceMailbox.Hash()
+	return r.rules[h]
+}
+
+// getRules returns all set rules.
+func (r *transferRules) getRules() []*Rule {
+	rules := []*Rule{}
+	for _, rule := range r.rules {
+		rules = append(rules, rule)
+	}
+	return rules
+}
+
+// reset wipes our all rules.
+func (r *transferRules) reset() {
+	r.rules = map[string]*Rule{}
+	r.save()
+}
+
+// save saves rules to file.
+func (r *transferRules) save() {
+	f, err := os.Create(r.filePath)
+	if err != nil {
+		log.WithError(err).Warn("Problem to write rules")
+		return
+	}
+	defer f.Close() //nolint[errcheck]
+
+	if err := json.NewEncoder(f).Encode(r.rules); err != nil {
+		log.WithError(err).Warn("Problem to marshal rules")
+	}
+}
+
+// Rule is data holder of rule for one source mailbox used by `transferRules`.
+type Rule struct {
+	Active          bool      `json:"active"`
+	SourceMailbox   Mailbox   `json:"source"`
+	TargetMailboxes []Mailbox `json:"targets"`
+	FromTime        int64     `json:"from"`
+	ToTime          int64     `json:"to"`
+}
+
+// String returns textual representation for log purposes.
+func (r *Rule) String() string {
+	return fmt.Sprintf(
+		"%s -> %s (%d - %d)",
+		r.SourceMailbox.Name,
+		strings.Join(r.TargetMailboxNames(), ", "),
+		r.FromTime,
+		r.ToTime,
+	)
+}
+
+func (r *Rule) isTimeInRange(t int64) bool {
+	if !r.HasTimeLimit() {
+		return true
+	}
+	return r.FromTime <= t && t <= r.ToTime
+}
+
+// HasTimeLimit returns whether rule defines time limit.
+func (r *Rule) HasTimeLimit() bool {
+	return r.FromTime != 0 || r.ToTime != 0
+}
+
+// FromDate returns time struct based on `FromTime`.
+func (r *Rule) FromDate() time.Time {
+	return time.Unix(r.FromTime, 0)
+}
+
+// ToDate returns time struct based on `ToTime`.
+func (r *Rule) ToDate() time.Time {
+	return time.Unix(r.ToTime, 0)
+}
+
+// TargetMailboxNames returns array of target mailbox names.
+func (r *Rule) TargetMailboxNames() (names []string) {
+	for _, mailbox := range r.TargetMailboxes {
+		names = append(names, mailbox.Name)
+	}
+	return
+}

internal/transfer/rules_test.go

@@ -0,0 +1,210 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	r "github.com/stretchr/testify/require"
+)
+
+func newTestRules(t *testing.T) (transferRules, func()) {
+	path, err := ioutil.TempDir("", "rules")
+	r.NoError(t, err)
+
+	ruleID := "rule"
+	rules := loadRules(path, ruleID)
+	return rules, func() {
+		_ = os.RemoveAll(path)
+	}
+}
+
+func TestLoadRules(t *testing.T) {
+	path, err := ioutil.TempDir("", "rules")
+	r.NoError(t, err)
+	defer os.RemoveAll(path) //nolint[errcheck]
+
+	ruleID := "rule"
+	rules := loadRules(path, ruleID)
+
+	mailboxA := Mailbox{ID: "1", Name: "One", Color: "orange", IsExclusive: true}
+	mailboxB := Mailbox{ID: "2", Name: "Two", Color: "", IsExclusive: true}
+	mailboxC := Mailbox{ID: "3", Name: "Three", Color: "", IsExclusive: false}
+
+	r.NoError(t, rules.setRule(mailboxA, []Mailbox{mailboxB, mailboxC}, 0, 0))
+	r.NoError(t, rules.setRule(mailboxB, []Mailbox{mailboxB}, 10, 20))
+	r.NoError(t, rules.setRule(mailboxC, []Mailbox{}, 0, 30))
+
+	rules2 := loadRules(path, ruleID)
+	r.Equal(t, map[string]*Rule{
+		mailboxA.Hash(): {Active: true, SourceMailbox: mailboxA, TargetMailboxes: []Mailbox{mailboxB, mailboxC}, FromTime: 0, ToTime: 0},
+		mailboxB.Hash(): {Active: true, SourceMailbox: mailboxB, TargetMailboxes: []Mailbox{mailboxB}, FromTime: 10, ToTime: 20},
+		mailboxC.Hash(): {Active: true, SourceMailbox: mailboxC, TargetMailboxes: []Mailbox{}, FromTime: 0, ToTime: 30},
+	}, rules2.rules)
+
+	rules2.unsetRule(mailboxA)
+	rules2.unsetRule(mailboxC)
+
+	rules3 := loadRules(path, ruleID)
+	r.Equal(t, map[string]*Rule{
+		mailboxA.Hash(): {Active: false, SourceMailbox: mailboxA, TargetMailboxes: []Mailbox{mailboxB, mailboxC}, FromTime: 0, ToTime: 0},
+		mailboxB.Hash(): {Active: true, SourceMailbox: mailboxB, TargetMailboxes: []Mailbox{mailboxB}, FromTime: 10, ToTime: 20},
+		mailboxC.Hash(): {Active: false, SourceMailbox: mailboxC, TargetMailboxes: []Mailbox{}, FromTime: 0, ToTime: 30},
+	}, rules3.rules)
+}
+
+func TestSetGlobalTimeLimit(t *testing.T) {
+	path, err := ioutil.TempDir("", "rules")
+	r.NoError(t, err)
+	defer os.RemoveAll(path) //nolint[errcheck]
+
+	rules := loadRules(path, "rule")
+
+	mailboxA := Mailbox{Name: "One"}
+	mailboxB := Mailbox{Name: "Two"}
+
+	r.NoError(t, rules.setRule(mailboxA, []Mailbox{}, 10, 20))
+	r.NoError(t, rules.setRule(mailboxB, []Mailbox{}, 0, 0))
+
+	rules.setGlobalTimeLimit(30, 40)
+
+	r.Equal(t, map[string]*Rule{
+		mailboxA.Hash(): {Active: true, SourceMailbox: mailboxA, TargetMailboxes: []Mailbox{}, FromTime: 10, ToTime: 20},
+		mailboxB.Hash(): {Active: true, SourceMailbox: mailboxB, TargetMailboxes: []Mailbox{}, FromTime: 30, ToTime: 40},
+	}, rules.rules)
+}
+
+func TestSetDefaultRules(t *testing.T) {
+	path, err := ioutil.TempDir("", "rules")
+	r.NoError(t, err)
+	defer os.RemoveAll(path) //nolint[errcheck]
+
+	rules := loadRules(path, "rule")
+
+	mailbox1 := Mailbox{Name: "One"}                       // Set manually, default will not override it.
+	mailbox2 := Mailbox{Name: "Two"}                       // Matched by `targetMailboxes`.
+	mailbox3 := Mailbox{Name: "Three"}                     // Matched by `defaultCallback`, not included in `targetMailboxes`.
+	mailbox4 := Mailbox{Name: "Four"}                      // Matched by nothing, will not be active.
+	mailbox5 := Mailbox{Name: "Spam", ID: pmapi.SpamLabel} // Spam is inactive by default (ID found in source).
+	mailbox6a := Mailbox{Name: "Draft"}                    // Draft is inactive by default (ID found in target, mailbox6b).
+	mailbox6b := Mailbox{Name: "Draft", ID: pmapi.DraftLabel}
+
+	sourceMailboxes := []Mailbox{mailbox1, mailbox2, mailbox3, mailbox4, mailbox5, mailbox6a}
+	targetMailboxes := []Mailbox{mailbox1, mailbox2, mailbox6b}
+
+	r.NoError(t, rules.setRule(mailbox1, []Mailbox{mailbox3}, 0, 0))
+
+	defaultCallback := func(mailbox Mailbox) []Mailbox {
+		if mailbox.Name == "Three" {
+			return []Mailbox{mailbox3}
+		}
+		return []Mailbox{}
+	}
+
+	rules.setDefaultRules(sourceMailboxes, targetMailboxes, defaultCallback)
+
+	r.Equal(t, map[string]*Rule{
+		mailbox1.Hash():  {Active: true, SourceMailbox: mailbox1, TargetMailboxes: []Mailbox{mailbox3}},
+		mailbox2.Hash():  {Active: true, SourceMailbox: mailbox2, TargetMailboxes: []Mailbox{mailbox2}},
+		mailbox3.Hash():  {Active: true, SourceMailbox: mailbox3, TargetMailboxes: []Mailbox{mailbox3}},
+		mailbox4.Hash():  {Active: false, SourceMailbox: mailbox4, TargetMailboxes: []Mailbox{}},
+		mailbox5.Hash():  {Active: false, SourceMailbox: mailbox5, TargetMailboxes: []Mailbox{}},
+		mailbox6a.Hash(): {Active: false, SourceMailbox: mailbox6a, TargetMailboxes: []Mailbox{mailbox6b}},
+	}, rules.rules)
+}
+
+func TestSetDefaultRulesDeactivateMissing(t *testing.T) {
+	path, err := ioutil.TempDir("", "rules")
+	r.NoError(t, err)
+	defer os.RemoveAll(path) //nolint[errcheck]
+
+	rules := loadRules(path, "rule")
+
+	mailboxA := Mailbox{ID: "1", Name: "One", Color: "", IsExclusive: true}
+	mailboxB := Mailbox{ID: "2", Name: "Two", Color: "", IsExclusive: true}
+
+	r.NoError(t, rules.setRule(mailboxA, []Mailbox{mailboxB}, 0, 0))
+	r.NoError(t, rules.setRule(mailboxB, []Mailbox{mailboxB}, 0, 0))
+
+	sourceMailboxes := []Mailbox{mailboxA}
+	targetMailboxes := []Mailbox{mailboxA, mailboxB}
+	defaultCallback := func(mailbox Mailbox) (mailboxes []Mailbox) {
+		return
+	}
+	rules.setDefaultRules(sourceMailboxes, targetMailboxes, defaultCallback)
+
+	r.Equal(t, map[string]*Rule{
+		mailboxA.Hash(): {Active: true, SourceMailbox: mailboxA, TargetMailboxes: []Mailbox{mailboxB}, FromTime: 0, ToTime: 0},
+		mailboxB.Hash(): {Active: false, SourceMailbox: mailboxB, TargetMailboxes: []Mailbox{mailboxB}, FromTime: 0, ToTime: 0},
+	}, rules.rules)
+}
+
+func TestIsTimeInRange(t *testing.T) {
+	tests := []struct {
+		rule Rule
+		time int64
+		want bool
+	}{
+		{generateTimeRule(0, 0), 0, true},
+		{generateTimeRule(0, 0), 10, true},
+		{generateTimeRule(0, 15), 10, true},
+		{generateTimeRule(5, 15), 10, true},
+		{generateTimeRule(0, 5), 10, false},
+		{generateTimeRule(5, 7), 10, false},
+		{generateTimeRule(15, 30), 10, false},
+		{generateTimeRule(15, 0), 10, false},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(fmt.Sprintf("%v / %d", tc.rule, tc.time), func(t *testing.T) {
+			got := tc.rule.isTimeInRange(tc.time)
+			r.Equal(t, tc.want, got)
+		})
+	}
+}
+
+func TestHasTimeLimit(t *testing.T) {
+	tests := []struct {
+		rule Rule
+		want bool
+	}{
+		{generateTimeRule(0, 0), false},
+		{generateTimeRule(0, 1), true},
+		{generateTimeRule(1, 2), true},
+		{generateTimeRule(1, 0), true},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(fmt.Sprintf("%v", tc.rule), func(t *testing.T) {
+			r.Equal(t, tc.want, tc.rule.HasTimeLimit())
+		})
+	}
+}
+
+func generateTimeRule(from, to int64) Rule {
+	return Rule{
+		SourceMailbox:   Mailbox{},
+		TargetMailboxes: []Mailbox{},
+		FromTime:        from,
+		ToTime:          to,
+	}
+}

internal/transfer/testdata/eml/Foo/msg.eml

@@ -0,0 +1,4 @@
+From: Bridge Test <bridgetest@pm.test>
+To: Bridge Test <bridgetest@protonmail.com>
+
+hello

internal/transfer/testdata/eml/Inbox/msg.eml

@@ -0,0 +1,4 @@
+From: Bridge Test <bridgetest@pm.test>
+To: Bridge Test <bridgetest@protonmail.com>
+
+hello

internal/transfer/testdata/emlmbox/Foo/msg.eml

@@ -0,0 +1,4 @@
+From: Bridge Test <bridgetest@pm.test>
+To: Bridge Test <bridgetest@protonmail.com>
+
+hello

internal/transfer/testdata/emlmbox/Inbox.mbox

@@ -0,0 +1,5 @@
+From - Mon May  4 16:40:31 2020
+From: Bridge Test <bridgetest@pm.test>
+To: Bridge Test <bridgetest@protonmail.com>
+
+hello

internal/transfer/testdata/keyring_userKey

@@ -0,0 +1,62 @@
+-----BEGIN PGP PRIVATE KEY BLOCK-----
+Version: OpenPGP.js v4.4.5
+Comment: testpassphrase
+
+xcLYBFzGzhEBCADBxfqTFMqfQzT77A5tuuhPFwPq8dfC2evs8u1OvTqFbztY
+5FOuSxzduyeDqQ1Fx6dKEOKgcYE8t1Uh4VSS7z6bTdY8j9yrL81kCVB46sE1
+OzStzyx/5l7OdH/pM4F+aKslnLvqlw0UeJr+UNizVtOCEUaNfVjPK3cc1ocx
+v+36K4RnnyfEtjUW9gDZbhgaF02G5ILHmWmbgM7I+77gCd2wI0EdY9s/JZQ+
+VmkMFqoMdY9PyBchoOIPUkkGQi1SaF4IEzMaAUSbnCYkHHY/SbfDTcR46VGq
+cXlkB1rq5xskaUQ9r+giCC/K4pc7bBkI1lQ7ADVuWvdrWnWapK0FO6CfABEB
+AAEAB/0YPhPJ0phA/EWviN+16bmGVOZNaVapjt2zMMybWmrtEQv3OeWgO3nP
+4cohRi/zaCBCphcm+dxbLhftW7AFi/9PVcR09436MB+oTCQFugpUWw+4TmA5
+BidxTpDxf4X2vH3rquQLBufWL6U7JlPeKAGL1xZ2aCq0DIeOk5D+xTjZizV2
+GIyQRVCLWb+LfDmvvcp3Y94X60KXdBAMuS1ZMKcY3Sl8VAXNB4KQsC/kByzf
+6FCB097XZRYV7lvJJQ7+6Wisb3yVi8sEQx2sFm5fAp+0qi3a6zRTEp49r6Hr
+gyWViH5zOOpA7DcNwx1Bwhi7GG0tak6EUnnKUNLfOupglcphBADmpXCgT4nc
+uSBYTiZSVcB/ICCkTxVsHL1WcXtPK2Ikzussx2n9kb0rapvuC0YLipX9lUkQ
+fyeC3jQJeCyN79AkDGkOfWaESueT2hM0Po+RwDgMibKn6yJ1zebz4Lc2J3C9
+oVFcAnql+9KyGsAPn03fyQzDnvhNnJvHJi4Hx8AWoQQA1xLoXeVBjRi0IjjU
+E6Mqaq5RLEog4kXRp86VSSEGHBwyIYnDiM//gjseo/CXuVyHwL7UXitp8s1B
+D1uE3APrhqUS66fD5pkF+z+RcSqiIv7I76NJ24Cdg38L6seGSjOHrq7/dEeG
+K6WqfQUCEjta3yNSg7pXb2wn2WZqKIK+rz8EALZRuMXeql/FtO3Cjb0sv7oT
+9dLP4cn1bskGRJ+Vok9lfCERbfXGccoAk3V+qSfpHgKxsebkRbUhf+trOGnw
+tW+kBWo/5hYGQuN+A9JogSJViT+nuZyE+x1/rKswDFmlMSdf2GIDARWIV0gc
+b1yOEwUmNBSthPcnFXvBr4BG3XTtNPTNLSJhcm9uMjEtM0BzYWRlbWJlLm9y
+ZyIgPGFyb24yMS0zQHNhZGVtYmUub3JnPsLAdQQQAQgAHwUCXMbOEQYLCQcI
+AwIEFQgKAgMWAgECGQECGwMCHgEACgkQZ/B4v2b2xB6XUgf/dHGRHimyMR78
+QYbEm2cuaEvOtq4a+J6Zv3P4VOWAbvkGWS9LDKSvVi60vq4oYOmF54HgPzur
+nA4OtZDf0HKwQK45VZ7CYD693o70jkKPrAAJG3yTsbesfiS7RbFyGKzKJ7EL
+nsUIJkfgm/SlKmXU/u8MOBO5Wg7/TcsS33sRWHl90j+9jbhqdl92R+vY/CwC
+ieFkQA7/TDv1u+NAalH+Lpkd8AIuEcki+TAogZ7oi/SnofwnoB7BxRm+mIkp
+ZZhIDSCaPOzLG8CSZ81d3HVHhqbf8dh0DFKFoUYyKdbOqIkNWWASf+c/ZEme
+IWcekY8hqwf/raZ56tGM/bRwYPcotMfC1wRcxs4RAQgAsMb5/ELWmrfPy3ba
+5qif+RXhGSbjitATNgHpoPUHrfTC7cn4JWHqehoXLAQpFAoKd+O/ZNpZozK9
+ilpqGUx05yMw06jNQEhYIbgIF4wzPpz02Lp6YeMwdF5LF+Rw83PHdHrA/wRV
+/QjL04+kZnN+G5HmzMlhFY+oZSpL+Gp1bTXgtAVDkhCnMB5tP2VwULMGyJ+X
+vRYxwTK2CrLjIVZv5n1VYY+caCowU6j/XFqvlCJj+G5oV+UhFOWffaMRXhOh
+a64RrhqT1Np7wCLvLMP2wpys9xlMcLQJLqDNxqOTp504V7dm67ncC0fKUsT4
+m4oTktnxKPd6MU+4VYveaLCquwARAQABAAf4u9s7gpGErs1USxmDO9TlyGZK
+aBlri8nMf3s+hOJCOo3cRaRHJBfdY6pu/baG6H6JTsWzeY4MHwr6N+dhVIEh
+FPMa9EZAjagyc4GugxWGiMVTfU+2AEfdrdynhQKMgXSctnnNCdkRuX0nwqb3
+nlupm1hsz2ze4+Wg0BKSLS0FQdoUbITdJUR69OHr4dNJVHWYI0JSBx4SdhV3
+y9163dDvmc+lW9AEaD53vyZWfzCHZxsR/gI32VmT0z5gn1t8w9AOdXo2lA1H
+bf7wh4/qCyujGu64ToZtiEny/GCyM6PofLtiZuJNLw3s/y+B2tKv22aTJ760
++Gib1xB9WcWjKyrxBADoeCyq+nHGrl0CwOkmjanlFymgo7mnBOXuiFOvGrKk
+M1meMU1TI4TEBWkVnDVMcSejgjAf/bX1dtouba1tMAMu7DlaV/0EwbSADRel
+RSqEbIzIOys+y9TY/BMI/uCKNyEKHvu1KUXADb+CBpdBpCfMBWDANFlo9xLz
+Ajcmu2dyawQAwquwC0VXQcvzfs+Hd5au5XvHdm1KidOiAdu6PH1SrOgenIN4
+lkEjHrJD9jmloO2/GVcxDBB2pmf0B4HEg7DuY9LXBrksP5eSbbRc5+UH1HUv
+u82AqQnfNKTd/jae+lLwaOS++ohtwMkkD6W0LdWnHPjyyXg4Oi9zPID3asRu
+3PED/3CYyjl6S8GTMY4FNH7Yxu9+NV2xpKE92Hf8K/hnYlmSSVKDCEeOJtLt
+BkkcSqY6liCNSMmJdVyAF2GrR+zmDac7UQRssf57oOWtSsGozt0aqJXuspMT
+6aB+P1UhZ8Ly9rWZNiJ0jwyfnQNOLCYDaqjFmiSpqrNnJ2Q1Xge3+k80P9DC
+wF8EGAEIAAkFAlzGzhECGwwACgkQZ/B4v2b2xB5wlwgAjZA1zdv5irFjyWVo
+4/itONtyO1NbdpyYpcct7vD0oV+a4wahQP0J3Kk1GhZ5tvAoZF/jakQQOM5o
+GjUYpXAGnr09Mv9EiQ2pDwXc2yq0WfXnGxNrpzOqdtV+IqY9NYkl55Tme7x+
+WRvrkPSUeUsyEGvxwR1stdv8eg9jUmxdl8Io3PYoFJJlrM/6aXeC1r3KOj7q
+XAnR0XHJ+QBSNKCWLlQv5hui9BKfcLiVKFK/dNhs82nRyhPr4sWFw6MTqdAK
+4zkn7l0jmy6Evi1AiiGPiHPnxeNErnofOIEh4REQj00deZADHrixTLtx2FuR
+uaSC3IcBmBsj1fNb4eYXElILjQ==
+=fMOl
+-----END PGP PRIVATE KEY BLOCK-----

internal/transfer/testdata/mbox/Foo.mbox

@@ -0,0 +1,5 @@
+From - Mon May  4 16:40:31 2020
+From: Bridge Test <bridgetest@pm.test>
+To: Bridge Test <bridgetest@protonmail.com>
+
+hello

internal/transfer/testdata/mbox/Inbox.mbox

@@ -0,0 +1,5 @@
+From - Mon May  4 16:40:31 2020
+From: Bridge Test <bridgetest@pm.test>
+To: Bridge Test <bridgetest@protonmail.com>
+
+hello

internal/transfer/transfer.go

@@ -0,0 +1,177 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+// Package transfer provides tools to export messages from one provider and
+// import them to another provider. Provider can be EML, MBOX, IMAP or PMAPI.
+package transfer
+
+import (
+	"crypto/sha256"
+	"fmt"
+
+	"github.com/sirupsen/logrus"
+)
+
+var log = logrus.WithField("pkg", "transfer") //nolint[gochecknoglobals]
+
+// Transfer is facade on top of import rules, progress manager and source
+// and target providers. This is the main object which should be used.
+type Transfer struct {
+	panicHandler PanicHandler
+	id           string
+	dir          string
+	rules        transferRules
+	source       SourceProvider
+	target       TargetProvider
+}
+
+// New creates Transfer for specific source and target. Usage:
+//
+//   source := transfer.NewEMLProvider(...)
+//   target := transfer.NewPMAPIProvider(...)
+//   transfer.New(source, target, ...)
+func New(panicHandler PanicHandler, transferDir string, source SourceProvider, target TargetProvider) (*Transfer, error) {
+	transferID := fmt.Sprintf("%x", sha256.Sum256([]byte(source.ID()+"-"+target.ID())))
+	rules := loadRules(transferDir, transferID)
+	transfer := &Transfer{
+		panicHandler: panicHandler,
+		id:           transferID,
+		dir:          transferDir,
+		rules:        rules,
+		source:       source,
+		target:       target,
+	}
+	if err := transfer.setDefaultRules(); err != nil {
+		return nil, err
+	}
+	return transfer, nil
+}
+
+// SetDefaultRules sets missing rules for source mailboxes with matching
+// target mailboxes. In case no matching mailbox is found, `defaultCallback`
+// with a source mailbox as a parameter is used.
+func (t *Transfer) setDefaultRules() error {
+	sourceMailboxes, err := t.SourceMailboxes()
+	if err != nil {
+		return err
+	}
+
+	targetMailboxes, err := t.TargetMailboxes()
+	if err != nil {
+		return err
+	}
+
+	defaultCallback := func(sourceMailbox Mailbox) []Mailbox {
+		return t.target.DefaultMailboxes(sourceMailbox)
+	}
+
+	t.rules.setDefaultRules(sourceMailboxes, targetMailboxes, defaultCallback)
+	return nil
+}
+
+// SetSkipEncryptedMessages sets whether message which cannot be decrypted
+// should be exported or skipped.
+func (t *Transfer) SetSkipEncryptedMessages(skip bool) {
+	t.rules.setSkipEncryptedMessages(skip)
+}
+
+// SetGlobalMailbox sets mailbox that is applied to every message in
+// the import phase.
+func (t *Transfer) SetGlobalMailbox(mailbox *Mailbox) {
+	t.rules.setGlobalMailbox(mailbox)
+}
+
+// SetGlobalTimeLimit sets time limit that is applied to rules without any
+// specified time limit.
+func (t *Transfer) SetGlobalTimeLimit(fromTime, toTime int64) {
+	t.rules.setGlobalTimeLimit(fromTime, toTime)
+}
+
+// SetRule sets sourceMailbox for transfer.
+func (t *Transfer) SetRule(sourceMailbox Mailbox, targetMailboxes []Mailbox, fromTime, toTime int64) error {
+	return t.rules.setRule(sourceMailbox, targetMailboxes, fromTime, toTime)
+}
+
+// UnsetRule unsets sourceMailbox from transfer.
+func (t *Transfer) UnsetRule(sourceMailbox Mailbox) {
+	t.rules.unsetRule(sourceMailbox)
+}
+
+// ResetRules unsets all rules.
+func (t *Transfer) ResetRules() {
+	t.rules.reset()
+}
+
+// GetRule returns rule for given mailbox.
+func (t *Transfer) GetRule(sourceMailbox Mailbox) *Rule {
+	return t.rules.getRule(sourceMailbox)
+}
+
+// GetRules returns all set transfer rules.
+func (t *Transfer) GetRules() []*Rule {
+	return t.rules.getRules()
+}
+
+// SourceMailboxes returns mailboxes available at source side.
+func (t *Transfer) SourceMailboxes() ([]Mailbox, error) {
+	return t.source.Mailboxes(false, true)
+}
+
+// TargetMailboxes returns mailboxes available at target side.
+func (t *Transfer) TargetMailboxes() ([]Mailbox, error) {
+	return t.target.Mailboxes(true, false)
+}
+
+// CreateTargetMailbox creates mailbox in target provider.
+func (t *Transfer) CreateTargetMailbox(mailbox Mailbox) (Mailbox, error) {
+	return t.target.CreateMailbox(mailbox)
+}
+
+// ChangeTarget allows to change target. Ideally should not be used.
+// Useful for situration after user changes mind where to export files and similar.
+func (t *Transfer) ChangeTarget(target TargetProvider) {
+	t.target = target
+}
+
+// Start starts the transfer from source to target.
+func (t *Transfer) Start() *Progress {
+	log.Debug("Transfer started")
+	t.rules.save()
+
+	log := log.WithField("id", t.id)
+	reportFile := newFileReport(t.dir, t.id)
+	progress := newProgress(log, reportFile)
+
+	ch := make(chan Message)
+
+	go func() {
+		defer t.panicHandler.HandlePanic()
+
+		progress.start()
+		t.source.TransferTo(t.rules, &progress, ch)
+		close(ch)
+	}()
+
+	go func() {
+		defer t.panicHandler.HandlePanic()
+
+		t.target.TransferFrom(t.rules, &progress, ch)
+		progress.finish()
+	}()
+
+	return &progress
+}

internal/transfer/transfer_test.go

@@ -0,0 +1,73 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"bytes"
+	"io/ioutil"
+	"testing"
+
+	"github.com/ProtonMail/gopenpgp/crypto"
+	transfermocks "github.com/ProtonMail/proton-bridge/internal/transfer/mocks"
+	pmapimocks "github.com/ProtonMail/proton-bridge/pkg/pmapi/mocks"
+	gomock "github.com/golang/mock/gomock"
+)
+
+type mocks struct {
+	t *testing.T
+
+	ctrl          *gomock.Controller
+	panicHandler  *transfermocks.MockPanicHandler
+	clientManager *transfermocks.MockClientManager
+	pmapiClient   *pmapimocks.MockClient
+
+	keyring *crypto.KeyRing
+}
+
+func initMocks(t *testing.T) mocks {
+	mockCtrl := gomock.NewController(t)
+
+	m := mocks{
+		t: t,
+
+		ctrl:          mockCtrl,
+		panicHandler:  transfermocks.NewMockPanicHandler(mockCtrl),
+		clientManager: transfermocks.NewMockClientManager(mockCtrl),
+		pmapiClient:   pmapimocks.NewMockClient(mockCtrl),
+		keyring:       newTestKeyring(),
+	}
+
+	m.clientManager.EXPECT().GetClient("user").Return(m.pmapiClient).AnyTimes()
+
+	return m
+}
+
+func newTestKeyring() *crypto.KeyRing {
+	data, err := ioutil.ReadFile("testdata/keyring_userKey")
+	if err != nil {
+		panic(err)
+	}
+	userKey, err := crypto.ReadArmoredKeyRing(bytes.NewReader(data))
+	if err != nil {
+		panic(err)
+	}
+	if err := userKey.Unlock([]byte("testpassphrase")); err != nil {
+		panic(err)
+	}
+	return userKey
+}

internal/transfer/types.go

@@ -0,0 +1,31 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+)
+
+type PanicHandler interface {
+	HandlePanic()
+}
+
+type ClientManager interface {
+	GetClient(userID string) pmapi.Client
+	CheckConnection() error
+}

internal/transfer/utils.go

@@ -0,0 +1,141 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"bufio"
+	"bytes"
+	"io/ioutil"
+	"net/mail"
+	"net/textproto"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+// getFolderNames collects all folder names under `root`.
+// Folder names will be without a path.
+func getFolderNames(root string) ([]string, error) {
+	return getFolderNamesWithFileSuffix(root, "")
+}
+
+// getFolderNamesWithFileSuffix collects all folder names under `root`, which
+// contains some file with a give `fileSuffix`. Names will be without a path.
+func getFolderNamesWithFileSuffix(root, fileSuffix string) ([]string, error) {
+	folders := []string{}
+
+	files, err := ioutil.ReadDir(root)
+	if err != nil {
+		return nil, err
+	}
+
+	hasFileWithSuffix := fileSuffix == ""
+	for _, file := range files {
+		if file.IsDir() {
+			subfolders, err := getFolderNamesWithFileSuffix(filepath.Join(root, file.Name()), fileSuffix)
+			if err != nil {
+				return nil, err
+			}
+			for _, subfolder := range subfolders {
+				match := false
+				for _, folder := range folders {
+					if folder == subfolder {
+						match = true
+						break
+					}
+				}
+				if !match {
+					folders = append(folders, subfolder)
+				}
+			}
+		} else if fileSuffix == "" || strings.HasSuffix(file.Name(), fileSuffix) {
+			hasFileWithSuffix = true
+		}
+	}
+
+	if hasFileWithSuffix {
+		folders = append(folders, filepath.Base(root))
+	}
+
+	sort.Strings(folders)
+	return folders, nil
+}
+
+// getFilePathsWithSuffix collects all file names with `suffix` under `root`.
+// File names will be with relative path based to `root`.
+func getFilePathsWithSuffix(root, suffix string) ([]string, error) {
+	fileNames, err := getFilePathsWithSuffixInner("", root, suffix)
+	if err != nil {
+		return nil, err
+	}
+	sort.Strings(fileNames)
+	return fileNames, err
+}
+
+func getFilePathsWithSuffixInner(prefix, root, suffix string) ([]string, error) {
+	fileNames := []string{}
+
+	files, err := ioutil.ReadDir(root)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, file := range files {
+		if !file.IsDir() {
+			if strings.HasSuffix(file.Name(), suffix) {
+				fileNames = append(fileNames, filepath.Join(prefix, file.Name()))
+			}
+		} else {
+			subfolderFileNames, err := getFilePathsWithSuffixInner(
+				filepath.Join(prefix, file.Name()),
+				filepath.Join(root, file.Name()),
+				suffix,
+			)
+			if err != nil {
+				return nil, err
+			}
+			fileNames = append(fileNames, subfolderFileNames...)
+		}
+	}
+
+	return fileNames, nil
+}
+
+// getMessageTime returns time of the message specified in the message header.
+func getMessageTime(body []byte) (int64, error) {
+	mailHeader, err := getMessageHeader(body)
+	if err != nil {
+		return 0, err
+	}
+	if t, err := mailHeader.Date(); err == nil && !t.IsZero() {
+		return t.Unix(), nil
+	}
+	return 0, nil
+}
+
+// getMessageHeader returns headers of the message body.
+func getMessageHeader(body []byte) (mail.Header, error) {
+	tpr := textproto.NewReader(bufio.NewReader(bytes.NewBuffer(body)))
+	header, err := tpr.ReadMIMEHeader()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read headers")
+	}
+	return mail.Header(header), nil
+}

internal/transfer/utils_test.go

@@ -0,0 +1,190 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	r "github.com/stretchr/testify/require"
+)
+
+func TestGetFolderNames(t *testing.T) {
+	root, clean := createTestingFolderStructure(t)
+	defer clean()
+
+	tests := []struct {
+		suffix    string
+		wantNames []string
+	}{
+		{
+			"",
+			[]string{
+				"bar",
+				"baz",
+				filepath.Base(root),
+				"foo",
+				"qwerty",
+				"test",
+			},
+		},
+		{
+			".eml",
+			[]string{
+				"bar",
+				"baz",
+				filepath.Base(root),
+				"foo",
+			},
+		},
+		{
+			".txt",
+			[]string{
+				filepath.Base(root),
+			},
+		},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.suffix, func(t *testing.T) {
+			names, err := getFolderNamesWithFileSuffix(root, tc.suffix)
+			r.NoError(t, err)
+			r.Equal(t, tc.wantNames, names)
+		})
+	}
+}
+
+func TestGetFilePathsWithSuffix(t *testing.T) {
+	root, clean := createTestingFolderStructure(t)
+	defer clean()
+
+	tests := []struct {
+		suffix    string
+		wantPaths []string
+	}{
+		{
+			".eml",
+			[]string{
+				"foo/bar/baz/msg1.eml",
+				"foo/bar/baz/msg2.eml",
+				"foo/bar/baz/msg3.eml",
+				"foo/bar/msg4.eml",
+				"foo/bar/msg5.eml",
+				"foo/baz/msg6.eml",
+				"foo/msg7.eml",
+				"msg10.eml",
+				"test/foo/msg8.eml",
+				"test/foo/msg9.eml",
+			},
+		},
+		{
+			".txt",
+			[]string{
+				"info.txt",
+			},
+		},
+		{
+			".hello",
+			[]string{},
+		},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.suffix, func(t *testing.T) {
+			paths, err := getFilePathsWithSuffix(root, tc.suffix)
+			r.NoError(t, err)
+			r.Equal(t, tc.wantPaths, paths)
+		})
+	}
+}
+
+func createTestingFolderStructure(t *testing.T) (string, func()) {
+	root, err := ioutil.TempDir("", "folderstructure")
+	r.NoError(t, err)
+
+	for _, path := range []string{
+		"foo/bar/baz",
+		"foo/baz",
+		"test/foo",
+		"qwerty",
+	} {
+		err = os.MkdirAll(filepath.Join(root, path), os.ModePerm)
+		r.NoError(t, err)
+	}
+
+	for _, path := range []string{
+		"foo/bar/baz/msg1.eml",
+		"foo/bar/baz/msg2.eml",
+		"foo/bar/baz/msg3.eml",
+		"foo/bar/msg4.eml",
+		"foo/bar/msg5.eml",
+		"foo/baz/msg6.eml",
+		"foo/msg7.eml",
+		"test/foo/msg8.eml",
+		"test/foo/msg9.eml",
+		"msg10.eml",
+		"info.txt",
+	} {
+		f, err := os.Create(filepath.Join(root, path))
+		r.NoError(t, err)
+		err = f.Close()
+		r.NoError(t, err)
+	}
+
+	return root, func() {
+		_ = os.RemoveAll(root)
+	}
+}
+
+func TestGetMessageTime(t *testing.T) {
+	tests := []struct {
+		body     string
+		wantTime int64
+		wantErr  string
+	}{
+		{"", 0, "failed to read headers: EOF"},
+		{"Subject: hello\n\n", 0, ""},
+		{"Date: Thu, 23 Apr 2020 04:52:44 +0000\n\n", 1587617564, ""},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.body, func(t *testing.T) {
+			time, err := getMessageTime([]byte(tc.body))
+			if tc.wantErr == "" {
+				r.NoError(t, err)
+			} else {
+				r.EqualError(t, err, tc.wantErr)
+			}
+			r.Equal(t, tc.wantTime, time)
+		})
+	}
+}
+
+func TestGetMessageHeader(t *testing.T) {
+	body := `Subject: Hello
+From: user@example.com
+
+Body
+`
+	header, err := getMessageHeader([]byte(body))
+	r.NoError(t, err)
+	r.Equal(t, header.Get("subject"), "Hello")
+	r.Equal(t, header.Get("from"), "user@example.com")
+}

internal/users/user.go

@@ -359,14 +359,17 @@ func (u *User) GetAddressID(address string) (id string, err error) {
 	u.lock.RLock()
 	defer u.lock.RUnlock()
 
-	address = strings.ToLower(address)
+	if u.store != nil {
-
+		address = strings.ToLower(address)
-	if u.store == nil {
+		return u.store.GetAddressID(address)
-		err = errors.New("store is not initialised")
-		return
 	}
 
-	return u.store.GetAddressID(address)
+	addresses := u.client().Addresses()
+	pmapiAddress := addresses.ByEmail(address)
+	if pmapiAddress != nil {
+		return pmapiAddress.ID, nil
+	}
+	return "", errors.New("address not found")
 }
 
 // GetBridgePassword returns bridge password. This is not a password of the PM

internal/users/users.go

@@ -52,8 +52,14 @@ type Users struct {
 	// People are used to that and so we preserve that ordering here.
 	users []*User
 
-	// idleUpdates is a channel which the imap backend listens to and which it uses
+	// useOnlyActiveAddresses determines whether credentials keeps only active
-	// to send idle updates to the mail client (eg thunderbird).
+	// addresses or all of them. Each usage has to be consisteng, e.g., once
+	// user is added, it saves address list to credentials and next time loads
+	// as is, without requesting server again.
+	useOnlyActiveAddresses bool
+
+	// idleUpdates is a channel which the imap backend listens to and which it
+	// uses to send idle updates to the mail client (eg thunderbird).
 	// The user stores should send idle updates on this channel.
 	idleUpdates chan imapBackend.Update
 
@@ -70,19 +76,21 @@ func New(
 	clientManager ClientManager,
 	credStorer CredentialsStorer,
 	storeFactory StoreMaker,
+	useOnlyActiveAddresses bool,
 ) *Users {
 	log.Trace("Creating new users")
 
 	u := &Users{
-		config:        config,
+		config:                 config,
-		panicHandler:  panicHandler,
+		panicHandler:           panicHandler,
-		events:        eventListener,
+		events:                 eventListener,
-		clientManager: clientManager,
+		clientManager:          clientManager,
-		credStorer:    credStorer,
+		credStorer:             credStorer,
-		storeFactory:  storeFactory,
+		storeFactory:           storeFactory,
-		idleUpdates:   make(chan imapBackend.Update),
+		useOnlyActiveAddresses: useOnlyActiveAddresses,
-		lock:          sync.RWMutex{},
+		idleUpdates:            make(chan imapBackend.Update),
-		stopAll:       make(chan struct{}),
+		lock:                   sync.RWMutex{},
+		stopAll:                make(chan struct{}),
 	}
 
 	go func() {
@@ -291,9 +299,15 @@ func (u *Users) addNewUser(apiUser *pmapi.User, auth *pmapi.Auth, hashedPassphra
 		return errors.Wrap(err, "failed to update API user")
 	}
 
-	activeEmails := client.Addresses().ActiveEmails()
+	emails := []string{}
+	for _, address := range client.Addresses() {
+		if u.useOnlyActiveAddresses && address.Receive != pmapi.CanReceive {
+			continue
+		}
+		emails = append(emails, address.Email)
+	}
 
-	if _, err = u.credStorer.Add(apiUser.ID, apiUser.Name, auth.GenToken(), hashedPassphrase, activeEmails); err != nil {
+	if _, err = u.credStorer.Add(apiUser.ID, apiUser.Name, auth.GenToken(), hashedPassphrase, emails); err != nil {
 		return errors.Wrap(err, "failed to add user to credentials store")
 	}
 

internal/users/users_test.go

@@ -238,7 +238,7 @@ func testNewUsers(t *testing.T, m mocks) *Users { //nolint[unparam]
 	m.eventListener.EXPECT().Add(events.UpgradeApplicationEvent, gomock.Any())
 	m.clientManager.EXPECT().GetAuthUpdateChannel().Return(make(chan pmapi.ClientAuth))
 
-	users := New(m.config, m.PanicHandler, m.eventListener, m.clientManager, m.credentialsStore, m.storeMaker)
+	users := New(m.config, m.PanicHandler, m.eventListener, m.clientManager, m.credentialsStore, m.storeMaker, true)
 
 	waitForEvents()
 

pkg/config/config.go

@@ -228,6 +228,11 @@ func (c *Config) GetPreferencesPath() string {
 	return filepath.Join(c.appDirsVersion.UserCache(), "prefs.json")
 }
 
+// GetTransferDir returns folder for import/export rule and report files.
+func (c *Config) GetTransferDir() string {
+	return filepath.Join(c.appDirsVersion.UserCache())
+}
+
 // GetDefaultAPIPort returns default Bridge local API port.
 func (c *Config) GetDefaultAPIPort() int {
 	return 1042

pkg/message/body.go

@@ -55,7 +55,7 @@ func WriteAttachmentBody(w io.Writer, kr *crypto.KeyRing, m *pmapi.Message, att
 		dr = r
 		err = nil
 		att.Name += ".gpg"
-		att.MIMEType = "application/pgp-encrypted"
+		att.MIMEType = "application/pgp-encrypted" //nolint
 	} else if err != nil && err != openpgperrors.ErrSignatureExpired {
 		err = fmt.Errorf("cannot decrypt attachment: %v", err)
 		return

pkg/message/build.go

@@ -0,0 +1,348 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package message
+
+import (
+	"bytes"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"mime/multipart"
+	"mime/quotedprintable"
+	"net/textproto"
+
+	"github.com/ProtonMail/gopenpgp/crypto"
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/emersion/go-textwrapper"
+	openpgperrors "golang.org/x/crypto/openpgp/errors"
+)
+
+// Builder for converting PM message to RFC822. Builder will directly write
+// changes to message when fetching or building message.
+type Builder struct {
+	cl  pmapi.Client
+	msg *pmapi.Message
+
+	EncryptedToHTML bool
+	succDcrpt       bool
+}
+
+// NewBuilder initiated with client and message meta info.
+func NewBuilder(client pmapi.Client, message *pmapi.Message) *Builder {
+	return &Builder{cl: client, msg: message, EncryptedToHTML: true, succDcrpt: false}
+}
+
+// fetchMessage will update original PM message if successful
+func (bld *Builder) fetchMessage() (err error) {
+	if bld.msg.Body != "" {
+		return nil
+	}
+
+	complete, err := bld.cl.GetMessage(bld.msg.ID)
+	if err != nil {
+		return
+	}
+
+	*bld.msg = *complete
+
+	return
+}
+
+func (bld *Builder) writeMessageBody(w io.Writer) error {
+	if err := bld.fetchMessage(); err != nil {
+		return err
+	}
+
+	err := bld.WriteBody(w)
+	if err != nil {
+		_, _ = io.WriteString(w, "\r\n")
+		if bld.EncryptedToHTML {
+			_ = CustomMessage(bld.msg, err, true)
+		}
+		_, err = io.WriteString(w, bld.msg.Body)
+		_, _ = io.WriteString(w, "\r\n")
+	}
+
+	return err
+}
+
+func (bld *Builder) writeAttachmentBody(w io.Writer, att *pmapi.Attachment) error {
+	// Retrieve encrypted attachment
+	r, err := bld.cl.GetAttachment(att.ID)
+	if err != nil {
+		return err
+	}
+	defer r.Close() //nolint[errcheck]
+
+	if err := bld.WriteAttachmentBody(w, att, r); err != nil {
+		// Returning an error here makes e-mail clients like Thunderbird behave
+		// badly, trying to retrieve the message again and again
+		log.Warnln("Cannot write attachment body:", err)
+	}
+	return nil
+}
+
+func (bld *Builder) writeRelatedPart(p io.Writer, inlines []*pmapi.Attachment) error {
+	related := multipart.NewWriter(p)
+
+	_ = related.SetBoundary(GetRelatedBoundary(bld.msg))
+
+	buf := &bytes.Buffer{}
+	if err := bld.writeMessageBody(buf); err != nil {
+		return err
+	}
+
+	// Write the body part
+	h := GetBodyHeader(bld.msg)
+
+	var err error
+	if p, err = related.CreatePart(h); err != nil {
+		return err
+	}
+
+	_, _ = buf.WriteTo(p)
+
+	for _, inline := range inlines {
+		buf = &bytes.Buffer{}
+		if err = bld.writeAttachmentBody(buf, inline); err != nil {
+			return err
+		}
+
+		h := GetAttachmentHeader(inline)
+		if p, err = related.CreatePart(h); err != nil {
+			return err
+		}
+		_, _ = buf.WriteTo(p)
+	}
+
+	_ = related.Close()
+	return nil
+}
+
+// BuildMessage converts PM message to body structure (not RFC3501) and bytes
+// of RC822 message. If successful the original PM message will contain decrypted body.
+func (bld *Builder) BuildMessage() (structure *BodyStructure, message []byte, err error) { //nolint[funlen]
+	if err = bld.fetchMessage(); err != nil {
+		return nil, nil, err
+	}
+
+	bodyBuf := &bytes.Buffer{}
+
+	mainHeader := GetHeader(bld.msg)
+	mainHeader.Set("Content-Type", "multipart/mixed; boundary="+GetBoundary(bld.msg))
+	if err = WriteHeader(bodyBuf, mainHeader); err != nil {
+		return nil, nil, err
+	}
+	_, _ = io.WriteString(bodyBuf, "\r\n")
+
+	// NOTE: Do we really need extra encapsulation? i.e. Bridge-IMAP message is always multipart/mixed
+
+	if bld.msg.MIMEType == pmapi.ContentTypeMultipartMixed {
+		_, _ = io.WriteString(bodyBuf, "\r\n--"+GetBoundary(bld.msg)+"\r\n")
+		if err = bld.writeMessageBody(bodyBuf); err != nil {
+			return nil, nil, err
+		}
+		_, _ = io.WriteString(bodyBuf, "\r\n--"+GetBoundary(bld.msg)+"--\r\n")
+	} else {
+		mw := multipart.NewWriter(bodyBuf)
+		_ = mw.SetBoundary(GetBoundary(bld.msg))
+
+		var partWriter io.Writer
+		atts, inlines := SeparateInlineAttachments(bld.msg)
+
+		if len(inlines) > 0 {
+			relatedHeader := GetRelatedHeader(bld.msg)
+			if partWriter, err = mw.CreatePart(relatedHeader); err != nil {
+				return nil, nil, err
+			}
+			_ = bld.writeRelatedPart(partWriter, inlines)
+		} else {
+			buf := &bytes.Buffer{}
+			if err = bld.writeMessageBody(buf); err != nil {
+				return nil, nil, err
+			}
+
+			// Write the body part
+			bodyHeader := GetBodyHeader(bld.msg)
+			if partWriter, err = mw.CreatePart(bodyHeader); err != nil {
+				return nil, nil, err
+			}
+
+			_, _ = buf.WriteTo(partWriter)
+		}
+
+		// Write the attachments parts
+		for _, att := range atts {
+			buf := &bytes.Buffer{}
+			if err = bld.writeAttachmentBody(buf, att); err != nil {
+				return nil, nil, err
+			}
+
+			attachmentHeader := GetAttachmentHeader(att)
+			if partWriter, err = mw.CreatePart(attachmentHeader); err != nil {
+				return nil, nil, err
+			}
+
+			_, _ = buf.WriteTo(partWriter)
+		}
+
+		_ = mw.Close()
+	}
+
+	// wee need to copy buffer before building body structure
+	message = bodyBuf.Bytes()
+	structure, err = NewBodyStructure(bodyBuf)
+	return structure, message, err
+}
+
+// SuccessfullyDecrypted is true when message was fetched and decrypted successfully
+func (bld *Builder) SuccessfullyDecrypted() bool { return bld.succDcrpt }
+
+// WriteBody decrypts PM message and writes main body section. The external PGP
+// message is written as is (including attachments)
+func (bld *Builder) WriteBody(w io.Writer) error {
+	kr, err := bld.cl.KeyRingForAddressID(bld.msg.AddressID)
+	if err != nil {
+		return err
+	}
+	// decrypt body
+	if err := bld.msg.Decrypt(kr); err != nil && err != openpgperrors.ErrSignatureExpired {
+		return err
+	}
+	bld.succDcrpt = true
+	if bld.msg.MIMEType != pmapi.ContentTypeMultipartMixed {
+		// transfer encoding
+		qp := quotedprintable.NewWriter(w)
+		if _, err := io.WriteString(qp, bld.msg.Body); err != nil {
+			return err
+		}
+		return qp.Close()
+	}
+	_, err = io.WriteString(w, bld.msg.Body)
+	return err
+}
+
+// WriteAttachmentBody decrypts and writes the attachments
+func (bld *Builder) WriteAttachmentBody(w io.Writer, att *pmapi.Attachment, attReader io.Reader) (err error) {
+	kr, err := bld.cl.KeyRingForAddressID(bld.msg.AddressID)
+	if err != nil {
+		return err
+	}
+	// Decrypt it
+	var dr io.Reader
+	dr, err = att.Decrypt(attReader, kr)
+	if err == openpgperrors.ErrKeyIncorrect {
+		// Do not fail if attachment is encrypted with a different key
+		dr = attReader
+		err = nil
+		att.Name += ".gpg"
+		att.MIMEType = "application/pgp-encrypted"
+	} else if err != nil && err != openpgperrors.ErrSignatureExpired {
+		err = fmt.Errorf("cannot decrypt attachment: %v", err)
+		return err
+	}
+
+	// transfer encoding
+	ww := textwrapper.NewRFC822(w)
+	bw := base64.NewEncoder(base64.StdEncoding, ww)
+
+	var n int64
+	if n, err = io.Copy(bw, dr); err != nil {
+		err = fmt.Errorf("cannot write attachment: %v (wrote %v bytes)", err, n)
+	}
+
+	_ = bw.Close()
+	return err
+}
+
+func BuildEncrypted(m *pmapi.Message, readers []io.Reader, kr *crypto.KeyRing) ([]byte, error) { //nolint[funlen]
+	b := &bytes.Buffer{}
+
+	// Overwrite content for main header for import.
+	// Even if message has just simple body we should upload as multipart/mixed.
+	// Each part has encrypted body and header reflects the original header.
+	mainHeader := GetHeader(m)
+	mainHeader.Set("Content-Type", "multipart/mixed; boundary="+GetBoundary(m))
+	mainHeader.Del("Content-Disposition")
+	mainHeader.Del("Content-Transfer-Encoding")
+	if err := WriteHeader(b, mainHeader); err != nil {
+		return nil, err
+	}
+	mw := multipart.NewWriter(b)
+	if err := mw.SetBoundary(GetBoundary(m)); err != nil {
+		return nil, err
+	}
+
+	// Write the body part.
+	bodyHeader := make(textproto.MIMEHeader)
+	bodyHeader.Set("Content-Type", m.MIMEType+"; charset=utf-8")
+	bodyHeader.Set("Content-Disposition", "inline")
+	bodyHeader.Set("Content-Transfer-Encoding", "7bit")
+
+	p, err := mw.CreatePart(bodyHeader)
+	if err != nil {
+		return nil, err
+	}
+	// First, encrypt the message body.
+	if err := m.Encrypt(kr, kr); err != nil {
+		return nil, err
+	}
+	if _, err := io.WriteString(p, m.Body); err != nil {
+		return nil, err
+	}
+
+	// Write the attachments parts.
+	for i := 0; i < len(m.Attachments); i++ {
+		att := m.Attachments[i]
+		r := readers[i]
+		h := GetAttachmentHeader(att)
+		p, err := mw.CreatePart(h)
+		if err != nil {
+			return nil, err
+		}
+		// Create line wrapper writer.
+		ww := textwrapper.NewRFC822(p)
+
+		// Create base64 writer.
+		bw := base64.NewEncoder(base64.StdEncoding, ww)
+
+		data, err := ioutil.ReadAll(r)
+		if err != nil {
+			return nil, err
+		}
+
+		// Create encrypted writer.
+		pgpMessage, err := kr.Encrypt(crypto.NewPlainMessage(data), nil)
+		if err != nil {
+			return nil, err
+		}
+		if _, err := bw.Write(pgpMessage.GetBinary()); err != nil {
+			return nil, err
+		}
+		if err := bw.Close(); err != nil {
+			return nil, err
+		}
+	}
+
+	if err := mw.Close(); err != nil {
+		return nil, err
+	}
+
+	return b.Bytes(), nil
+}

pkg/message/message.go

@@ -23,8 +23,11 @@ import (
 	"strings"
 
 	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/sirupsen/logrus"
 )
 
+var log = logrus.WithField("pkg", "pkg/message") //nolint[gochecknoglobals]
+
 func GetBoundary(m *pmapi.Message) string {
 	// The boundary needs to be deterministic because messages are not supposed to
 	// change.

pkg/message/parser.go

@@ -37,7 +37,6 @@ import (
 	pmmime "github.com/ProtonMail/proton-bridge/pkg/mime"
 	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
 	"github.com/jaytaylor/html2text"
-	log "github.com/sirupsen/logrus"
 )
 
 func parseAttachment(filename string, mediaType string, h textproto.MIMEHeader) (att *pmapi.Attachment) {

pkg/message/utils.go

@@ -0,0 +1,85 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package message
+
+import (
+	"bytes"
+	"html/template"
+	"io"
+	"net/http"
+	"net/mail"
+	"net/textproto"
+
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+)
+
+func WriteHeader(w io.Writer, h textproto.MIMEHeader) (err error) {
+	if err = http.Header(h).Write(w); err != nil {
+		return
+	}
+	_, err = io.WriteString(w, "\r\n")
+	return
+}
+
+const customMessageTemplate = `
+<html>
+	<head></head>
+	<body style="font-family: Arial,'Helvetica Neue',Helvetica,sans-serif; font-size: 14px;">
+		<div style="color:#555; background-color:#cf9696; padding:20px; border-radius: 4px;">
+			<strong>Decryption error</strong><br/>
+			Decryption of this message's encrypted content failed.
+			<pre>{{.Error}}</pre>
+		</div>
+
+		{{if .AttachBody}}
+		<div style="color:#333; background-color:#f4f4f4;  border: 1px solid #acb0bf; border-radius: 2px; padding:1rem; margin:1rem 0; font-family:monospace; font-size: 1em;">
+			<pre>{{.Body}}</pre>
+		</div>
+		{{- end}}
+	</body>
+</html>
+`
+
+type customMessageData struct {
+	Error      string
+	AttachBody bool
+	Body       string
+}
+
+func CustomMessage(m *pmapi.Message, decodeError error, attachBody bool) error {
+	t := template.Must(template.New("customMessage").Parse(customMessageTemplate))
+
+	b := new(bytes.Buffer)
+
+	if err := t.Execute(b, customMessageData{
+		Error:      decodeError.Error(),
+		AttachBody: attachBody,
+		Body:       m.Body,
+	}); err != nil {
+		return err
+	}
+
+	m.MIMEType = pmapi.ContentTypeHTML
+	m.Body = b.String()
+
+	// NOTE: we need to set header in custom message header, so we check that is non-nil.
+	if m.Header == nil {
+		m.Header = make(mail.Header)
+	}
+	return nil
+}

pkg/pmapi/clientmanager.go

@@ -347,6 +347,14 @@ func (cm *ClientManager) CheckConnection() error {
 	return nil
 }
 
+// CheckConnection returns an error if there is no internet connection.
+func CheckConnection() error {
+	client := &http.Client{Timeout: time.Second * 10}
+	retStatus := make(chan error)
+	checkConnection(client, "http://protonstatus.com/vpn_status", retStatus)
+	return <-retStatus
+}
+
 func checkConnection(client *http.Client, url string, errorChannel chan error) {
 	resp, err := client.Get(url)
 	if err != nil {

pkg/pmapi/import.go

@@ -95,6 +95,11 @@ type ImportMsgReq struct {
 	LabelIDs []string
 }
 
+func (req ImportMsgReq) String() string {
+	data, _ := json.Marshal(req)
+	return string(data)
+}
+
 // ImportRes is a response to an import request.
 type ImportRes struct {
 	Res

pkg/updates/updates.go

@@ -120,7 +120,7 @@ func (u *Updates) CreateJSONAndSign(deployDir, goos string) error {
 	return nil
 }
 
-func (u *Updates) CheckIsBridgeUpToDate() (isUpToDate bool, latestVersion VersionInfo, err error) {
+func (u *Updates) CheckIsUpToDate() (isUpToDate bool, latestVersion VersionInfo, err error) {
 	localVersion := u.GetLocalVersion()
 	latestVersion, err = u.getLatestVersion()
 	if err != nil {

pkg/updates/updates_test.go

@@ -71,14 +71,14 @@ func startServer() {
 
 func TestCheckBridgeIsUpToDate(t *testing.T) {
 	updates := newTestUpdates("1.1.6")
-	isUpToDate, _, err := updates.CheckIsBridgeUpToDate()
+	isUpToDate, _, err := updates.CheckIsUpToDate()
 	require.NoError(t, err)
 	require.True(t, isUpToDate, "Bridge should be up to date")
 }
 
 func TestCheckBridgeIsNotUpToDate(t *testing.T) {
 	updates := newTestUpdates("1.1.5")
-	isUpToDate, _, err := updates.CheckIsBridgeUpToDate()
+	isUpToDate, _, err := updates.CheckIsUpToDate()
 	require.NoError(t, err)
 	require.True(t, !isUpToDate, "Bridge should not be up to date")
 }

utils/credits.sh

@@ -1,6 +1,5 @@
 #!/bin/bash
 
-
 # Copyright (c) 2020 Proton Technologies AG
 #
 # This file is part of ProtonMail Bridge.
@@ -20,6 +19,8 @@
 
 ## Generate credits from go.mod
 
+PACKAGE=$1
+
 # Vendor packages
 LOCKFILE=../go.mod
 egrep $'^\t[^=>]*$' $LOCKFILE  | sed -r 's/\t([^ ]*) v.*/\1/g' > tmp1
@@ -30,6 +31,6 @@ echo -e "\nFont Awesome 4.7.0\n\nQt 5.13 by Qt group\n" >> tmp
 # join lines
 sed -i -e ':a' -e 'N' -e '$!ba' -e 's|\n|;|g' tmp
 
-cat ../utils/license_header.txt > ../internal/bridge/credits.go
+cat ../utils/license_header.txt > ../internal/$PACKAGE/credits.go
-echo -e '// Code generated by '`echo $0`' at '`date`'. DO NOT EDIT.\n\npackage bridge\n\nconst Credits = "'$(cat tmp)'"' >> ../internal/bridge/credits.go
+echo -e '// Code generated by '`echo $0`' at '`date`'. DO NOT EDIT.\n\npackage '$PACKAGE'\n\nconst Credits = "'$(cat tmp)'"' >> ../internal/$PACKAGE/credits.go
 rm tmp1 tmp

utils/release-notes.sh

@@ -17,7 +17,8 @@
 # You should have received a copy of the GNU General Public License
 # along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
 
+PACKAGE=$1
 
 # Generate release notes information
-cat ../utils/license_header.txt > ../internal/bridge/release_notes.go
+cat ../utils/license_header.txt > ../internal/$PACKAGE/release_notes.go
-echo -e '// Code generated by '`echo $0`' at '`date`'. DO NOT EDIT.\n\npackage bridge\n\nconst ReleaseNotes = `'"$(cat ../release-notes/notes.txt)"'\n`\n\nconst ReleaseFixedBugs = `'"$(cat ../release-notes/bugs.txt)"'\n`' >> ../internal/bridge/release_notes.go
+echo -e "// Code generated by `echo $0` at '`date`'. DO NOT EDIT.\n\npackage ${PACKAGE}\n\nconst ReleaseNotes = \``cat ../release-notes/notes-${PACKAGE}.txt`\n\`\n\nconst ReleaseFixedBugs = \``cat ../release-notes/bugs-${PACKAGE}.txt`\n\`" >> ../internal/$PACKAGE/release_notes.go