Bridge GoldenGate 1.5.0

- Ensured better message flow by refactoring both address and date parsing
- Improved secure connectivity checks
- Better deb packaging
- More robust error handling

- Ensured that conversations are properly threaded
- Fixed Linux font issues (Fedora)
- Better handling of Mime encrypted messages

.gitignore

@@ -29,6 +29,7 @@ frontend/qml/*.qmlc
 # Build files
 bridge_darwin_*.tgz
 cmd/Desktop-Bridge/deploy
+cmd/Import-Export/deploy
 internal/frontend/qt*/moc.cpp
 internal/frontend/qt*/moc.go
 internal/frontend/qt*/moc.h
@@ -43,4 +44,4 @@ internal/frontend/rcc.qrc
 internal/frontend/rcc_cgo_*.go
 vendor-cache/
 
-/main.go
+/main.go

.gitlab-ci.yml

@@ -82,7 +82,9 @@ dependency-updates:
   script:
     - make build
   artifacts:
-    expire_in: 2 week
+    # Note: The latest artifacts for refs are locked against deletion, and kept regardless of the expiry time.
+    # Introduced in GitLab 13.0 behind a disabled feature flag, and made the default behavior in GitLab 13.4.
+    expire_in: 1 day
 
 build-linux:
   extends: .build-base
@@ -91,6 +93,17 @@ build-linux:
     paths:
       - bridge_*.tgz
 
+build-linux-qa:
+  extends: .build-base
+  only:
+    - web
+  script:
+    - BUILD_TAGS="build_qa pmapi_qa" make build
+  artifacts:
+    name: "bridge-linux-qa-$CI_COMMIT_SHORT_SHA"
+    paths:
+      - bridge_*.tgz
+
 build-ie-linux:
   extends: .build-base
   script:
@@ -124,6 +137,17 @@ build-darwin:
     paths:
       - bridge_*.tgz
 
+build-darwin-qa:
+  extends: .build-darwin-base
+  only:
+    - web
+  script:
+    - BUILD_TAGS="build_qa pmapi_qa" make build
+  artifacts:
+    name: "bridge-darwin-qa-$CI_COMMIT_SHORT_SHA"
+    paths:
+      - bridge_*.tgz
+
 build-ie-darwin:
   extends: .build-darwin-base
   script:
@@ -155,6 +179,23 @@ build-windows:
     paths:
       - bridge_*.tgz
 
+build-windows-qa:
+  extends: .build-windows-base
+  only:
+    - web
+  script:
+    # We need to install docker because qtdeploy builds for windows inside a docker container.
+    # Docker will connect to the dockerd daemon provided by the runner service docker:dind at tcp://docker:2375.
+    - curl -fsSL https://get.docker.com -o get-docker.sh && sh get-docker.sh
+    - apt-get update && apt-get -y install binutils-mingw-w64 tar gzip
+    - ln -s /usr/bin/x86_64-w64-mingw32-windres /usr/bin/windres
+    - go mod download
+    - TARGET_OS=windows BUILD_TAGS="build_qa pmapi_qa" make build
+  artifacts:
+    name: "bridge-windows-qa-$CI_COMMIT_SHORT_SHA"
+    paths:
+      - bridge_*.tgz
+
 build-ie-windows:
   extends: .build-windows-base
   script:

.golangci.yml

@@ -20,6 +20,9 @@ issues:
         - gochecknoglobals
         - gochecknoinits
         - gosec
+    - path: pkg/message/rfc5322
+      linters:
+        - dupl
 
 linters-settings:
   godox:

BUILDS.md

@@ -19,7 +19,6 @@ Otherwise, the sending of crash reports will be disabled.
 export MSYSTEM=
 ```
 
-
 ### Build Bridge
 * in project root run
 
@@ -44,6 +43,12 @@ make build-ie
     * for `windows`, the binary will have the file extension `.exe` (e.g `proton-bridge.exe`)
     * for `darwin`, the application will be created with name of the project directory (e.g `proton-bridge.app`)
 
+### Tags
+Note that repository contains both Bridge and Import-Export apps and they are
+not released together. Therefore, each app has own tag prefix. Bridge tags
+starts with `br-` and Import-Export tags starts with `ie-`. Both tags continue
+with semantic versioning `MAJOR.MINOR.PATCH`. An example of full tag is
+`br-1.4.4` or `ie-1.1.2` (current versions in October 2020).
 
 ## Useful tests, lints and checks
 In order to be able to run following commands please install the development dependencies: 

Changelog.md

@@ -4,12 +4,137 @@ Changelog [format](http://keepachangelog.com/en/1.0.0/)
 
 ## Unreleased
 
-## [IE 0.2.x] Congo
+### Changed
+* Updated go-mbox dependency back to upstream.
+
+### Fixed
+* GODT-847 Waiting for unilateral update during deleting the message.
+* GODT-849 Show in error counts in the end also lost messages.
+* GODT-835 Do not include conversation ID in references to show properly conversation threads in clients.
+
+## [IE 1.2.0] Elbe
+
+### Added
+* GODT-763 Detect Gmail labels from All Mail mbox export (using X-Gmail-Label header).
+* GODT-834 Info about tags in BUILDS.md and link to Import-Export page in README.md.
+* GODT-777 Support Apple Mail MBOX export format.
+
+### Fixed
+* GODT-677 Windows IE: global import settings not fit in window.
+* GODT-794 Congo fails to update to Danube
+* GODT-749 Don't force PGP/Inline when sending plaintext messages.
+* GODT-764 Fix deadlock in integration tests for Import-Export.
+* GODT-662 Do not resume paused transfer progress after dismissing cancel popup.
+* GODT-772 Sanitize mailbox names for exporting to follow OS restrictions.
+* GODT-771 Show fatal errors after export is terminated.
+* GODT-779 Do not propagate updates when progress is stopped.
+* GODT-779 Unpause progress during fatal error to properly stop progress.
+* GODT-779 Stop ongoing transfer calls sooner (re-check after import request is generated).
+* Fix measurement of uploading attachments during transfer.
+* GODT-827 Do not spam sentry with bad ID by integration test.
+* GODT-700 Fix UTF-7 incompatibility.
+* GODT-837 Fix flaky TestFailUnpauseAndStops.
+* GODT-782 Don't use TLS pinning when checking connectivity status.
+
+### Changed
+* TLS pins conform to official list.
+
+
+## [Bridge 1.4.5] Forth
+
+### Fixed
+* GODT-829 Remove `NoInferior` to display sub-folders in apple mail.
+
+## [Bridge 1.4.4] Forth
+
+### Fixed
+* GODT-798 Replace, don't add, transfer encoding when making body 7-bit clean.
+* Move/Copy duplicate for emails with References in Outlook
+* CSB-247 Cannot update from 1.4.0
+
+
+## [Bridge 1.4.3] Forth
+
+### Changed
+* Reverted sending IMAP updates to be not blocking again.
+
+### Fixed
+* GODT-783 Settings flags by FLAGS (not using +/-FLAGS) do not change spam state.
+
+
+## [Bridge 1.4.2] Forth
+
+### Changed
+* GODT-761 Use label.Path instead of Name to partially support subfolders for webapp beta release.
+* GODT-765 Improve speed of checking whether message is deleted.
+
+
+## [IE 1.1.2] Danube (beta 2020-09-xx)
+
+### Fixed
+* GODT-770 Better handling of extraneous end-of-mail indicator.
+* GODT-776 Fix crash when IMAP client connects while account is logging in.
+
+### Changed
+* Bump crypto version to v0.0.0-20200818122824-ed5d25e28db8
+* GODT-785 Clear separation of different message IDs in integration tests.
+### Changed
+* GODT-741 Import-Export shows "Unable to parse time" notice instead of zero time in error report window.
+
+* Bump crypto version to v0.0.0-20200818122824-ed5d25e28db8.
+* GODT-374 Allow to send calendar update multiple times.
+
+## [IE 1.1.1] Danube (beta 2020-09-xx) [Bridge 1.4.1] Forth (beta 2020-09-xx)
+
+### Fixed
+* GODT-752 Parsing message with empty addresses.
+* GODT-752 Parsing non-utf8 multipart/alternative message.
+* GODT-752 Parsing message with duplicate charset parameter.
+
+
+## [IE 1.1.0] Danube
+
+### Fixed
+* GODT-703 Import-Export showed always at least one total message.
+* GODT-738 Fix for mbox files with long lines.
+### Fixed
+* GODT-732 Do not mix font awesome icon with regular text to avoid issues on Fedora.
+
+
+## [Bridge 1.4.0] Forth
+
+### Added
+* GODT-682 Persistent anonymous API cookies for Import-Export.
+* GODT-357 Use go-message to make a better message parser.
+* GODT-720 Time measurement of progress for Import-Export.
+
+### Changed
+* GODT-511 User agent format changed.
+* Unsilent errors reading mbox files.
+* GODT-692 QA build with option to change API URL by ENV variable.
+* GODT-704 User agent detected by fake IMAP extension instead of AUTH callback (some clients use LOGIN instead of AUTH).
+* GODT-695 Parallel upload for ProtonMail target.
+
+### Removed
+* GODT-519 Unused AUTH scope parsing methods.
+
+### Fixed
+* GODT-698 Use correct package type for signed PGP/Inline messages.
+* Generic bug report window title.
+* Fix missing check for unencrypted recipients during sending.
+* Version checking for catalina.
+* GODT-730 Limit maximal TLS version for Yahoo IMAP server.
+
+
+## [IE 1.0.x] Congo (v1.0.0 live 2020-09-08)
 
 ### Added
 * GODT-633 Persistent anonymous API cookies for better load balancing and abuse detection. 
+* GODT-461 Add support for `\Deleted` flag.
 
 ### Changed
+* GODT-462 Pausing event loop while FETCHing to prevent EXPUNGE
+* Wait for unilateral response to be delivered
 * GODT-409 Set flags have to replace all flags.
 * GODT-531 Better way to add trusted certificate in macOS.
 * Bumped golangci-lint to v1.29.0
@@ -36,6 +161,8 @@ Changelog [format](http://keepachangelog.com/en/1.0.0/)
     * golang.org/x/text v0.3.2 -> v0.3.3
 * Set first-start to false in bridge, not in frontend.
 * GODT-400 Refactor sendingInfo.
+* GODT-513 Update routes to API v4.
+* GODT-551 Do not ignore errors during message flagging.
 * GODT-380 Adding IE GUI to Bridge repo and building
     * BR: extend functionality of PopupDialog
     * BR: makefile APP_VERSION instead of BRIDGE_VERSION
@@ -49,11 +176,14 @@ Changelog [format](http://keepachangelog.com/en/1.0.0/)
     * IE: Removed `onLoginFinished`
     * Structure for transfer rules in QML
 * GODT-213 Convert panics from message parser to error.
+* GODT-585 Do not allow deleting messages from All Mail.
 
 ### Fixed
 * GODT-655 Fix date picker with automatic Windows DST
 * GODT-454 Fix send on closed channel when receiving unencrypted send confirmation from GUI.
 * GODT-597 Duplicate sending when draft creation takes too long
+* GODT-634 Hover on links in popups.
+
 
 
 ## [v1.3.x] Emma (v1.3.2 beta 2020-08-04, v1.3.3 beta 2020-08-06, v1.3.3 live 2020-08-12)

Makefile

@@ -9,8 +9,9 @@ TARGET_OS?=${GOOS}
 ## Build
 .PHONY: build build-ie build-nogui build-ie-nogui check-has-go
 
-BRIDGE_APP_VERSION?=1.4.0-git
+# Keep version hardcoded so app build works also without Git repository.
-IE_APP_VERSION?=1.0.0-git
+BRIDGE_APP_VERSION?=1.5.0-git
+IE_APP_VERSION?=1.2.0-git
 APP_VERSION:=${BRIDGE_APP_VERSION}
 SRC_ICO:=logo.ico
 SRC_ICNS:=Bridge.icns
@@ -56,7 +57,6 @@ ifeq "${TARGET_CMD}" "Import-Export"
     TGZ_TARGET:=ie_${TARGET_OS}_${REVISION}.tgz
 endif
 
-
 build: ${TGZ_TARGET}
 build-ie:
 	TARGET_CMD=Import-Export $(MAKE) build
@@ -264,7 +264,6 @@ run-ie-qt:
 run-ie-nogui:
 	TARGET_CMD=Import-Export $(MAKE) run-nogui
 
-
 clean-frontend-qt:
 	$(MAKE) -C internal/frontend/qt -f Makefile.local clean
 clean-frontend-qt-ie:
@@ -281,3 +280,8 @@ clean: clean-vendor
 	rm -rf cmd/Import-Export/deploy
 	rm -f build last.log mem.pprof main.go
 	rm -rf logo.ico icon.rc icon_windows.syso internal/frontend/qt/icon_windows.syso
+
+.PHONY: generate
+generate:
+	go generate ./...
+	$(MAKE) add-license

README.md

@@ -1,7 +1,7 @@
 # ProtonMail Bridge and Import Export app
 Copyright (c) 2020 Proton Technologies AG
 
-This repository holds the ProtonMail Bridge application.
+This repository holds the ProtonMail Bridge and the ProtonMail Import-Export applications.
 For a detailed build information see [BUILDS](./BUILDS.md).
 For licensing information see [COPYING](./COPYING.md).
 For contribution policy see [CONTRIBUTING](./CONTRIBUTING.md).
@@ -35,6 +35,8 @@ configure transfer rules (match source and target mailboxes, set time
 range limits and so on) and hit start. Once the transfer is complete,
 check the results.
 
+More details [on the public website](https://protonmail.com/import-export).
+
 ## Keychain
 You need to have a keychain in order to run the ProtonMail Bridge. On Mac or
 Windows, Bridge uses native credential managers. On Linux, use

cmd/Import-Export/main.go

@@ -21,9 +21,11 @@ import (
 	"runtime/pprof"
 
 	"github.com/ProtonMail/proton-bridge/internal/cmd"
+	"github.com/ProtonMail/proton-bridge/internal/cookies"
 	"github.com/ProtonMail/proton-bridge/internal/events"
 	"github.com/ProtonMail/proton-bridge/internal/frontend"
 	"github.com/ProtonMail/proton-bridge/internal/importexport"
+	"github.com/ProtonMail/proton-bridge/internal/preferences"
 	"github.com/ProtonMail/proton-bridge/internal/updates"
 	"github.com/ProtonMail/proton-bridge/internal/users/credentials"
 	"github.com/ProtonMail/proton-bridge/pkg/config"
@@ -36,6 +38,10 @@ import (
 )
 
 const (
+	// cacheVersion is used for cache files such as lock, or preferences.
+	// Different number will drop old files and create new ones.
+	cacheVersion = "c11"
+
 	appName     = "importExport"
 	appNameDash = "import-export-app"
 )
@@ -58,7 +64,7 @@ func main() {
 // IMPORTANT: ***Read the comments before CHANGING the order ***
 func run(context *cli.Context) (contextError error) { // nolint[funlen]
 	// We need to have config instance to setup a logs, panic handler, etc ...
-	cfg := config.New(appName, constants.Version, constants.Revision, "")
+	cfg := config.New(appName, constants.Version, constants.Revision, cacheVersion)
 
 	// We want to know about any problem. Our PanicHandler calls sentry which is
 	// not dependent on anything else. If that fails, it tries to create crash
@@ -132,6 +138,16 @@ func run(context *cli.Context) (contextError error) { // nolint[funlen]
 	// implementation depending on whether build flag pmapi_prod is used or not.
 	cm.SetRoundTripper(cfg.GetRoundTripper(cm, eventListener))
 
+	pref := preferences.New(cfg)
+
+	// Cookies must be persisted across restarts.
+	jar, err := cookies.NewCookieJar(pref)
+	if err != nil {
+		logrus.WithError(err).Warn("Could not create cookie jar")
+	} else {
+		cm.SetCookieJar(jar)
+	}
+
 	importexportInstance := importexport.New(cfg, panicHandler, eventListener, cm, credentialsStore)
 
 	// Decide about frontend mode before initializing rest of import-export.

go.mod

@@ -13,16 +13,18 @@ require (
 
 require (
 	github.com/0xAX/notificator v0.0.0-20191016112426-3962a5ea8da1
+	github.com/Masterminds/semver/v3 v3.1.0
 	github.com/ProtonMail/go-appdir v1.1.0
 	github.com/ProtonMail/go-apple-mobileconfig v0.0.0-20160701194735-7ea9927a11f6
 	github.com/ProtonMail/go-autostart v0.0.0-20181114175602-c5272053443a
 	github.com/ProtonMail/go-imap-id v0.0.0-20190926060100-f94a56b9ecde
 	github.com/ProtonMail/go-vcard v0.0.0-20180326232728-33aaa0a0c8a5
 	github.com/ProtonMail/gopenpgp/v2 v2.0.1
+	github.com/PuerkitoBio/goquery v1.5.1
 	github.com/abiosoft/ishell v2.0.0+incompatible
 	github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db // indirect
 	github.com/allan-simon/go-singleinstance v0.0.0-20160830203053-79edcfdc2dfc
-	github.com/andybalholm/cascadia v1.2.0
+	github.com/antlr/antlr4 v0.0.0-20201020194047-0a7eaede42b0
 	github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 // indirect
 	github.com/chzyer/logex v1.1.10 // indirect
 	github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 // indirect
@@ -34,8 +36,8 @@ require (
 	github.com/emersion/go-imap-quota v0.0.0-20200423100218-dcfd1b7d2b41
 	github.com/emersion/go-imap-specialuse v0.0.0-20200722111535-598ff00e4075
 	github.com/emersion/go-imap-unselect v0.0.0-20171113212723-b985794e5f26
-	github.com/emersion/go-mbox v1.0.0
+	github.com/emersion/go-mbox v1.0.2
-	github.com/emersion/go-message v0.11.1
+	github.com/emersion/go-message v0.12.1-0.20200903165315-e1abe21f389a
 	github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21
 	github.com/emersion/go-textwrapper v0.0.0-20160606182133-d0e65e56babe
 	github.com/emersion/go-vcard v0.0.0-20190105225839-8856043f13c5 // indirect
@@ -46,11 +48,9 @@ require (
 	github.com/golang/mock v1.4.4
 	github.com/google/go-cmp v0.5.1
 	github.com/google/uuid v1.1.1
-	github.com/go-delve/delve v1.4.1 // indirect
 	github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c // indirect
 	github.com/hashicorp/go-multierror v1.1.0
 	github.com/jaytaylor/html2text v0.0.0-20200412013138-3577fbdbcff7
-	github.com/jhillyerd/enmime v0.8.1
 	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0
 	github.com/keybase/go-keychain v0.0.0-20200502122510-cda31fe0c86d
 	github.com/logrusorgru/aurora v2.0.3+incompatible
@@ -60,13 +60,11 @@ require (
 	github.com/nsf/jsondiff v0.0.0-20200515183724-f29ed568f4ce
 	github.com/olekukonko/tablewriter v0.0.4 // indirect
 	github.com/pkg/errors v0.9.1
-	github.com/psampaz/go-mod-outdated v0.6.0 // indirect
 	github.com/sirupsen/logrus v1.6.0
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
+	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
 	github.com/stretchr/testify v1.6.1
 	github.com/therecipe/qt v0.0.0-20200701200531-7f61353ee73e
-	github.com/therecipe/qt/internal/binding/files/docs/5.12.0 v0.0.0-20200603231648-26cdb75b6f22 // indirect
-	github.com/therecipe/qt/internal/binding/files/docs/5.13.0 v0.0.0-20200603231648-26cdb75b6f22 // indirect
 	github.com/twinj/uuid v1.0.0 // indirect
 	github.com/urfave/cli v1.22.4
 	go.etcd.io/bbolt v1.3.5
@@ -77,8 +75,8 @@ require (
 
 replace (
 	github.com/docker/docker-credential-helpers => github.com/ProtonMail/docker-credential-helpers v1.1.0
-	github.com/emersion/go-imap => github.com/jameshoulahan/go-imap v0.0.0-20200728140727-d57327f48843
+	github.com/emersion/go-imap => github.com/ProtonMail/go-imap v0.0.0-20201016095853-a7520cc904d3
 	github.com/emersion/go-smtp => github.com/ProtonMail/go-smtp v0.0.0-20181206232543-8261df20d309
 	github.com/jameskeane/bcrypt => github.com/ProtonMail/bcrypt v0.0.0-20170924085257-7509ea014998
-	golang.org/x/crypto => github.com/ProtonMail/crypto v0.0.0-20200416114516-1fa7f403fb9c
+	golang.org/x/crypto => github.com/ProtonMail/crypto v0.0.0-20200818122824-ed5d25e28db8
 )

go.sum

@@ -1,10 +1,12 @@
 github.com/0xAX/notificator v0.0.0-20191016112426-3962a5ea8da1 h1:j9HaafapDbPbGRDku6e/HRs6KBMcKHiWcm1/9Sbxnl4=
 github.com/0xAX/notificator v0.0.0-20191016112426-3962a5ea8da1/go.mod h1:NtXa9WwQsukMHZpjNakTTz0LArxvGYdPA9CjIcUSZ6s=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/Masterminds/semver/v3 v3.1.0 h1:Y2lUDsFKVRSYGojLJ1yLxSXdMmMYTYls0rCvoqmMUQk=
+github.com/Masterminds/semver/v3 v3.1.0/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
 github.com/ProtonMail/bcrypt v0.0.0-20170924085257-7509ea014998 h1:YT2uVwQiRQZxCaaahwfcgTq2j3j66w00n/27gb/zubs=
 github.com/ProtonMail/bcrypt v0.0.0-20170924085257-7509ea014998/go.mod h1:HecWFHognK8GfRDGnFQbW/LiV7A3MX3gZVs45vk5h8I=
-github.com/ProtonMail/crypto v0.0.0-20200416114516-1fa7f403fb9c h1:DAvlgde2Stu18slmjwikiMPs/CKPV35wSvmJS34z0FU=
+github.com/ProtonMail/crypto v0.0.0-20200818122824-ed5d25e28db8 h1:u1j0xLTrCHpNS40B6m4Sv3IVUz5m9jt+AnTIopT3IgM=
-github.com/ProtonMail/crypto v0.0.0-20200416114516-1fa7f403fb9c/go.mod h1:Pxr7w4gA2ikI4sWyYwEffm+oew1WAJHzG1SiDpQMkrI=
+github.com/ProtonMail/crypto v0.0.0-20200818122824-ed5d25e28db8/go.mod h1:Pxr7w4gA2ikI4sWyYwEffm+oew1WAJHzG1SiDpQMkrI=
 github.com/ProtonMail/docker-credential-helpers v1.1.0 h1:+kvUIpwWcbtP3WFv5sSvkFn/XLzSqPOB5AAthuk9xPk=
 github.com/ProtonMail/docker-credential-helpers v1.1.0/go.mod h1:mK0aBveCxhnQ756AmaTfXMZDeULvheYVhF/MWMErN5g=
 github.com/ProtonMail/go-appdir v1.1.0 h1:9hdNDlU9kTqRKVNzmoqah8qqrj5QZyLByQdwQNlFWig=
@@ -13,6 +15,8 @@ github.com/ProtonMail/go-apple-mobileconfig v0.0.0-20160701194735-7ea9927a11f6 h
 github.com/ProtonMail/go-apple-mobileconfig v0.0.0-20160701194735-7ea9927a11f6/go.mod h1:EtDfBMIDWmVe4viZCuBTEfe3OIIo0ghbpOaAZVO+hVg=
 github.com/ProtonMail/go-autostart v0.0.0-20181114175602-c5272053443a h1:fXK2KsfnkBV9Nh+9SKzHchYjuE9s0vI20JG1mbtEAcc=
 github.com/ProtonMail/go-autostart v0.0.0-20181114175602-c5272053443a/go.mod h1:oTGdE7/DlWIr23G0IKW3OXK9wZ5Hw1GGiaJFccTvZi4=
+github.com/ProtonMail/go-imap v0.0.0-20201016095853-a7520cc904d3 h1:Jvv9t3rSg/ID3Fh+uYsxgmvNI9fYnlab4vtBsbPtmq8=
+github.com/ProtonMail/go-imap v0.0.0-20201016095853-a7520cc904d3/go.mod h1:yKASt+C3ZiDAiCSssxg9caIckWF/JG7ZQTO7GAmvicU=
 github.com/ProtonMail/go-imap-id v0.0.0-20190926060100-f94a56b9ecde h1:5koQozTDELymYOyFbQ/VSubexAEXzDR8qGM5mO8GRdw=
 github.com/ProtonMail/go-imap-id v0.0.0-20190926060100-f94a56b9ecde/go.mod h1:795VPXcRUIQ9JyMNHP4el582VokQfippgjkQP3Gk0r0=
 github.com/ProtonMail/go-mime v0.0.0-20190923161245-9b5a4261663a h1:W6RrgN/sTxg1msqzFFb+G80MFmpjMw61IU+slm+wln4=
@@ -23,25 +27,24 @@ github.com/ProtonMail/go-vcard v0.0.0-20180326232728-33aaa0a0c8a5 h1:Uga1DHFN4GU
 github.com/ProtonMail/go-vcard v0.0.0-20180326232728-33aaa0a0c8a5/go.mod h1:oeP9CMN+ajWp5jKp1kue5daJNwMMxLF+ujPaUIoJWlA=
 github.com/ProtonMail/gopenpgp/v2 v2.0.1 h1:x0uvDhry5WzoHeJO4J3dgMLhG4Z9PeBJ2O+sDOY0LcU=
 github.com/ProtonMail/gopenpgp/v2 v2.0.1/go.mod h1:wQQCJo7DURO6S9VwH+kSDEYs/B63yZnAEfGlOg8YNBY=
+github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
+github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
 github.com/abiosoft/ishell v2.0.0+incompatible h1:zpwIuEHc37EzrsIYah3cpevrIc8Oma7oZPxr03tlmmw=
 github.com/abiosoft/ishell v2.0.0+incompatible/go.mod h1:HQR9AqF2R3P4XXpMpI0NAzgHf/aS6+zVXRj14cVk9qg=
 github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db h1:CjPUSXOiYptLbTdr1RceuZgSFDQ7U15ITERUGrUORx8=
 github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db/go.mod h1:rB3B4rKii8V21ydCbIzH5hZiCQE7f5E9SzUb/ZZx530=
 github.com/allan-simon/go-singleinstance v0.0.0-20160830203053-79edcfdc2dfc h1:mZca0/HZ/XWXP9txkfdl2GH6mUzBqAlyJz3u5Lg8fuA=
 github.com/allan-simon/go-singleinstance v0.0.0-20160830203053-79edcfdc2dfc/go.mod h1:qqsTQiwdyqxU05iDCsi0oN3P4nrVxAmn8xCtODDSf/U=
-github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5h18aE=
+github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
-github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY=
+github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
-github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a h1:MISbI8sU/PSK/ztvmWKFcI7UGb5/HQT7B+i3a2myKgI=
+github.com/antlr/antlr4 v0.0.0-20201020194047-0a7eaede42b0 h1:7RW94Pqb4Twsfpz42ALQ+sD0cUUpN8HF4uzKyQf2D8Y=
-github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a/go.mod h1:2GxOXOlEPAMFPfp014mK1SWq8G8BN8o7/dfYqJrVGn8=
+github.com/antlr/antlr4 v0.0.0-20201020194047-0a7eaede42b0/go.mod h1:T7PbCXFs94rrTttyxjbyT5+/1V8T2TYDejxUfHJjw1Y=
 github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 h1:JLaf/iINcLyjwbtTsCJjc6rtlASgHeIJPrB6QmwURnA=
 github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
 github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/cosiner/argv v0.0.0-20170225145430-13bacc38a0a5/go.mod h1:p/NrK5tF6ICIly4qwEDsf6VDirFiWWz0FenfYBwJaKQ=
-github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk=
-github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
@@ -64,11 +67,11 @@ github.com/emersion/go-imap-specialuse v0.0.0-20200722111535-598ff00e4075 h1:z8T
 github.com/emersion/go-imap-specialuse v0.0.0-20200722111535-598ff00e4075/go.mod h1:/nybxhI8kXom8Tw6BrHMl42usALvka6meORflnnYwe4=
 github.com/emersion/go-imap-unselect v0.0.0-20171113212723-b985794e5f26 h1:FiSb8+XBQQSkcX3ubr+1tAtlRJBYaFmRZqOAweZ9Wy8=
 github.com/emersion/go-imap-unselect v0.0.0-20171113212723-b985794e5f26/go.mod h1:+gnnZx3Mg3MnCzZrv0eZdp5puxXQUgGT/6N6L7ShKfM=
-github.com/emersion/go-mbox v1.0.0 h1:HN6aKbyqmgIfK9fS/gen+NRr2wXLSxZXWfdAIAnzQPc=
+github.com/emersion/go-mbox v1.0.2 h1:tE/rT+lEugK9y0myEymCCHnwlZN04hlXPrbKkxRBA5I=
-github.com/emersion/go-mbox v1.0.0/go.mod h1:Yp9IVuuOYLEuMv4yjgDHvhb5mHOcYH6x92Oas3QqEZI=
+github.com/emersion/go-mbox v1.0.2/go.mod h1:Yp9IVuuOYLEuMv4yjgDHvhb5mHOcYH6x92Oas3QqEZI=
-github.com/emersion/go-message v0.11.1 h1:0C/S4JIXDTSfXB1vpqdimAYyK4+79fgEAMQ0dSL+Kac=
 github.com/emersion/go-message v0.11.1/go.mod h1:C4jnca5HOTo4bGN9YdqNQM9sITuT3Y0K6bSUw9RklvY=
-github.com/emersion/go-sasl v0.0.0-20191210011802-430746ea8b9b h1:uhWtEWBHgop1rqEk2klKaxPAkVDCXexai6hSuRQ7Nvs=
+github.com/emersion/go-message v0.12.1-0.20200903165315-e1abe21f389a h1:3C6qIGgPr1qAT0ikRD5NbyKpME/iHCDeXhpv/JJsFsE=
+github.com/emersion/go-message v0.12.1-0.20200903165315-e1abe21f389a/go.mod h1:kYIioST9GDHte9/BRWgi93rpqbDuFftMjKSMaXS8ABo=
 github.com/emersion/go-sasl v0.0.0-20191210011802-430746ea8b9b/go.mod h1:G/dpzLu16WtQpBfQ/z3LYiYJn3ZhKSGWn83fyoyQe/k=
 github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 h1:OJyUGMJTzHTd1XQp98QTaHernxMYzRaOasRir9hUlFQ=
 github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ=
@@ -84,25 +87,12 @@ github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JY
 github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
 github.com/go-resty/resty/v2 v2.3.0 h1:JOOeAvjSlapTT92p8xiS19Zxev1neGikoHsXJeOq8So=
 github.com/go-resty/resty/v2 v2.3.0/go.mod h1:UpN9CgLZNsv4e9XG50UU8xdI0F43UQ4HmxLBDwaroHU=
-github.com/go-delve/delve v1.4.1 h1:kZs0umEv+VKnK84kY9/ZXWrakdLTeRTyYjFdgLelZCQ=
-github.com/go-delve/delve v1.4.1/go.mod h1:vmy6iObn7zg8FQ5KOCIe6TruMNsqpoZO8uMiRea+97k=
-github.com/go-resty/resty/v2 v2.2.0 h1:vgZ1cdblp8Aw4jZj3ZsKh6yKAlMg3CHMrqFSFFd+jgY=
-github.com/go-resty/resty/v2 v2.2.0/go.mod h1:nYW/8rxqQCmI3bPz9Fsmjbr2FBjGuR2Mzt6kDh3zZ7w=
-github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw=
-github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
-github.com/gogs/chardet v0.0.0-20150115103509-2404f7772561 h1:aBzukfDxQlCTVS0NBUjI5YA3iVeaZ9Tb5PxNrrIP1xs=
-github.com/gogs/chardet v0.0.0-20150115103509-2404f7772561/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14=
 github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc=
 github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
 github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/golang/mock v1.4.3 h1:GV+pQPG/EUUbkh47niozDcADz6go/dUwhVzdUQHIVRw=
-github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
-github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-dap v0.2.0/go.mod h1:5q8aYQFnHOAZEMP+6vmq25HKYAEwE+LF5yh7JKrrhSQ=
 github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4=
 github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
@@ -110,24 +100,14 @@ github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/U
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI=
 github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
-github.com/jameshoulahan/go-imap v0.0.0-20200728140727-d57327f48843 h1:suxlO4AC4E4bjueAsL0m+qp8kmkxRWMGj+5bBU/KJ8g=
-github.com/jameshoulahan/go-imap v0.0.0-20200728140727-d57327f48843/go.mod h1:yKASt+C3ZiDAiCSssxg9caIckWF/JG7ZQTO7GAmvicU=
-github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
-github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
-github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
-github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/jaytaylor/html2text v0.0.0-20190408195923-01ec452cbe43/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk=
 github.com/jaytaylor/html2text v0.0.0-20200412013138-3577fbdbcff7 h1:g0fAGBisHaEQ0TRq1iBvemFRf+8AEWEmBESSiWB3Vsc=
 github.com/jaytaylor/html2text v0.0.0-20200412013138-3577fbdbcff7/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk=
-github.com/jhillyerd/enmime v0.8.1 h1:Kz4xj3sJJ4Ju8e+w/7v9H4Matv5ijPgv7UkhPf+C15I=
-github.com/jhillyerd/enmime v0.8.1/go.mod h1:MBHs3ugk03NGjMM6PuRynlKf+HA5eSillZ+TRCm73AE=
 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA=
 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
 github.com/keybase/go-keychain v0.0.0-20200502122510-cda31fe0c86d h1:gVjhBCfVGl32RIBooOANzfw+0UqX8HU+yPlMv8vypcg=
 github.com/keybase/go-keychain v0.0.0-20200502122510-cda31fe0c86d/go.mod h1:W6EbaYmb4RldPn0N3gvVHjY1wmU59kbymhW9NATWhwY=
 github.com/keybase/go.dbus v0.0.0-20200324223359-a94be52c0b03/go.mod h1:a8clEhrrGV/d76/f9r2I41BwANMihfZYV9C223vaxqE=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -137,61 +117,41 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
 github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
-github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs=
-github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
 github.com/martinlindhe/base36 v1.0.0 h1:eYsumTah144C0A8P1T/AVSUk5ZoLnhfYFM3OGQxB52A=
 github.com/martinlindhe/base36 v1.0.0/go.mod h1:+AtEs8xrBpCeYgSLoY/aJ6Wf37jtBuR0s35750M27+8=
-github.com/mattn/go-colorable v0.0.0-20170327083344-ded68f7a9561/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
-github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM=
 github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
-github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
-github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/miekg/dns v1.1.30 h1:Qww6FseFn8PRfw07jueqIXqodm0JKiiKuK0DeXSqfyo=
 github.com/miekg/dns v1.1.30/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
-github.com/miekg/dns v1.1.29 h1:xHBEhR+t5RzcFJjBLJlax2daXOrTYtr9z4WdKEfWFzg=
-github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/myesui/uuid v1.0.0 h1:xCBmH4l5KuvLYc5L7AS7SZg9/jKdIFubM7OVoLqaQUI=
 github.com/myesui/uuid v1.0.0/go.mod h1:2CDfNgU0LR8mIdO8vdWd8i9gWWxLlcoIGGpSNgafq84=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nsf/jsondiff v0.0.0-20200515183724-f29ed568f4ce h1:RPclfga2SEJmgMmz2k+Mg7cowZ8yv4Trqw9UsJby758=
 github.com/nsf/jsondiff v0.0.0-20200515183724-f29ed568f4ce/go.mod h1:uFMI8w+ref4v2r9jz+c9i1IfIttS/OkmLfrk1jne5hs=
-github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88=
-github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8=
 github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
-github.com/peterh/liner v0.0.0-20170317030525-88609521dc4b/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/psampaz/go-mod-outdated v0.6.0 h1:DXS6rdsz4rpezbPsckQflqrYSEBvsF5GAmUWP+UvnQo=
-github.com/psampaz/go-mod-outdated v0.6.0/go.mod h1:r78NYWd1z+F9Zdsfy70svgXOz363B08BWnTyFSgEESs=
-github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
-github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
-github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
 github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
-github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA=
 github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog=
-github.com/spf13/cobra v0.0.0-20170417170307-b6cb39589372/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
-github.com/spf13/pflag v0.0.0-20170417173400-9e4c21054fa1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo=
 github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -201,40 +161,25 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/therecipe/qt v0.0.0-20200701200531-7f61353ee73e h1:G0DQ/TRQyrEZjtLlLwevFjaRiG8eeCMlq9WXQ2OO2bk=
 github.com/therecipe/qt v0.0.0-20200701200531-7f61353ee73e/go.mod h1:SUUR2j3aE1z6/g76SdD6NwACEpvCxb3fvG82eKbD6us=
-github.com/therecipe/qt v0.0.0-20200126204426-5074eb6d8c41 h1:yBVcrpbaQYJBdKT2pxTdlL4hBE/eM4UPcyj9YpyvSok=
-github.com/therecipe/qt v0.0.0-20200126204426-5074eb6d8c41/go.mod h1:SUUR2j3aE1z6/g76SdD6NwACEpvCxb3fvG82eKbD6us=
-github.com/therecipe/qt v0.0.0-20200603231648-26cdb75b6f22 h1:UrNr8EZueA1eREFmG5gVHBeeOuwW2GbzI9VfdB5uK+c=
-github.com/therecipe/qt/internal/binding/files/docs v0.0.0-20191019224306-1097424d656c h1:/VhcwU7WuFEVgDHZ9V8PIYAyYqQ6KNxFUjBMOf2aFZM=
-github.com/therecipe/qt/internal/binding/files/docs/5.12.0 v0.0.0-20200603231648-26cdb75b6f22 h1:FumuOkCw78iheUI3eIYhAgtsj/0HQBAib/jXk1cslJw=
-github.com/therecipe/qt/internal/binding/files/docs/5.12.0 v0.0.0-20200603231648-26cdb75b6f22/go.mod h1:7m8PDYDEtEVqfjoUQc2UrFqhG0CDmoVJjRlQxexndFc=
-github.com/therecipe/qt/internal/binding/files/docs/5.13.0 v0.0.0-20200603231648-26cdb75b6f22 h1:aYzTBQ/hC6FtbaRnyylxlhbSGMPnyD5lAzVO3Ae6emA=
-github.com/therecipe/qt/internal/binding/files/docs/5.13.0 v0.0.0-20200603231648-26cdb75b6f22/go.mod h1:mH55Ek7AZcdns5KPp99O0bg+78el64YCYWHiQKrOdt4=
 github.com/twinj/uuid v1.0.0 h1:fzz7COZnDrXGTAOHGuUGYd6sG+JMq+AoE7+Jlu0przk=
 github.com/twinj/uuid v1.0.0/go.mod h1:mMgcE1RHFUFqe5AfiwlINXisXfDGro23fWdPUfOMjRY=
 github.com/urfave/cli v1.22.4 h1:u7tSpNPPswAFymm8IehJhy4uJMlUuU/GmqSkvJ1InXA=
 github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
 go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
-github.com/urfave/cli v1.22.3 h1:FpNT6zq26xNpHZy08emi755QwzLPs6Pukqjlc7RfOMU=
-github.com/urfave/cli v1.22.3/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
-go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
-go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.starlark.net v0.0.0-20190702223751-32f345186213/go.mod h1:c1/X6cHgvdXj6pUlmWKMkuqRnW4K8x2vwt6JAaaircg=
-golang.org/x/arch v0.0.0-20190927153633-4e8777c89be4/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190420063019-afa5a82059c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
@@ -246,41 +191,28 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20191127201027-ecd32218bd7f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191216052735-49a3e744a425 h1:VvQyQJN0tSuecqgcIxMWnnfG5kSmgy9KZR9sW3W5QeA=
 golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/stretchr/testify.v1 v1.2.2 h1:yhQC6Uy5CqibAIlk1wlusa/MJ3iAN49/BsR/dCCKz3M=
 gopkg.in/stretchr/testify.v1 v1.2.2/go.mod h1:QI5V/q6UbPmuhtm10CaFZxED9NreB8PnFYN9JcR6TxU=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
-rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
-rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

internal/bridge/credits.go

@@ -15,8 +15,8 @@
 // You should have received a copy of the GNU General Public License
 // along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
 
-// Code generated by ./credits.sh at Fri 04 Sep 2020 01:57:36 PM CEST. DO NOT EDIT.
+// Code generated by ./credits.sh at Wed Nov 4 12:24:36 PM CET 2020. DO NOT EDIT.
 
 package bridge
 
-const Credits = "github.com/0xAX/notificator;github.com/abiosoft/ishell;github.com/abiosoft/readline;github.com/allan-simon/go-singleinstance;github.com/andybalholm/cascadia;github.com/certifi/gocertifi;github.com/chzyer/logex;github.com/chzyer/test;github.com/cucumber/godog;github.com/docker/docker-credential-helpers;github.com/emersion/go-imap;github.com/emersion/go-imap-appendlimit;github.com/emersion/go-imap-idle;github.com/emersion/go-imap-move;github.com/emersion/go-imap-quota;github.com/emersion/go-imap-specialuse;github.com/emersion/go-imap-unselect;github.com/emersion/go-mbox;github.com/emersion/go-message;github.com/emersion/go-sasl;github.com/emersion/go-smtp;github.com/emersion/go-textwrapper;github.com/emersion/go-vcard;github.com/fatih/color;github.com/flynn-archive/go-shlex;github.com/getsentry/raven-go;github.com/go-delve/delve;github.com/golang/mock;github.com/google/go-cmp;github.com/google/uuid;github.com/gopherjs/gopherjs;github.com/go-resty/resty/v2;github.com/hashicorp/go-multierror;github.com/jameshoulahan/go-imap;github.com/jameskeane/bcrypt;github.com/jaytaylor/html2text;github.com/jhillyerd/enmime;github.com/kardianos/osext;github.com/keybase/go-keychain;github.com/logrusorgru/aurora;github.com/mattn/go-runewidth;github.com/miekg/dns;github.com/myesui/uuid;github.com/nsf/jsondiff;github.com/olekukonko/tablewriter;github.com/pkg/errors;github.com/ProtonMail/bcrypt;github.com/ProtonMail/crypto;github.com/ProtonMail/docker-credential-helpers;github.com/ProtonMail/go-appdir;github.com/ProtonMail/go-apple-mobileconfig;github.com/ProtonMail/go-autostart;github.com/ProtonMail/go-imap-id;github.com/ProtonMail/gopenpgp/v2;github.com/ProtonMail/go-smtp;github.com/ProtonMail/go-vcard;github.com/psampaz/go-mod-outdated;github.com/sirupsen/logrus;github.com/skratchdot/open-golang;github.com/stretchr/testify;github.com/therecipe/qt;github.com/twinj/uuid;github.com/urfave/cli;go.etcd.io/bbolt;golang.org/x/crypto;golang.org/x/net;golang.org/x/text;gopkg.in/stretchr/testify.v1;;Font Awesome 4.7.0;;Qt 5.13 by Qt group;"
+const Credits = "github.com/0xAX/notificator;github.com/abiosoft/ishell;github.com/abiosoft/readline;github.com/allan-simon/go-singleinstance;github.com/antlr/antlr4;github.com/certifi/gocertifi;github.com/chzyer/logex;github.com/chzyer/test;github.com/cucumber/godog;github.com/docker/docker-credential-helpers;github.com/emersion/go-imap;github.com/emersion/go-imap-appendlimit;github.com/emersion/go-imap-idle;github.com/emersion/go-imap-move;github.com/emersion/go-imap-quota;github.com/emersion/go-imap-specialuse;github.com/emersion/go-imap-unselect;github.com/emersion/go-mbox;github.com/emersion/go-message;github.com/emersion/go-sasl;github.com/emersion/go-smtp;github.com/emersion/go-textwrapper;github.com/emersion/go-vcard;github.com/fatih/color;github.com/flynn-archive/go-shlex;github.com/getsentry/raven-go;github.com/golang/mock;github.com/google/go-cmp;github.com/google/uuid;github.com/gopherjs/gopherjs;github.com/go-resty/resty/v2;github.com/hashicorp/go-multierror;github.com/jameskeane/bcrypt;github.com/jaytaylor/html2text;github.com/kardianos/osext;github.com/keybase/go-keychain;github.com/logrusorgru/aurora;github.com/Masterminds/semver/v3;github.com/mattn/go-runewidth;github.com/miekg/dns;github.com/myesui/uuid;github.com/nsf/jsondiff;github.com/olekukonko/tablewriter;github.com/pkg/errors;github.com/ProtonMail/bcrypt;github.com/ProtonMail/crypto;github.com/ProtonMail/docker-credential-helpers;github.com/ProtonMail/go-appdir;github.com/ProtonMail/go-apple-mobileconfig;github.com/ProtonMail/go-autostart;github.com/ProtonMail/go-imap;github.com/ProtonMail/go-imap-id;github.com/ProtonMail/gopenpgp/v2;github.com/ProtonMail/go-smtp;github.com/ProtonMail/go-vcard;github.com/PuerkitoBio/goquery;github.com/sirupsen/logrus;github.com/skratchdot/open-golang;github.com/ssor/bom;github.com/stretchr/testify;github.com/therecipe/qt;github.com/twinj/uuid;github.com/urfave/cli;go.etcd.io/bbolt;golang.org/x/crypto;golang.org/x/net;golang.org/x/text;gopkg.in/stretchr/testify.v1;;Font Awesome 4.7.0;;Qt 5.13 by Qt group;"

internal/bridge/release_notes.go

@@ -15,17 +15,17 @@
 // You should have received a copy of the GNU General Public License
 // along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
 
-// Code generated by ./release-notes.sh at 'Fri 04 Sep 2020 01:57:36 PM CEST'. DO NOT EDIT.
+// Code generated by ./release-notes.sh at 'Wed Nov  4 12:24:35 PM CET 2020'. DO NOT EDIT.
 
 package bridge
 
-const ReleaseNotes = `• Improvements to Alternative Routing: Version two of this feature is now more resilient to unstable internet connections, which results in a smoother experience using this feature. Also includes fixes to previous implementation of Alternative Routing when first starting the application or when turning it off.
+const ReleaseNotes = `• Ensured better message flow by refactoring both address and date parsing
-• Email parsing improvements: Improved detection of email encodings embedded in html/xml in addition to message header; add a fallback option if encoding is not specified and decoding as UTF8 fails (ISO-8859-1) ; tweaked logic of parsing "References" header.
+• Improved secure connectivity checks
-• User interaction improvements: Some smaller improvements in specific cases to make the interaction with Proton Bridge clearer for the user
+• Better deb packaging
-• Code updates & maintenance: Migrated to GopenPGP v2, updates to GoIMAPv1, increased bbolt version to 1.3.5 and various improvements regarding extensibility and maintainability for upcoming work.
+• More robust error handling
-• General stability improvements: Improvements to the behavior of the application under various unstable internet conditions.
 `
 
-const ReleaseFixedBugs = `• Fixed a slew of smaller bugs and some conditions which could cause the application to crash.
+const ReleaseFixedBugs = `• Ensured that conversations are properly threaded
-• The full changelog can be found at https://github.com/ProtonMail/proton-bridge/blob/master/Changelog.md
+• Fixed Linux font issues (Fedora)
+• Better handling of Mime encrypted messages
 `

internal/frontend/cli-ie/accounts.go

@@ -79,7 +79,7 @@ func (f *frontendCLI) loginAccount(c *ishell.Context) { // nolint[funlen]
 			return
 		}
 
-		_, err = client.Auth2FA(twoFactor, auth)
+		err = client.Auth2FA(twoFactor, auth)
 		if err != nil {
 			f.processAPIError(err)
 			return

internal/frontend/cli/accounts.go

@@ -126,7 +126,7 @@ func (f *frontendCLI) loginAccount(c *ishell.Context) { // nolint[funlen]
 			return
 		}
 
-		_, err = client.Auth2FA(twoFactor, auth)
+		err = client.Auth2FA(twoFactor, auth)
 		if err != nil {
 			f.processAPIError(err)
 			return

internal/frontend/qml/BridgeUI/MainWindow.qml

@@ -354,7 +354,7 @@ Window {
                 } else {
                     return qsTr('A new version of Bridge is available.<br>
                     Check <a href="%1">release notes</a> to learn what is new in %2.<br>
-                    You can continue with the update or download and install new version manually from<br><br>
+                    You can continue with the update or download and install the new version manually from<br><br>
                     <a href="%3">%3</a>',
                     "Message for update in Win/Mac").arg("releaseNotes").arg(go.newversion).arg(go.landingPage)
                 }

internal/frontend/qml/BridgeUI/SettingsView.qml

@@ -48,6 +48,7 @@ Item {
                     text : qsTr("Clear", "clickable link next to clear cache button in settings")
                     color: Style.main.text
                     font {
+                        family : cacheClear.font.family // use default font, not font-awesome
                         pointSize : Style.settings.fontSize * Style.pt
                         underline : true
                     }
@@ -66,6 +67,7 @@ Item {
                     text : qsTr("Clear", "clickable link next to clear keychain button in settings")
                     color: Style.main.text
                     font {
+                        family : cacheKeychain.font.family // use default font, not font-awesome
                         pointSize : Style.settings.fontSize * Style.pt
                         underline : true
                     }
@@ -125,6 +127,7 @@ Item {
                     text : qsTr("Change", "clickable link next to change ports button in settings")
                     color: Style.main.text
                     font {
+                        family : changePort.font.family // use default font, not font-awesome
                         pointSize : Style.settings.fontSize * Style.pt
                         underline : true
                     }

internal/frontend/qml/GuiIE.qml

@@ -276,6 +276,10 @@ Item {
                 winMain.dialogExport.hide()
             }
         }
+
+        onUpdateFinished : {
+            winMain.dialogUpdate.finished(hasError)
+        }
     }
 
     function folderIcon(folderName, folderType) { // translations

internal/frontend/qml/ImportExportUI/DialogExport.qml

@@ -217,7 +217,10 @@ Dialog {
                 Text {
                     anchors.centerIn: parent
                     text: {
-                        if (progressbarExport.isFinished) return qsTr("Export finished","todo")
+                        if (progressbarExport.isFinished) {
+                            if (go.progressDescription=="") return qsTr("Export finished","todo")
+                            else return qsTr("Export failed: %1").arg(go.progressDescription)
+                        }
                         if (
                             go.progressDescription == gui.enums.progressInit || 
                             (go.progress==0 && go.description=="")
@@ -450,7 +453,6 @@ Dialog {
             errorPopup.hide()
         }
         onClickedNo  : {
-            go.resumeProcess()
             errorPopup.hide()
         }
     }

internal/frontend/qml/ImportExportUI/DialogImport.qml

@@ -279,9 +279,8 @@ Dialog {
             titleTo        : root.address
         }
 
-        Rectangle {
+        Column {
             id: masterImportSettings
-            height: 150 // fixme
             anchors {
                 right  : parent.right
                 left   : parent.left
@@ -291,45 +290,47 @@ Dialog {
                 rightMargin  : Style.main.leftMargin
                 bottomMargin : Style.main.bottomMargin
             }
-            color: Style.dialog.background
 
-            Text {
+            spacing: Style.main.bottomMargin
-                id: labelMasterImportSettings
-                text: qsTr("Master import settings:")
 
-                font {
+            Row {
-                    bold: true
+                spacing: masterImportSettings.width - labelMasterImportSettings.width - resetSourceButton.width
-                    family: Style.fontawesome.name
-                    pointSize: Style.main.fontSize * Style.pt
-                }
-                color: Style.main.text
 
-                InfoToolTip {
+                Text {
-                    info: qsTr(
+                    id: labelMasterImportSettings
-                        "If master import date range is selected only emails within this range will be imported, unless it is specified differently in folder date range.",
+                    text: qsTr("Master import settings:")
-                        "Text in master import settings tooltip."
+
-                    )
+                    font {
-                    anchors {
+                        bold: true
-                        left: parent.right
+                        family: Style.fontawesome.name
-                        bottom: parent.bottom
+                        pointSize: Style.main.fontSize * Style.pt
-                        leftMargin : Style.dialog.leftMargin
+                    }
+                    color: Style.main.text
+
+                    InfoToolTip {
+                        anchors {
+                            left: parent.right
+                            bottom: parent.bottom
+                            leftMargin : Style.dialog.leftMargin
+                        }
+                        info: qsTr(
+                            "If master import date range is selected only emails within this range will be imported, unless it is specified differently in folder date range.",
+                            "Text in master import settings tooltip."
+                        )
                     }
                 }
-            }
 
-            // Reset all to default
+                // Reset all to default
-            ClickIconText {
+                ClickIconText {
-                anchors {
+                    id: resetSourceButton
-                    right: parent.right
+                    text:qsTr("Reset all settings to default")
-                    bottom: labelMasterImportSettings.bottom
+                    iconText: Style.fa.refresh
-                }
+                    textColor: Style.main.textBlue
-                text:qsTr("Reset all settings to default")
+                    onClicked: {
-                iconText: Style.fa.refresh
+                        go.resetSource()
-                textColor: Style.main.textBlue
+                        root.decrementCurrentIndex()
-                onClicked: {
+                        timer.start()
-                    go.resetSource()
+                    }
-                    root.decrementCurrentIndex()
-                    timer.start()
                 }
             }
 
@@ -348,49 +349,40 @@ Dialog {
 
             InlineDateRange {
                 id: globalDateRange
-                anchors {
-                    left      : parent.left
-                    top       : line.bottom
-                    topMargin : Style.dialog.topMargin
-                }
             }
 
             // Add global label (inline)
             InlineLabelSelect {
                 id: globalLabels
-                anchors {
+            }
-                    left      : parent.left
+        }
-                    top       : globalDateRange.bottom
+
-                    topMargin : Style.dialog.topMargin
+        // Buttons
-                }
+        Row {
-                //labelWidth : globalDateRange.labelWidth
+            spacing: Style.dialog.spacing
+            anchors {
+                right: parent.right
+                bottom: parent.bottom
+                rightMargin: Style.main.leftMargin
+                bottomMargin: Style.main.bottomMargin
             }
 
-            // Buttons
+            ButtonRounded {
-            Row {
+                id: buttonCancelThree
-                spacing: Style.dialog.spacing
+                fa_icon    : Style.fa.times
-                anchors{
+                text       : qsTr("Cancel", "todo")
-                    bottom       : parent.bottom
+                color_main : Style.dialog.textBlue
-                    right        : parent.right
+                onClicked  : root.cancel()
-                }
+            }
 
-                ButtonRounded {
+            ButtonRounded {
-                    id: buttonCancelThree
+                id: buttonNextThree
-                    fa_icon    : Style.fa.times
+                fa_icon     : Style.fa.check
-                    text       : qsTr("Cancel", "todo")
+                text        : qsTr("Import", "todo")
-                    color_main : Style.dialog.textBlue
+                color_main  : Style.dialog.background
-                    onClicked  : root.cancel()
+                color_minor : Style.dialog.textBlue
-                }
+                isOpaque    : true
-
+                onClicked   : root.okay()
-                ButtonRounded {
-                    id: buttonNextThree
-                    fa_icon     : Style.fa.check
-                    text        : qsTr("Import", "todo")
-                    color_main  : Style.dialog.background
-                    color_minor : Style.dialog.textBlue
-                    isOpaque    : true
-                    onClicked   : root.okay()
-                }
             }
         }
     }
@@ -483,18 +475,30 @@ Dialog {
                 }
             }
 
-            Text {
+            Row {
                 property int fails: go.progressFails
                 visible: fails > 0
-                color : Style.main.textRed
-                font.family: Style.fontawesome.name
-                font.pointSize: Style.main.fontSize * Style.pt
                 anchors.horizontalCenter: parent.horizontalCenter
-                text: Style.fa.exclamation_circle + " " + (
+
-                    fails == 1 ?
+                Text {
-                    qsTr("%1 message failed to be imported").arg(fails) :
+                    color: Style.main.textRed
-                    qsTr("%1 messages failed to be imported").arg(fails)
+                    font {
-                )
+                        pointSize : Style.dialog.fontSize * Style.pt
+                        family    : Style.fontawesome.name
+                    }
+                    text: Style.fa.exclamation_circle
+                }
+
+                Text {
+                    property int fails: go.progressFails
+                    color: Style.main.textRed
+                    font.pointSize: Style.main.fontSize * Style.pt
+                    text: " " + (
+                        fails == 1 ?
+                        qsTr("%1 message failed to be imported").arg(fails) :
+                        qsTr("%1 messages failed to be imported").arg(fails)
+                    )
+                }
             }
 
             Row { // buttons
@@ -575,12 +579,23 @@ Dialog {
             anchors.centerIn : finalReport
             spacing          : Style.dialog.heightSeparator
 
-            Text {
+            Row {
-                text: go.progressDescription!="" ? qsTr("Import failed: %1").arg(go.progressDescription) : Style.fa.check_circle + " " + qsTr("Import completed successfully")
                 anchors.horizontalCenter: parent.horizontalCenter
-                color: go.progressDescription!="" ? Style.main.textRed : Style.main.textGreen
+
-                font.bold : true
+                Text {
-                font.family: Style.fontawesome.name
+                    font {
+                        pointSize: Style.dialog.fontSize * Style.pt
+                        family: Style.fontawesome.name
+                    }
+                    color: Style.main.textGreen
+                    text: go.progressDescription!="" ? "" : Style.fa.check_circle
+                }
+
+                Text {
+                    text: go.progressDescription!="" ? qsTr("Import failed: %1").arg(go.progressDescription) : " " + qsTr("Import completed successfully")
+                    color: go.progressDescription!="" ? Style.main.textRed : Style.main.textGreen
+                    font.bold : true
+                }
             }
 
             Text {
@@ -773,11 +788,6 @@ Dialog {
             errorPopup.hide()
         }
         onClickedNo  : {
-            if (errorPopup.msgID == "ask_send_report") {
-                errorPopup.hide()
-                return
-            }
-            go.resumeProcess()
             errorPopup.hide()
         }
 

internal/frontend/qml/ImportExportUI/HelpView.qml

@@ -74,7 +74,7 @@ Item {
                 )
                 width: wrapper.width
                 color : Style.transparent
-                Text {
+                AccessibleText {
                     id: aboutText
                     anchors {
                         bottom: parent.bottom
@@ -82,8 +82,8 @@ Item {
                     }
                     color: Style.main.textDisabled
                     horizontalAlignment: Qt.AlignHCenter
-                    font.family : Style.fontawesome.name
+                    font.pointSize :  Style.main.fontSize * Style.pt
-                    text: "ProtonMail Import-Export app Version "+go.getBackendVersion()+"\n"+Style.fa.copyright + " 2020 Proton Technologies AG"
+                    text: "ProtonMail Import-Export app Version "+go.getBackendVersion()+"\n© 2020 Proton Technologies AG"
                 }
             }
 

internal/frontend/qml/ImportExportUI/MainWindow.qml

@@ -274,15 +274,15 @@ Window {
                 }
             } else {
                 if (go.goos=="linux") {
-                    return qsTr('New version of %1 is available.<br>
+                    return qsTr('A new version of %1 is available.<br>
                     Check <a href="%2">release notes</a> to learn what is new in %3.<br>
-                    Use your package manager to update or download and install new version manually from<br><br>
+                    Use your package manager to update or download and install new the version manually from<br><br>
                     <a href="%4">%4</a>',
                     "Message for update in Linux").arg(go.programTitle).arg("releaseNotes").arg(go.newversion).arg(go.landingPage)
                 } else {
-                    return qsTr('New version of %1 is available.<br>
+                    return qsTr('A new version of %1 is available.<br>
                     Check <a href="%2">release notes</a> to learn what is new in %3.<br>
-                    You can continue with update or download and install new version manually from<br><br>
+                    You can continue with update or download and install new the version manually from<br><br>
                     <a href="%4">%4</a>',
                     "Message for update in Win/Mac").arg(go.programTitle).arg("releaseNotes").arg(go.newversion).arg(go.landingPage)
                 }

internal/frontend/qml/ImportExportUI/SelectFolderMenu.qml

@@ -42,34 +42,50 @@ ComboBox {
         root.below = popup.y>0
     }
 
-    contentItem : Text {
+    contentItem : Row {
         id: boxText
-        verticalAlignment: Text.AlignVCenter
-        font {
-            family: Style.fontawesome.name
-            pointSize : Style.dialog.fontSize * Style.pt
-            bold: root.down
-        }
-        elide: Text.ElideRight
-        textFormat: Text.StyledText
 
-        text : root.displayText
+        Text {
-        color: !root.enabled ? Style.main.textDisabled : ( root.down ? Style.main.background : Style.main.text )
+            anchors.verticalCenter: parent.verticalCenter
+            font {
+                pointSize: Style.dialog.fontSize * Style.pt
+                family: Style.fontawesome.name
+            }
+            text: {
+                if (view.currentIndex >= 0) {
+                    if (!root.isFolderType) {
+                        return Style.fa.tags + " "
+                    }
+                    var tgtIcon = view.currentItem.folderIcon
+                    var tgtColor = view.currentItem.folderColor
+                    if (tgtIcon != Style.fa.folder_open) {
+                        return tgtIcon + " "
+                    }
+                    return '<font color="'+tgtColor+'">'+ tgtIcon + "</font> "
+                }
+                return ""
+            }
+            color: !root.enabled ? Style.main.textDisabled : ( root.down ? Style.main.background : Style.main.text )
+        }
+
+        Text {
+            anchors.verticalCenter: parent.verticalCenter
+            font {
+                pointSize : Style.dialog.fontSize * Style.pt
+                bold: root.down
+            }
+            elide: Text.ElideRight
+            textFormat: Text.StyledText
+
+            text : root.displayText
+            color: !root.enabled ? Style.main.textDisabled : ( root.down ? Style.main.background : Style.main.text )
+        }
     }
 
     displayText: {
         if (view.currentIndex >= 0) {
-            if (!root.isFolderType) return Style.fa.tags + " " + qsTr("Add/Remove labels")
+            if (!root.isFolderType) return qsTr("Add/Remove labels")
-
+            return view.currentItem.folderName
-            var tgtName = view.currentItem.folderName
-            var tgtIcon = view.currentItem.folderIcon
-            var tgtColor = view.currentItem.folderColor
-
-            if (tgtIcon != Style.fa.folder_open) {
-                return tgtIcon + " " + tgtName
-            }
-
-            return '<font color="'+tgtColor+'">'+ tgtIcon + "</font> " + tgtName
         }
         if (root.isFolderType) return qsTr("No folder selected")
         return qsTr("No labels selected")

internal/frontend/qml/ImportExportUI/SettingsView.qml

@@ -44,6 +44,7 @@ Item {
                     text : qsTr("Clear")
                     color: Style.main.text
                     font {
+                        family : cacheKeychain.font.family // use default font, not font-awesome
                         pointSize : Style.settings.fontSize * Style.pt
                         underline : true
                     }

internal/frontend/qml/ProtonUI/BubbleNote.qml

@@ -106,6 +106,7 @@ Rectangle {
             }
             MouseArea {
                 anchors.fill: mainText
+                cursorShape: mainText.hoveredLink ? Qt.PointingHandCursor : Qt.ArrowCursor
                 acceptedButtons: Qt.NoButton
             }
 

internal/frontend/qml/ProtonUI/BugReportWindow.qml

@@ -39,7 +39,7 @@ Window {
 
     color   : "transparent"
     flags   : Qt.Window | Qt.Dialog | Qt.FramelessWindowHint
-    title   : "ProtonMail Bridge - Bug report"
+    title   : "Bug report"
     visible : false
 
     WindowTitleBar {
@@ -84,7 +84,7 @@ Window {
             height: content.height - (
                 (clientVersion.visible ?  clientVersion.height + Style.dialog.fontSize : 0) +
                 userAddress.height  + Style.dialog.fontSize +
-                securityNote.contentHeight + Style.dialog.fontSize +
+                securityNoteText.contentHeight + Style.dialog.fontSize +
                 cancelButton.height + Style.dialog.fontSize
             )
             clip: true
@@ -215,7 +215,7 @@ Window {
         }
 
         // Note
-        AccessibleText {
+        Row {
             id: securityNote
             anchors {
                 left: parent.left
@@ -223,14 +223,32 @@ Window {
                 top: userAddress.bottom
                 topMargin: Style.dialog.fontSize
             }
-            wrapMode: Text.Wrap
+
-            color: Style.dialog.text
+            Text {
-            font.pointSize : Style.dialog.fontSize * Style.pt
+                id: securityNoteIcon
-            text:
+                font {
-            "<span style='font-family: " + Style.fontawesome.name + "'>" + Style.fa.exclamation_triangle + "</span> " +
+                    pointSize : Style.dialog.fontSize * Style.pt
-            qsTr("Bug reports are not end-to-end encrypted!", "The first part of warning in bug report form") + " " +
+                    family    : Style.fontawesome.name
-            qsTr("Please do not send any sensitive information.", "The second part of warning in bug report form") + " " +
+                }
-            qsTr("Contact us at security@protonmail.com for critical security issues.", "The third part of warning in bug report form")
+                color: Style.dialog.text
+                text  : Style.fa.exclamation_triangle
+            }
+
+            AccessibleText {
+                id: securityNoteText
+                anchors {
+                    left: securityNoteIcon.right
+                    leftMargin: 5 * Style.pt
+                    right: parent.right
+                }
+                wrapMode: Text.Wrap
+                color: Style.dialog.text
+                font.pointSize : Style.dialog.fontSize * Style.pt
+                text:
+                qsTr("Bug reports are not end-to-end encrypted!", "The first part of warning in bug report form") + " " +
+                qsTr("Please do not send any sensitive information.", "The second part of warning in bug report form") + " " +
+                qsTr("Contact us at security@protonmail.com for critical security issues.", "The third part of warning in bug report form")
+            }
         }
 
         // buttons

internal/frontend/qt-common/accounts.go

@@ -208,7 +208,7 @@ func (a *Accounts) Auth2FA(twoFacAuth string) int {
 	if a.auth == nil || a.authClient == nil {
 		err = fmt.Errorf("missing authentication in auth2FA %p %p", a.auth, a.authClient)
 	} else {
-		_, err = a.authClient.Auth2FA(twoFacAuth, a.auth)
+		err = a.authClient.Auth2FA(twoFacAuth, a.auth)
 	}
 
 	if a.showLoginError(err, "auth2FA") {

internal/frontend/qt-ie/error_list.go

@@ -72,6 +72,9 @@ func (e *ErrorListModel) data(index *core.QModelIndex, role int) *core.QVariant
 	case MailSubject:
 		return qtcommon.NewQVariantString(r.Subject)
 	case MailDate:
+		if r.Time.IsZero() {
+			return qtcommon.NewQVariantString("Unavailable")
+		}
 		return qtcommon.NewQVariantString(r.Time.String())
 	case MailFrom:
 		return qtcommon.NewQVariantString(r.From)

internal/frontend/qt-ie/frontend.go

@@ -338,9 +338,7 @@ func (f *FrontendQt) setProgressManager(progress *transfer.Progress) {
 				break
 			}
 			failed, imported, _, _, total := progress.GetCounts()
-			if total != 0 {
+			f.Qml.SetTotal(int(total))
-				f.Qml.SetTotal(int(total))
-			}
 			f.Qml.SetProgressFails(int(failed))
 			f.Qml.SetProgressDescription(progress.PauseReason())
 			if total > 0 {
@@ -351,6 +349,9 @@ func (f *FrontendQt) setProgressManager(progress *transfer.Progress) {
 				}
 			}
 		}
+		// Counts will add lost messages only once the progress is completeled.
+		failed, _, _, _, _ := progress.GetCounts()
+		f.Qml.SetProgressFails(int(failed))
 
 		if err := progress.GetFatalError(); err != nil {
 			f.Qml.SetProgressDescription(err.Error())

internal/frontend/qt/accounts.go

@@ -164,7 +164,7 @@ func (s *FrontendQt) auth2FA(twoFacAuth string) int {
 	if s.auth == nil || s.authClient == nil {
 		err = fmt.Errorf("missing authentication in auth2FA %p %p", s.auth, s.authClient)
 	} else {
-		_, err = s.authClient.Auth2FA(twoFacAuth, s.auth)
+		err = s.authClient.Auth2FA(twoFacAuth, s.auth)
 	}
 
 	if s.showLoginError(err, "auth2FA") {

internal/imap/backend.go

@@ -23,7 +23,6 @@ import (
 	"sync"
 	"time"
 
-	imapid "github.com/ProtonMail/go-imap-id"
 	"github.com/ProtonMail/proton-bridge/internal/bridge"
 	"github.com/ProtonMail/proton-bridge/internal/events"
 	"github.com/ProtonMail/proton-bridge/pkg/listener"
@@ -45,9 +44,6 @@ type imapBackend struct {
 	users       map[string]*imapUser
 	usersLocker sync.Locker
 
-	lastMailClient       imapid.ID
-	lastMailClientLocker sync.Locker
-
 	imapCache     map[string]map[string]string
 	imapCachePath string
 	imapCacheLock *sync.RWMutex
@@ -87,9 +83,6 @@ func newIMAPBackend(
 		users:       map[string]*imapUser{},
 		usersLocker: &sync.Mutex{},
 
-		lastMailClient:       imapid.ID{imapid.FieldName: clientNone},
-		lastMailClientLocker: &sync.Mutex{},
-
 		imapCachePath: cfg.GetIMAPCachePath(),
 		imapCacheLock: &sync.RWMutex{},
 	}
@@ -164,7 +157,9 @@ func (ib *imapBackend) Login(_ *imap.ConnInfo, username, password string) (goIMA
 
 	if err := imapUser.user.CheckBridgeLogin(password); err != nil {
 		log.WithError(err).Error("Could not check bridge password")
-		_ = imapUser.Logout()
+		if err := imapUser.Logout(); err != nil {
+			log.WithError(err).Warn("Could not logout user after unsuccessful login check")
+		}
 		// Apple Mail sometimes generates a lot of requests very quickly.
 		// It's therefore good to have a timeout after a bad login so that we can slow
 		// those requests down a little bit.
@@ -192,23 +187,6 @@ func (ib *imapBackend) CreateMessageLimit() *uint32 {
 	return nil
 }
 
-func (ib *imapBackend) setLastMailClient(id imapid.ID) {
-	ib.lastMailClientLocker.Lock()
-	defer ib.lastMailClientLocker.Unlock()
-
-	if name, ok := id[imapid.FieldName]; ok && ib.lastMailClient[imapid.FieldName] != name {
-		ib.lastMailClient = imapid.ID{}
-		for k, v := range id {
-			ib.lastMailClient[k] = v
-		}
-		log.Warn("Mail Client ID changed to ", ib.lastMailClient)
-		ib.bridge.SetCurrentClient(
-			ib.lastMailClient[imapid.FieldName],
-			ib.lastMailClient[imapid.FieldVersion],
-		)
-	}
-}
-
 // monitorDisconnectedUsers removes users when it receives a close connection event for them.
 func (ib *imapBackend) monitorDisconnectedUsers() {
 	ch := make(chan string)

internal/imap/backend_cache.go

@@ -80,7 +80,7 @@ func (ib *imapBackend) removeFromCache(userID, label, toRemove string) {
 
 func (ib *imapBackend) getCacheList(userID, label string) (list string) {
 	if err := ib.loadIMAPCache(); err != nil {
-		log.Warn("Could not load cache: ", err)
+		log.WithError(err).Warn("Could not load cache")
 	}
 
 	ib.imapCacheLock.Lock()
@@ -97,7 +97,9 @@ func (ib *imapBackend) getCacheList(userID, label string) (list string) {
 
 	ib.imapCacheLock.Unlock()
 
-	_ = ib.saveIMAPCache()
+	if err := ib.saveIMAPCache(); err != nil {
+		log.WithError(err).Warn("Could not save cache")
+	}
 	return
 }
 

internal/imap/bridge.go

@@ -76,5 +76,9 @@ func newBridgeUserWrap(bridgeUser *users.User) *bridgeUserWrap {
 }
 
 func (u *bridgeUserWrap) GetStore() storeUserProvider {
-	return newStoreUserWrap(u.User.GetStore())
+	store := u.User.GetStore()
+	if store == nil {
+		return nil
+	}
+	return newStoreUserWrap(store)
 }

internal/imap/id/extension.go

@@ -0,0 +1,90 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package id
+
+import (
+	imapid "github.com/ProtonMail/go-imap-id"
+	imapserver "github.com/emersion/go-imap/server"
+)
+
+type currentClientSetter interface {
+	SetCurrentClient(name, version string)
+}
+
+// Extension for IMAP server
+type extension struct {
+	extID  imapserver.ConnExtension
+	setter currentClientSetter
+}
+
+func (ext *extension) Capabilities(conn imapserver.Conn) []string {
+	return ext.extID.Capabilities(conn)
+}
+
+func (ext *extension) Command(name string) imapserver.HandlerFactory {
+	newIDHandler := ext.extID.Command(name)
+	if newIDHandler == nil {
+		return nil
+	}
+	return func() imapserver.Handler {
+		if hdlrID, ok := newIDHandler().(*imapid.Handler); ok {
+			return &handler{
+				hdlrID: hdlrID,
+				setter: ext.setter,
+			}
+		}
+		return nil
+	}
+}
+
+func (ext *extension) NewConn(conn imapserver.Conn) imapserver.Conn {
+	return ext.extID.NewConn(conn)
+}
+
+type handler struct {
+	hdlrID *imapid.Handler
+	setter currentClientSetter
+}
+
+func (hdlr *handler) Parse(fields []interface{}) error {
+	return hdlr.hdlrID.Parse(fields)
+}
+
+func (hdlr *handler) Handle(conn imapserver.Conn) error {
+	err := hdlr.hdlrID.Handle(conn)
+	if err == nil {
+		id := hdlr.hdlrID.Command.ID
+		hdlr.setter.SetCurrentClient(
+			id[imapid.FieldName],
+			id[imapid.FieldVersion],
+		)
+	}
+	return err
+}
+
+// NewExtension returns extension which is adding RFC2871 ID capability, with
+// direct interface to set information about email client to backend.
+func NewExtension(serverID imapid.ID, setter currentClientSetter) imapserver.Extension {
+	if conExtID, ok := imapid.NewExtension(serverID).(imapserver.ConnExtension); ok {
+		return &extension{
+			extID:  conExtID,
+			setter: setter,
+		}
+	}
+	return nil
+}

internal/imap/imap.go

@@ -22,12 +22,6 @@ import "github.com/sirupsen/logrus"
 const (
 	fetchMessagesWorkers    = 5 // In how many workers to fetch message (group list on IMAP).
 	fetchAttachmentsWorkers = 5 // In how many workers to fetch attachments (for one message).
-
-	clientAppleMail   = "Mac OS X Mail"             //nolint[deadcode]
-	clientThunderbird = "Thunderbird"               //nolint[deadcode]
-	clientOutlookMac  = "Microsoft Outlook for Mac" //nolint[deadcode]
-	clientOutlookWin  = "Microsoft Outlook"         //nolint[deadcode]
-	clientNone        = ""
 )
 
 var (

internal/imap/mailbox.go

@@ -80,7 +80,10 @@ func (im *imapMailbox) Info() (*imap.MailboxInfo, error) {
 }
 
 func (im *imapMailbox) getFlags() []string {
-	flags := []string{imap.NoInferiorsAttr} // Subfolders are not yet supported by API.
+	flags := []string{}
+	if !im.storeMailbox.IsFolder() || im.storeMailbox.IsSystem() {
+		flags = append(flags, imap.NoInferiorsAttr) // Subfolders are not supported for System or Label
+	}
 	switch im.storeMailbox.LabelID() {
 	case pmapi.SentLabel:
 		flags = append(flags, specialuse.Sent)
@@ -173,9 +176,8 @@ func (im *imapMailbox) Check() error {
 
 // Expunge permanently removes all messages that have the \Deleted flag set
 // from the currently selected mailbox.
-// Our messages do not have \Deleted flag, nothing to do here.
 func (im *imapMailbox) Expunge() error {
-	return nil
+	return im.storeMailbox.RemoveDeleted()
 }
 
 func (im *imapMailbox) ListQuotas() ([]string, error) {

internal/imap/mailbox_message.go

@@ -24,7 +24,6 @@ import (
 	"mime/multipart"
 	"net/mail"
 	"net/textproto"
-	"regexp"
 	"sort"
 	"strings"
 	"time"
@@ -37,7 +36,6 @@ import (
 	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
 	"github.com/emersion/go-imap"
 	"github.com/hashicorp/go-multierror"
-	enmime "github.com/jhillyerd/enmime"
 	"github.com/pkg/errors"
 	openpgperrors "golang.org/x/crypto/openpgp/errors"
 )
@@ -142,18 +140,19 @@ func (im *imapMailbox) CreateMessage(flags []string, date time.Time, body imap.L
 	references := m.Header.Get("References")
 	referenceList := strings.Fields(references)
 
-	if len(referenceList) > 0 {
+	// In case there is a mail client which corrupts headers, try
+	// "References" too.
+	if internalID == "" && len(referenceList) > 0 {
 		lastReference := referenceList[len(referenceList)-1]
-		// In case we are using a mail client which corrupts headers, try "References" too.
+		match := pmapi.RxInternalReferenceFormat.FindStringSubmatch(lastReference)
-		re := regexp.MustCompile(pmapi.InternalReferenceFormat)
+		if len(match) == 2 {
-		match := re.FindStringSubmatch(lastReference)
+			internalID = match[1]
-		if len(match) > 0 {
-			internalID = match[0]
 		}
 	}
 
-	// Avoid appending a message which is already on the server. Apply the new
+	// Avoid appending a message which is already on the server. Apply the
-	// label instead. This sometimes happens which Outlook (it uses APPEND instead of COPY).
+	// new label instead. This always happens with Outlook (it uses APPEND
+	// instead of COPY).
 	if internalID != "" {
 		// Check to see if this belongs to a different address in split mode or another ProtonMail account.
 		msg, err := im.storeMailbox.GetMessage(internalID)
@@ -221,6 +220,9 @@ func (im *imapMailbox) getMessage(storeMessage storeMessageProvider, items []ima
 			}
 		case imap.FetchFlags:
 			msg.Flags = message.GetFlags(m)
+			if storeMessage.IsMarkedDeleted() {
+				msg.Flags = append(msg.Flags, imap.DeletedFlag)
+			}
 		case imap.FetchInternalDate:
 			msg.InternalDate = time.Unix(m.Time, 0)
 		case imap.FetchRFC822Size:
@@ -238,26 +240,30 @@ func (im *imapMailbox) getMessage(storeMessage storeMessageProvider, items []ima
 				return nil, err
 			}
 		default:
-			s := item
+			if err = im.getLiteralForSection(item, msg, storeMessage); err != nil {
-
-			var section *imap.BodySectionName
-			if section, err = imap.ParseBodySectionName(s); err != nil {
-				err = nil // Ignore error
-				break
-			}
-
-			var literal imap.Literal
-			if literal, err = im.getMessageBodySection(storeMessage, section); err != nil {
 				return
 			}
-
-			msg.Body[section] = literal
 		}
 	}
 
 	return msg, err
 }
 
+func (im *imapMailbox) getLiteralForSection(itemSection imap.FetchItem, msg *imap.Message, storeMessage storeMessageProvider) error {
+	section, err := imap.ParseBodySectionName(itemSection)
+	if err != nil { // Ignore error
+		return nil
+	}
+
+	var literal imap.Literal
+	if literal, err = im.getMessageBodySection(storeMessage, section); err != nil {
+		return err
+	}
+
+	msg.Body[section] = literal
+	return nil
+}
+
 func (im *imapMailbox) getBodyStructure(storeMessage storeMessageProvider) (
 	structure *message.BodyStructure,
 	bodyReader *bytes.Reader, err error,
@@ -446,17 +452,6 @@ func (im *imapMailbox) writeMessageBody(w io.Writer, m *pmapi.Message) (err erro
 	return
 }
 
-func (im *imapMailbox) writeAndParseMIMEBody(m *pmapi.Message) (mime *enmime.Envelope, err error) { //nolint[unused]
-	b := &bytes.Buffer{}
-	if err = im.writeMessageBody(b, m); err != nil {
-		return
-	}
-
-	mime, err = enmime.ReadEnvelope(b)
-
-	return
-}
-
 func (im *imapMailbox) writeAttachmentBody(w io.Writer, m *pmapi.Message, att *pmapi.Attachment) (err error) {
 	// Retrieve encrypted attachment.
 	r, err := im.user.client().GetAttachment(att.ID)

internal/imap/mailbox_messages.go

@@ -57,7 +57,11 @@ func (im *imapMailbox) UpdateMessagesFlags(uid bool, seqSet *imap.SeqSet, operat
 	return im.addOrRemoveFlags(operation, messageIDs, flags)
 }
 
-func (im *imapMailbox) setFlags(messageIDs, flags []string) error {
+// setFlags is used for FLAGS command (not +FLAGS or -FLAGS), which means
+// to set flags passed as an argument and unset the rest. For example,
+// if message is not read, is flagged and is not deleted, call FLAGS \Seen
+// should flag message as read, unflagged and keep undeleted.
+func (im *imapMailbox) setFlags(messageIDs, flags []string) error { //nolint
 	seen := false
 	flagged := false
 	deleted := false
@@ -77,29 +81,48 @@ func (im *imapMailbox) setFlags(messageIDs, flags []string) error {
 	}
 
 	if seen {
-		_ = im.storeMailbox.MarkMessagesRead(messageIDs)
+		if err := im.storeMailbox.MarkMessagesRead(messageIDs); err != nil {
+			return err
+		}
 	} else {
-		_ = im.storeMailbox.MarkMessagesUnread(messageIDs)
+		if err := im.storeMailbox.MarkMessagesUnread(messageIDs); err != nil {
+			return err
+		}
 	}
 
 	if flagged {
-		_ = im.storeMailbox.MarkMessagesStarred(messageIDs)
+		if err := im.storeMailbox.MarkMessagesStarred(messageIDs); err != nil {
+			return err
+		}
 	} else {
-		_ = im.storeMailbox.MarkMessagesUnstarred(messageIDs)
+		if err := im.storeMailbox.MarkMessagesUnstarred(messageIDs); err != nil {
+			return err
+		}
 	}
 
 	if deleted {
-		_ = im.storeMailbox.DeleteMessages(messageIDs)
+		if err := im.storeMailbox.MarkMessagesDeleted(messageIDs); err != nil {
+			return err
+		}
+	} else {
+		if err := im.storeMailbox.MarkMessagesUndeleted(messageIDs); err != nil {
+			return err
+		}
 	}
 
-	spamMailbox, err := im.storeAddress.GetMailbox("Spam")
+	// Spam should not be taken into action here as Outlook is using FLAGS
-	if err != nil {
+	// without preserving junk flag. Probably it's because junk is not standard
-		return err
+	// in the rfc3501 and thus Outlook expects calling FLAGS \Seen will not
-	}
+	// change the state of junk or other non-standard flags.
+	// Still, its safe to label as spam once any client sends the request.
 	if spam {
-		_ = spamMailbox.LabelMessages(messageIDs)
+		spamMailbox, err := im.storeAddress.GetMailbox("Spam")
-	} else {
+		if err != nil {
-		_ = spamMailbox.UnlabelMessages(messageIDs)
+			return err
+		}
+		if err := spamMailbox.LabelMessages(messageIDs); err != nil {
+			return err
+		}
 	}
 
 	return nil
@@ -111,22 +134,36 @@ func (im *imapMailbox) addOrRemoveFlags(operation imap.FlagsOp, messageIDs, flag
 		case imap.SeenFlag:
 			switch operation {
 			case imap.AddFlags:
-				_ = im.storeMailbox.MarkMessagesRead(messageIDs)
+				if err := im.storeMailbox.MarkMessagesRead(messageIDs); err != nil {
+					return err
+				}
 			case imap.RemoveFlags:
-				_ = im.storeMailbox.MarkMessagesUnread(messageIDs)
+				if err := im.storeMailbox.MarkMessagesUnread(messageIDs); err != nil {
+					return err
+				}
 			}
 		case imap.FlaggedFlag:
 			switch operation {
 			case imap.AddFlags:
-				_ = im.storeMailbox.MarkMessagesStarred(messageIDs)
+				if err := im.storeMailbox.MarkMessagesStarred(messageIDs); err != nil {
+					return err
+				}
 			case imap.RemoveFlags:
-				_ = im.storeMailbox.MarkMessagesUnstarred(messageIDs)
+				if err := im.storeMailbox.MarkMessagesUnstarred(messageIDs); err != nil {
+					return err
+				}
 			}
 		case imap.DeletedFlag:
-			if operation == imap.RemoveFlags {
+			switch operation {
-				break // Nothing to do, no message has the \Deleted flag.
+			case imap.AddFlags:
+				if err := im.storeMailbox.MarkMessagesDeleted(messageIDs); err != nil {
+					return err
+				}
+			case imap.RemoveFlags:
+				if err := im.storeMailbox.MarkMessagesUndeleted(messageIDs); err != nil {
+					return err
+				}
 			}
-			_ = im.storeMailbox.DeleteMessages(messageIDs)
 		case imap.AnsweredFlag, imap.DraftFlag, imap.RecentFlag:
 			// Not supported.
 		case message.AppleMailJunkFlag, message.ThunderbirdJunkFlag:
@@ -140,9 +177,13 @@ func (im *imapMailbox) addOrRemoveFlags(operation imap.FlagsOp, messageIDs, flag
 			// No label removal is necessary because Spam and Inbox are both exclusive labels so the backend
 			// will automatically take care of label removal.
 			case imap.AddFlags:
-				_ = storeMailbox.LabelMessages(messageIDs)
+				if err := storeMailbox.LabelMessages(messageIDs); err != nil {
+					return err
+				}
 			case imap.RemoveFlags:
-				_ = storeMailbox.UnlabelMessages(messageIDs)
+				if err := storeMailbox.UnlabelMessages(messageIDs); err != nil {
+					return err
+				}
 			}
 		}
 	}
@@ -186,6 +227,20 @@ func (im *imapMailbox) labelMessages(uid bool, seqSet *imap.SeqSet, targetLabel
 		return err
 	}
 
+	deletedIDs := []string{}
+	allDeletedIDs, err := im.storeMailbox.GetDeletedAPIIDs()
+	if err != nil {
+		log.WithError(err).Warn("Problem to get deleted API IDs")
+	} else {
+		for _, messageID := range messageIDs {
+			for _, deletedID := range allDeletedIDs {
+				if messageID == deletedID {
+					deletedIDs = append(deletedIDs, deletedID)
+				}
+			}
+		}
+	}
+
 	// Label messages first to not lose them. If message is only in trash and we unlabel
 	// it, it will be removed completely and we cannot label it back.
 	if err := targetStoreMailbox.LabelMessages(messageIDs); err != nil {
@@ -197,6 +252,13 @@ func (im *imapMailbox) labelMessages(uid bool, seqSet *imap.SeqSet, targetLabel
 		}
 	}
 
+	// Preserve \Deleted flag at target location.
+	if len(deletedIDs) > 0 {
+		if err := targetStoreMailbox.MarkMessagesDeleted(deletedIDs); err != nil {
+			log.WithError(err).Warn("Problem to preserve deleted flag for copied messages")
+		}
+	}
+
 	targetSeqSet := targetStoreMailbox.GetUIDList(messageIDs)
 	return uidplus.CopyResponse(targetStoreMailbox.UIDValidity(), sourceSeqSet, targetSeqSet)
 }
@@ -321,6 +383,9 @@ func (im *imapMailbox) SearchMessages(isUID bool, criteria *imap.SearchCriteria)
 		if !m.Has(pmapi.FlagOpened) {
 			messageFlagsMap[imap.RecentFlag] = true
 		}
+		if storeMessage.IsMarkedDeleted() {
+			messageFlagsMap[imap.DeletedFlag] = true
+		}
 
 		flagMatch := true
 		for _, flag := range criteria.WithFlags {
@@ -383,6 +448,12 @@ func (im *imapMailbox) ListMessages(isUID bool, seqSet *imap.SeqSet, items []ima
 		im.panicHandler.HandlePanic()
 	}()
 
+	// EXPUNGE cannot be sent during listing and can come only from
+	// the event loop, so we prevent any server side update to avoid
+	// the problem.
+	im.storeUser.PauseEventLoop(true)
+	defer im.storeUser.PauseEventLoop(false)
+
 	var markAsReadIDs []string
 	markAsReadMutex := &sync.Mutex{}
 

internal/imap/server.go

@@ -28,6 +28,7 @@ import (
 	imapid "github.com/ProtonMail/go-imap-id"
 	"github.com/ProtonMail/proton-bridge/internal/bridge"
 	"github.com/ProtonMail/proton-bridge/internal/events"
+	"github.com/ProtonMail/proton-bridge/internal/imap/id"
 	"github.com/ProtonMail/proton-bridge/internal/imap/uidplus"
 	"github.com/ProtonMail/proton-bridge/pkg/listener"
 	"github.com/emersion/go-imap"
@@ -60,34 +61,12 @@ func NewIMAPServer(debugClient, debugServer bool, port int, tls *tls.Config, ima
 	s.UpgradeError = imapBackend.upgradeError
 
 	serverID := imapid.ID{
-		imapid.FieldName:       "ProtonMail",
+		imapid.FieldName:       "ProtonMail Bridge",
 		imapid.FieldVendor:     "Proton Technologies AG",
 		imapid.FieldSupportURL: "https://protonmail.com/support",
 	}
 
 	s.EnableAuth(sasl.Login, func(conn imapserver.Conn) sasl.Server {
-		conn.Server().ForEachConn(func(candidate imapserver.Conn) {
-			if id, ok := candidate.(imapid.Conn); ok {
-				if conn.Context() == candidate.Context() {
-					// ID is not available right at the beginning of the connection.
-					// Clients send ID quickly after AUTH. We need to wait for it.
-					go func() {
-						start := time.Now()
-						for {
-							if id.ID() != nil {
-								imapBackend.setLastMailClient(id.ID())
-								break
-							}
-							if time.Since(start) > 10*time.Second {
-								break
-							}
-							time.Sleep(100 * time.Millisecond)
-						}
-					}()
-				}
-			}
-		})
-
 		return sasl.NewLoginServer(func(address, password string) error {
 			user, err := conn.Server().Backend.Login(nil, address, password)
 			if err != nil {
@@ -105,7 +84,7 @@ func NewIMAPServer(debugClient, debugServer bool, port int, tls *tls.Config, ima
 		imapidle.NewExtension(),
 		imapmove.NewExtension(),
 		imapspecialuse.NewExtension(),
-		imapid.NewExtension(serverID),
+		id.NewExtension(serverID, imapBackend.bridge),
 		imapquota.NewExtension(),
 		imapappendlimit.NewExtension(),
 		imapunselect.NewExtension(),

internal/imap/store.go

@@ -41,6 +41,8 @@ type storeUserProvider interface {
 		attachedPublicKey,
 		attachedPublicKeyName string,
 		parentID string) (*pmapi.Message, []*pmapi.Attachment, error)
+
+	PauseEventLoop(bool)
 }
 
 type storeAddressProvider interface {
@@ -68,6 +70,7 @@ type storeMailboxProvider interface {
 	GetAPIIDsFromSequenceRange(start, stop uint32) ([]string, error)
 	GetLatestAPIID() (string, error)
 	GetNextUID() (uint32, error)
+	GetDeletedAPIIDs() ([]string, error)
 	GetCounts() (dbTotal, dbUnread, dbUnreadSeqNum uint, err error)
 	GetUIDList(apiIDs []string) *uidplus.OrderedSeq
 	GetUIDByHeader(header *mail.Header) uint32
@@ -81,8 +84,10 @@ type storeMailboxProvider interface {
 	MarkMessagesUnread(apiID []string) error
 	MarkMessagesStarred(apiID []string) error
 	MarkMessagesUnstarred(apiID []string) error
+	MarkMessagesDeleted(apiID []string) error
+	MarkMessagesUndeleted(apiID []string) error
 	ImportMessage(msg *pmapi.Message, body []byte, labelIDs []string) error
-	DeleteMessages(apiID []string) error
+	RemoveDeleted() error
 }
 
 type storeMessageProvider interface {
@@ -90,6 +95,7 @@ type storeMessageProvider interface {
 	UID() (uint32, error)
 	SequenceNumber() (uint32, error)
 	Message() *pmapi.Message
+	IsMarkedDeleted() bool
 
 	SetSize(int64) error
 	SetContentTypeAndHeader(string, mail.Header) error

internal/imap/uidplus/extension.go

@@ -25,6 +25,7 @@
 package uidplus
 
 import (
+	"errors"
 	"fmt"
 
 	"github.com/emersion/go-imap"
@@ -113,18 +114,43 @@ func (os *OrderedSeq) String() string {
 	return out
 }
 
-// UIDExpunge implements server.Handler but has no effect because Bridge is not
+// UIDExpunge implements server.Handler but Bridge is not supporting
-// using EXPUNGE at all. The message is deleted right after it was flagged as
+// UID EXPUNGE with specific UIDs.
-// \Deleted Bridge should simply ignore this command with empty `OK` response.
+type UIDExpunge struct {
-//
+	expunge *server.Expunge
-// If not implemented it would cause harmless IMAP error.
+}
-//
-// This overrides the standard EXPUNGE functionality.
-type UIDExpunge struct{}
 
-func (e *UIDExpunge) Parse(fields []interface{}) error { log.Traceln("parse", fields); return nil }
+func newUIDExpunge() *UIDExpunge {
-func (e *UIDExpunge) Handle(conn server.Conn) error    { log.Traceln("handle"); return nil }
+	return &UIDExpunge{expunge: &server.Expunge{}}
-func (e *UIDExpunge) UidHandle(conn server.Conn) error { log.Traceln("uid handle"); return nil } //nolint[golint]
+}
+
+func (e *UIDExpunge) Parse(fields []interface{}) error {
+	if len(fields) < 1 {
+		return e.expunge.Parse(fields)
+	}
+
+	// RFC4315#section-2.1
+	// The UID EXPUNGE command permanently removes all messages that both
+	// have the \Deleted flag set and have a UID that is included in the
+	// specified sequence set from the currently selected mailbox. If a
+	// message either does not have the \Deleted flag set or has a UID
+	// that is not included in the specified sequence set, it is not
+	// affected.
+	//
+	// Current implementation supports only deletion of all messages
+	// marked as deleted. It will probably need mailbox interface change:
+	// ExpungeUIDs(seqSet). Not sure how to combine with original
+	// e.expunge.Handle().
+	return errors.New("UID EXPUNGE with UIDs is not supported")
+}
+
+func (e *UIDExpunge) Handle(conn server.Conn) error {
+	return e.expunge.Handle(conn)
+}
+
+func (e *UIDExpunge) UidHandle(conn server.Conn) error { //nolint[golint]
+	return e.expunge.Handle(conn)
+}
 
 type extension struct{}
 
@@ -143,7 +169,7 @@ func (ext *extension) Capabilities(c server.Conn) []string {
 func (ext *extension) Command(name string) server.HandlerFactory {
 	if name == "EXPUNGE" {
 		return func() server.Handler {
-			return &UIDExpunge{}
+			return newUIDExpunge()
 		}
 	}
 

internal/importexport/credits.go

@@ -15,8 +15,8 @@
 // You should have received a copy of the GNU General Public License
 // along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
 
-// Code generated by ./credits.sh at Fri 04 Sep 2020 01:57:36 PM CEST. DO NOT EDIT.
+// Code generated by ./credits.sh at Wed Nov 4 12:24:36 PM CET 2020. DO NOT EDIT.
 
 package importexport
 
-const Credits = "github.com/0xAX/notificator;github.com/abiosoft/ishell;github.com/abiosoft/readline;github.com/allan-simon/go-singleinstance;github.com/andybalholm/cascadia;github.com/certifi/gocertifi;github.com/chzyer/logex;github.com/chzyer/test;github.com/cucumber/godog;github.com/docker/docker-credential-helpers;github.com/emersion/go-imap;github.com/emersion/go-imap-appendlimit;github.com/emersion/go-imap-idle;github.com/emersion/go-imap-move;github.com/emersion/go-imap-quota;github.com/emersion/go-imap-specialuse;github.com/emersion/go-imap-unselect;github.com/emersion/go-mbox;github.com/emersion/go-message;github.com/emersion/go-sasl;github.com/emersion/go-smtp;github.com/emersion/go-textwrapper;github.com/emersion/go-vcard;github.com/fatih/color;github.com/flynn-archive/go-shlex;github.com/getsentry/raven-go;github.com/go-delve/delve;github.com/golang/mock;github.com/google/go-cmp;github.com/google/uuid;github.com/gopherjs/gopherjs;github.com/go-resty/resty/v2;github.com/hashicorp/go-multierror;github.com/jameshoulahan/go-imap;github.com/jameskeane/bcrypt;github.com/jaytaylor/html2text;github.com/jhillyerd/enmime;github.com/kardianos/osext;github.com/keybase/go-keychain;github.com/logrusorgru/aurora;github.com/mattn/go-runewidth;github.com/miekg/dns;github.com/myesui/uuid;github.com/nsf/jsondiff;github.com/olekukonko/tablewriter;github.com/pkg/errors;github.com/ProtonMail/bcrypt;github.com/ProtonMail/crypto;github.com/ProtonMail/docker-credential-helpers;github.com/ProtonMail/go-appdir;github.com/ProtonMail/go-apple-mobileconfig;github.com/ProtonMail/go-autostart;github.com/ProtonMail/go-imap-id;github.com/ProtonMail/gopenpgp/v2;github.com/ProtonMail/go-smtp;github.com/ProtonMail/go-vcard;github.com/psampaz/go-mod-outdated;github.com/sirupsen/logrus;github.com/skratchdot/open-golang;github.com/stretchr/testify;github.com/therecipe/qt;github.com/twinj/uuid;github.com/urfave/cli;go.etcd.io/bbolt;golang.org/x/crypto;golang.org/x/net;golang.org/x/text;gopkg.in/stretchr/testify.v1;;Font Awesome 4.7.0;;Qt 5.13 by Qt group;"
+const Credits = "github.com/0xAX/notificator;github.com/abiosoft/ishell;github.com/abiosoft/readline;github.com/allan-simon/go-singleinstance;github.com/antlr/antlr4;github.com/certifi/gocertifi;github.com/chzyer/logex;github.com/chzyer/test;github.com/cucumber/godog;github.com/docker/docker-credential-helpers;github.com/emersion/go-imap;github.com/emersion/go-imap-appendlimit;github.com/emersion/go-imap-idle;github.com/emersion/go-imap-move;github.com/emersion/go-imap-quota;github.com/emersion/go-imap-specialuse;github.com/emersion/go-imap-unselect;github.com/emersion/go-mbox;github.com/emersion/go-message;github.com/emersion/go-sasl;github.com/emersion/go-smtp;github.com/emersion/go-textwrapper;github.com/emersion/go-vcard;github.com/fatih/color;github.com/flynn-archive/go-shlex;github.com/getsentry/raven-go;github.com/golang/mock;github.com/google/go-cmp;github.com/google/uuid;github.com/gopherjs/gopherjs;github.com/go-resty/resty/v2;github.com/hashicorp/go-multierror;github.com/jameskeane/bcrypt;github.com/jaytaylor/html2text;github.com/kardianos/osext;github.com/keybase/go-keychain;github.com/logrusorgru/aurora;github.com/Masterminds/semver/v3;github.com/mattn/go-runewidth;github.com/miekg/dns;github.com/myesui/uuid;github.com/nsf/jsondiff;github.com/olekukonko/tablewriter;github.com/pkg/errors;github.com/ProtonMail/bcrypt;github.com/ProtonMail/crypto;github.com/ProtonMail/docker-credential-helpers;github.com/ProtonMail/go-appdir;github.com/ProtonMail/go-apple-mobileconfig;github.com/ProtonMail/go-autostart;github.com/ProtonMail/go-imap;github.com/ProtonMail/go-imap-id;github.com/ProtonMail/gopenpgp/v2;github.com/ProtonMail/go-smtp;github.com/ProtonMail/go-vcard;github.com/PuerkitoBio/goquery;github.com/sirupsen/logrus;github.com/skratchdot/open-golang;github.com/ssor/bom;github.com/stretchr/testify;github.com/therecipe/qt;github.com/twinj/uuid;github.com/urfave/cli;go.etcd.io/bbolt;golang.org/x/crypto;golang.org/x/net;golang.org/x/text;gopkg.in/stretchr/testify.v1;;Font Awesome 4.7.0;;Qt 5.13 by Qt group;"

internal/importexport/release_notes.go

@@ -15,19 +15,18 @@
 // You should have received a copy of the GNU General Public License
 // along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
 
-// Code generated by ./release-notes.sh at 'Fri 04 Sep 2020 01:57:36 PM CEST'. DO NOT EDIT.
+// Code generated by ./release-notes.sh at 'Wed Nov  4 12:24:35 PM CET 2020'. DO NOT EDIT.
 
 package importexport
 
-const ReleaseNotes = `***Note: If you were using the Import-Export app before, you need to uninstall the older version and log in again to the new version***
+const ReleaseNotes = `• Improvements to the import from large mbox files with multiple labels
-
+• Not allow to run multiple instances of the app or transfers at the same time
-• Complete code refactor in preparation of stable and open source release of the Import-Export app
+• Various enhancements of the import process related to parsing
-• Increased number of supported mail providers by changing the way the folder structures are handled (NIL hierarchy delimiter)
+• Cosmetic GUI changes
-• Improved handling for unstable internet and pause and resume behavior
+• Better error handling
 `
 
-const ReleaseFixedBugs = `• Fixed rare cases where the application freezes when starting/stopping imports
+const ReleaseFixedBugs = `• Linux font issues - Fedora specific
-• Allowed current date to be included in the selected date range for both import and export
+• App response to the user pausing and canceling import or export
-• Improved manual update process
+• Handling errors during update
-• Limit space usage by on device application logs
 `

internal/smtp/preferences.go

@@ -95,9 +95,8 @@ func (b *sendPreferencesBuilder) shouldEncrypt() bool {
 	return false
 }
 
-func (b *sendPreferencesBuilder) withSign() {
+func (b *sendPreferencesBuilder) withSign(sign bool) {
-	v := true
+	b.sign = &sign
-	b.sign = &v
 }
 
 func (b *sendPreferencesBuilder) withSignDefault() {
@@ -192,7 +191,7 @@ func (b *sendPreferencesBuilder) build() (p SendPreferences) {
 			p.Scheme = pmapi.PGPMIMEPackage
 		}
 
-	case b.shouldSign() && !b.shouldEncrypt():
+	case b.shouldSign() && !b.shouldEncrypt() && b.getScheme() == pgpMIME:
 		p.Scheme = pmapi.ClearMIMEPackage
 
 	default:
@@ -258,7 +257,7 @@ func (b *sendPreferencesBuilder) setInternalPGPSettings(
 
 	// We always encrypt and sign internal mail.
 	b.withEncrypt(true)
-	b.withSign()
+	b.withSign(true)
 
 	// We use a custom scheme for internal messages.
 	b.withScheme(pmInternal)
@@ -369,7 +368,7 @@ func (b *sendPreferencesBuilder) setExternalPGPSettingsWithWKDKeys(
 
 	// We always encrypt and sign external mail if WKD keys are present.
 	b.withEncrypt(true)
-	b.withSign()
+	b.withSign(true)
 
 	// If the contact has a specific Scheme preference, we set it (otherwise we
 	// leave it unset to allow it to be filled in with the default value later).
@@ -402,9 +401,13 @@ func (b *sendPreferencesBuilder) setExternalPGPSettingsWithoutWKDKeys(
 ) (err error) {
 	b.withEncrypt(vCardData.Encrypt)
 
+	if vCardData.SignIsSet {
+		b.withSign(vCardData.Sign)
+	}
+
 	// Sign must be enabled whenever encrypt is.
-	if vCardData.Sign || vCardData.Encrypt {
+	if vCardData.Encrypt {
-		b.withSign()
+		b.withSign(true)
 	}
 
 	// If the contact has a specific Scheme preference, we set it (otherwise we
@@ -475,7 +478,7 @@ func (b *sendPreferencesBuilder) setEncryptionPreferences(mailSettings pmapi.Mai
 	}
 
 	if b.shouldEncrypt() {
-		b.withSign()
+		b.withSign(true)
 	}
 
 	// If undefined, default to the user mail setting "Default PGP scheme".
@@ -495,23 +498,11 @@ func (b *sendPreferencesBuilder) setEncryptionPreferences(mailSettings pmapi.Mai
 	if b.shouldSign() && b.getScheme() == pgpInline {
 		b.withMIMEType("text/plain")
 	} else {
-		switch mailSettings.ComposerMode {
+		b.withMIMETypeDefault(mailSettings.DraftMIMEType)
-		case pmapi.ComposerModeNormal:
-			b.withMIMETypeDefault("text/html")
-		case pmapi.ComposerModePlain:
-			b.withMIMETypeDefault("text/plain")
-		}
 	}
 }
 
 func (b *sendPreferencesBuilder) setMIMEPreferences(composerMIMEType string) {
-	// If the sign flag (that we just determined above) is true we use the scheme
-	// in the encryption preferences, unless the plain text format has been
-	// selected in the composer, in which case we must enforce PGP/INLINE.
-	if !b.isInternal() && b.shouldSign() && composerMIMEType == "text/plain" {
-		b.withScheme(pgpInline)
-	}
-
 	// If the sign flag (that we just determined above) is true, then the MIME
 	// type is determined by the PGP scheme (also determined above): we should
 	// use 'text/plain' for a PGP/Inline scheme, and 'multipart/mixed' otherwise.

internal/smtp/preferences_test.go

@@ -51,7 +51,7 @@ func TestPreferencesBuilder(t *testing.T) {
 			contactMeta:  &ContactMetadata{},
 			receivedKeys: []pmapi.PublicKey{{PublicKey: testPublicKey}},
 			isInternal:   true,
-			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, ComposerMode: pmapi.ComposerModeNormal},
+			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, DraftMIMEType: "text/html"},
 
 			wantEncrypt:   true,
 			wantSign:      true,
@@ -66,7 +66,7 @@ func TestPreferencesBuilder(t *testing.T) {
 			contactMeta:  &ContactMetadata{MIMEType: "text/plain"},
 			receivedKeys: []pmapi.PublicKey{{PublicKey: testPublicKey}},
 			isInternal:   true,
-			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, ComposerMode: pmapi.ComposerModeNormal},
+			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, DraftMIMEType: "text/html"},
 
 			wantEncrypt:   true,
 			wantSign:      true,
@@ -81,7 +81,7 @@ func TestPreferencesBuilder(t *testing.T) {
 			contactMeta:  &ContactMetadata{Keys: []string{testContactKey}},
 			receivedKeys: []pmapi.PublicKey{{PublicKey: testPublicKey}},
 			isInternal:   true,
-			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, ComposerMode: pmapi.ComposerModeNormal},
+			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, DraftMIMEType: "text/html"},
 
 			wantEncrypt:   true,
 			wantSign:      true,
@@ -97,7 +97,7 @@ func TestPreferencesBuilder(t *testing.T) {
 			contactMeta:  &ContactMetadata{Keys: []string{testOtherContactKey}},
 			receivedKeys: []pmapi.PublicKey{{PublicKey: testPublicKey}},
 			isInternal:   true,
-			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, ComposerMode: pmapi.ComposerModeNormal},
+			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, DraftMIMEType: "text/html"},
 
 			wantEncrypt:   true,
 			wantSign:      true,
@@ -112,7 +112,7 @@ func TestPreferencesBuilder(t *testing.T) {
 			contactMeta:  &ContactMetadata{},
 			receivedKeys: []pmapi.PublicKey{{PublicKey: testPublicKey}},
 			isInternal:   false,
-			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, ComposerMode: pmapi.ComposerModeNormal},
+			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, DraftMIMEType: "text/html"},
 
 			wantEncrypt:   true,
 			wantSign:      true,
@@ -127,7 +127,7 @@ func TestPreferencesBuilder(t *testing.T) {
 			contactMeta:  &ContactMetadata{MIMEType: "text/plain"},
 			receivedKeys: []pmapi.PublicKey{{PublicKey: testPublicKey}},
 			isInternal:   false,
-			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, ComposerMode: pmapi.ComposerModeNormal},
+			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, DraftMIMEType: "text/html"},
 
 			wantEncrypt:   true,
 			wantSign:      true,
@@ -142,7 +142,7 @@ func TestPreferencesBuilder(t *testing.T) {
 			contactMeta:  &ContactMetadata{},
 			receivedKeys: []pmapi.PublicKey{{PublicKey: testPublicKey}},
 			isInternal:   false,
-			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPInlinePackage, ComposerMode: pmapi.ComposerModeNormal},
+			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPInlinePackage, DraftMIMEType: "text/html"},
 
 			wantEncrypt:   true,
 			wantSign:      true,
@@ -157,7 +157,7 @@ func TestPreferencesBuilder(t *testing.T) {
 			contactMeta:  &ContactMetadata{Scheme: pgpInline},
 			receivedKeys: []pmapi.PublicKey{{PublicKey: testPublicKey}},
 			isInternal:   false,
-			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, ComposerMode: pmapi.ComposerModeNormal},
+			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, DraftMIMEType: "text/html"},
 
 			wantEncrypt:   true,
 			wantSign:      true,
@@ -172,7 +172,7 @@ func TestPreferencesBuilder(t *testing.T) {
 			contactMeta:  &ContactMetadata{Scheme: pgpMIME},
 			receivedKeys: []pmapi.PublicKey{{PublicKey: testPublicKey}},
 			isInternal:   false,
-			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPInlinePackage, ComposerMode: pmapi.ComposerModeNormal},
+			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPInlinePackage, DraftMIMEType: "text/html"},
 
 			wantEncrypt:   true,
 			wantSign:      true,
@@ -187,7 +187,7 @@ func TestPreferencesBuilder(t *testing.T) {
 			contactMeta:  &ContactMetadata{Keys: []string{testContactKey}},
 			receivedKeys: []pmapi.PublicKey{{PublicKey: testPublicKey}},
 			isInternal:   false,
-			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, ComposerMode: pmapi.ComposerModeNormal},
+			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, DraftMIMEType: "text/html"},
 
 			wantEncrypt:   true,
 			wantSign:      true,
@@ -203,7 +203,7 @@ func TestPreferencesBuilder(t *testing.T) {
 			contactMeta:  &ContactMetadata{Keys: []string{testOtherContactKey}},
 			receivedKeys: []pmapi.PublicKey{{PublicKey: testPublicKey}},
 			isInternal:   false,
-			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, ComposerMode: pmapi.ComposerModeNormal},
+			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, DraftMIMEType: "text/html"},
 
 			wantEncrypt:   true,
 			wantSign:      true,
@@ -218,7 +218,7 @@ func TestPreferencesBuilder(t *testing.T) {
 			contactMeta:  &ContactMetadata{},
 			receivedKeys: []pmapi.PublicKey{},
 			isInternal:   false,
-			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, ComposerMode: pmapi.ComposerModeNormal},
+			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, DraftMIMEType: "text/html"},
 
 			wantEncrypt:  false,
 			wantSign:     false,
@@ -232,7 +232,7 @@ func TestPreferencesBuilder(t *testing.T) {
 			contactMeta:  &ContactMetadata{MIMEType: "text/plain"},
 			receivedKeys: []pmapi.PublicKey{},
 			isInternal:   false,
-			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, ComposerMode: pmapi.ComposerModeNormal},
+			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, DraftMIMEType: "text/html"},
 
 			wantEncrypt:  false,
 			wantSign:     false,
@@ -243,10 +243,24 @@ func TestPreferencesBuilder(t *testing.T) {
 		{
 			name: "external with sign enabled",
 
-			contactMeta:  &ContactMetadata{Sign: true},
+			contactMeta:  &ContactMetadata{Sign: true, SignIsSet: true},
 			receivedKeys: []pmapi.PublicKey{},
 			isInternal:   false,
-			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, ComposerMode: pmapi.ComposerModeNormal},
+			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, DraftMIMEType: "text/html"},
+
+			wantEncrypt:  false,
+			wantSign:     true,
+			wantScheme:   pmapi.ClearMIMEPackage,
+			wantMIMEType: "multipart/mixed",
+		},
+
+		{
+			name: "external with sign enabled, sending plaintext, should still send as ClearMIME",
+
+			contactMeta:  &ContactMetadata{Sign: true, SignIsSet: true},
+			receivedKeys: []pmapi.PublicKey{},
+			isInternal:   false,
+			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, DraftMIMEType: "text/plain"},
 
 			wantEncrypt:  false,
 			wantSign:     true,
@@ -260,7 +274,7 @@ func TestPreferencesBuilder(t *testing.T) {
 			contactMeta:  &ContactMetadata{Keys: []string{testContactKey}},
 			receivedKeys: []pmapi.PublicKey{},
 			isInternal:   false,
-			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, ComposerMode: pmapi.ComposerModeNormal},
+			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, DraftMIMEType: "text/html"},
 
 			wantEncrypt:   false,
 			wantSign:      false,
@@ -272,10 +286,10 @@ func TestPreferencesBuilder(t *testing.T) {
 		{
 			name: "external with pinned contact public key, encrypted and signed",
 
-			contactMeta:  &ContactMetadata{Keys: []string{testContactKey}, Encrypt: true, Sign: true},
+			contactMeta:  &ContactMetadata{Keys: []string{testContactKey}, Encrypt: true, Sign: true, SignIsSet: true},
 			receivedKeys: []pmapi.PublicKey{},
 			isInternal:   false,
-			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, ComposerMode: pmapi.ComposerModeNormal},
+			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, DraftMIMEType: "text/html"},
 
 			wantEncrypt:   true,
 			wantSign:      true,
@@ -287,10 +301,10 @@ func TestPreferencesBuilder(t *testing.T) {
 		{
 			name: "external with pinned contact public key, encrypted and signed using contact-specific pgp-inline",
 
-			contactMeta:  &ContactMetadata{Keys: []string{testContactKey}, Encrypt: true, Sign: true, Scheme: pgpInline},
+			contactMeta:  &ContactMetadata{Keys: []string{testContactKey}, Encrypt: true, Sign: true, Scheme: pgpInline, SignIsSet: true},
 			receivedKeys: []pmapi.PublicKey{},
 			isInternal:   false,
-			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, ComposerMode: pmapi.ComposerModeNormal},
+			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, DraftMIMEType: "text/html"},
 
 			wantEncrypt:   true,
 			wantSign:      true,
@@ -302,10 +316,10 @@ func TestPreferencesBuilder(t *testing.T) {
 		{
 			name: "external with pinned contact public key, encrypted and signed using global pgp-inline",
 
-			contactMeta:  &ContactMetadata{Keys: []string{testContactKey}, Encrypt: true, Sign: true},
+			contactMeta:  &ContactMetadata{Keys: []string{testContactKey}, Encrypt: true, Sign: true, SignIsSet: true},
 			receivedKeys: []pmapi.PublicKey{},
 			isInternal:   false,
-			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPInlinePackage, ComposerMode: pmapi.ComposerModeNormal},
+			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPInlinePackage, DraftMIMEType: "text/html"},
 
 			wantEncrypt:   true,
 			wantSign:      true,

internal/smtp/send_recorder.go

@@ -20,6 +20,7 @@ package smtp
 import (
 	"crypto/sha256"
 	"fmt"
+	"strings"
 	"sync"
 	"time"
 
@@ -48,6 +49,15 @@ func newSendRecorder() *sendRecorder {
 }
 
 func (q *sendRecorder) getMessageHash(message *pmapi.Message) string {
+	// Outlook Calendar updates has only headers (no body) and thus have always
+	// the same hash. If the message is type of calendar, the "is sending"
+	// check to avoid potential duplicates is skipped. Duplicates should not
+	// be a problem in this case as calendar updates are small.
+	contentType := message.Header.Get("Content-Type")
+	if strings.HasPrefix(contentType, "text/calendar") {
+		return ""
+	}
+
 	h := sha256.New()
 	_, _ = h.Write([]byte(message.AddressID + message.Subject))
 	if message.Sender != nil {
@@ -101,6 +111,10 @@ func (q *sendRecorder) isSendingOrSent(client messageGetter, hash string) (isSen
 	q.lock.Lock()
 	defer q.lock.Unlock()
 
+	if hash == "" {
+		return false, false
+	}
+
 	q.deleteExpiredKeys()
 	value, ok := q.hashes[hash]
 	if !ok {

internal/smtp/send_recorder_test.go

@@ -349,6 +349,32 @@ func TestSendRecorder_getMessageHash(t *testing.T) {
 			},
 			false,
 		},
+		{ // Different content type - calendar
+			&pmapi.Message{
+				Header: mail.Header{
+					"Content-Type": []string{"text/calendar"},
+				},
+				AddressID: "address123",
+				Subject:   "Subject #1",
+				Sender: &mail.Address{
+					Address: "from@pm.me",
+				},
+				ToList: []*mail.Address{
+					{Address: "to@pm.me"},
+				},
+				CCList:  []*mail.Address{},
+				BCCList: []*mail.Address{},
+				Body:    "body",
+				Attachments: []*pmapi.Attachment{
+					{
+						Name:     "att1",
+						MIMEType: "image/png",
+						Size:     12345,
+					},
+				},
+			},
+			false,
+		},
 	}
 	for i, tc := range testCases {
 		tc := tc // bind
@@ -382,12 +408,13 @@ func TestSendRecorder_isSendingOrSent(t *testing.T) {
 		{"hash", &pmapi.Message{Type: pmapi.MessageTypeDraft, Time: time.Now().Unix()}, nil, true, false},
 		{"hash", &pmapi.Message{Type: pmapi.MessageTypeSent}, nil, false, true},
 		{"hash", &pmapi.Message{Type: pmapi.MessageTypeInboxAndSent}, nil, false, true},
+		{"", &pmapi.Message{Type: pmapi.MessageTypeInboxAndSent}, nil, false, false},
 	}
 	for i, tc := range testCases {
 		tc := tc // bind
 		t.Run(fmt.Sprintf("%d / %v / %v / %v", i, tc.hash, tc.message, tc.err), func(t *testing.T) {
 			messageGetter := &testSendRecorderGetMessageMock{message: tc.message, err: tc.err}
-			isSending, wasSent := q.isSendingOrSent(messageGetter, "hash")
+			isSending, wasSent := q.isSendingOrSent(messageGetter, tc.hash)
 			assert.Equal(t, tc.wantIsSending, isSending, "isSending does not match")
 			assert.Equal(t, tc.wantWasSent, wasSent, "wasSent does not match")
 		})

internal/smtp/user.go

@@ -21,10 +21,10 @@ package smtp
 
 import (
 	"encoding/base64"
+	"fmt"
 	"io"
 	"mime"
 	"net/mail"
-	"regexp"
 	"strings"
 	"time"
 
@@ -179,11 +179,12 @@ func (su *smtpUser) Send(from string, to []string, messageReader io.Reader) (err
 			return err
 		}
 
-		attachedPublicKeyName = "publickey - " + kr.GetIdentities()[0].Name
+		attachedPublicKeyName = fmt.Sprintf("publickey - %v - %v", kr.GetIdentities()[0].Name, firstKey.GetFingerprint()[:8])
 	}
 
 	message, mimeBody, plainBody, attReaders, err := message.Parse(messageReader, attachedPublicKey, attachedPublicKeyName)
 	if err != nil {
+		log.WithError(err).Error("Failed to parse message")
 		return
 	}
 	clearBody := message.Body
@@ -290,6 +291,9 @@ func (su *smtpUser) Send(from string, to []string, messageReader io.Reader) (err
 		}
 
 		sendPreferences, err := su.getSendPreferences(email, message.MIMEType, mailSettings)
+		if !sendPreferences.Encrypt {
+			containsUnencryptedRecipients = true
+		}
 		if err != nil {
 			return err
 		}
@@ -359,7 +363,9 @@ func (su *smtpUser) Send(from string, to []string, messageReader io.Reader) (err
 			return errors.New("error decoding subject message " + message.Header.Get("Subject"))
 		}
 		if !su.continueSendingUnencryptedMail(subject) {
-			_ = su.client().DeleteMessages([]string{message.ID})
+			if err := su.client().DeleteMessages([]string{message.ID}); err != nil {
+				log.WithError(err).Warn("Failed to delete canceled messages")
+			}
 			return errors.New("sending was canceled by user")
 		}
 	}
@@ -401,9 +407,9 @@ func (su *smtpUser) handleReferencesHeader(m *pmapi.Message) (draftID, parentID
 		if !strings.Contains(reference, "@"+pmapi.InternalIDDomain) {
 			newReferences = append(newReferences, reference)
 		} else { // internalid is the parentID.
-			idMatch := regexp.MustCompile(pmapi.InternalReferenceFormat).FindStringSubmatch(reference)
+			idMatch := pmapi.RxInternalReferenceFormat.FindStringSubmatch(reference)
-			if len(idMatch) > 0 {
+			if len(idMatch) == 2 {
-				lastID := strings.TrimSuffix(strings.Trim(idMatch[0], "<>"), "@protonmail.internalid")
+				lastID := idMatch[1]
 				filter := &pmapi.MessagesFilter{ID: []string{lastID}}
 				if su.addressID != "" {
 					filter.AddressID = su.addressID

internal/smtp/vcard_tools.go

@@ -27,13 +27,13 @@ import (
 )
 
 type ContactMetadata struct {
-	Email       string
+	Email     string
-	Keys        []string
+	Keys      []string
-	Scheme      string
+	Scheme    string
-	Sign        bool
+	Sign      bool
-	SignMissing bool
+	SignIsSet bool
-	Encrypt     bool
+	Encrypt   bool
-	MIMEType    string
+	MIMEType  string
 }
 
 const (
@@ -72,22 +72,22 @@ func GetContactMetadataFromVCards(cards []pmapi.Card, email string) (contactMeta
 		// Warn: ParseBool treats 1, T, True, true as true and 0, F, Fale, false as false.
 		//       However PMEL declares 'true' is true, 'false' is false. every other string is true
 		encrypt, _ := strconv.ParseBool(parsedCard.GetValueByGroup(FieldPMEncrypt, group))
-		var sign, signMissing bool
+		var sign, signIsSet bool
 		if len(parsedCard[FieldPMSign]) == 0 {
-			signMissing = true
+			signIsSet = false
 		} else {
 			sign, _ = strconv.ParseBool(parsedCard.GetValueByGroup(FieldPMSign, group))
-			signMissing = false
+			signIsSet = true
 		}
 		mimeType := parsedCard.GetValueByGroup(FieldPMMIMEType, group)
 		return &ContactMetadata{
-			Email:       email,
+			Email:     email,
-			Keys:        keys,
+			Keys:      keys,
-			Scheme:      scheme,
+			Scheme:    scheme,
-			Sign:        sign,
+			Sign:      sign,
-			SignMissing: signMissing,
+			SignIsSet: signIsSet,
-			Encrypt:     encrypt,
+			Encrypt:   encrypt,
-			MIMEType:    mimeType,
+			MIMEType:  mimeType,
 		}, nil
 	}
 	return &ContactMetadata{}, nil

internal/store/address.go

@@ -69,7 +69,7 @@ func (storeAddress *Address) init(foldersAndLabels []*pmapi.Label) (err error) {
 			prefix := getLabelPrefix(label)
 
 			var mailbox *Mailbox
-			if mailbox, err = txNewMailbox(tx, storeAddress, label.ID, prefix, label.Name, label.Color); err != nil {
+			if mailbox, err = txNewMailbox(tx, storeAddress, label.ID, prefix, label.Path, label.Color); err != nil {
 				storeAddress.log.
 					WithError(err).
 					WithField("labelID", label.ID).

internal/store/address_mailbox.go

@@ -73,14 +73,14 @@ func (storeAddress *Address) createOrUpdateMailboxEvent(label *pmapi.Label) erro
 	prefix := getLabelPrefix(label)
 	mailbox, ok := storeAddress.mailboxes[label.ID]
 	if !ok {
-		mailbox, err := newMailbox(storeAddress, label.ID, prefix, label.Name, label.Color)
+		mailbox, err := newMailbox(storeAddress, label.ID, prefix, label.Path, label.Color)
 		if err != nil {
 			return err
 		}
 		storeAddress.mailboxes[label.ID] = mailbox
 		mailbox.store.imapMailboxCreated(storeAddress.address, mailbox.labelName)
 	} else {
-		mailbox.labelName = prefix + label.Name
+		mailbox.labelName = prefix + label.Path
 		mailbox.color = label.Color
 	}
 	return nil

internal/store/cache.go

@@ -45,7 +45,9 @@ func (c *Cache) getEventID(userID string) string {
 	c.lock.Lock()
 	defer c.lock.Unlock()
 
-	_ = c.loadCache()
+	if err := c.loadCache(); err != nil {
+		log.WithError(err).Warn("Problem to load store cache")
+	}
 
 	if c.cache == nil {
 		c.cache = map[string]map[string]string{}

internal/store/change.go

@@ -37,7 +37,7 @@ func (store *Store) SetIMAPUpdateChannel(updates chan imapBackend.Update) {
 	}
 }
 
-func (store *Store) imapNotice(address, notice string) {
+func (store *Store) imapNotice(address, notice string) *imapBackend.StatusUpdate {
 	update := new(imapBackend.StatusUpdate)
 	update.Update = imapBackend.NewUpdate(address, "")
 	update.StatusResp = &imap.StatusResp{
@@ -46,25 +46,35 @@ func (store *Store) imapNotice(address, notice string) {
 		Info: notice,
 	}
 	store.imapSendUpdate(update)
+	return update
 }
 
-func (store *Store) imapUpdateMessage(address, mailboxName string, uid, sequenceNumber uint32, msg *pmapi.Message) {
+func (store *Store) imapUpdateMessage(
+	address, mailboxName string,
+	uid, sequenceNumber uint32,
+	msg *pmapi.Message, hasDeletedFlag bool,
+) *imapBackend.MessageUpdate {
 	store.log.WithFields(logrus.Fields{
 		"address": address,
 		"mailbox": mailboxName,
 		"seqNum":  sequenceNumber,
 		"uid":     uid,
 		"flags":   message.GetFlags(msg),
+		"deleted": hasDeletedFlag,
 	}).Trace("IDLE update")
 	update := new(imapBackend.MessageUpdate)
 	update.Update = imapBackend.NewUpdate(address, mailboxName)
 	update.Message = imap.NewMessage(sequenceNumber, []imap.FetchItem{imap.FetchFlags, imap.FetchUid})
 	update.Message.Flags = message.GetFlags(msg)
+	if hasDeletedFlag {
+		update.Message.Flags = append(update.Message.Flags, imap.DeletedFlag)
+	}
 	update.Message.Uid = uid
 	store.imapSendUpdate(update)
+	return update
 }
 
-func (store *Store) imapDeleteMessage(address, mailboxName string, sequenceNumber uint32) {
+func (store *Store) imapDeleteMessage(address, mailboxName string, sequenceNumber uint32) *imapBackend.ExpungeUpdate {
 	store.log.WithFields(logrus.Fields{
 		"address": address,
 		"mailbox": mailboxName,
@@ -74,9 +84,10 @@ func (store *Store) imapDeleteMessage(address, mailboxName string, sequenceNumbe
 	update.Update = imapBackend.NewUpdate(address, mailboxName)
 	update.SeqNum = sequenceNumber
 	store.imapSendUpdate(update)
+	return update
 }
 
-func (store *Store) imapMailboxCreated(address, mailboxName string) {
+func (store *Store) imapMailboxCreated(address, mailboxName string) *imapBackend.MailboxInfoUpdate {
 	store.log.WithFields(logrus.Fields{
 		"address": address,
 		"mailbox": mailboxName,
@@ -89,9 +100,10 @@ func (store *Store) imapMailboxCreated(address, mailboxName string) {
 		Name:       mailboxName,
 	}
 	store.imapSendUpdate(update)
+	return update
 }
 
-func (store *Store) imapMailboxStatus(address, mailboxName string, total, unread, unreadSeqNum uint) {
+func (store *Store) imapMailboxStatus(address, mailboxName string, total, unread, unreadSeqNum uint) *imapBackend.MailboxUpdate {
 	store.log.WithFields(logrus.Fields{
 		"address":      address,
 		"mailbox":      mailboxName,
@@ -106,6 +118,7 @@ func (store *Store) imapMailboxStatus(address, mailboxName string, total, unread
 	update.MailboxStatus.Unseen = uint32(unread)
 	update.MailboxStatus.UnseenSeqNum = uint32(unreadSeqNum)
 	store.imapSendUpdate(update)
+	return update
 }
 
 func (store *Store) imapSendUpdate(update imapBackend.Update) {
@@ -116,7 +129,7 @@ func (store *Store) imapSendUpdate(update imapBackend.Update) {
 
 	select {
 	case <-time.After(1 * time.Second):
-		store.log.Error("Could not send IMAP update (timeout)")
+		store.log.Warn("IMAP update could not be sent (timeout)")
 		return
 	case store.imapUpdates <- update:
 	}

internal/store/event_loop.go

@@ -38,7 +38,8 @@ type eventLoop struct {
 	pollCh         chan chan struct{}
 	stopCh         chan struct{}
 	notifyStopCh   chan struct{}
-	isRunning      bool
+	isRunning      bool // The whole event loop is running.
+	isTickerPaused bool // The periodic loop is paused (but the event loop itself is still running).
 	hasInternet    bool
 
 	pollCounter int
@@ -59,6 +60,7 @@ func newEventLoop(cache *Cache, store *Store, user BridgeUser, events listener.L
 		currentEventID: cache.getEventID(user.ID()),
 		pollCh:         make(chan chan struct{}),
 		isRunning:      false,
+		isTickerPaused: false,
 
 		log: eventLog,
 
@@ -68,10 +70,6 @@ func newEventLoop(cache *Cache, store *Store, user BridgeUser, events listener.L
 	}
 }
 
-func (loop *eventLoop) IsRunning() bool {
-	return loop.isRunning
-}
-
 func (loop *eventLoop) client() pmapi.Client {
 	return loop.store.client()
 }
@@ -156,6 +154,10 @@ func (loop *eventLoop) loop() {
 			close(loop.notifyStopCh)
 			return
 		case <-t.C:
+			if loop.isTickerPaused {
+				loop.log.Trace("Event loop paused, skipping")
+				continue
+			}
 			// Randomise periodic calls within range pollInterval ± pollSpread to reduces potential load spikes on API.
 			time.Sleep(time.Duration(rand.Intn(2*int(pollIntervalSpread.Milliseconds()))) * time.Millisecond)
 		case eventProcessedCh = <-loop.pollCh:

internal/store/event_loop_test.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
 	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -102,10 +103,13 @@ func TestEventLoopUpdateMessageFromLoop(t *testing.T) {
 	// Event loop runs in goroutine and will be stopped by deferred mock clearing.
 	go m.store.eventLoop.start()
 
-	require.Eventually(t, func() bool {
+	var err error
-		msg, err := m.store.getMessageFromDB("msg1")
+	assert.Eventually(t, func() bool {
+		var msg *pmapi.Message
+		msg, err = m.store.getMessageFromDB("msg1")
 		return err == nil && msg.Subject == newSubject
 	}, time.Second, 10*time.Millisecond)
+	require.NoError(t, err)
 }
 
 func TestEventLoopUpdateMessage(t *testing.T) {

internal/store/mailbox.go

@@ -41,7 +41,7 @@ type Mailbox struct {
 }
 
 func newMailbox(storeAddress *Address, labelID, labelPrefix, labelName, color string) (mb *Mailbox, err error) {
-	_ = storeAddress.store.db.Update(func(tx *bolt.Tx) error {
+	err = storeAddress.store.db.Update(func(tx *bolt.Tx) error {
 		mb, err = txNewMailbox(tx, storeAddress, labelID, labelPrefix, labelName, color)
 		return err
 	})
@@ -142,6 +142,9 @@ func initMailboxBucket(tx *bolt.Tx, bucketName []byte) error {
 	if _, err := bucket.CreateBucketIfNotExists(apiIDsBucket); err != nil {
 		return err
 	}
+	if _, err := bucket.CreateBucketIfNotExists(deletedIDsBucket); err != nil {
+		return err
+	}
 
 	return nil
 }
@@ -238,6 +241,11 @@ func (storeMailbox *Mailbox) txGetAPIIDsBucket(tx *bolt.Tx) *bolt.Bucket {
 	return storeMailbox.txGetBucket(tx).Bucket(apiIDsBucket)
 }
 
+// txGetDeletedIDsBucket returns the bucket with messagesID marked as deleted
+func (storeMailbox *Mailbox) txGetDeletedIDsBucket(tx *bolt.Tx) *bolt.Bucket {
+	return storeMailbox.txGetBucket(tx).Bucket(deletedIDsBucket)
+}
+
 // txGetBucket returns the bucket of mailbox containing mapping buckets.
 func (storeMailbox *Mailbox) txGetBucket(tx *bolt.Tx) *bolt.Bucket {
 	return tx.Bucket(mailboxesBucket).Bucket(storeMailbox.getBucketName())

internal/store/mailbox_counts.go

@@ -125,6 +125,7 @@ func (mc *mailboxCounts) getPMLabel() *pmapi.Label {
 	return &pmapi.Label{
 		ID:        mc.LabelID,
 		Name:      mc.LabelName,
+		Path:      mc.LabelName,
 		Color:     mc.Color,
 		Order:     mc.Order,
 		Type:      pmapi.LabelTypeMailbox,
@@ -158,7 +159,7 @@ func (store *Store) createOrUpdateMailboxCountsBuckets(labels []*pmapi.Label) er
 			}
 
 			// Update mailbox info, but dont change on-API-counts.
-			mailbox.LabelName = label.Name
+			mailbox.LabelName = label.Path
 			mailbox.Color = label.Color
 			mailbox.Order = label.Order
 			mailbox.IsFolder = label.Exclusive == 1

internal/store/mailbox_ids.go

@@ -129,7 +129,11 @@ func (storeMailbox *Mailbox) getUID(apiID string) (uid uint32, err error) {
 }
 
 func (storeMailbox *Mailbox) txGetUID(tx *bolt.Tx, apiID string) (uint32, error) {
-	b := storeMailbox.txGetAPIIDsBucket(tx)
+	return storeMailbox.txGetUIDFromBucket(storeMailbox.txGetAPIIDsBucket(tx), apiID)
+}
+
+// txGetUIDFromBucket expects pointer to API bucket.
+func (storeMailbox *Mailbox) txGetUIDFromBucket(b *bolt.Bucket, apiID string) (uint32, error) {
 	v := b.Get([]byte(apiID))
 	if v == nil {
 		return 0, ErrNoSuchAPIID
@@ -137,6 +141,19 @@ func (storeMailbox *Mailbox) txGetUID(tx *bolt.Tx, apiID string) (uint32, error)
 	return btoi(v), nil
 }
 
+// GetDeletedAPIIDs returns API IDs in this mailbox for message ID.
+func (storeMailbox *Mailbox) GetDeletedAPIIDs() (apiIDs []string, err error) {
+	err = storeMailbox.db().Update(func(tx *bolt.Tx) error {
+		b := storeMailbox.txGetDeletedIDsBucket(tx)
+		c := b.Cursor()
+		for k, _ := c.First(); k != nil; k, _ = c.Next() {
+			apiIDs = append(apiIDs, string(k))
+		}
+		return nil
+	})
+	return
+}
+
 // getSequenceNumber returns IMAP sequence number in the mailbox for the message with the given API ID `apiID`.
 func (storeMailbox *Mailbox) getSequenceNumber(apiID string) (seqNum uint32, err error) {
 	err = storeMailbox.db().View(func(tx *bolt.Tx) error {

internal/store/mailbox_message.go

@@ -18,12 +18,16 @@
 package store
 
 import (
+	"time"
+
 	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	bolt "go.etcd.io/bbolt"
 )
 
+// ErrAllMailOpNotAllowed is error user when user tries to do unsupported
+// operation on All Mail folder
 var ErrAllMailOpNotAllowed = errors.New("operation not allowed for 'All Mail' folder")
 
 // GetMessage returns the `pmapi.Message` struct wrapped in `StoreMessage`
@@ -96,11 +100,8 @@ func (storeMailbox *Mailbox) LabelMessages(apiIDs []string) error {
 // It has to be propagated to all the same messages in all mailboxes.
 // The propagation is processed by the event loop.
 func (storeMailbox *Mailbox) UnlabelMessages(apiIDs []string) error {
-	log.WithFields(logrus.Fields{
+	storeMailbox.log.WithField("messages", apiIDs).
-		"messages": apiIDs,
+		Trace("Unlabeling messages")
-		"label":    storeMailbox.labelID,
-		"mailbox":  storeMailbox.Name,
-	}).Trace("Unlabeling messages")
 	if storeMailbox.labelID == pmapi.AllMailLabel {
 		return ErrAllMailOpNotAllowed
 	}
@@ -129,6 +130,9 @@ func (storeMailbox *Mailbox) MarkMessagesRead(apiIDs []string) error {
 			ids = append(ids, apiID)
 		}
 	}
+	if len(ids) == 0 {
+		return nil
+	}
 	return storeMailbox.client().MarkMessagesRead(ids)
 }
 
@@ -170,54 +174,63 @@ func (storeMailbox *Mailbox) MarkMessagesUnstarred(apiIDs []string) error {
 	return storeMailbox.client().UnlabelMessages(apiIDs, pmapi.StarredLabel)
 }
 
-// DeleteMessages deletes messages.
+// MarkMessagesDeleted adds local flag \Deleted. This is not propagated to API
-// If the mailbox is All Mail or All Sent, it does nothing.
+// until RemoveDeleted is called
-// If the mailbox is Trash or Spam and message is not in any other mailbox, messages is deleted.
+func (storeMailbox *Mailbox) MarkMessagesDeleted(apiIDs []string) error {
-// In all other cases the message is only removed from the mailbox.
-func (storeMailbox *Mailbox) DeleteMessages(apiIDs []string) error {
 	log.WithFields(logrus.Fields{
 		"messages": apiIDs,
 		"label":    storeMailbox.labelID,
 		"mailbox":  storeMailbox.Name,
-	}).Trace("Deleting messages")
+	}).Trace("Marking messages as deleted")
+	if storeMailbox.labelID == pmapi.AllMailLabel {
+		return ErrAllMailOpNotAllowed
+	}
+	return storeMailbox.store.db.Update(func(tx *bolt.Tx) error {
+		return storeMailbox.txMarkMessagesAsDeleted(tx, apiIDs, true)
+	})
+}
+
+// MarkMessagesUndeleted removes local flag \Deleted. This is not propagated to
+// API.
+func (storeMailbox *Mailbox) MarkMessagesUndeleted(apiIDs []string) error {
+	log.WithFields(logrus.Fields{
+		"messages": apiIDs,
+		"label":    storeMailbox.labelID,
+		"mailbox":  storeMailbox.Name,
+	}).Trace("Marking messages as undeleted")
+	if storeMailbox.labelID == pmapi.AllMailLabel {
+		return ErrAllMailOpNotAllowed
+	}
+	return storeMailbox.store.db.Update(func(tx *bolt.Tx) error {
+		return storeMailbox.txMarkMessagesAsDeleted(tx, apiIDs, false)
+	})
+}
+
+// RemoveDeleted sends request to API to remove message from mailbox.
+// If the mailbox is All Mail or All Sent, it does nothing.
+// If the mailbox is Trash or Spam and message is not in any other mailbox, messages is deleted.
+// In all other cases the message is only removed from the mailbox.
+func (storeMailbox *Mailbox) RemoveDeleted() error {
+	storeMailbox.log.Trace("Deleting messages")
+
+	apiIDs, err := storeMailbox.GetDeletedAPIIDs()
+	if err != nil {
+		return err
+	}
+
+	if len(apiIDs) == 0 {
+		storeMailbox.log.Debug("List to expunge is empty")
+		return nil
+	}
+
 	defer storeMailbox.pollNow()
 
 	switch storeMailbox.labelID {
 	case pmapi.AllMailLabel, pmapi.AllSentLabel:
 		break
 	case pmapi.TrashLabel, pmapi.SpamLabel:
-		messageIDsToDelete := []string{}
+		if err := storeMailbox.deleteFromTrashOrSpam(apiIDs); err != nil {
-		messageIDsToUnlabel := []string{}
+			return err
-		for _, apiID := range apiIDs {
-			msg, err := storeMailbox.store.getMessageFromDB(apiID)
-			if err != nil {
-				return err
-			}
-
-			otherLabels := false
-			// If the message has any custom label, we don't want to delete it, only remove trash/spam label.
-			for _, label := range msg.LabelIDs {
-				if label != pmapi.SpamLabel && label != pmapi.TrashLabel && label != pmapi.AllMailLabel && label != pmapi.AllSentLabel && label != pmapi.DraftLabel && label != pmapi.AllDraftsLabel {
-					otherLabels = true
-					break
-				}
-			}
-
-			if otherLabels {
-				messageIDsToUnlabel = append(messageIDsToUnlabel, apiID)
-			} else {
-				messageIDsToDelete = append(messageIDsToDelete, apiID)
-			}
-		}
-		if len(messageIDsToUnlabel) > 0 {
-			if err := storeMailbox.client().UnlabelMessages(messageIDsToUnlabel, storeMailbox.labelID); err != nil {
-				log.WithError(err).Warning("Cannot unlabel before deleting")
-			}
-		}
-		if len(messageIDsToDelete) > 0 {
-			if err := storeMailbox.client().DeleteMessages(messageIDsToDelete); err != nil {
-				return err
-			}
 		}
 	case pmapi.DraftLabel:
 		if err := storeMailbox.client().DeleteMessages(apiIDs); err != nil {
@@ -231,6 +244,50 @@ func (storeMailbox *Mailbox) DeleteMessages(apiIDs []string) error {
 	return nil
 }
 
+// deleteFromTrashOrSpam will remove messages from API forever. If messages
+// still has some custom label the message will not be deleted. Instead it will
+// be removed from Trash or Spam.
+func (storeMailbox *Mailbox) deleteFromTrashOrSpam(apiIDs []string) error {
+	l := storeMailbox.log.WithField("messages", apiIDs)
+	l.Trace("Deleting messages from trash")
+
+	messageIDsToDelete := []string{}
+	messageIDsToUnlabel := []string{}
+	for _, apiID := range apiIDs {
+		msg, err := storeMailbox.store.getMessageFromDB(apiID)
+		if err != nil {
+			return err
+		}
+
+		otherLabels := false
+		// If the message has any custom label, we don't want to delete it, only remove trash/spam label.
+		for _, label := range msg.LabelIDs {
+			if label != pmapi.SpamLabel && label != pmapi.TrashLabel && label != pmapi.AllMailLabel && label != pmapi.AllSentLabel && label != pmapi.DraftLabel && label != pmapi.AllDraftsLabel {
+				otherLabels = true
+				break
+			}
+		}
+
+		if otherLabels {
+			messageIDsToUnlabel = append(messageIDsToUnlabel, apiID)
+		} else {
+			messageIDsToDelete = append(messageIDsToDelete, apiID)
+		}
+	}
+	if len(messageIDsToUnlabel) > 0 {
+		if err := storeMailbox.client().UnlabelMessages(messageIDsToUnlabel, storeMailbox.labelID); err != nil {
+			l.WithError(err).Warning("Cannot unlabel before deleting")
+		}
+	}
+	if len(messageIDsToDelete) > 0 {
+		if err := storeMailbox.client().DeleteMessages(messageIDsToDelete); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func (storeMailbox *Mailbox) txSkipAndRemoveFromMailbox(tx *bolt.Tx, msg *pmapi.Message) (skipAndRemove bool) {
 	defer func() {
 		if skipAndRemove {
@@ -270,7 +327,7 @@ func (storeMailbox *Mailbox) txCreateOrUpdateMessages(tx *bolt.Tx, msgs []*pmapi
 
 	// Buckets are not initialized right away because it's a heavy operation.
 	// The best option is to get the same bucket only once and only when needed.
-	var apiBucket, imapBucket *bolt.Bucket
+	var apiBucket, imapBucket, deletedBucket *bolt.Bucket
 	for _, msg := range msgs {
 		if storeMailbox.txSkipAndRemoveFromMailbox(tx, msg) {
 			continue
@@ -289,12 +346,15 @@ func (storeMailbox *Mailbox) txCreateOrUpdateMessages(tx *bolt.Tx, msgs []*pmapi
 			}
 		} else {
 			uidb := apiBucket.Get([]byte(msg.ID))
-
 			if uidb != nil {
 				if imapBucket == nil {
 					imapBucket = storeMailbox.txGetIMAPIDsBucket(tx)
 				}
 				seqNum, seqErr := storeMailbox.txGetSequenceNumberOfUID(imapBucket, uidb)
+				if deletedBucket == nil {
+					deletedBucket = storeMailbox.txGetDeletedIDsBucket(tx)
+				}
+				isMarkedAsDeleted := deletedBucket.Get([]byte(msg.ID)) != nil
 				if seqErr == nil {
 					storeMailbox.store.imapUpdateMessage(
 						storeMailbox.storeAddress.address,
@@ -302,6 +362,7 @@ func (storeMailbox *Mailbox) txCreateOrUpdateMessages(tx *bolt.Tx, msgs []*pmapi
 						btoi(uidb),
 						seqNum,
 						msg,
+						isMarkedAsDeleted,
 					)
 				}
 				continue
@@ -335,6 +396,7 @@ func (storeMailbox *Mailbox) txCreateOrUpdateMessages(tx *bolt.Tx, msgs []*pmapi
 			uid,
 			seqNum,
 			msg,
+			false, // new message is never marked as deleted
 		)
 		shouldSendMailboxUpdate = true
 	}
@@ -359,6 +421,7 @@ func (storeMailbox *Mailbox) txDeleteMessage(tx *bolt.Tx, apiID string) error {
 	}
 
 	imapBucket := storeMailbox.txGetIMAPIDsBucket(tx)
+	deletedBucket := storeMailbox.txGetDeletedIDsBucket(tx)
 
 	seqNum, seqNumErr := storeMailbox.txGetSequenceNumberOfUID(imapBucket, uidb)
 	if seqNumErr != nil {
@@ -373,6 +436,10 @@ func (storeMailbox *Mailbox) txDeleteMessage(tx *bolt.Tx, apiID string) error {
 		return errors.Wrap(err, "cannot delete from API bucket")
 	}
 
+	if err := deletedBucket.Delete(apiIDb); err != nil {
+		return errors.Wrap(err, "cannot delete from mark-as-deleted bucket")
+	}
+
 	if seqNumErr == nil {
 		storeMailbox.store.imapDeleteMessage(
 			storeMailbox.storeAddress.address,
@@ -401,3 +468,58 @@ func (storeMailbox *Mailbox) txMailboxStatusUpdate(tx *bolt.Tx) error {
 	)
 	return nil
 }
+
+func (storeMailbox *Mailbox) txMarkMessagesAsDeleted(tx *bolt.Tx, apiIDs []string, markAsDeleted bool) error {
+	// Load all buckets before looping over apiIDs
+	metaBucket := tx.Bucket(metadataBucket)
+	apiBucket := storeMailbox.txGetAPIIDsBucket(tx)
+	uidBucket := storeMailbox.txGetIMAPIDsBucket(tx)
+	deletedBucket := storeMailbox.txGetDeletedIDsBucket(tx)
+	for _, apiID := range apiIDs {
+		if markAsDeleted {
+			if err := deletedBucket.Put([]byte(apiID), []byte{1}); err != nil {
+				return err
+			}
+		} else {
+			if err := deletedBucket.Delete([]byte(apiID)); err != nil {
+				return err
+			}
+		}
+
+		msg, err := storeMailbox.store.txGetMessageFromBucket(metaBucket, apiID)
+		if err != nil {
+			return err
+		}
+
+		uid, err := storeMailbox.txGetUIDFromBucket(apiBucket, apiID)
+		if err != nil {
+			return err
+		}
+
+		seqNum, err := storeMailbox.txGetSequenceNumberOfUID(uidBucket, itob(uid))
+		if err != nil {
+			return err
+		}
+
+		// In order to send flags in format
+		// S: * 2 FETCH (FLAGS (\Deleted \Seen))
+		update := storeMailbox.store.imapUpdateMessage(
+			storeMailbox.storeAddress.address,
+			storeMailbox.labelName,
+			uid,
+			seqNum,
+			msg,
+			markAsDeleted,
+		)
+
+		// txMarkMessagesAsDeleted is called only during processing request
+		// from IMAP call (i.e., not from event loop) and in such cases we
+		// have to wait to propagate update back before closing the response.
+		select {
+		case <-time.After(1 * time.Second):
+		case <-update.Done():
+		}
+	}
+
+	return nil
+}

internal/store/message.go

@@ -62,6 +62,21 @@ func (message *Message) Message() *pmapi.Message {
 	return message.msg
 }
 
+// IsMarkedDeleted returns true if message is marked as deleted for specific
+// mailbox
+func (message *Message) IsMarkedDeleted() bool {
+	isMarkedAsDeleted := false
+	err := message.storeMailbox.db().View(func(tx *bolt.Tx) error {
+		isMarkedAsDeleted = message.storeMailbox.txGetDeletedIDsBucket(tx).Get([]byte(message.msg.ID)) != nil
+		return nil
+	})
+	if err != nil {
+		message.storeMailbox.log.WithError(err).Error("Not able to retrieve deleted mark, assuming false.")
+		return false
+	}
+	return isMarkedAsDeleted
+}
+
 // SetSize updates the information about size of decrypted message which can be
 // used for IMAP. This should not trigger any IMAP update.
 // NOTE: The size from the server corresponds to pure body bytes. Hence it

internal/store/store.go

@@ -70,6 +70,8 @@ var (
 	//       * {imapUID} -> string messageID
 	//     * api_ids
 	//       * {messageID} -> uint32 imapUID
+	//     * deleted_ids (can be missing or have no keys)
+	//       * {messageID} -> true
 	metadataBucket    = []byte("metadata")          //nolint[gochecknoglobals]
 	countsBucket      = []byte("counts")            //nolint[gochecknoglobals]
 	addressInfoBucket = []byte("address_info")      //nolint[gochecknoglobals]
@@ -78,6 +80,7 @@ var (
 	mailboxesBucket   = []byte("mailboxes")         //nolint[gochecknoglobals]
 	imapIDsBucket     = []byte("imap_ids")          //nolint[gochecknoglobals]
 	apiIDsBucket      = []byte("api_ids")           //nolint[gochecknoglobals]
+	deletedIDsBucket  = []byte("deleted_ids")       //nolint[gochecknoglobals]
 	mboxVersionBucket = []byte("mailboxes_version") //nolint[gochecknoglobals]
 
 	// ErrNoSuchAPIID when mailbox does not have API ID.
@@ -348,6 +351,18 @@ func (store *Store) addAddress(address, addressID string, labels []*pmapi.Label)
 	return
 }
 
+// PauseEventLoop sets whether the ticker is periodically polling or not.
+func (store *Store) PauseEventLoop(pause bool) {
+	store.lock.Lock()
+	defer store.lock.Unlock()
+
+	store.log.WithField("pause", pause).Info("Pausing event loop")
+
+	if store.eventLoop != nil {
+		store.eventLoop.isTickerPaused = pause
+	}
+}
+
 // Close stops the event loop and closes the database to free the file.
 func (store *Store) Close() error {
 	store.lock.Lock()

internal/store/store_test_exports.go

@@ -26,6 +26,10 @@ import (
 	bolt "go.etcd.io/bbolt"
 )
 
+func (loop *eventLoop) IsRunning() bool {
+	return loop.isRunning
+}
+
 // TestSync triggers a sync of the store.
 func (store *Store) TestSync() {
 	store.lock.Lock()
@@ -102,11 +106,13 @@ func txDumpMailsFactory(tb assert.TestingT) func(tx *bolt.Tx) error {
 		err := mailboxes.ForEach(func(mboxName, mboxData []byte) error {
 			fmt.Println("mbox:", string(mboxName))
 			b := mailboxes.Bucket(mboxName).Bucket(imapIDsBucket)
+			deletedMailboxes := mailboxes.Bucket(mboxName).Bucket(deletedIDsBucket)
 			c := b.Cursor()
 			i := 0
 			for imapID, apiID := c.First(); imapID != nil; imapID, apiID = c.Next() {
 				i++
-				fmt.Println("  ", i, "imap", btoi(imapID), "api", string(apiID))
+				isDeleted := deletedMailboxes != nil && deletedMailboxes.Get(apiID) != nil
+				fmt.Println("  ", i, "imap", btoi(imapID), "api", string(apiID), "isDeleted", isDeleted)
 				data := metadata.Get(apiID)
 				if !assert.NotNil(tb, data) {
 					continue

internal/store/user_mailbox.go

@@ -210,7 +210,9 @@ func (store *Store) deleteMailboxEvent(labelID string) error {
 	store.lock.Lock()
 	defer store.lock.Unlock()
 
-	_ = store.removeMailboxCount(labelID)
+	if err := store.removeMailboxCount(labelID); err != nil {
+		log.WithError(err).Warn("Problem to remove mailbox counts while deleting mailbox")
+	}
 
 	for _, a := range store.addresses {
 		if err := a.deleteMailboxEvent(labelID); err != nil {

internal/store/user_message.go

@@ -63,7 +63,7 @@ func (store *Store) CreateDraft(
 		attachmentReaders = append(attachmentReaders, strings.NewReader(attachedPublicKey))
 		publicKeyAttachment := &pmapi.Attachment{
 			Name:     attachedPublicKeyName + ".asc",
-			MIMEType: "application/pgp-key",
+			MIMEType: "application/pgp-keys",
 			Header:   textproto.MIMEHeader{},
 		}
 		attachments = append(attachments, publicKeyAttachment)
@@ -143,8 +143,10 @@ func (store *Store) getMessageFromDB(apiID string) (msg *pmapi.Message, err erro
 }
 
 func (store *Store) txGetMessage(tx *bolt.Tx, apiID string) (*pmapi.Message, error) {
-	b := tx.Bucket(metadataBucket)
+	return store.txGetMessageFromBucket(tx.Bucket(metadataBucket), apiID)
+}
 
+func (store *Store) txGetMessageFromBucket(b *bolt.Bucket, apiID string) (*pmapi.Message, error) {
 	msgb := b.Get([]byte(apiID))
 	if msgb == nil {
 		return nil, ErrNoSuchAPIID

internal/transfer/message.go

@@ -29,17 +29,34 @@ type Message struct {
 	ID      string
 	Unread  bool
 	Body    []byte
-	Source  Mailbox
+	Sources []Mailbox
 	Targets []Mailbox
 }
 
+// sourceNames returns array of source mailbox names.
+func (msg Message) sourceNames() (names []string) {
+	for _, mailbox := range msg.Sources {
+		names = append(names, mailbox.Name)
+	}
+	return
+}
+
+// targetNames returns array of target mailbox names.
+func (msg Message) targetNames() (names []string) {
+	for _, mailbox := range msg.Targets {
+		names = append(names, mailbox.Name)
+	}
+	return
+}
+
 // MessageStatus holds status for message used by progress manager.
 type MessageStatus struct {
-	eventTime time.Time // Time of adding message to the process.
+	eventTime   time.Time // Time of adding message to the process.
-	rule      *Rule     // Rule with source and target mailboxes.
+	sourceNames []string  // Source mailbox names message is in.
-	SourceID  string    // Message ID at the source.
+	SourceID    string    // Message ID at the source.
-	targetID  string    // Message ID at the target (if any).
+	targetNames []string  // Target mailbox names message is in.
-	bodyHash  string    // Hash of the message body.
+	targetID    string    // Message ID at the target (if any).
+	bodyHash    string    // Hash of the message body.
 
 	exported  bool
 	imported  bool

internal/transfer/progress.go

@@ -62,11 +62,20 @@ func (p *Progress) update() {
 		return
 	}
 
-	// In case no one listens for an update, do not block the progress.
+	// In case no one listens for an update, do not block the whole progress.
-	select {
+	go func() {
-	case p.updateCh <- struct{}{}:
+		defer func() {
-	case <-time.After(100 * time.Millisecond):
+			// updateCh can be closed at the end of progress which is fine.
-	}
+			if r := recover(); r != nil {
+				log.WithField("r", r).Warn("Failed to send update")
+			}
+		}()
+
+		select {
+		case p.updateCh <- struct{}{}:
+		case <-time.After(5 * time.Millisecond):
+		}
+	}()
 }
 
 // finish should be called as the last call once everything is done.
@@ -84,7 +93,7 @@ func (p *Progress) fatal(err error) {
 	defer p.lock.Unlock()
 
 	log.WithError(err).Error("Progress finished")
-	p.isStopped = true
+	p.setStop()
 	p.fatalError = err
 	p.cleanUpdateCh()
 }
@@ -117,16 +126,17 @@ func (p *Progress) updateCount(mailbox string, count uint) {
 }
 
 // addMessage should be called as soon as there is ID of the message.
-func (p *Progress) addMessage(messageID string, rule *Rule) {
+func (p *Progress) addMessage(messageID string, sourceNames, targetNames []string) {
 	p.lock.Lock()
 	defer p.lock.Unlock()
 	defer p.update()
 
 	p.log.WithField("id", messageID).Trace("Message added")
 	p.messageStatuses[messageID] = &MessageStatus{
-		eventTime: time.Now(),
+		eventTime:   time.Now(),
-		rule:      rule,
+		sourceNames: sourceNames,
-		SourceID:  messageID,
+		SourceID:    messageID,
+		targetNames: targetNames,
 	}
 }
 
@@ -273,6 +283,15 @@ func (p *Progress) Stop() {
 	defer p.update()
 
 	p.log.Info("Progress stopped")
+	p.setStop()
+
+	// Once progress is stopped, some calls might be in progress. Results from
+	// those calls are irrelevant so we can close update channel sooner to not
+	// propagate any progress to user interface anymore.
+	p.cleanUpdateCh()
+}
+
+func (p *Progress) setStop() {
 	p.isStopped = true
 	p.pauseReason = "" // Clear pause to run paused code and stop it.
 }

internal/transfer/progress_test.go

@@ -19,6 +19,7 @@ package transfer
 
 import (
 	"testing"
+	"time"
 
 	"github.com/pkg/errors"
 	a "github.com/stretchr/testify/assert"
@@ -47,21 +48,21 @@ func TestProgressAddingMessages(t *testing.T) {
 	drainProgressUpdateChannel(&progress)
 
 	// msg1 has no problem.
-	progress.addMessage("msg1", nil)
+	progress.addMessage("msg1", []string{}, []string{})
 	progress.messageExported("msg1", []byte(""), nil)
 	progress.messageImported("msg1", "", nil)
 
 	// msg2 has an import problem.
-	progress.addMessage("msg2", nil)
+	progress.addMessage("msg2", []string{}, []string{})
 	progress.messageExported("msg2", []byte(""), nil)
 	progress.messageImported("msg2", "", errors.New("failed import"))
 
 	// msg3 has an export problem.
-	progress.addMessage("msg3", nil)
+	progress.addMessage("msg3", []string{}, []string{})
 	progress.messageExported("msg3", []byte(""), errors.New("failed export"))
 
 	// msg4 has an export problem and import is also called.
-	progress.addMessage("msg4", nil)
+	progress.addMessage("msg4", []string{}, []string{})
 	progress.messageExported("msg4", []byte(""), errors.New("failed export"))
 	progress.messageImported("msg4", "", nil)
 
@@ -91,7 +92,7 @@ func TestProgressFinish(t *testing.T) {
 	progress.finish()
 	r.Nil(t, progress.updateCh)
 
-	r.NotPanics(t, func() { progress.addMessage("msg", nil) })
+	r.NotPanics(t, func() { progress.addMessage("msg", []string{}, []string{}) })
 }
 
 func TestProgressFatalError(t *testing.T) {
@@ -101,7 +102,29 @@ func TestProgressFatalError(t *testing.T) {
 	progress.fatal(errors.New("fatal error"))
 	r.Nil(t, progress.updateCh)
 
-	r.NotPanics(t, func() { progress.addMessage("msg", nil) })
+	r.NotPanics(t, func() { progress.addMessage("msg", []string{}, []string{}) })
+}
+
+func TestFailUnpauseAndStops(t *testing.T) {
+	progress := newProgress(log, nil)
+	drainProgressUpdateChannel(&progress)
+
+	progress.Pause("pausing")
+	progress.fatal(errors.New("fatal error"))
+
+	r.Nil(t, progress.updateCh)
+	r.True(t, progress.isStopped)
+	r.False(t, progress.IsPaused())
+	r.Eventually(t, progress.shouldStop, time.Second, 10*time.Millisecond)
+}
+
+func TestStopClosesUpdates(t *testing.T) {
+	progress := newProgress(log, nil)
+	ch := progress.updateCh
+
+	progress.Stop()
+	r.Nil(t, progress.updateCh)
+	r.PanicsWithError(t, "send on closed channel", func() { ch <- struct{}{} })
 }
 
 func drainProgressUpdateChannel(progress *Progress) {

internal/transfer/provider_eml_source.go

@@ -109,7 +109,7 @@ func (p *EMLProvider) exportMessages(rule *Rule, filePaths []string, progress *P
 		// addMessage is called after time check to not report message
 		// which should not be exported but any error from reading body
 		// or parsing time is reported as an error.
-		progress.addMessage(filePath, rule)
+		progress.addMessage(filePath, msg.sourceNames(), msg.targetNames())
 		progress.messageExported(filePath, msg.Body, err)
 		if err == nil {
 			ch <- msg
@@ -134,7 +134,7 @@ func (p *EMLProvider) exportMessage(rule *Rule, filePath string) (Message, error
 		ID:      filePath,
 		Unread:  false,
 		Body:    body,
-		Source:  rule.SourceMailbox,
+		Sources: []Mailbox{rule.SourceMailbox},
 		Targets: rule.TargetMailboxes,
 	}, nil
 }

internal/transfer/provider_eml_target.go

@@ -61,7 +61,7 @@ func (p *EMLProvider) TransferFrom(rules transferRules, progress *Progress, ch <
 func (p *EMLProvider) createFolders(rules transferRules) error {
 	for rule := range rules.iterateActiveRules() {
 		for _, mailbox := range rule.TargetMailboxes {
-			path := filepath.Join(p.root, mailbox.Name)
+			path := filepath.Join(p.root, sanitizeFileName(mailbox.Name))
 			if err := os.MkdirAll(path, os.ModePerm); err != nil {
 				return err
 			}
@@ -71,7 +71,7 @@ func (p *EMLProvider) createFolders(rules transferRules) error {
 }
 
 func (p *EMLProvider) writeFile(msg Message) error {
-	fileName := filepath.Base(msg.ID)
+	fileName := sanitizeFileName(filepath.Base(msg.ID))
 	if filepath.Ext(fileName) != ".eml" {
 		fileName += ".eml"
 	}

internal/transfer/provider_imap.go

@@ -31,6 +31,8 @@ type IMAPProvider struct {
 	addr     string
 
 	client *imapClient.Client
+
+	timeIt *timeIt
 }
 
 // NewIMAPProvider returns new IMAPProvider.
@@ -39,6 +41,8 @@ func NewIMAPProvider(username, password, host, port string) (*IMAPProvider, erro
 		username: username,
 		password: password,
 		addr:     net.JoinHostPort(host, port),
+
+		timeIt: newTimeIt("imap"),
 	}
 
 	if err := p.auth(); err != nil {

internal/transfer/provider_imap_source.go

@@ -40,6 +40,9 @@ func (p *IMAPProvider) TransferTo(rules transferRules, progress *Progress, ch ch
 	log.Info("Started transfer from IMAP to channel")
 	defer log.Info("Finished transfer from IMAP to channel")
 
+	p.timeIt.clear()
+	defer p.timeIt.logResults()
+
 	imapMessageInfoMap := p.loadMessageInfoMap(rules, progress)
 
 	for rule := range rules.iterateActiveRules() {
@@ -78,6 +81,9 @@ func (p *IMAPProvider) loadMessageInfoMap(rules transferRules, progress *Progres
 }
 
 func (p *IMAPProvider) loadMessagesInfo(rule *Rule, progress *Progress, uidValidity, count uint32) map[string]imapMessageInfo {
+	p.timeIt.start("load", rule.SourceMailbox.Name)
+	defer p.timeIt.stop("load", rule.SourceMailbox.Name)
+
 	messagesInfo := map[string]imapMessageInfo{}
 
 	pageStart := uint32(1)
@@ -118,7 +124,7 @@ func (p *IMAPProvider) loadMessagesInfo(rule *Rule, progress *Progress, uidValid
 				uid:  imapMessage.Uid,
 				size: imapMessage.Size,
 			}
-			progress.addMessage(id, rule)
+			progress.addMessage(id, []string{rule.SourceMailbox.Name}, rule.TargetMailboxNames())
 		}
 
 		progress.callWrap(func() error {
@@ -199,13 +205,18 @@ func (p *IMAPProvider) exportMessages(rule *Rule, progress *Progress, ch chan<-
 		progress.messageExported(id, body, err)
 		if err == nil {
 			msg := p.exportMessage(rule, id, imapMessage, body)
+
+			p.timeIt.stop("fetch", rule.SourceMailbox.Name)
 			ch <- msg
+			p.timeIt.start("fetch", rule.SourceMailbox.Name)
 		}
 	}
 
+	p.timeIt.start("fetch", rule.SourceMailbox.Name)
 	progress.callWrap(func() error {
 		return p.uidFetch(rule.SourceMailbox.Name, seqSet, items, processMessageCallback)
 	})
+	p.timeIt.stop("fetch", rule.SourceMailbox.Name)
 }
 
 func (p *IMAPProvider) exportMessage(rule *Rule, id string, imapMessage *imap.Message, body []byte) Message {
@@ -220,7 +231,7 @@ func (p *IMAPProvider) exportMessage(rule *Rule, id string, imapMessage *imap.Me
 		ID:      id,
 		Unread:  unread,
 		Body:    body,
-		Source:  rule.SourceMailbox,
+		Sources: []Mailbox{rule.SourceMailbox},
 		Targets: rule.TargetMailboxes,
 	}
 }

internal/transfer/provider_imap_utils.go

@@ -18,7 +18,9 @@
 package transfer
 
 import (
+	"crypto/tls"
 	"net"
+	"strings"
 	"time"
 
 	imapID "github.com/ProtonMail/go-imap-id"
@@ -146,7 +148,19 @@ func (p *IMAPProvider) auth() error { //nolint[funlen]
 	if host == "127.0.0.1" {
 		client, err = imapClient.Dial(p.addr)
 	} else {
-		client, err = imapClient.DialTLS(p.addr, nil)
+		// IMAP.mail.yahoo.com have problem with golang TLS1.3
+		// implementation with weird behaviour i.e. Yahoo
+		// no error during dial or handshake but server logs out right
+		// after successful login leaving no time to perform any
+		// action. It was discovered that limiting to maximum TLS
+		// version 1.2 for yahoo servers is working solution.
+
+		var tlsConf *tls.Config
+		if strings.Contains(strings.ToLower(host), "yahoo") {
+			log.Warning("Yahoo server detected: limiting maximal TLS version to 1.2.")
+			tlsConf = &tls.Config{MaxVersion: tls.VersionTLS12}
+		}
+		client, err = imapClient.DialTLS(p.addr, tlsConf)
 	}
 	if err != nil {
 		return ErrIMAPConnection{imapError{Err: err, Message: "failed to connect to server"}}

internal/transfer/provider_mbox.go

@@ -18,8 +18,11 @@
 package transfer
 
 import (
+	"os"
 	"path/filepath"
 	"strings"
+
+	"github.com/pkg/errors"
 )
 
 // MBOXProvider implements import and export to/from MBOX structure.
@@ -44,16 +47,35 @@ func (p *MBOXProvider) ID() string {
 // In case the same folder name is used more than once (for example root/a/foo
 // and root/b/foo), it's treated as the same folder.
 func (p *MBOXProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) {
-	filePaths, err := getFilePathsWithSuffix(p.root, "mbox")
+	filePaths, err := getAllPathsWithSuffix(p.root, ".mbox")
 	if err != nil {
 		return nil, err
 	}
 
-	mailboxes := []Mailbox{}
+	mailboxNames := map[string]bool{}
 	for _, filePath := range filePaths {
 		fileName := filepath.Base(filePath)
-		mailboxName := strings.TrimSuffix(fileName, ".mbox")
+		filePath, err := p.handleAppleMailMBOXStructure(filePath)
+		if err != nil {
+			log.WithError(err).Warn("Failed to handle MBOX structure")
+			continue
+		}
 
+		mailboxName := strings.TrimSuffix(fileName, ".mbox")
+		mailboxNames[mailboxName] = true
+
+		labels, err := getGmailLabelsFromMboxFile(filepath.Join(p.root, filePath))
+		if err != nil {
+			log.WithError(err).Error("Failed to get gmail labels from mbox file")
+			continue
+		}
+		for label := range labels {
+			mailboxNames[label] = true
+		}
+	}
+
+	mailboxes := []Mailbox{}
+	for mailboxName := range mailboxNames {
 		mailboxes = append(mailboxes, Mailbox{
 			ID:          "",
 			Name:        mailboxName,
@@ -61,6 +83,20 @@ func (p *MBOXProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox,
 			IsExclusive: false,
 		})
 	}
-
 	return mailboxes, nil
 }
+
+// handleAppleMailMBOXStructure changes the path of mailbox directory to
+// the path of mbox file. Apple Mail MBOX exports has this structure:
+// `Folder.mbox` directory with `mbox` file inside.
+// Example: `Folder.mbox/mbox` (and this function converts `Folder.mbox`
+// to `Folder.mbox/mbox`).
+func (p *MBOXProvider) handleAppleMailMBOXStructure(filePath string) (string, error) {
+	if info, err := os.Stat(filepath.Join(p.root, filePath)); err == nil && info.IsDir() {
+		if _, err := os.Stat(filepath.Join(p.root, filePath, "mbox")); err != nil {
+			return "", errors.Wrap(err, "wrong mbox structure")
+		}
+		return filepath.Join(filePath, "mbox"), nil
+	}
+	return filePath, nil
+}

internal/transfer/provider_mbox_gmail_labels.go

@@ -0,0 +1,118 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"mime"
+	"os"
+	"strings"
+)
+
+type stringSet map[string]bool
+
+const xGmailLabelsHeader = "X-Gmail-Labels"
+
+// filteredOutGmailLabels is set of labels which we don't want to show to users
+// as they might be auto-generated by Gmail and unwanted.
+var filteredOutGmailLabels = []string{ //nolint[gochecknoglobals]
+	"Unread",
+	"Opened",
+	"IMAP_Junk",
+	"IMAP_NonJunk",
+	"IMAP_NotJunk",
+	"IMAP_$NotJunk",
+}
+
+func getGmailLabelsFromMboxFile(filePath string) (stringSet, error) {
+	f, err := os.Open(filePath) //nolint[gosec]
+	if err != nil {
+		return nil, err
+	}
+	return getGmailLabelsFromMboxReader(f)
+}
+
+func getGmailLabelsFromMboxReader(f io.Reader) (stringSet, error) {
+	allLabels := stringSet{}
+
+	// Scanner is not used as it does not support long lines and some mbox
+	// files contain very long lines even though that should not be happening.
+	r := bufio.NewReader(f)
+	for {
+		b, isPrefix, err := r.ReadLine()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		for isPrefix {
+			_, isPrefix, err = r.ReadLine()
+			if err != nil {
+				break
+			}
+		}
+		if bytes.HasPrefix(b, []byte(xGmailLabelsHeader)) {
+			for label := range getGmailLabelsFromValue(string(b)) {
+				allLabels[label] = true
+			}
+		}
+	}
+
+	return allLabels, nil
+}
+
+func getGmailLabelsFromMessage(body []byte) (stringSet, error) {
+	header, err := getMessageHeader(body)
+	if err != nil {
+		return nil, err
+	}
+	labels := header.Get(xGmailLabelsHeader)
+	return getGmailLabelsFromValue(labels), nil
+}
+
+func getGmailLabelsFromValue(value string) stringSet {
+	value = strings.TrimPrefix(value, xGmailLabelsHeader+":")
+	if decoded, err := new(mime.WordDecoder).DecodeHeader(value); err != nil {
+		log.WithError(err).Error("Failed to decode header")
+	} else {
+		value = decoded
+	}
+
+	labels := stringSet{}
+	for _, label := range strings.Split(value, ",") {
+		label = strings.TrimSpace(label)
+		if label == "" {
+			continue
+		}
+		skip := false
+		for _, filteredOutLabel := range filteredOutGmailLabels {
+			if label == filteredOutLabel {
+				skip = true
+				break
+			}
+		}
+		if skip {
+			continue
+		}
+		labels[label] = true
+	}
+	return labels
+}

internal/transfer/provider_mbox_gmail_labels_test.go

@@ -0,0 +1,135 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	r "github.com/stretchr/testify/require"
+)
+
+func TestGetGmailLabelsFromMboxReader(t *testing.T) {
+	mboxFile := `From - Mon May  4 16:40:31 2020
+Subject: Test 1
+X-Gmail-Labels: Foo,Bar
+
+hello
+
+From - Mon May  4 16:40:31 2020
+Subject: Test 2
+X-Gmail-Labels: Foo , Baz
+
+hello
+
+From - Mon May  4 16:40:31 2020
+Subject: Test 3
+X-Gmail-Labels: ,
+
+hello
+
+From - Mon May  4 16:40:31 2020
+Subject: Test 4
+X-Gmail-Labels:
+
+hello
+
+From - Mon May  4 16:40:31 2020
+Subject: Test 5
+
+hello
+
+`
+	mboxReader := strings.NewReader(mboxFile)
+	labels, err := getGmailLabelsFromMboxReader(mboxReader)
+	r.NoError(t, err)
+	r.Equal(t, toSet("Foo", "Bar", "Baz"), labels)
+}
+
+func TestGetGmailLabelsFromMessage(t *testing.T) {
+	tests := []struct {
+		body       string
+		wantLabels stringSet
+	}{
+		{`Subject: One
+X-Gmail-Labels: Foo,Bar
+
+Hello
+`, toSet("Foo", "Bar")},
+		{`Subject: Two
+X-Gmail-Labels: Foo , Bar ,
+
+Hello
+`, toSet("Foo", "Bar")},
+		{`Subject: Three
+X-Gmail-Labels: ,
+
+Hello
+`, toSet()},
+		{`Subject: Four
+X-Gmail-Labels:
+
+Hello
+`, toSet()},
+		{`Subject: Five
+
+Hello
+`, toSet()},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(fmt.Sprintf("%v", tc.body), func(t *testing.T) {
+			labels, err := getGmailLabelsFromMessage([]byte(tc.body))
+			r.NoError(t, err)
+			r.Equal(t, tc.wantLabels, labels)
+		})
+	}
+}
+
+func TestGetGmailLabelsFromValue(t *testing.T) {
+	tests := []struct {
+		value      string
+		wantLabels stringSet
+	}{
+		{"Foo,Bar", toSet("Foo", "Bar")},
+		{" Foo , Bar ", toSet("Foo", "Bar")},
+		{" Foo , Bar , ", toSet("Foo", "Bar")},
+		{" Foo Bar ", toSet("Foo Bar")},
+		{" , ", toSet()},
+		{" ", toSet()},
+		{"", toSet()},
+		{"=?UTF-8?Q?Archived,Category_personal,test_=F0=9F=98=80=F0=9F=99=83?=", toSet("Archived", "Category personal", "test 😀🙃")},
+		{"IMAP_NotJunk,Foo,Opened,bar,Unread", toSet("Foo", "bar")},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(fmt.Sprintf("%v", tc.value), func(t *testing.T) {
+			labels := getGmailLabelsFromValue(tc.value)
+			r.Equal(t, tc.wantLabels, labels)
+		})
+	}
+}
+
+func toSet(items ...string) stringSet {
+	set := map[string]bool{}
+	for _, item := range items {
+		set[item] = true
+	}
+	return set
+}

internal/transfer/provider_mbox_source.go

@@ -34,7 +34,7 @@ func (p *MBOXProvider) TransferTo(rules transferRules, progress *Progress, ch ch
 	log.Info("Started transfer from MBOX to channel")
 	defer log.Info("Finished transfer from MBOX to channel")
 
-	filePathsPerFolder, err := p.getFilePathsPerFolder(rules)
+	filePathsPerFolder, err := p.getFilePathsPerFolder()
 	if err != nil {
 		progress.fatal(err)
 		return
@@ -45,32 +45,29 @@ func (p *MBOXProvider) TransferTo(rules transferRules, progress *Progress, ch ch
 	}
 
 	for folderName, filePaths := range filePathsPerFolder {
-		// No error guaranteed by getFilePathsPerFolder.
+		log.WithField("folder", folderName).Debug("Estimating folder counts")
-		rule, _ := rules.getRuleBySourceMailboxName(folderName)
 		for _, filePath := range filePaths {
 			if progress.shouldStop() {
 				break
 			}
-			p.updateCount(rule, progress, filePath)
+			p.updateCount(progress, filePath)
 		}
 	}
 	progress.countsFinal()
 
 	for folderName, filePaths := range filePathsPerFolder {
-		// No error guaranteed by getFilePathsPerFolder.
+		log.WithField("folder", folderName).Debug("Processing folder")
-		rule, _ := rules.getRuleBySourceMailboxName(folderName)
-		log.WithField("rule", rule).Debug("Processing rule")
 		for _, filePath := range filePaths {
 			if progress.shouldStop() {
 				break
 			}
-			p.transferTo(rule, progress, ch, filePath)
+			p.transferTo(rules, progress, ch, folderName, filePath)
 		}
 	}
 }
 
-func (p *MBOXProvider) getFilePathsPerFolder(rules transferRules) (map[string][]string, error) {
+func (p *MBOXProvider) getFilePathsPerFolder() (map[string][]string, error) {
-	filePaths, err := getFilePathsWithSuffix(p.root, ".mbox")
+	filePaths, err := getAllPathsWithSuffix(p.root, ".mbox")
 	if err != nil {
 		return nil, err
 	}
@@ -78,19 +75,19 @@ func (p *MBOXProvider) getFilePathsPerFolder(rules transferRules) (map[string][]
 	filePathsMap := map[string][]string{}
 	for _, filePath := range filePaths {
 		fileName := filepath.Base(filePath)
-		folder := strings.TrimSuffix(fileName, ".mbox")
+		filePath, err := p.handleAppleMailMBOXStructure(filePath)
-		_, err := rules.getRuleBySourceMailboxName(folder)
+		// Skip unsupported MBOX structures. It was already filtered out in configuration step.
 		if err != nil {
-			log.WithField("msg", filePath).Trace("Mailbox skipped due to folder name")
 			continue
 		}
 
+		folder := strings.TrimSuffix(fileName, ".mbox")
 		filePathsMap[folder] = append(filePathsMap[folder], filePath)
 	}
 	return filePathsMap, nil
 }
 
-func (p *MBOXProvider) updateCount(rule *Rule, progress *Progress, filePath string) {
+func (p *MBOXProvider) updateCount(progress *Progress, filePath string) {
 	mboxReader := p.openMbox(progress, filePath)
 	if mboxReader == nil {
 		return
@@ -99,15 +96,18 @@ func (p *MBOXProvider) updateCount(rule *Rule, progress *Progress, filePath stri
 	count := 0
 	for {
 		_, err := mboxReader.NextMessage()
-		if err != nil {
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			progress.fatal(err)
 			break
 		}
 		count++
 	}
-	progress.updateCount(rule.SourceMailbox.Name, uint(count))
+	progress.updateCount(filePath, uint(count))
 }
 
-func (p *MBOXProvider) transferTo(rule *Rule, progress *Progress, ch chan<- Message, filePath string) {
+func (p *MBOXProvider) transferTo(rules transferRules, progress *Progress, ch chan<- Message, folderName, filePath string) {
 	mboxReader := p.openMbox(progress, filePath)
 	if mboxReader == nil {
 		return
@@ -131,50 +131,122 @@ func (p *MBOXProvider) transferTo(rule *Rule, progress *Progress, ch chan<- Mess
 			break
 		}
 
-		msg, err := p.exportMessage(rule, id, msgReader)
+		msg, err := p.exportMessage(rules, folderName, id, msgReader)
 
-		// Read and check time in body only if the rule specifies it
+		if err == nil && len(msg.Targets) == 0 {
-		// to not waste energy.
+			// Here should be called progress.messageSkipped(id) once we have
-		if err == nil && rule.HasTimeLimit() {
+			// this feature, and following progress.updateCount can be removed.
-			msgTime, msgTimeErr := getMessageTime(msg.Body)
+			continue
-			if msgTimeErr != nil {
-				err = msgTimeErr
-			} else if !rule.isTimeInRange(msgTime) {
-				log.WithField("msg", id).Debug("Message skipped due to time")
-				continue
-			}
 		}
 
-		// Counting only messages filtered by time to update count to correct total.
 		count++
 
 		// addMessage is called after time check to not report message
 		// which should not be exported but any error from reading body
 		// or parsing time is reported as an error.
-		progress.addMessage(id, rule)
+		progress.addMessage(id, msg.sourceNames(), msg.targetNames())
 		progress.messageExported(id, msg.Body, err)
 		if err == nil {
 			ch <- msg
 		}
 	}
-	progress.updateCount(rule.SourceMailbox.Name, uint(count))
+	progress.updateCount(filePath, uint(count))
 }
 
-func (p *MBOXProvider) exportMessage(rule *Rule, id string, msgReader io.Reader) (Message, error) {
+func (p *MBOXProvider) exportMessage(rules transferRules, folderName, id string, msgReader io.Reader) (Message, error) {
 	body, err := ioutil.ReadAll(msgReader)
 	if err != nil {
 		return Message{}, errors.Wrap(err, "failed to read message")
 	}
 
+	msgRules := p.getMessageRules(rules, folderName, id, body)
+	sources := p.getMessageSources(msgRules)
+	targets := p.getMessageTargets(msgRules, id, body)
 	return Message{
 		ID:      id,
 		Unread:  false,
 		Body:    body,
-		Source:  rule.SourceMailbox,
+		Sources: sources,
-		Targets: rule.TargetMailboxes,
+		Targets: targets,
 	}, nil
 }
 
+func (p *MBOXProvider) getMessageRules(rules transferRules, folderName, id string, body []byte) []*Rule {
+	msgRules := []*Rule{}
+
+	folderRule, err := rules.getRuleBySourceMailboxName(folderName)
+	if err != nil {
+		log.WithField("msg", id).WithField("source", folderName).Debug("Message source doesn't have a rule")
+	} else {
+		msgRules = append(msgRules, folderRule)
+	}
+
+	gmailLabels, err := getGmailLabelsFromMessage(body)
+	if err != nil {
+		log.WithError(err).Error("Failed to get gmail labels, ")
+	} else {
+		for label := range gmailLabels {
+			rule, err := rules.getRuleBySourceMailboxName(label)
+			if err != nil {
+				log.WithField("msg", id).WithField("source", label).Debug("Message source doesn't have a rule")
+				continue
+			}
+			msgRules = append(msgRules, rule)
+		}
+	}
+
+	return msgRules
+}
+
+func (p *MBOXProvider) getMessageSources(msgRules []*Rule) []Mailbox {
+	sources := []Mailbox{}
+	for _, rule := range msgRules {
+		sources = append(sources, rule.SourceMailbox)
+	}
+	return sources
+}
+
+func (p *MBOXProvider) getMessageTargets(msgRules []*Rule, id string, body []byte) []Mailbox {
+	targets := []Mailbox{}
+	haveExclusiveMailbox := false
+	for _, rule := range msgRules {
+		// Read and check time in body only if the rule specifies it
+		// to not waste energy.
+		if rule.HasTimeLimit() {
+			msgTime, err := getMessageTime(body)
+			if err != nil {
+				log.WithError(err).Error("Failed to parse time, time check skipped")
+			} else if !rule.isTimeInRange(msgTime) {
+				log.WithField("msg", id).WithField("source", rule.SourceMailbox.Name).Debug("Message skipped due to time")
+				continue
+			}
+		}
+		for _, newTarget := range rule.TargetMailboxes {
+			// msgRules is sorted. The first rule is based on the folder name,
+			// followed by the order from X-Gmail-Labels. The rule based on
+			// the folder name should have priority for exclusive target.
+			if newTarget.IsExclusive && haveExclusiveMailbox {
+				continue
+			}
+			found := false
+			for _, target := range targets {
+				if target.Hash() == newTarget.Hash() {
+					found = true
+					break
+				}
+			}
+			if found {
+				continue
+			}
+			if newTarget.IsExclusive {
+				haveExclusiveMailbox = true
+			}
+			targets = append(targets, newTarget)
+		}
+	}
+	return targets
+}
+
 func (p *MBOXProvider) openMbox(progress *Progress, mboxPath string) *mbox.Reader {
 	mboxPath = filepath.Join(p.root, mboxPath)
 	mboxFile, err := os.Open(mboxPath) //nolint[gosec]

internal/transfer/provider_mbox_target.go

@@ -57,7 +57,7 @@ func (p *MBOXProvider) TransferFrom(rules transferRules, progress *Progress, ch
 func (p *MBOXProvider) writeMessage(msg Message) error {
 	var multiErr error
 	for _, mailbox := range msg.Targets {
-		mboxName := filepath.Base(mailbox.Name)
+		mboxName := sanitizeFileName(mailbox.Name)
 		if !strings.HasSuffix(mboxName, ".mbox") {
 			mboxName += ".mbox"
 		}

internal/transfer/provider_mbox_test.go

@@ -35,27 +35,35 @@ func newTestMBOXProvider(path string) *MBOXProvider {
 }
 
 func TestMBOXProviderMailboxes(t *testing.T) {
-	provider := newTestMBOXProvider("")
-
 	tests := []struct {
+		provider      *MBOXProvider
 		includeEmpty  bool
 		wantMailboxes []Mailbox
 	}{
-		{true, []Mailbox{
+		{newTestMBOXProvider(""), true, []Mailbox{
+			{Name: "All Mail"},
 			{Name: "Foo"},
+			{Name: "Bar"},
 			{Name: "Inbox"},
 		}},
-		{false, []Mailbox{
+		{newTestMBOXProvider(""), false, []Mailbox{
+			{Name: "All Mail"},
 			{Name: "Foo"},
+			{Name: "Bar"},
 			{Name: "Inbox"},
 		}},
+		{newTestMBOXProvider("testdata/mbox-applemail"), true, []Mailbox{
+			{Name: "All Mail"},
+			{Name: "Foo"},
+			{Name: "Bar"},
+		}},
 	}
 	for _, tc := range tests {
 		tc := tc
 		t.Run(fmt.Sprintf("%v", tc.includeEmpty), func(t *testing.T) {
-			mailboxes, err := provider.Mailboxes(tc.includeEmpty, false)
+			mailboxes, err := tc.provider.Mailboxes(tc.includeEmpty, false)
 			r.NoError(t, err)
-			r.Equal(t, tc.wantMailboxes, mailboxes)
+			r.ElementsMatch(t, tc.wantMailboxes, mailboxes)
 		})
 	}
 }
@@ -67,14 +75,47 @@ func TestMBOXProviderTransferTo(t *testing.T) {
 	defer rulesClose()
 	setupMBOXRules(rules)
 
-	testTransferTo(t, rules, provider, []string{
+	msgs := testTransferTo(t, rules, provider, []string{
+		"All Mail.mbox:1",
+		"All Mail.mbox:2",
 		"Foo.mbox:1",
 		"Inbox.mbox:1",
 	})
+	got := map[string][]string{}
+	for _, msg := range msgs {
+		got[msg.ID] = msg.targetNames()
+	}
+	r.Equal(t, map[string][]string{
+		"All Mail.mbox:1": {"Archive", "Foo"}, // Bar is not in rules.
+		"All Mail.mbox:2": {"Archive", "Foo"},
+		"Foo.mbox:1":      {"Foo"},
+		"Inbox.mbox:1":    {"Inbox"},
+	}, got)
+}
+
+func TestMBOXProviderTransferToAppleMail(t *testing.T) {
+	provider := newTestMBOXProvider("testdata/mbox-applemail")
+
+	rules, rulesClose := newTestRules(t)
+	defer rulesClose()
+	setupMBOXRules(rules)
+
+	msgs := testTransferTo(t, rules, provider, []string{
+		"All Mail.mbox/mbox:1",
+		"All Mail.mbox/mbox:2",
+	})
+	got := map[string][]string{}
+	for _, msg := range msgs {
+		got[msg.ID] = msg.targetNames()
+	}
+	r.Equal(t, map[string][]string{
+		"All Mail.mbox/mbox:1": {"Archive", "Foo"}, // Bar is not in rules.
+		"All Mail.mbox/mbox:2": {"Archive", "Foo"},
+	}, got)
 }
 
 func TestMBOXProviderTransferFrom(t *testing.T) {
-	dir, err := ioutil.TempDir("", "eml")
+	dir, err := ioutil.TempDir("", "mbox")
 	r.NoError(t, err)
 	defer os.RemoveAll(dir) //nolint[errcheck]
 
@@ -94,7 +135,7 @@ func TestMBOXProviderTransferFrom(t *testing.T) {
 }
 
 func TestMBOXProviderTransferFromTo(t *testing.T) {
-	dir, err := ioutil.TempDir("", "eml")
+	dir, err := ioutil.TempDir("", "mbox")
 	r.NoError(t, err)
 	defer os.RemoveAll(dir) //nolint[errcheck]
 
@@ -103,23 +144,57 @@ func TestMBOXProviderTransferFromTo(t *testing.T) {
 
 	rules, rulesClose := newTestRules(t)
 	defer rulesClose()
-	setupEMLRules(rules)
+	setupMBOXRules(rules)
 
 	testTransferFromTo(t, rules, source, target, 5*time.Second)
 
 	checkMBOXFileStructure(t, dir, []string{
+		"Archive.mbox",
 		"Foo.mbox",
 		"Inbox.mbox",
 	})
 }
 
+func TestMBOXProviderGetMessageTargetsReturnsOnlyOneFolder(t *testing.T) {
+	provider := newTestMBOXProvider("")
+
+	folderA := Mailbox{Name: "Folder A", IsExclusive: true}
+	folderB := Mailbox{Name: "Folder B", IsExclusive: true}
+	labelA := Mailbox{Name: "Label A", IsExclusive: false}
+	labelB := Mailbox{Name: "Label B", IsExclusive: false}
+	labelC := Mailbox{Name: "Label C", IsExclusive: false}
+
+	rule1 := &Rule{TargetMailboxes: []Mailbox{folderA, labelA, labelB}}
+	rule2 := &Rule{TargetMailboxes: []Mailbox{folderB, labelC}}
+	rule3 := &Rule{TargetMailboxes: []Mailbox{folderB}}
+
+	tests := []struct {
+		rules         []*Rule
+		wantMailboxes []Mailbox
+	}{
+		{[]*Rule{}, []Mailbox{}},
+		{[]*Rule{rule1}, []Mailbox{folderA, labelA, labelB}},
+		{[]*Rule{rule1, rule2}, []Mailbox{folderA, labelA, labelB, labelC}},
+		{[]*Rule{rule1, rule3}, []Mailbox{folderA, labelA, labelB}},
+		{[]*Rule{rule3, rule1}, []Mailbox{folderB, labelA, labelB}},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(fmt.Sprintf("%v", tc.rules), func(t *testing.T) {
+			mailboxes := provider.getMessageTargets(tc.rules, "", []byte(""))
+			r.Equal(t, tc.wantMailboxes, mailboxes)
+		})
+	}
+}
+
 func setupMBOXRules(rules transferRules) {
+	_ = rules.setRule(Mailbox{Name: "All Mail"}, []Mailbox{{Name: "Archive"}}, 0, 0)
 	_ = rules.setRule(Mailbox{Name: "Inbox"}, []Mailbox{{Name: "Inbox"}}, 0, 0)
 	_ = rules.setRule(Mailbox{Name: "Foo"}, []Mailbox{{Name: "Foo"}}, 0, 0)
 }
 
 func checkMBOXFileStructure(t *testing.T, root string, expectedFiles []string) {
-	files, err := getFilePathsWithSuffix(root, ".mbox")
+	files, err := getAllPathsWithSuffix(root, ".mbox")
 	r.NoError(t, err)
 	r.Equal(t, expectedFiles, files)
 }

internal/transfer/provider_pmapi.go

@@ -33,8 +33,10 @@ type PMAPIProvider struct {
 	addressID     string
 	keyRing       *crypto.KeyRing
 
-	importMsgReqMap  map[string]*pmapi.ImportMsgReq // Key is msg transfer ID.
+	nextImportRequests     map[string]*pmapi.ImportMsgReq // Key is msg transfer ID.
-	importMsgReqSize int
+	nextImportRequestsSize int
+
+	timeIt *timeIt
 }
 
 // NewPMAPIProvider returns new PMAPIProvider.
@@ -45,8 +47,10 @@ func NewPMAPIProvider(config *pmapi.ClientConfig, clientManager ClientManager, u
 		userID:        userID,
 		addressID:     addressID,
 
-		importMsgReqMap:  map[string]*pmapi.ImportMsgReq{},
+		nextImportRequests:     map[string]*pmapi.ImportMsgReq{},
-		importMsgReqSize: 0,
+		nextImportRequestsSize: 0,
+
+		timeIt: newTimeIt("pmapi"),
 	}
 
 	if addressID != "" {

internal/transfer/provider_pmapi_source.go

@@ -34,6 +34,9 @@ func (p *PMAPIProvider) TransferTo(rules transferRules, progress *Progress, ch c
 	log.Info("Started transfer from PMAPI to channel")
 	defer log.Info("Finished transfer from PMAPI to channel")
 
+	p.timeIt.clear()
+	defer p.timeIt.logResults()
+
 	// TransferTo cannot end sooner than loadCounts goroutine because
 	// loadCounts writes to channel in progress which would be closed.
 	// That can happen for really small accounts.
@@ -120,7 +123,7 @@ func (p *PMAPIProvider) transferTo(rule *Rule, progress *Progress, ch chan<- Mes
 				}
 
 				msgID := fmt.Sprintf("%s_%s", rule.SourceMailbox.ID, pmapiMessage.ID)
-				progress.addMessage(msgID, rule)
+				progress.addMessage(msgID, []string{rule.SourceMailbox.Name}, rule.TargetMailboxNames())
 				msg, err := p.exportMessage(rule, progress, pmapiMessage.ID, msgID, skipEncryptedMessages)
 				progress.messageExported(msgID, msg.Body, err)
 				if err == nil {
@@ -147,6 +150,9 @@ func (p *PMAPIProvider) exportMessage(rule *Rule, progress *Progress, pmapiMsgID
 		return err
 	})
 
+	p.timeIt.start("build", msgID)
+	defer p.timeIt.stop("build", msgID)
+
 	msgBuilder := pkgMessage.NewBuilder(p.client(), msg)
 	msgBuilder.EncryptedToHTML = false
 	_, body, err := msgBuilder.BuildMessage()
@@ -171,7 +177,7 @@ func (p *PMAPIProvider) exportMessage(rule *Rule, progress *Progress, pmapiMsgID
 		ID:      msgID,
 		Unread:  unread,
 		Body:    body,
-		Source:  rule.SourceMailbox,
+		Sources: []Mailbox{rule.SourceMailbox},
 		Targets: rule.TargetMailboxes,
 	}, nil
 }

internal/transfer/provider_pmapi_target.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"sync"
 
 	pkgMessage "github.com/ProtonMail/proton-bridge/pkg/message"
 	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
@@ -32,6 +33,7 @@ import (
 const (
 	pmapiImportBatchMaxItems = 10
 	pmapiImportBatchMaxSize  = 25 * 1000 * 1000 // 25 MB
+	pmapiImportWorkers       = 4                // To keep memory under 1 GB.
 )
 
 // DefaultMailboxes returns the default mailboxes for default rules if no other is found.
@@ -72,10 +74,16 @@ func (p *PMAPIProvider) TransferFrom(rules transferRules, progress *Progress, ch
 	log.Info("Started transfer from channel to PMAPI")
 	defer log.Info("Finished transfer from channel to PMAPI")
 
+	p.timeIt.clear()
+	defer p.timeIt.logResults()
+
 	// Cache has to be cleared before each transfer to not contain
 	// old stuff from previous cancelled run.
-	p.importMsgReqMap = map[string]*pmapi.ImportMsgReq{}
+	p.nextImportRequests = map[string]*pmapi.ImportMsgReq{}
-	p.importMsgReqSize = 0
+	p.nextImportRequestsSize = 0
+
+	preparedImportRequestsCh := make(chan map[string]*pmapi.ImportMsgReq)
+	wg := p.startImportWorkers(progress, preparedImportRequestsCh)
 
 	for msg := range ch {
 		if progress.shouldStop() {
@@ -85,13 +93,15 @@ func (p *PMAPIProvider) TransferFrom(rules transferRules, progress *Progress, ch
 		if p.isMessageDraft(msg) {
 			p.transferDraft(rules, progress, msg)
 		} else {
-			p.transferMessage(rules, progress, msg)
+			p.transferMessage(rules, progress, msg, preparedImportRequestsCh)
 		}
 	}
 
-	if len(p.importMsgReqMap) > 0 {
+	if len(p.nextImportRequests) > 0 {
-		p.importMessages(progress)
+		preparedImportRequestsCh <- p.nextImportRequests
 	}
+	close(preparedImportRequestsCh)
+	wg.Wait()
 }
 
 func (p *PMAPIProvider) isMessageDraft(msg Message) bool {
@@ -114,7 +124,10 @@ func (p *PMAPIProvider) importDraft(msg Message, globalMailbox *Mailbox) (string
 		return "", errors.Wrap(err, "failed to parse message")
 	}
 
-	if err := message.Encrypt(p.keyRing, nil); err != nil {
+	p.timeIt.start("encrypt", msg.ID)
+	err = message.Encrypt(p.keyRing, nil)
+	p.timeIt.stop("encrypt", msg.ID)
+	if err != nil {
 		return "", errors.Wrap(err, "failed to encrypt draft")
 	}
 
@@ -125,7 +138,7 @@ func (p *PMAPIProvider) importDraft(msg Message, globalMailbox *Mailbox) (string
 	attachments := message.Attachments
 	message.Attachments = nil
 
-	draft, err := p.createDraft(message, "", pmapi.DraftActionReply)
+	draft, err := p.createDraft(msg.ID, message, "", pmapi.DraftActionReply)
 	if err != nil {
 		return "", errors.Wrap(err, "failed to create draft")
 	}
@@ -140,13 +153,15 @@ func (p *PMAPIProvider) importDraft(msg Message, globalMailbox *Mailbox) (string
 			return "", errors.Wrap(err, "failed to sign attachment")
 		}
 
+		p.timeIt.start("encrypt", msg.ID)
 		r = bytes.NewReader(attachmentBody)
 		encReader, err := attachment.Encrypt(p.keyRing, r)
+		p.timeIt.stop("encrypt", msg.ID)
 		if err != nil {
 			return "", errors.Wrap(err, "failed to encrypt attachment")
 		}
 
-		_, err = p.createAttachment(attachment, encReader, sigReader)
+		_, err = p.createAttachment(msg.ID, attachment, encReader, sigReader)
 		if err != nil {
 			return "", errors.Wrap(err, "failed to create attachment")
 		}
@@ -155,19 +170,25 @@ func (p *PMAPIProvider) importDraft(msg Message, globalMailbox *Mailbox) (string
 	return draft.ID, nil
 }
 
-func (p *PMAPIProvider) transferMessage(rules transferRules, progress *Progress, msg Message) {
+func (p *PMAPIProvider) transferMessage(rules transferRules, progress *Progress, msg Message, preparedImportRequestsCh chan map[string]*pmapi.ImportMsgReq) {
 	importMsgReq, err := p.generateImportMsgReq(msg, rules.globalMailbox)
 	if err != nil {
 		progress.messageImported(msg.ID, "", err)
 		return
 	}
 
-	importMsgReqSize := len(importMsgReq.Body)
+	if progress.shouldStop() {
-	if p.importMsgReqSize+importMsgReqSize > pmapiImportBatchMaxSize || len(p.importMsgReqMap) == pmapiImportBatchMaxItems {
+		return
-		p.importMessages(progress)
 	}
-	p.importMsgReqMap[msg.ID] = importMsgReq
+
-	p.importMsgReqSize += importMsgReqSize
+	importMsgReqSize := len(importMsgReq.Body)
+	if p.nextImportRequestsSize+importMsgReqSize > pmapiImportBatchMaxSize || len(p.nextImportRequests) == pmapiImportBatchMaxItems {
+		preparedImportRequestsCh <- p.nextImportRequests
+		p.nextImportRequests = map[string]*pmapi.ImportMsgReq{}
+		p.nextImportRequestsSize = 0
+	}
+	p.nextImportRequests[msg.ID] = importMsgReq
+	p.nextImportRequestsSize += importMsgReqSize
 }
 
 func (p *PMAPIProvider) generateImportMsgReq(msg Message, globalMailbox *Mailbox) (*pmapi.ImportMsgReq, error) {
@@ -176,7 +197,9 @@ func (p *PMAPIProvider) generateImportMsgReq(msg Message, globalMailbox *Mailbox
 		return nil, errors.Wrap(err, "failed to parse message")
 	}
 
+	p.timeIt.start("encrypt", msg.ID)
 	body, err := p.encryptMessage(message, attachmentReaders)
+	p.timeIt.stop("encrypt", msg.ID)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to encrypt message")
 	}
@@ -208,6 +231,9 @@ func (p *PMAPIProvider) generateImportMsgReq(msg Message, globalMailbox *Mailbox
 }
 
 func (p *PMAPIProvider) parseMessage(msg Message) (m *pmapi.Message, r []io.Reader, err error) {
+	p.timeIt.start("parse", msg.ID)
+	defer p.timeIt.stop("parse", msg.ID)
+
 	// Old message parser is panicking in some cases.
 	// Instead of crashing we try to convert to regular error.
 	defer func() {
@@ -254,26 +280,39 @@ func computeMessageFlags(labels []string) (flag int64) {
 	return flag
 }
 
-func (p *PMAPIProvider) importMessages(progress *Progress) {
+func (p *PMAPIProvider) startImportWorkers(progress *Progress, preparedImportRequestsCh chan map[string]*pmapi.ImportMsgReq) *sync.WaitGroup {
+	var wg sync.WaitGroup
+	wg.Add(pmapiImportWorkers)
+	for i := 0; i < pmapiImportWorkers; i++ {
+		go func() {
+			for importRequests := range preparedImportRequestsCh {
+				p.importMessages(progress, importRequests)
+			}
+			wg.Done()
+		}()
+	}
+	return &wg
+}
+
+func (p *PMAPIProvider) importMessages(progress *Progress, importRequests map[string]*pmapi.ImportMsgReq) {
 	if progress.shouldStop() {
 		return
 	}
 
 	importMsgIDs := []string{}
 	importMsgRequests := []*pmapi.ImportMsgReq{}
-	for msgID, req := range p.importMsgReqMap {
+	for msgID, req := range importRequests {
 		importMsgIDs = append(importMsgIDs, msgID)
 		importMsgRequests = append(importMsgRequests, req)
 	}
-
+	log.WithField("msgIDs", importMsgIDs).Trace("Importing messages")
-	log.WithField("msgIDs", importMsgIDs).WithField("size", p.importMsgReqSize).Debug("Importing messages")
+	results, err := p.importRequest(importMsgIDs[0], importMsgRequests)
-	results, err := p.importRequest(importMsgRequests)
 
 	// In case the whole request failed, try to import every message one by one.
 	if err != nil || len(results) == 0 {
 		log.WithError(err).Warning("Importing messages failed, trying one by one")
-		for msgID, req := range p.importMsgReqMap {
+		for msgID, req := range importRequests {
-			importedID, err := p.importMessage(progress, req)
+			importedID, err := p.importMessage(msgID, progress, req)
 			progress.messageImported(msgID, importedID, err)
 		}
 		return
@@ -285,20 +324,17 @@ func (p *PMAPIProvider) importMessages(progress *Progress) {
 		if result.Error != nil {
 			log.WithError(result.Error).WithField("msg", msgID).Warning("Importing message failed, trying alone")
 			req := importMsgRequests[index]
-			importedID, err := p.importMessage(progress, req)
+			importedID, err := p.importMessage(msgID, progress, req)
 			progress.messageImported(msgID, importedID, err)
 		} else {
 			progress.messageImported(msgID, result.MessageID, nil)
 		}
 	}
-
-	p.importMsgReqMap = map[string]*pmapi.ImportMsgReq{}
-	p.importMsgReqSize = 0
 }
 
-func (p *PMAPIProvider) importMessage(progress *Progress, req *pmapi.ImportMsgReq) (importedID string, importedErr error) {
+func (p *PMAPIProvider) importMessage(msgSourceID string, progress *Progress, req *pmapi.ImportMsgReq) (importedID string, importedErr error) {
 	progress.callWrap(func() error {
-		results, err := p.importRequest([]*pmapi.ImportMsgReq{req})
+		results, err := p.importRequest(msgSourceID, []*pmapi.ImportMsgReq{req})
 		if err != nil {
 			return errors.Wrap(err, "failed to import messages")
 		}

internal/transfer/provider_pmapi_utils.go

@@ -18,6 +18,7 @@
 package transfer
 
 import (
+	"fmt"
 	"io"
 	"time"
 
@@ -71,6 +72,11 @@ func (p *PMAPIProvider) tryReconnect() error {
 
 func (p *PMAPIProvider) listMessages(filter *pmapi.MessagesFilter) (messages []*pmapi.Message, count int, err error) {
 	err = p.ensureConnection(func() error {
+		// Sort is used in the key so the filter is different for estimating and real fetching.
+		key := fmt.Sprintf("%s_%s_%d", filter.LabelID, filter.Sort, filter.Page)
+		p.timeIt.start("listing", key)
+		defer p.timeIt.stop("listing", key)
+
 		messages, count, err = p.client().ListMessages(filter)
 		return err
 	})
@@ -79,30 +85,44 @@ func (p *PMAPIProvider) listMessages(filter *pmapi.MessagesFilter) (messages []*
 
 func (p *PMAPIProvider) getMessage(msgID string) (message *pmapi.Message, err error) {
 	err = p.ensureConnection(func() error {
+		p.timeIt.start("download", msgID)
+		defer p.timeIt.stop("download", msgID)
+
 		message, err = p.client().GetMessage(msgID)
 		return err
 	})
 	return
 }
 
-func (p *PMAPIProvider) importRequest(req []*pmapi.ImportMsgReq) (res []*pmapi.ImportMsgRes, err error) {
+func (p *PMAPIProvider) importRequest(msgSourceID string, req []*pmapi.ImportMsgReq) (res []*pmapi.ImportMsgRes, err error) {
 	err = p.ensureConnection(func() error {
+		p.timeIt.start("upload", msgSourceID)
+		defer p.timeIt.stop("upload", msgSourceID)
+
 		res, err = p.client().Import(req)
 		return err
 	})
 	return
 }
 
-func (p *PMAPIProvider) createDraft(message *pmapi.Message, parent string, action int) (draft *pmapi.Message, err error) {
+func (p *PMAPIProvider) createDraft(msgSourceID string, message *pmapi.Message, parent string, action int) (draft *pmapi.Message, err error) {
 	err = p.ensureConnection(func() error {
+		p.timeIt.start("upload", msgSourceID)
+		defer p.timeIt.stop("upload", msgSourceID)
+
 		draft, err = p.client().CreateDraft(message, parent, action)
 		return err
 	})
 	return
 }
 
-func (p *PMAPIProvider) createAttachment(att *pmapi.Attachment, r io.Reader, sig io.Reader) (created *pmapi.Attachment, err error) {
+func (p *PMAPIProvider) createAttachment(msgSourceID string, att *pmapi.Attachment, r io.Reader, sig io.Reader) (created *pmapi.Attachment, err error) {
 	err = p.ensureConnection(func() error {
+		// Use some attributes from attachment to have unique key for each call.
+		key := fmt.Sprintf("%s_%s_%d", msgSourceID, att.Name, att.Size)
+		p.timeIt.start("upload", key)
+		defer p.timeIt.stop("upload", key)
+
 		created, err = p.client().CreateAttachment(att, r, sig)
 		return err
 	})

internal/transfer/provider_test.go

@@ -43,7 +43,7 @@ hello
 `, subject))
 }
 
-func testTransferTo(t *testing.T, rules transferRules, provider SourceProvider, expectedMessageIDs []string) {
+func testTransferTo(t *testing.T, rules transferRules, provider SourceProvider, expectedMessageIDs []string) []Message {
 	progress := newProgress(log, nil)
 	drainProgressUpdateChannel(&progress)
 
@@ -53,13 +53,17 @@ func testTransferTo(t *testing.T, rules transferRules, provider SourceProvider,
 		close(ch)
 	}()
 
+	msgs := []Message{}
 	gotMessageIDs := []string{}
 	for msg := range ch {
+		msgs = append(msgs, msg)
 		gotMessageIDs = append(gotMessageIDs, msg.ID)
 	}
 	r.ElementsMatch(t, expectedMessageIDs, gotMessageIDs)
 
 	r.Empty(t, progress.GetFailedMessages())
+
+	return msgs
 }
 
 func testTransferFrom(t *testing.T, rules transferRules, provider TargetProvider, messages []Message) {
@@ -69,7 +73,7 @@ func testTransferFrom(t *testing.T, rules transferRules, provider TargetProvider
 	ch := make(chan Message)
 	go func() {
 		for _, message := range messages {
-			progress.addMessage(message.ID, nil)
+			progress.addMessage(message.ID, []string{}, []string{})
 			progress.messageExported(message.ID, []byte(""), nil)
 			ch <- message
 		}

internal/transfer/report.go

@@ -114,7 +114,7 @@ type messageReport struct {
 	SourceID        string
 	TargetID        string
 	BodyHash        string
-	SourceMailbox   string
+	SourceMailboxes []string
 	TargetMailboxes []string
 	Error           string
 
@@ -130,8 +130,8 @@ func newMessageReportFromMessageStatus(messageStatus *MessageStatus, includePriv
 		SourceID:        messageStatus.SourceID,
 		TargetID:        messageStatus.targetID,
 		BodyHash:        messageStatus.bodyHash,
-		SourceMailbox:   messageStatus.rule.SourceMailbox.Name,
+		SourceMailboxes: messageStatus.sourceNames,
-		TargetMailboxes: messageStatus.rule.TargetMailboxNames(),
+		TargetMailboxes: messageStatus.targetNames,
 		Error:           messageStatus.GetErrorMessage(),
 	}
 

internal/transfer/testdata/mbox-applemail/All Mail.mbox/mbox

@@ -0,0 +1,16 @@
+From - Mon May  4 16:40:31 2020
+From: Bridge Test <bridgetest@pm.test>
+To: Bridge Test <bridgetest@protonmail.com>
+Subject: Test 1
+X-Gmail-Labels: Foo,Bar
+
+hello
+
+
+From - Mon May  4 16:40:31 2020
+From: Bridge Test <bridgetest@pm.test>
+To: Bridge Test <bridgetest@protonmail.com>
+Subject: Test 2
+X-Gmail-Labels: Foo
+
+hello

internal/transfer/testdata/mbox/All Mail.mbox

@@ -0,0 +1,16 @@
+From - Mon May  4 16:40:31 2020
+From: Bridge Test <bridgetest@pm.test>
+To: Bridge Test <bridgetest@protonmail.com>
+Subject: Test 1
+X-Gmail-Labels: Foo,Bar
+
+hello
+
+
+From - Mon May  4 16:40:31 2020
+From: Bridge Test <bridgetest@pm.test>
+To: Bridge Test <bridgetest@protonmail.com>
+Subject: Test 2
+X-Gmail-Labels: Foo
+
+hello

internal/transfer/timeit.go

@@ -0,0 +1,80 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"sync"
+	"time"
+)
+
+type timeIt struct {
+	lock    sync.Locker
+	name    string
+	groups  map[string]int64
+	ongoing map[string]time.Time
+}
+
+func newTimeIt(name string) *timeIt {
+	return &timeIt{
+		lock:    &sync.Mutex{},
+		name:    name,
+		groups:  map[string]int64{},
+		ongoing: map[string]time.Time{},
+	}
+}
+
+func (t *timeIt) clear() {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	t.groups = map[string]int64{}
+	t.ongoing = map[string]time.Time{}
+}
+
+func (t *timeIt) start(group, id string) {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	t.ongoing[group+"/"+id] = time.Now()
+}
+
+func (t *timeIt) stop(group, id string) {
+	endTime := time.Now()
+
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	startTime, ok := t.ongoing[group+"/"+id]
+	if !ok {
+		log.WithField("group", group).WithField("id", id).Error("Stop called before start")
+		return
+	}
+	delete(t.ongoing, group+"/"+id)
+
+	diff := endTime.Sub(startTime).Milliseconds()
+	t.groups[group] += diff
+}
+
+func (t *timeIt) logResults() {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	// Print also ongoing to be sure that nothing was left out.
+	// Basically ongoing should be empty.
+	log.WithField("name", t.name).WithField("result", t.groups).WithField("ongoing", t.ongoing).Debug("Time measurement")
+}

internal/transfer/transfer.go

@@ -181,7 +181,10 @@ func (t *Transfer) Start() *Progress {
 	reportFile := newFileReport(t.logDir, t.id)
 	progress := newProgress(log, reportFile)
 
-	ch := make(chan Message)
+	// Small queue to prevent having idle source while target is blocked.
+	// E.g., when upload to PM is in progress, we can in meantime download
+	// the next batch from remote IMAP server.
+	ch := make(chan Message, 10)
 
 	go func() {
 		defer t.panicHandler.HandlePanic()

internal/transfer/utils.go

@@ -24,9 +24,11 @@ import (
 	"net/mail"
 	"net/textproto"
 	"path/filepath"
+	"runtime"
 	"sort"
 	"strings"
 
+	"github.com/ProtonMail/proton-bridge/pkg/message/rfc5322"
 	"github.com/pkg/errors"
 )
 
@@ -81,7 +83,7 @@ func getFolderNamesWithFileSuffix(root, fileSuffix string) ([]string, error) {
 // getFilePathsWithSuffix collects all file names with `suffix` under `root`.
 // File names will be with relative path based to `root`.
 func getFilePathsWithSuffix(root, suffix string) ([]string, error) {
-	fileNames, err := getFilePathsWithSuffixInner("", root, suffix)
+	fileNames, err := getFilePathsWithSuffixInner("", root, suffix, false)
 	if err != nil {
 		return nil, err
 	}
@@ -89,7 +91,18 @@ func getFilePathsWithSuffix(root, suffix string) ([]string, error) {
 	return fileNames, err
 }
 
-func getFilePathsWithSuffixInner(prefix, root, suffix string) ([]string, error) {
+// getAllPathsWithSuffix is the same as getFilePathsWithSuffix but includes
+// also directories.
+func getAllPathsWithSuffix(root, suffix string) ([]string, error) {
+	fileNames, err := getFilePathsWithSuffixInner("", root, suffix, true)
+	if err != nil {
+		return nil, err
+	}
+	sort.Strings(fileNames)
+	return fileNames, err
+}
+
+func getFilePathsWithSuffixInner(prefix, root, suffix string, includeDir bool) ([]string, error) {
 	fileNames := []string{}
 
 	files, err := ioutil.ReadDir(root)
@@ -103,10 +116,14 @@ func getFilePathsWithSuffixInner(prefix, root, suffix string) ([]string, error)
 				fileNames = append(fileNames, filepath.Join(prefix, file.Name()))
 			}
 		} else {
+			if includeDir && strings.HasSuffix(file.Name(), suffix) {
+				fileNames = append(fileNames, filepath.Join(prefix, file.Name()))
+			}
 			subfolderFileNames, err := getFilePathsWithSuffixInner(
 				filepath.Join(prefix, file.Name()),
 				filepath.Join(root, file.Name()),
 				suffix,
+				includeDir,
 			)
 			if err != nil {
 				return nil, err
@@ -120,14 +137,21 @@ func getFilePathsWithSuffixInner(prefix, root, suffix string) ([]string, error)
 
 // getMessageTime returns time of the message specified in the message header.
 func getMessageTime(body []byte) (int64, error) {
-	mailHeader, err := getMessageHeader(body)
+	hdr, err := getMessageHeader(body)
 	if err != nil {
 		return 0, err
 	}
-	if t, err := mailHeader.Date(); err == nil && !t.IsZero() {
+
-		return t.Unix(), nil
+	t, err := rfc5322.ParseDateTime(hdr.Get("Date"))
+	if err != nil {
+		return 0, err
 	}
-	return 0, nil
+
+	if t.IsZero() {
+		return 0, nil
+	}
+
+	return t.Unix(), nil
 }
 
 // getMessageHeader returns headers of the message body.
@@ -139,3 +163,24 @@ func getMessageHeader(body []byte) (mail.Header, error) {
 	}
 	return mail.Header(header), nil
 }
+
+// sanitizeFileName replaces problematic special characters with underscore.
+func sanitizeFileName(fileName string) string {
+	if len(fileName) == 0 {
+		return fileName
+	}
+	if runtime.GOOS != "windows" && (fileName[0] == '-' || fileName[0] == '.') { //nolint[goconst]
+		fileName = "_" + fileName[1:]
+	}
+	return strings.Map(func(r rune) rune {
+		switch r {
+		case '\\', '/', ':', '*', '?', '"', '<', '>', '|':
+			return '_'
+		case '[', ']', '(', ')', '{', '}', '^', '#', '%', '&', '!', '@', '+', '=', '\'', '~':
+			if runtime.GOOS != "windows" {
+				return '_'
+			}
+		}
+		return r
+	}, fileName)
+}

internal/transfer/utils_test.go

@@ -21,6 +21,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"runtime"
 	"testing"
 
 	r "github.com/stretchr/testify/require"
@@ -38,6 +39,7 @@ func TestGetFolderNames(t *testing.T) {
 			"",
 			[]string{
 				"bar",
+				"bar.mbox",
 				"baz",
 				filepath.Base(root),
 				"foo",
@@ -94,6 +96,13 @@ func TestGetFilePathsWithSuffix(t *testing.T) {
 				"test/foo/msg9.eml",
 			},
 		},
+		{
+			".mbox",
+			[]string{
+				"bar.mbox",
+				"foo.mbox",
+			},
+		},
 		{
 			".txt",
 			[]string{
@@ -108,7 +117,7 @@ func TestGetFilePathsWithSuffix(t *testing.T) {
 	for _, tc := range tests {
 		tc := tc
 		t.Run(tc.suffix, func(t *testing.T) {
-			paths, err := getFilePathsWithSuffix(root, tc.suffix)
+			paths, err := getAllPathsWithSuffix(root, tc.suffix)
 			r.NoError(t, err)
 			r.Equal(t, tc.wantPaths, paths)
 		})
@@ -124,6 +133,7 @@ func createTestingFolderStructure(t *testing.T) (string, func()) {
 		"foo/baz",
 		"test/foo",
 		"qwerty",
+		"bar.mbox",
 	} {
 		err = os.MkdirAll(filepath.Join(root, path), os.ModePerm)
 		r.NoError(t, err)
@@ -141,6 +151,8 @@ func createTestingFolderStructure(t *testing.T) (string, func()) {
 		"test/foo/msg9.eml",
 		"msg10.eml",
 		"info.txt",
+		"foo.mbox",
+		"bar.mbox/mbox", // Apple Mail mbox export format.
 	} {
 		f, err := os.Create(filepath.Join(root, path))
 		r.NoError(t, err)
@@ -188,3 +200,26 @@ Body
 	r.Equal(t, header.Get("subject"), "Hello")
 	r.Equal(t, header.Get("from"), "user@example.com")
 }
+
+func TestSanitizeFileName(t *testing.T) {
+	tests := map[string]string{
+		"hello":            "hello",
+		"a\\b/c:*?d\"<>|e": "a_b_c___d____e",
+	}
+	if runtime.GOOS == "darwin" || runtime.GOOS == "linux" {
+		tests[".hello"] = "_hello"
+		tests["-hello"] = "_hello"
+	}
+	if runtime.GOOS == "windows" {
+		tests["[hello]&@=~~"] = "_hello______"
+	}
+
+	for path, wantPath := range tests {
+		path := path
+		wantPath := wantPath
+		t.Run(path, func(t *testing.T) {
+			gotPath := sanitizeFileName(path)
+			r.Equal(t, wantPath, gotPath)
+		})
+	}
+}

internal/updates/sync.go

@@ -45,7 +45,7 @@ func syncFolders(localPath, updatePath string) (err error) {
 }
 
 func removeMissing(folderToCleanPath, itemsToKeepPath string) (err error) {
-	log.Debug("remove missing")
+	log.WithField("from", folderToCleanPath).Debug("Remove missing.")
 	// Create list of files.
 	existingRelPaths := map[string]bool{}
 	err = filepath.Walk(itemsToKeepPath, func(keepThis string, _ os.FileInfo, walkErr error) error {
@@ -56,7 +56,7 @@ func removeMissing(folderToCleanPath, itemsToKeepPath string) (err error) {
 		if walkErr != nil {
 			return walkErr
 		}
-		log.Debug("path to keep ", relPath)
+		log.WithField("path", relPath).Trace("Keep the path.")
 		existingRelPaths[relPath] = true
 		return nil
 	})
@@ -95,12 +95,18 @@ func removeMissing(folderToCleanPath, itemsToKeepPath string) (err error) {
 }
 
 func restoreFromBackup(backupDir, localPath string) {
-	log.Error("recovering from ", backupDir, " to ", localPath)
+	log.WithField("from", backupDir).
-	_ = copyRecursively(backupDir, localPath)
+		WithField("to", localPath).
+		Error("recovering")
+	if err := copyRecursively(backupDir, localPath); err != nil {
+		log.WithField("from", backupDir).
+			WithField("to", localPath).
+			Error("Not able to recover.")
+	}
 }
 
 func createBackup(srcFile, dstDir string) (err error) {
-	log.Debug("backup ", srcFile, " in ", dstDir)
+	log.WithField("from", srcFile).WithField("to", dstDir).Debug("Create backup")
 	if err = mkdirAllClear(dstDir); err != nil {
 		return
 	}

internal/updates/updates.go

@@ -107,7 +107,7 @@ func NewImportExport(updateTempDir string) *Updates {
 		versionFileBaseName: "current_version_ie",
 		updateFileBaseName:  "ie/ie_upgrade",
 		linuxFileBaseName:   "ie/protonmail-import-export-app",
-		macAppBundleName:    "Import-Export app.app",
+		macAppBundleName:    "ProtonMail Import-Export app.app",
 	}
 }
 
@@ -310,7 +310,9 @@ func (u *Updates) StartUpgrade(currentStatus chan<- Progress) { // nolint[funlen
 	status.UpdateDescription(InfoUpgrading)
 	switch runtime.GOOS {
 	case "windows": //nolint[goconst]
-		installerFile := strings.Split(u.winInstallerFile, "/")[1]
+		// Cannot use filepath.Base on windows it has different delimiter
+		split := strings.Split(u.winInstallerFile, "/")
+		installerFile := split[len(split)-1]
 		cmd := exec.Command("./" + installerFile) // nolint[gosec]
 		cmd.Dir = u.updateTempDir
 		status.Err = cmd.Start()
@@ -326,10 +328,15 @@ func (u *Updates) StartUpgrade(currentStatus chan<- Progress) { // nolint[funlen
 		localPath = filepath.Dir(localPath) // .app
 
 		updatePath := filepath.Join(u.updateTempDir, u.macAppBundleName)
-		log.Warn("localPath ", localPath)
+		log.WithField("local", localPath).
-		log.Warn("updatePath ", updatePath)
+			WithField("update", updatePath).
+			Info("Syncing folders..")
 		status.Err = syncFolders(localPath, updatePath)
 		if status.Err != nil {
+			log.WithField("from", localPath).
+				WithField("to", updatePath).
+				WithError(status.Err).
+				Error("Sync failed.")
 			return
 		}
 		status.UpdateDescription(InfoRestartApp)

pkg/message/header.go

@@ -19,7 +19,6 @@ package message
 
 import (
 	"mime"
-	"net/mail"
 	"net/textproto"
 	"strings"
 	"time"
@@ -85,10 +84,6 @@ func GetHeader(msg *pmapi.Message) textproto.MIMEHeader { //nolint[funlen]
 	}
 	if msg.ConversationID != "" {
 		h.Set("X-Pm-ConversationID-Id", msg.ConversationID)
-		if references := h.Get("References"); !strings.Contains(references, msg.ConversationID) {
-			references += " <" + msg.ConversationID + "@" + pmapi.ConversationIDDomain + ">"
-			h.Set("References", references)
-		}
 	}
 
 	return h
@@ -140,75 +135,3 @@ func GetAttachmentHeader(att *pmapi.Attachment) textproto.MIMEHeader {
 
 	return h
 }
-
-// ========= Header parsing and sanitizing functions =========
-
-func parseHeader(h mail.Header) (m *pmapi.Message, err error) { //nolint[unparam]
-	m = pmapi.NewMessage()
-
-	if subject, err := pmmime.DecodeHeader(h.Get("Subject")); err == nil {
-		m.Subject = subject
-	}
-	if addrs, err := sanitizeAddressList(h, "From"); err == nil && len(addrs) > 0 {
-		m.Sender = addrs[0]
-	}
-	if addrs, err := sanitizeAddressList(h, "Reply-To"); err == nil && len(addrs) > 0 {
-		m.ReplyTos = addrs
-	}
-	if addrs, err := sanitizeAddressList(h, "To"); err == nil {
-		m.ToList = addrs
-	}
-	if addrs, err := sanitizeAddressList(h, "Cc"); err == nil {
-		m.CCList = addrs
-	}
-	if addrs, err := sanitizeAddressList(h, "Bcc"); err == nil {
-		m.BCCList = addrs
-	}
-	m.Time = 0
-	if t, err := h.Date(); err == nil && !t.IsZero() {
-		m.Time = t.Unix()
-	}
-
-	m.Header = h
-	return
-}
-
-func sanitizeAddressList(h mail.Header, field string) (addrs []*mail.Address, err error) {
-	raw := h.Get(field)
-	if raw == "" {
-		err = mail.ErrHeaderNotPresent
-		return
-	}
-	var decoded string
-	decoded, err = pmmime.DecodeHeader(raw)
-	if err != nil {
-		return
-	}
-	addrs, err = mail.ParseAddressList(parseAddressComment(decoded))
-	if err == nil {
-		if addrs == nil {
-			addrs = []*mail.Address{}
-		}
-		return
-	}
-	// Probably missing encoding error -- try to at least parse addresses in brackets.
-	addrStr := h.Get(field)
-	first := strings.Index(addrStr, "<")
-	last := strings.LastIndex(addrStr, ">")
-	if first < 0 || last < 0 || first >= last {
-		return
-	}
-	var addrList []string
-	open := first
-	for open < last && 0 <= open {
-		addrStr = addrStr[open:]
-		close := strings.Index(addrStr, ">")
-		addrList = append(addrList, addrStr[:close+1])
-		addrStr = addrStr[close:]
-		open = strings.Index(addrStr, "<")
-		last = strings.LastIndex(addrStr, ">")
-	}
-	addrStr = strings.Join(addrList, ", ")
-	//
-	return mail.ParseAddressList(addrStr)
-}

pkg/message/html.go

@@ -1,76 +0,0 @@
-// Copyright (c) 2020 Proton Technologies AG
-//
-// This file is part of ProtonMail Bridge.
-//
-// ProtonMail Bridge is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// ProtonMail Bridge is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
-
-package message
-
-import (
-	"bytes"
-	"errors"
-	escape "html"
-	"strings"
-
-	"github.com/andybalholm/cascadia"
-	"golang.org/x/net/html"
-)
-
-func plaintextToHTML(text string) (output string) {
-	text = escape.EscapeString(text)
-	text = strings.Replace(text, "\n\r", "<br>", -1)
-	text = strings.Replace(text, "\r\n", "<br>", -1)
-	text = strings.Replace(text, "\n", "<br>", -1)
-	text = strings.Replace(text, "\r", "<br>", -1)
-
-	return "<div>" + text + "</div>"
-}
-
-func stripHTML(input string) (stripped string, err error) {
-	reader := strings.NewReader(input)
-	doc, _ := html.Parse(reader)
-	body := cascadia.MustCompile("body").MatchFirst(doc)
-	if body == nil {
-		err = errors.New("failed to find necessary html element")
-		return
-	}
-	var buf1 bytes.Buffer
-	if err = html.Render(&buf1, body); err != nil {
-		stripped = input
-		return
-	}
-	stripped = buf1.String()
-	// Handle double body tags edge case.
-	if strings.Index(stripped, "<body") == 0 {
-		startIndex := strings.Index(stripped, ">")
-		if startIndex < 5 {
-			return
-		}
-		stripped = stripped[startIndex+1:]
-		// Closing body tag is optional.
-		closingIndex := strings.Index(stripped, "</body>")
-		if closingIndex > -1 {
-			stripped = stripped[:closingIndex]
-		}
-	}
-	return
-}
-
-func addOuterHTMLTags(input string) (output string) {
-	return "<html><head></head><body>" + input + "</body></html>"
-}
-
-func makeEmbeddedImageHTML(cid, name string) (output string) {
-	return "<img class=\"proton-embedded\" alt=\"" + name + "\" src=\"cid:" + cid + "\">"
-}

pkg/message/parser.go

@@ -19,455 +19,506 @@ package message
 
 import (
 	"bytes"
-	"crypto/sha256"
-	"encoding/base64"
-	"encoding/hex"
 	"fmt"
 	"io"
-	"io/ioutil"
-	"math/rand"
 	"mime"
-	"mime/quotedprintable"
 	"net/mail"
 	"net/textproto"
-	"regexp"
-	"strconv"
 	"strings"
 
+	"github.com/ProtonMail/proton-bridge/pkg/message/parser"
+	"github.com/ProtonMail/proton-bridge/pkg/message/rfc5322"
 	pmmime "github.com/ProtonMail/proton-bridge/pkg/mime"
 	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/emersion/go-message"
 	"github.com/jaytaylor/html2text"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
-func parseAttachment(filename string, mediaType string, h textproto.MIMEHeader) (att *pmapi.Attachment) {
+func Parse(r io.Reader, key, keyName string) (m *pmapi.Message, mimeBody, plainBody string, attReaders []io.Reader, err error) {
-	if decoded, err := pmmime.DecodeHeader(filename); err == nil {
+	logrus.Trace("Parsing message")
-		filename = decoded
+
-	}
+	p, err := parser.New(r)
-	if filename == "" {
+	if err != nil {
-		ext, err := mime.ExtensionsByType(mediaType)
+		err = errors.Wrap(err, "failed to create new parser")
-		if err == nil && len(ext) > 0 {
+		return
-			filename = "attachment" + ext[0]
-		}
 	}
 
-	att = &pmapi.Attachment{
+	if err = convertForeignEncodings(p); err != nil {
-		Name:     filename,
+		err = errors.Wrap(err, "failed to convert foreign encodings")
-		MIMEType: mediaType,
+		return
-		Header:   h,
 	}
 
-	headerContentID := strings.Trim(h.Get("Content-Id"), " <>")
+	m = pmapi.NewMessage()
 
-	if headerContentID != "" {
+	if err = parseMessageHeader(m, p.Root().Header); err != nil {
-		att.ContentID = headerContentID
+		err = errors.Wrap(err, "failed to parse message header")
+		return
 	}
 
-	return
+	if m.Attachments, attReaders, err = collectAttachments(p); err != nil {
+		err = errors.Wrap(err, "failed to collect attachments")
+		return
+	}
+
+	if m.Body, plainBody, err = buildBodies(p); err != nil {
+		err = errors.Wrap(err, "failed to build bodies")
+		return
+	}
+
+	if m.MIMEType, err = determineMIMEType(p); err != nil {
+		err = errors.Wrap(err, "failed to determine mime type")
+		return
+	}
+
+	// We only attach the public key manually to the MIME body for
+	// signed/encrypted external recipients. It's not important for it to be
+	// collected as an attachment; that's already done when we upload the draft.
+	if key != "" {
+		attachPublicKey(p.Root(), key, keyName)
+	}
+
+	mimeBodyBuffer := new(bytes.Buffer)
+
+	if err = p.NewWriter().Write(mimeBodyBuffer); err != nil {
+		err = errors.Wrap(err, "failed to write out mime message")
+		return
+	}
+
+	return m, mimeBodyBuffer.String(), plainBody, attReaders, nil
 }
 
-var reEmailComment = regexp.MustCompile("[(][^)]*[)]") //nolint[gochecknoglobals]
+func convertForeignEncodings(p *parser.Parser) error {
+	logrus.Trace("Converting foreign encodings")
 
-// parseAddressComment removes the comments completely even though they should be allowed
+	return p.NewWalker().
-// http://tools.wordtothewise.com/rfc/822
+		RegisterContentTypeHandler("text/html", func(p *parser.Part) error {
-// NOTE: This should be supported in go>1.10 but it seems it's not ¯\_(ツ)_/¯
+			if err := p.ConvertToUTF8(); err != nil {
-func parseAddressComment(raw string) string {
+				return err
-	return reEmailComment.ReplaceAllString(raw, "")
-}
-
-// Some clients incorrectly format messages with embedded attachments to have a format like
-// I. text/plain II. attachment III. text/plain
-// which we need to convert to a single HTML part with an embedded attachment.
-func combineParts(m *pmapi.Message, parts []io.Reader, headers []textproto.MIMEHeader, convertPlainToHTML bool, atts *[]io.Reader) (isHTML bool, err error) { //nolint[funlen]
-	isHTML = true
-	foundText := false
-
-	for i := len(parts) - 1; i >= 0; i-- {
-		part := parts[i]
-		h := headers[i]
-
-		disp, dispParams, _ := pmmime.ParseMediaType(h.Get("Content-Disposition"))
-
-		d := pmmime.DecodeContentEncoding(part, h.Get("Content-Transfer-Encoding"))
-		if d == nil {
-			log.Warnf("Unsupported Content-Transfer-Encoding '%v'", h.Get("Content-Transfer-Encoding"))
-			d = part
-		}
-
-		contentType := h.Get("Content-Type")
-		if contentType == "" {
-			contentType = "text/plain"
-		}
-		mediaType, params, _ := pmmime.ParseMediaType(contentType)
-
-		if strings.HasPrefix(mediaType, "text/") && mediaType != "text/calendar" && disp != "attachment" {
-			// This is text.
-			var b []byte
-			if b, err = ioutil.ReadAll(d); err != nil {
-				continue
 			}
-			b, err = pmmime.DecodeCharset(b, contentType)
+
+			return p.ConvertMetaCharset()
+		}).
+		RegisterContentTypeHandler("text/.*", func(p *parser.Part) error {
+			return p.ConvertToUTF8()
+		}).
+		RegisterDefaultHandler(func(p *parser.Part) error {
+			t, params, _ := p.ContentType()
+			// multipart/alternative, for example, can contain extra charset.
+			if params != nil && params["charset"] != "" {
+				return p.ConvertToUTF8()
+			}
+			logrus.WithField("type", t).Trace("Not converting part to utf-8")
+			return nil
+		}).
+		Walk()
+}
+
+func collectAttachments(p *parser.Parser) ([]*pmapi.Attachment, []io.Reader, error) {
+	var (
+		atts []*pmapi.Attachment
+		data []io.Reader
+		err  error
+	)
+
+	w := p.NewWalker().
+		RegisterContentDispositionHandler("attachment", func(p *parser.Part) error {
+			att, err := parseAttachment(p.Header)
 			if err != nil {
-				log.Warn("Decode charset error: ", err)
+				return err
-				return false, err
-			}
-			contents := string(b)
-			if strings.Contains(mediaType, "text/plain") && len(contents) > 0 {
-				if !convertPlainToHTML {
-					isHTML = false
-				} else {
-					contents = plaintextToHTML(contents)
-				}
-			} else if strings.Contains(mediaType, "text/html") && len(contents) > 0 {
-				contents, err = stripHTML(contents)
-				if err != nil {
-					return isHTML, err
-				}
-			}
-			m.Body = contents + m.Body
-			foundText = true
-		} else {
-			// This is an attachment.
-			filename := dispParams["filename"]
-			if filename == "" {
-				// Using "name" in Content-Type is discouraged.
-				filename = params["name"]
-			}
-			if filename == "" && mediaType == "text/calendar" {
-				filename = "event.ics"
 			}
 
-			att := parseAttachment(filename, mediaType, h)
+			atts = append(atts, att)
+			data = append(data, bytes.NewReader(p.Body))
 
-			b := &bytes.Buffer{}
+			return nil
-			if d == nil {
+		}).
-				continue
+		RegisterContentTypeHandler("text/calendar", func(p *parser.Part) error {
-			}
+			att, err := parseAttachment(p.Header)
-			if _, err = io.Copy(b, d); err != nil {
+			if err != nil {
-				continue
+				return err
-			}
-			if foundText && att.ContentID == "" && strings.Contains(mediaType, "image") {
-				// Treat this as an inline attachment even though it is not marked as one.
-				hasher := sha256.New()
-				_, _ = hasher.Write([]byte(att.Name + strconv.Itoa(b.Len())))
-				bytes := hasher.Sum(nil)
-				cid := hex.EncodeToString(bytes) + "@protonmail.com"
-
-				att.ContentID = cid
-				embeddedHTML := makeEmbeddedImageHTML(cid, att.Name)
-				m.Body = embeddedHTML + m.Body
 			}
 
-			m.Attachments = append(m.Attachments, att)
+			atts = append(atts, att)
-			*atts = append(*atts, b)
+			data = append(data, bytes.NewReader(p.Body))
-		}
-	}
-	if isHTML {
-		m.Body = addOuterHTMLTags(m.Body)
-	}
-	return isHTML, nil
-}
 
-func checkHeaders(headers []textproto.MIMEHeader) bool {
+			return nil
-	foundAttachment := false
+		}).
+		RegisterContentTypeHandler("text/.*", func(p *parser.Part) error {
+			return nil
+		}).
+		RegisterDefaultHandler(func(p *parser.Part) error {
+			if len(p.Children()) > 0 {
+				return nil
+			}
 
-	for i := 0; i < len(headers); i++ {
+			att, err := parseAttachment(p.Header)
-		h := headers[i]
+			if err != nil {
+				return err
+			}
 
-		mediaType, _, _ := pmmime.ParseMediaType(h.Get("Content-Type"))
+			atts = append(atts, att)
+			data = append(data, bytes.NewReader(p.Body))
 
-		if !strings.HasPrefix(mediaType, "text/") {
+			return nil
-			foundAttachment = true
+		})
-		} else if foundAttachment {
-			// This means that there is a text part after the first attachment,
-			// so we will have to convert the body from plain->HTML.
-			return true
-		}
-	}
-	return false
-}
 
-// ============================== 7bit Filter ==========================
+	if err = w.Walk(); err != nil {
-// For every MIME part in the tree that has "8bit" or "binary" content
+		return nil, nil, err
-// transfer encoding: transcode it to "quoted-printable".
-
-type SevenBitFilter struct {
-	target pmmime.VisitAcceptor
-}
-
-func NewSevenBitFilter(targetAccepter pmmime.VisitAcceptor) *SevenBitFilter {
-	return &SevenBitFilter{
-		target: targetAccepter,
-	}
-}
-
-func decodePart(partReader io.Reader, header textproto.MIMEHeader) (decodedPart io.Reader) {
-	decodedPart = pmmime.DecodeContentEncoding(partReader, header.Get("Content-Transfer-Encoding"))
-	if decodedPart == nil {
-		log.Warnf("Unsupported Content-Transfer-Encoding '%v'", header.Get("Content-Transfer-Encoding"))
-		decodedPart = partReader
-	}
-	return
-}
-
-func (sd SevenBitFilter) Accept(partReader io.Reader, header textproto.MIMEHeader, hasPlainSibling bool, isFirst, isLast bool) error {
-	cte := strings.ToLower(header.Get("Content-Transfer-Encoding"))
-	if isFirst && pmmime.IsLeaf(header) && cte != "quoted-printable" && cte != "base64" && cte != "7bit" {
-		decodedPart := decodePart(partReader, header)
-
-		filteredHeader := textproto.MIMEHeader{}
-		for k, v := range header {
-			filteredHeader[k] = v
-		}
-		filteredHeader.Set("Content-Transfer-Encoding", "quoted-printable")
-
-		filteredBuffer := &bytes.Buffer{}
-		decodedSlice, _ := ioutil.ReadAll(decodedPart)
-		w := quotedprintable.NewWriter(filteredBuffer)
-		if _, err := w.Write(decodedSlice); err != nil {
-			log.Errorf("cannot write quotedprintable from %q: %v", cte, err)
-		}
-		if err := w.Close(); err != nil {
-			log.Errorf("cannot close quotedprintable from %q: %v", cte, err)
-		}
-
-		_ = sd.target.Accept(filteredBuffer, filteredHeader, hasPlainSibling, true, isLast)
-	} else {
-		_ = sd.target.Accept(partReader, header, hasPlainSibling, isFirst, isLast)
-	}
-	return nil
-}
-
-// =================== HTML Only convertor ==================================
-// In any part of MIME tree structure, replace standalone text/html with
-// multipart/alternative containing both text/html and text/plain.
-
-type HTMLOnlyConvertor struct {
-	target pmmime.VisitAcceptor
-}
-
-func NewHTMLOnlyConvertor(targetAccepter pmmime.VisitAcceptor) *HTMLOnlyConvertor {
-	return &HTMLOnlyConvertor{
-		target: targetAccepter,
-	}
-}
-
-func randomBoundary() string {
-	buf := make([]byte, 30)
-
-	// We specifically use `math/rand` here to allow the generator to be seeded for test purposes.
-	// The random numbers need not be cryptographically secure; we are simply generating random part boundaries.
-	if _, err := rand.Read(buf); err != nil { // nolint[gosec]
-		panic(err)
 	}
 
-	return fmt.Sprintf("%x", buf)
+	return atts, data, nil
 }
 
-func (hoc HTMLOnlyConvertor) Accept(partReader io.Reader, header textproto.MIMEHeader, hasPlainSiblings bool, isFirst, isLast bool) error {
+// buildBodies collects all text/html and text/plain parts and returns two bodies,
-	mediaType, _, err := pmmime.ParseMediaType(header.Get("Content-Type"))
+//  - a rich text body (in which html is allowed), and
-	if isFirst && err == nil && mediaType == "text/html" && !hasPlainSiblings {
+//  - a plaintext body (in which html is converted to plaintext).
-		multiPartHeaders := make(textproto.MIMEHeader)
+//
-		for k, v := range header {
+// text/html parts are converted to plaintext in order to build the plaintext body,
-			multiPartHeaders[k] = v
+// unless there is already a plaintext part provided via multipart/alternative,
-		}
+// in which case the provided alternative is chosen.
-		boundary := randomBoundary()
+func buildBodies(p *parser.Parser) (richBody, plainBody string, err error) {
-		multiPartHeaders.Set("Content-Type", "multipart/alternative; boundary=\""+boundary+"\"")
+	richParts, err := collectBodyParts(p, "text/html")
-		childCte := header.Get("Content-Transfer-Encoding")
+	if err != nil {
+		return
+	}
 
-		_ = hoc.target.Accept(partReader, multiPartHeaders, false, true, false)
+	plainParts, err := collectBodyParts(p, "text/plain")
+	if err != nil {
+		return
+	}
 
-		partData, _ := ioutil.ReadAll(partReader)
+	richBuilder, plainBuilder := strings.Builder{}, strings.Builder{}
 
-		htmlChildHeaders := make(textproto.MIMEHeader)
+	for _, richPart := range richParts {
-		htmlChildHeaders.Set("Content-Transfer-Encoding", childCte)
+		_, _ = richBuilder.Write(richPart.Body)
-		htmlChildHeaders.Set("Content-Type", "text/html")
+	}
-		htmlReader := bytes.NewReader(partData)
-		_ = hoc.target.Accept(htmlReader, htmlChildHeaders, false, true, false)
 
-		_ = hoc.target.Accept(partReader, multiPartHeaders, hasPlainSiblings, false, false)
+	for _, plainPart := range plainParts {
+		_, _ = plainBuilder.Write(getPlainBody(plainPart))
+	}
 
-		plainChildHeaders := make(textproto.MIMEHeader)
+	return richBuilder.String(), plainBuilder.String(), nil
-		plainChildHeaders.Set("Content-Transfer-Encoding", childCte)
+}
-		plainChildHeaders.Set("Content-Type", "text/plain")
+
-		unHtmlized, err := html2text.FromReader(bytes.NewReader(partData))
+// collectBodyParts collects all body parts in the parse tree, preferring
+// parts of the given content type if alternatives exist.
+func collectBodyParts(p *parser.Parser, preferredContentType string) (parser.Parts, error) {
+	v := p.
+		NewVisitor(func(p *parser.Part, visit parser.Visit) (interface{}, error) {
+			childParts, err := collectChildParts(p, visit)
+			if err != nil {
+				return nil, err
+			}
+
+			return joinChildParts(childParts), nil
+		}).
+		RegisterRule("multipart/alternative", func(p *parser.Part, visit parser.Visit) (interface{}, error) {
+			childParts, err := collectChildParts(p, visit)
+			if err != nil {
+				return nil, err
+			}
+
+			return bestChoice(childParts, preferredContentType), nil
+		}).
+		RegisterRule("text/plain", func(p *parser.Part, visit parser.Visit) (interface{}, error) {
+			disp, _, err := p.Header.ContentDisposition()
+			if err != nil {
+				disp = ""
+			}
+
+			if disp == "attachment" {
+				return parser.Parts{}, nil
+			}
+
+			return parser.Parts{p}, nil
+		}).
+		RegisterRule("text/html", func(p *parser.Part, visit parser.Visit) (interface{}, error) {
+			disp, _, err := p.Header.ContentDisposition()
+			if err != nil {
+				disp = ""
+			}
+
+			if disp == "attachment" {
+				return parser.Parts{}, nil
+			}
+
+			return parser.Parts{p}, nil
+		})
+
+	res, err := v.Visit()
+	if err != nil {
+		return nil, err
+	}
+
+	return res.(parser.Parts), nil
+}
+
+func collectChildParts(p *parser.Part, visit parser.Visit) ([]parser.Parts, error) {
+	childParts := []parser.Parts{}
+
+	for _, child := range p.Children() {
+		res, err := visit(child)
 		if err != nil {
-			unHtmlized = string(partData)
+			return nil, err
 		}
-		plainReader := strings.NewReader(unHtmlized)
-		_ = hoc.target.Accept(plainReader, plainChildHeaders, false, true, true)
 
-		_ = hoc.target.Accept(partReader, multiPartHeaders, hasPlainSiblings, false, true)
+		childParts = append(childParts, res.(parser.Parts))
-	} else {
-		_ = hoc.target.Accept(partReader, header, hasPlainSiblings, isFirst, isLast)
 	}
-	return nil
+
+	return childParts, nil
 }
 
-// ======= Public Key Attacher ========
+func joinChildParts(childParts []parser.Parts) parser.Parts {
+	res := parser.Parts{}
 
-type PublicKeyAttacher struct {
+	for _, parts := range childParts {
-	target                pmmime.VisitAcceptor
+		res = append(res, parts...)
-	attachedPublicKey     string
+	}
-	attachedPublicKeyName string
+
-	appendToMultipart     bool
+	return res
-	depth                 int
 }
 
-func NewPublicKeyAttacher(targetAccepter pmmime.VisitAcceptor, attachedPublicKey, attachedPublicKeyName string) *PublicKeyAttacher {
+func bestChoice(childParts []parser.Parts, preferredContentType string) parser.Parts {
-	return &PublicKeyAttacher{
+	// If one of the parts has preferred content type, use that.
-		target:                targetAccepter,
+	for i := len(childParts) - 1; i >= 0; i-- {
-		attachedPublicKey:     attachedPublicKey,
+		if allPartsHaveContentType(childParts[i], preferredContentType) {
-		attachedPublicKeyName: attachedPublicKeyName,
+			return childParts[i]
-		appendToMultipart:     false,
-		depth:                 0,
-	}
-}
-
-func min(a, b int) int {
-	if a < b {
-		return a
-	}
-	return b
-}
-
-func split(input string, sliceLength int) string {
-	processed := input
-	result := ""
-	for len(processed) > 0 {
-		cutPoint := min(sliceLength, len(processed))
-		part := processed[0:cutPoint]
-		result = result + part + "\n"
-		processed = processed[cutPoint:]
-	}
-	return result
-}
-
-func createKeyAttachment(publicKey, publicKeyName string) (headers textproto.MIMEHeader, contents io.Reader) {
-	attachmentHeaders := make(textproto.MIMEHeader)
-	attachmentHeaders.Set("Content-Type", "application/pgp-key; name=\""+publicKeyName+"\"")
-	attachmentHeaders.Set("Content-Transfer-Encoding", "base64")
-	attachmentHeaders.Set("Content-Disposition", "attachment; filename=\""+publicKeyName+".asc.pgp\"")
-
-	buffer := &bytes.Buffer{}
-	w := base64.NewEncoder(base64.StdEncoding, buffer)
-	_, _ = w.Write([]byte(publicKey))
-	_ = w.Close()
-
-	return attachmentHeaders, strings.NewReader(split(buffer.String(), 73))
-}
-
-func (pka *PublicKeyAttacher) Accept(partReader io.Reader, header textproto.MIMEHeader, hasPlainSiblings bool, isFirst, isLast bool) error {
-	if isFirst && !pmmime.IsLeaf(header) {
-		pka.depth++
-	}
-	if isLast && !pmmime.IsLeaf(header) {
-		defer func() {
-			pka.depth--
-		}()
-	}
-	isRoot := (header.Get("From") != "")
-
-	// NOTE: This should also work for unspecified Content-Type (in which case us-ascii text/plain is assumed)!
-	mediaType, _, err := pmmime.ParseMediaType(header.Get("Content-Type"))
-	if isRoot && isFirst && err == nil && pka.attachedPublicKey != "" { //nolint[gocritic]
-		if strings.HasPrefix(mediaType, "multipart/mixed") {
-			pka.appendToMultipart = true
-			_ = pka.target.Accept(partReader, header, hasPlainSiblings, isFirst, isLast)
-		} else {
-			// Create two siblings with attachment in the case toplevel is not multipart/mixed.
-			multiPartHeaders := make(textproto.MIMEHeader)
-			for k, v := range header {
-				multiPartHeaders[k] = v
-			}
-			boundary := randomBoundary()
-			multiPartHeaders.Set("Content-Type", "multipart/mixed; boundary=\""+boundary+"\"")
-			multiPartHeaders.Del("Content-Transfer-Encoding")
-
-			_ = pka.target.Accept(partReader, multiPartHeaders, false, true, false)
-
-			originalHeader := make(textproto.MIMEHeader)
-			originalHeader.Set("Content-Type", header.Get("Content-Type"))
-			if header.Get("Content-Transfer-Encoding") != "" {
-				originalHeader.Set("Content-Transfer-Encoding", header.Get("Content-Transfer-Encoding"))
-			}
-
-			_ = pka.target.Accept(partReader, originalHeader, false, true, false)
-			_ = pka.target.Accept(partReader, multiPartHeaders, hasPlainSiblings, false, false)
-
-			attachmentHeaders, attachmentReader := createKeyAttachment(pka.attachedPublicKey, pka.attachedPublicKeyName)
-
-			_ = pka.target.Accept(attachmentReader, attachmentHeaders, false, true, true)
-			_ = pka.target.Accept(partReader, multiPartHeaders, hasPlainSiblings, false, true)
 		}
-	} else if isLast && pka.depth == 1 && pka.attachedPublicKey != "" {
-		_ = pka.target.Accept(partReader, header, hasPlainSiblings, isFirst, false)
-		attachmentHeaders, attachmentReader := createKeyAttachment(pka.attachedPublicKey, pka.attachedPublicKeyName)
-		_ = pka.target.Accept(attachmentReader, attachmentHeaders, hasPlainSiblings, true, true)
-		_ = pka.target.Accept(partReader, header, hasPlainSiblings, isFirst, true)
-	} else {
-		_ = pka.target.Accept(partReader, header, hasPlainSiblings, isFirst, isLast)
 	}
-	return nil
+
+	// Otherwise, choose the last one.
+	return childParts[len(childParts)-1]
 }
 
-// ======= Parser ==========
+func allPartsHaveContentType(parts parser.Parts, contentType string) bool {
-
+	if len(parts) == 0 {
-func Parse(r io.Reader, attachedPublicKey, attachedPublicKeyName string) (m *pmapi.Message, mimeBody string, plainContents string, atts []io.Reader, err error) {
+		return false
-	secondReader := new(bytes.Buffer)
-	_, _ = secondReader.ReadFrom(r)
-
-	mimeBody = secondReader.String()
-
-	mm, err := mail.ReadMessage(secondReader)
-	if err != nil {
-		return
 	}
 
-	if m, err = parseHeader(mm.Header); err != nil {
+	for _, part := range parts {
-		return
+		t, _, err := part.ContentType()
+		if err != nil {
+			return false
+		}
+
+		if t != contentType {
+			return false
+		}
 	}
 
-	h := textproto.MIMEHeader(m.Header)
+	return true
-	mmBodyData, err := ioutil.ReadAll(mm.Body)
+}
-	if err != nil {
+
-		return
+func determineMIMEType(p *parser.Parser) (string, error) {
+	var isHTML bool
+
+	w := p.NewWalker().
+		RegisterContentTypeHandler("text/html", func(p *parser.Part) (err error) {
+			isHTML = true
+			return
+		})
+
+	if err := w.Walk(); err != nil {
+		return "", err
 	}
 
-	printAccepter := pmmime.NewMIMEPrinter()
-
-	publicKeyAttacher := NewPublicKeyAttacher(printAccepter, attachedPublicKey, attachedPublicKeyName)
-	sevenBitFilter := NewSevenBitFilter(publicKeyAttacher)
-
-	plainTextCollector := pmmime.NewPlainTextCollector(sevenBitFilter)
-	htmlOnlyConvertor := NewHTMLOnlyConvertor(plainTextCollector)
-
-	visitor := pmmime.NewMimeVisitor(htmlOnlyConvertor)
-	err = pmmime.VisitAll(bytes.NewReader(mmBodyData), h, visitor)
-	/*
-		err = visitor.VisitAll(h, bytes.NewReader(mmBodyData))
-	*/
-	if err != nil {
-		return
-	}
-
-	mimeBody = printAccepter.String()
-
-	plainContents = plainTextCollector.GetPlainText()
-
-	parts, headers, err := pmmime.GetAllChildParts(bytes.NewReader(mmBodyData), h)
-
-	if err != nil {
-		return
-	}
-
-	convertPlainToHTML := checkHeaders(headers)
-	isHTML, err := combineParts(m, parts, headers, convertPlainToHTML, &atts)
-
 	if isHTML {
-		m.MIMEType = "text/html"
+		return "text/html", nil
-	} else {
-		m.MIMEType = "text/plain"
 	}
 
-	return m, mimeBody, plainContents, atts, err
+	return "text/plain", nil
+}
+
+// getPlainBody returns the body of the given part, converting html to
+// plaintext where possible.
+func getPlainBody(part *parser.Part) []byte {
+	contentType, _, err := part.ContentType()
+	if err != nil {
+		return part.Body
+	}
+
+	switch contentType {
+	case "text/html":
+		text, err := html2text.FromReader(bytes.NewReader(part.Body))
+		if err != nil {
+			return part.Body
+		}
+
+		return []byte(text)
+
+	default:
+		return part.Body
+	}
+}
+
+func attachPublicKey(p *parser.Part, key, keyName string) {
+	h := message.Header{}
+
+	h.Set("Content-Type", fmt.Sprintf(`application/pgp-keys; name="%v.asc"; filename="%v.asc"`, keyName, keyName))
+	h.Set("Content-Disposition", fmt.Sprintf(`attachment; name="%v.asc"; filename="%v.asc"`, keyName, keyName))
+	h.Set("Content-Transfer-Encoding", "base64")
+
+	p.AddChild(&parser.Part{
+		Header: h,
+		Body:   []byte(key),
+	})
+}
+
+func parseMessageHeader(m *pmapi.Message, h message.Header) error { // nolint[funlen]
+	mimeHeader, err := toMailHeader(h)
+	if err != nil {
+		return err
+	}
+	m.Header = mimeHeader
+
+	fields := h.Fields()
+
+	for fields.Next() {
+		switch strings.ToLower(fields.Key()) {
+		case "subject":
+			s, err := fields.Text()
+			if err != nil {
+				if s, err = pmmime.DecodeHeader(fields.Value()); err != nil {
+					return errors.Wrap(err, "failed to parse subject")
+				}
+			}
+
+			m.Subject = s
+
+		case "from":
+			sender, err := rfc5322.ParseAddressList(fields.Value())
+			if err != nil {
+				return errors.Wrap(err, "failed to parse from")
+			}
+			if len(sender) > 0 {
+				m.Sender = sender[0]
+			}
+
+		case "to":
+			toList, err := rfc5322.ParseAddressList(fields.Value())
+			if err != nil {
+				return errors.Wrap(err, "failed to parse to")
+			}
+			m.ToList = toList
+
+		case "reply-to":
+			replyTos, err := rfc5322.ParseAddressList(fields.Value())
+			if err != nil {
+				return errors.Wrap(err, "failed to parse reply-to")
+			}
+			m.ReplyTos = replyTos
+
+		case "cc":
+			ccList, err := rfc5322.ParseAddressList(fields.Value())
+			if err != nil {
+				return errors.Wrap(err, "failed to parse cc")
+			}
+			m.CCList = ccList
+
+		case "bcc":
+			bccList, err := rfc5322.ParseAddressList(fields.Value())
+			if err != nil {
+				return errors.Wrap(err, "failed to parse bcc")
+			}
+			m.BCCList = bccList
+
+		case "date":
+			date, err := rfc5322.ParseDateTime(fields.Value())
+			if err != nil {
+				return errors.Wrap(err, "failed to parse date")
+			}
+			m.Time = date.Unix()
+		}
+	}
+
+	return nil
+}
+
+func parseAttachment(h message.Header) (*pmapi.Attachment, error) {
+	att := &pmapi.Attachment{}
+
+	mimeHeader, err := toMIMEHeader(h)
+	if err != nil {
+		return nil, err
+	}
+	att.Header = mimeHeader
+
+	mimeType, _, err := h.ContentType()
+	if err != nil {
+		return nil, err
+	}
+	att.MIMEType = mimeType
+
+	_, dispParams, dispErr := h.ContentDisposition()
+	if dispErr != nil {
+		ext, err := mime.ExtensionsByType(att.MIMEType)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(ext) > 0 {
+			att.Name = "attachment" + ext[0]
+		}
+	} else {
+		att.Name = dispParams["filename"]
+
+		if att.Name == "" {
+			att.Name = "attachment.bin"
+		}
+	}
+
+	att.ContentID = strings.Trim(h.Get("Content-Id"), " <>")
+
+	return att, nil
+}
+
+func toMailHeader(h message.Header) (mail.Header, error) {
+	mimeHeader := make(mail.Header)
+
+	if err := forEachDecodedHeaderField(h, func(key, val string) error {
+		mimeHeader[key] = []string{val}
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return mimeHeader, nil
+}
+
+func toMIMEHeader(h message.Header) (textproto.MIMEHeader, error) {
+	mimeHeader := make(textproto.MIMEHeader)
+
+	if err := forEachDecodedHeaderField(h, func(key, val string) error {
+		mimeHeader[key] = []string{val}
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return mimeHeader, nil
+}
+
+func forEachDecodedHeaderField(h message.Header, fn func(string, string) error) error {
+	fields := h.Fields()
+
+	for fields.Next() {
+		text, err := fields.Text()
+		if err != nil {
+			if !message.IsUnknownCharset(err) {
+				return err
+			}
+
+			if text, err = pmmime.DecodeHeader(fields.Value()); err != nil {
+				return err
+			}
+		}
+
+		if err := fn(fields.Key(), text); err != nil {
+			return err
+		}
+	}
+
+	return nil
 }

pkg/message/parser/handler.go

@@ -0,0 +1,59 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package parser
+
+import (
+	"regexp"
+)
+
+type HandlerFunc func(*Part) error
+
+type handler struct {
+	typeRegExp, dispRegExp *regexp.Regexp
+	fn                     HandlerFunc
+}
+
+func (h *handler) matchPart(p *Part) bool {
+	return h.matchType(p) || h.matchDisp(p)
+}
+
+func (h *handler) matchType(p *Part) bool {
+	if h.typeRegExp == nil {
+		return false
+	}
+
+	t, _, err := p.ContentType()
+	if err != nil {
+		t = ""
+	}
+
+	return h.typeRegExp.MatchString(t)
+}
+
+func (h *handler) matchDisp(p *Part) bool {
+	if h.dispRegExp == nil {
+		return false
+	}
+
+	disp, _, err := p.Header.ContentDisposition()
+	if err != nil {
+		disp = ""
+	}
+
+	return h.dispRegExp.MatchString(disp)
+}

pkg/message/parser/parser.go

@@ -0,0 +1,154 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package parser
+
+import (
+	"io"
+	"io/ioutil"
+
+	"github.com/emersion/go-message"
+)
+
+type Parser struct {
+	stack []*Part
+	root  *Part
+}
+
+func New(r io.Reader) (*Parser, error) {
+	p := new(Parser)
+
+	entity, err := message.Read(newEndOfMailTrimmer(r))
+	if err != nil && !message.IsUnknownCharset(err) {
+		return nil, err
+	}
+
+	if err := p.parseEntity(entity); err != nil {
+		return nil, err
+	}
+
+	return p, nil
+}
+
+func (p *Parser) NewWalker() *Walker {
+	return newWalker(p.root)
+}
+
+func (p *Parser) NewVisitor(defaultRule VisitorRule) *Visitor {
+	return newVisitor(p.root, defaultRule)
+}
+
+func (p *Parser) NewWriter() *Writer {
+	return newWriter(p.root)
+}
+
+func (p *Parser) Root() *Part {
+	return p.root
+}
+
+// Section returns the message part referred to by the given section. A section
+// is zero or more integers. For example, section 1.2.3 will return the third
+// part of the second part of the first part of the message.
+func (p *Parser) Section(section []int) (part *Part, err error) {
+	part = p.root
+
+	for _, n := range section {
+		if part, err = part.Child(n); err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+func (p *Parser) beginPart() {
+	p.stack = append(p.stack, &Part{})
+}
+
+func (p *Parser) endPart() {
+	var part *Part
+
+	p.stack, part = p.stack[:len(p.stack)-1], p.stack[len(p.stack)-1]
+
+	if len(p.stack) > 0 {
+		p.top().children = append(p.top().children, part)
+	} else {
+		p.root = part
+	}
+}
+
+func (p *Parser) top() *Part {
+	if len(p.stack) == 0 {
+		return nil
+	}
+
+	return p.stack[len(p.stack)-1]
+}
+
+func (p *Parser) withHeader(h message.Header) {
+	p.top().Header = h
+}
+
+func (p *Parser) withBody(bytes []byte) {
+	p.top().Body = bytes
+}
+
+func (p *Parser) parseEntity(e *message.Entity) error {
+	p.beginPart()
+	defer p.endPart()
+
+	p.withHeader(e.Header)
+
+	if mr := e.MultipartReader(); mr != nil {
+		return p.parseMultipart(mr)
+	}
+
+	return p.parsePart(e)
+}
+
+func (p *Parser) parsePart(e *message.Entity) (err error) {
+	bytes, err := ioutil.ReadAll(e.Body)
+	if err != nil {
+		return
+	}
+
+	p.withBody(bytes)
+
+	return
+}
+
+func (p *Parser) parseMultipart(r message.MultipartReader) (err error) {
+	for {
+		var child *message.Entity
+
+		if child, err = r.NextPart(); err != nil && !message.IsUnknownCharset(err) {
+			return ignoreEOF(err)
+		}
+
+		if err = p.parseEntity(child); err != nil {
+			return
+		}
+	}
+}
+
+func ignoreEOF(err error) error {
+	if err == io.EOF {
+		return nil
+	}
+
+	return err
+}

pkg/message/parser/parser_test.go

@@ -0,0 +1,54 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package parser
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func newTestParser(t *testing.T, msg string) *Parser {
+	p, err := New(getFileReader(msg))
+	require.NoError(t, err)
+
+	return p
+}
+
+func getFileReader(filename string) io.ReadCloser {
+	f, err := os.Open(filepath.Join("testdata", filename))
+
+	if err != nil {
+		panic(err)
+	}
+
+	return f
+}
+
+func getFileAsString(filename string) string {
+	b, err := ioutil.ReadAll(getFileReader(filename))
+	if err != nil {
+		panic(err)
+	}
+
+	return string(b)
+}

pkg/message/parser/part.go

@@ -0,0 +1,200 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package parser
+
+import (
+	"bytes"
+	"errors"
+	"mime"
+	"unicode/utf8"
+
+	pmmime "github.com/ProtonMail/proton-bridge/pkg/mime"
+	"github.com/PuerkitoBio/goquery"
+	"github.com/emersion/go-message"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/charset"
+	"golang.org/x/text/encoding"
+)
+
+type Parts []*Part
+
+type Part struct {
+	Header   message.Header
+	Body     []byte
+	children Parts
+}
+
+func (p *Part) ContentType() (string, map[string]string, error) {
+	t, params, err := p.Header.ContentType()
+	if err != nil {
+		// go-message's implementation of ContentType() doesn't handle duplicate parameters
+		// e.g. Content-Type: text/plain; charset=utf-8; charset=UTF-8
+		// so if it fails, we try again with pmmime's implementation, which does.
+		t, params, err = pmmime.ParseMediaType(p.Header.Get("Content-Type"))
+	}
+
+	return t, params, err
+}
+
+func (p *Part) Child(n int) (part *Part, err error) {
+	if len(p.children) < n {
+		return nil, errors.New("no such part")
+	}
+
+	return p.children[n-1], nil
+}
+
+func (p *Part) Children() Parts {
+	return p.children
+}
+
+func (p *Part) AddChild(child *Part) {
+	if p.isMultipartMixed() {
+		p.children = append(p.children, child)
+	} else {
+		root := &Part{
+			Header:   getContentHeaders(p.Header),
+			Body:     p.Body,
+			children: p.children,
+		}
+
+		p.Body = nil
+		p.children = Parts{root, child}
+		stripContentHeaders(&p.Header)
+		p.Header.Set("Content-Type", "multipart/mixed")
+	}
+}
+
+func (p *Part) ConvertToUTF8() error {
+	logrus.Trace("Converting part to utf-8")
+
+	t, params, err := p.ContentType()
+	if err != nil {
+		return err
+	}
+
+	decoder := selectSuitableDecoder(p, t, params)
+
+	if p.Body, err = decoder.Bytes(p.Body); err != nil {
+		return err
+	}
+
+	if params == nil {
+		params = make(map[string]string)
+	}
+
+	params["charset"] = "UTF-8"
+
+	p.Header.SetContentType(t, params)
+
+	return nil
+}
+
+func (p *Part) ConvertMetaCharset() error {
+	doc, err := html.Parse(bytes.NewReader(p.Body))
+	if err != nil {
+		return err
+	}
+
+	goquery.NewDocumentFromNode(doc).Find("meta").Each(func(n int, sel *goquery.Selection) {
+		if val, ok := sel.Attr("content"); ok {
+			t, params, err := pmmime.ParseMediaType(val)
+			if err != nil {
+				return
+			}
+
+			params["charset"] = "UTF-8"
+
+			sel.SetAttr("content", mime.FormatMediaType(t, params))
+		}
+
+		if _, ok := sel.Attr("charset"); ok {
+			sel.SetAttr("charset", "UTF-8")
+		}
+	})
+
+	buf := new(bytes.Buffer)
+
+	if err := html.Render(buf, doc); err != nil {
+		return err
+	}
+
+	p.Body = buf.Bytes()
+
+	return nil
+}
+
+func selectSuitableDecoder(p *Part, t string, params map[string]string) *encoding.Decoder {
+	if charset, ok := params["charset"]; ok {
+		logrus.WithField("charset", charset).Trace("The part has a specified charset")
+
+		if decoder, err := pmmime.SelectDecoder(charset); err == nil {
+			logrus.Trace("The charset is known; decoder has been selected")
+			return decoder
+		}
+
+		logrus.Warn("The charset is unknown; no decoder could be selected")
+	}
+
+	if utf8.Valid(p.Body) {
+		logrus.Trace("The part is already valid utf-8, returning noop encoder")
+		return encoding.Nop.NewDecoder()
+	}
+
+	encoding, name, _ := charset.DetermineEncoding(p.Body, t)
+
+	logrus.WithField("name", name).Warn("Determined encoding by reading body")
+
+	return encoding.NewDecoder()
+}
+
+func (p *Part) is7BitClean() bool {
+	for _, b := range p.Body {
+		if b > 1<<7 {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (p *Part) isMultipartMixed() bool {
+	t, _, err := p.ContentType()
+	if err != nil {
+		return false
+	}
+
+	return t == "multipart/mixed"
+}
+
+func getContentHeaders(header message.Header) message.Header {
+	var res message.Header
+
+	res.Set("Content-Type", header.Get("Content-Type"))
+	res.Set("Content-Disposition", header.Get("Content-Disposition"))
+	res.Set("Content-Transfer-Encoding", header.Get("Content-Transfer-Encoding"))
+
+	return res
+}
+
+func stripContentHeaders(header *message.Header) {
+	header.Del("Content-Type")
+	header.Del("Content-Disposition")
+	header.Del("Content-Transfer-Encoding")
+}

pkg/message/parser/part_test.go

@@ -0,0 +1,73 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package parser
+
+import (
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestPart(t *testing.T) {
+	p := newTestParser(t, "complex_structure.eml")
+
+	wantParts := map[string]string{
+		"":        "multipart/mixed",
+		"1":       "text/plain",
+		"2":       "application/octet-stream",
+		"3":       "multipart/mixed",
+		"3.1":     "text/plain",
+		"3.2":     "application/octet-stream",
+		"4":       "multipart/mixed",
+		"4.1":     "image/gif",
+		"4.2":     "multipart/mixed",
+		"4.2.1":   "text/plain",
+		"4.2.2":   "multipart/alternative",
+		"4.2.2.1": "text/plain",
+		"4.2.2.2": "text/html",
+	}
+
+	for partNumber, wantContType := range wantParts {
+		part, err := p.Section(getSectionNumber(partNumber))
+		require.NoError(t, err)
+
+		contType, _, err := part.ContentType()
+		require.NoError(t, err)
+		assert.Equal(t, wantContType, contType)
+	}
+}
+
+func getSectionNumber(s string) (part []int) {
+	if s == "" {
+		return
+	}
+
+	for _, number := range strings.Split(s, ".") {
+		i64, err := strconv.ParseInt(number, 10, 64)
+		if err != nil {
+			panic(err)
+		}
+
+		part = append(part, int(i64))
+	}
+
+	return
+}