Release Danube: notes, version bump, change log

Added missing signal, corrected the update name, log tweaks.

.gitlab-ci.yml

@@ -82,7 +82,9 @@ dependency-updates:
   script:
     - make build
   artifacts:
-    expire_in: 2 week
+    # Note: The latest artifacts for refs are locked against deletion, and kept regardless of the expiry time.
+    # Introduced in GitLab 13.0 behind a disabled feature flag, and made the default behavior in GitLab 13.4.
+    expire_in: 1 day
 
 build-linux:
   extends: .build-base

BUILDS.md

@@ -19,7 +19,6 @@ Otherwise, the sending of crash reports will be disabled.
 export MSYSTEM=
 ```
 
-
 ### Build Bridge
 * in project root run
 
@@ -44,6 +43,12 @@ make build-ie
     * for `windows`, the binary will have the file extension `.exe` (e.g `proton-bridge.exe`)
     * for `darwin`, the application will be created with name of the project directory (e.g `proton-bridge.app`)
 
+### Tags
+Note that repository contains both Bridge and Import-Export apps and they are
+not released together. Therefore, each app has own tag prefix. Bridge tags
+starts with `br-` and Import-Export tags starts with `ie-`. Both tags continue
+with semantic versioning `MAJOR.MINOR.PATCH`. An example of full tag is
+`br-1.4.4` or `ie-1.1.2` (current versions in October 2020).
 
 ## Useful tests, lints and checks
 In order to be able to run following commands please install the development dependencies: 

Changelog.md

@@ -4,9 +4,37 @@ Changelog [format](http://keepachangelog.com/en/1.0.0/)
 
 ## Unreleased
 
+## [IE 1.2.0] Elbe
+
+### Added
+* GODT-763 Detect Gmail labels from All Mail mbox export (using X-Gmail-Label header).
+* GODT-834 Info about tags in BUILDS.md and link to Import-Export page in README.md.
+* GODT-777 Support Apple Mail MBOX export format.
+
+### Fixed
+* GODT-677 Windows IE: global import settings not fit in window.
+* GODT-794 Congo fails to update to Danube
+* GODT-749 Don't force PGP/Inline when sending plaintext messages.
+* GODT-764 Fix deadlock in integration tests for Import-Export.
+* GODT-662 Do not resume paused transfer progress after dismissing cancel popup.
+* GODT-772 Sanitize mailbox names for exporting to follow OS restrictions.
+* GODT-771 Show fatal errors after export is terminated.
+* GODT-779 Do not propagate updates when progress is stopped.
+* GODT-779 Unpause progress during fatal error to properly stop progress.
+* GODT-779 Stop ongoing transfer calls sooner (re-check after import request is generated).
+* Fix measurement of uploading attachments during transfer.
+* GODT-827 Do not spam sentry with bad ID by integration test.
+* GODT-700 Fix UTF-7 incompatibility.
+* GODT-837 Fix flaky TestFailUnpauseAndStops.
+
+## [Bridge 1.4.5] Forth
+
+### Fixed
+* GODT-829 Remove `NoInferior` to display sub-folders in apple mail.
+
 ## [Bridge 1.4.4] Forth
 
-### Fixed
+### Fixed
 * GODT-798 Replace, don't add, transfer encoding when making body 7-bit clean.
 * Move/Copy duplicate for emails with References in Outlook
 * CSB-247 Cannot update from 1.4.0
@@ -17,7 +45,7 @@ Changelog [format](http://keepachangelog.com/en/1.0.0/)
 ### Changed
 * Reverted sending IMAP updates to be not blocking again.
 
-### Fixed
+### Fixed
 * GODT-783 Settings flags by FLAGS (not using +/-FLAGS) do not change spam state.
 
 
@@ -28,6 +56,21 @@ Changelog [format](http://keepachangelog.com/en/1.0.0/)
 * GODT-765 Improve speed of checking whether message is deleted.
 
 
+## [IE 1.1.2] Danube (beta 2020-09-xx)
+
+### Fixed
+* GODT-770 Better handling of extraneous end-of-mail indicator.
+* GODT-776 Fix crash when IMAP client connects while account is logging in.
+
+### Changed
+* Bump crypto version to v0.0.0-20200818122824-ed5d25e28db8
+* GODT-785 Clear separation of different message IDs in integration tests.
+### Changed
+* GODT-741 Import-Export shows "Unable to parse time" notice instead of zero time in error report window.
+
+* Bump crypto version to v0.0.0-20200818122824-ed5d25e28db8.
+* GODT-374 Allow to send calendar update multiple times.
+
 ## [IE 1.1.1] Danube (beta 2020-09-xx) [Bridge 1.4.1] Forth (beta 2020-09-xx)
 
 ### Fixed
@@ -41,6 +84,8 @@ Changelog [format](http://keepachangelog.com/en/1.0.0/)
 ### Fixed
 * GODT-703 Import-Export showed always at least one total message.
 * GODT-738 Fix for mbox files with long lines.
+### Fixed
+* GODT-732 Do not mix font awesome icon with regular text to avoid issues on Fedora.
 
 
 ## [Bridge 1.4.0] Forth

Makefile

@@ -10,8 +10,8 @@ TARGET_OS?=${GOOS}
 .PHONY: build build-ie build-nogui build-ie-nogui check-has-go
 
 # Keep version hardcoded so app build works also without Git repository.
-BRIDGE_APP_VERSION?=1.4.0-git
+BRIDGE_APP_VERSION?=1.4.5-git
-IE_APP_VERSION?=1.1.0-git
+IE_APP_VERSION?=1.2.0-git
 APP_VERSION:=${BRIDGE_APP_VERSION}
 SRC_ICO:=logo.ico
 SRC_ICNS:=Bridge.icns

README.md

@@ -1,7 +1,7 @@
 # ProtonMail Bridge and Import Export app
 Copyright (c) 2020 Proton Technologies AG
 
-This repository holds the ProtonMail Bridge application.
+This repository holds the ProtonMail Bridge and the ProtonMail Import-Export applications.
 For a detailed build information see [BUILDS](./BUILDS.md).
 For licensing information see [COPYING](./COPYING.md).
 For contribution policy see [CONTRIBUTING](./CONTRIBUTING.md).
@@ -35,6 +35,8 @@ configure transfer rules (match source and target mailboxes, set time
 range limits and so on) and hit start. Once the transfer is complete,
 check the results.
 
+More details [on the public website](https://protonmail.com/import-export).
+
 ## Keychain
 You need to have a keychain in order to run the ProtonMail Bridge. On Mac or
 Windows, Bridge uses native credential managers. On Linux, use

go.mod

@@ -74,9 +74,9 @@ require (
 
 replace (
 	github.com/docker/docker-credential-helpers => github.com/ProtonMail/docker-credential-helpers v1.1.0
-	github.com/emersion/go-imap => github.com/ProtonMail/go-imap v0.0.0-20200828124548-d04b0dc1f399
+	github.com/emersion/go-imap => github.com/ProtonMail/go-imap v0.0.0-20201016095853-a7520cc904d3
 	github.com/emersion/go-mbox => github.com/ProtonMail/mbox v0.0.0-20200918064939-909a18c9af45
 	github.com/emersion/go-smtp => github.com/ProtonMail/go-smtp v0.0.0-20181206232543-8261df20d309
 	github.com/jameskeane/bcrypt => github.com/ProtonMail/bcrypt v0.0.0-20170924085257-7509ea014998
-	golang.org/x/crypto => github.com/ProtonMail/crypto v0.0.0-20200416114516-1fa7f403fb9c
+	golang.org/x/crypto => github.com/ProtonMail/crypto v0.0.0-20200818122824-ed5d25e28db8
 )

go.sum

@@ -1,13 +1,12 @@
 github.com/0xAX/notificator v0.0.0-20191016112426-3962a5ea8da1 h1:j9HaafapDbPbGRDku6e/HRs6KBMcKHiWcm1/9Sbxnl4=
 github.com/0xAX/notificator v0.0.0-20191016112426-3962a5ea8da1/go.mod h1:NtXa9WwQsukMHZpjNakTTz0LArxvGYdPA9CjIcUSZ6s=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
 github.com/Masterminds/semver/v3 v3.1.0 h1:Y2lUDsFKVRSYGojLJ1yLxSXdMmMYTYls0rCvoqmMUQk=
 github.com/Masterminds/semver/v3 v3.1.0/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
 github.com/ProtonMail/bcrypt v0.0.0-20170924085257-7509ea014998 h1:YT2uVwQiRQZxCaaahwfcgTq2j3j66w00n/27gb/zubs=
 github.com/ProtonMail/bcrypt v0.0.0-20170924085257-7509ea014998/go.mod h1:HecWFHognK8GfRDGnFQbW/LiV7A3MX3gZVs45vk5h8I=
-github.com/ProtonMail/crypto v0.0.0-20200416114516-1fa7f403fb9c h1:DAvlgde2Stu18slmjwikiMPs/CKPV35wSvmJS34z0FU=
+github.com/ProtonMail/crypto v0.0.0-20200818122824-ed5d25e28db8 h1:u1j0xLTrCHpNS40B6m4Sv3IVUz5m9jt+AnTIopT3IgM=
-github.com/ProtonMail/crypto v0.0.0-20200416114516-1fa7f403fb9c/go.mod h1:Pxr7w4gA2ikI4sWyYwEffm+oew1WAJHzG1SiDpQMkrI=
+github.com/ProtonMail/crypto v0.0.0-20200818122824-ed5d25e28db8/go.mod h1:Pxr7w4gA2ikI4sWyYwEffm+oew1WAJHzG1SiDpQMkrI=
 github.com/ProtonMail/docker-credential-helpers v1.1.0 h1:+kvUIpwWcbtP3WFv5sSvkFn/XLzSqPOB5AAthuk9xPk=
 github.com/ProtonMail/docker-credential-helpers v1.1.0/go.mod h1:mK0aBveCxhnQ756AmaTfXMZDeULvheYVhF/MWMErN5g=
 github.com/ProtonMail/go-appdir v1.1.0 h1:9hdNDlU9kTqRKVNzmoqah8qqrj5QZyLByQdwQNlFWig=
@@ -16,8 +15,8 @@ github.com/ProtonMail/go-apple-mobileconfig v0.0.0-20160701194735-7ea9927a11f6 h
 github.com/ProtonMail/go-apple-mobileconfig v0.0.0-20160701194735-7ea9927a11f6/go.mod h1:EtDfBMIDWmVe4viZCuBTEfe3OIIo0ghbpOaAZVO+hVg=
 github.com/ProtonMail/go-autostart v0.0.0-20181114175602-c5272053443a h1:fXK2KsfnkBV9Nh+9SKzHchYjuE9s0vI20JG1mbtEAcc=
 github.com/ProtonMail/go-autostart v0.0.0-20181114175602-c5272053443a/go.mod h1:oTGdE7/DlWIr23G0IKW3OXK9wZ5Hw1GGiaJFccTvZi4=
-github.com/ProtonMail/go-imap v0.0.0-20200828124548-d04b0dc1f399 h1:wBo/Xgb/Dn2loU47D+PJaOoIZ67i3AqYp51gLn8YE5U=
+github.com/ProtonMail/go-imap v0.0.0-20201016095853-a7520cc904d3 h1:Jvv9t3rSg/ID3Fh+uYsxgmvNI9fYnlab4vtBsbPtmq8=
-github.com/ProtonMail/go-imap v0.0.0-20200828124548-d04b0dc1f399/go.mod h1:yKASt+C3ZiDAiCSssxg9caIckWF/JG7ZQTO7GAmvicU=
+github.com/ProtonMail/go-imap v0.0.0-20201016095853-a7520cc904d3/go.mod h1:yKASt+C3ZiDAiCSssxg9caIckWF/JG7ZQTO7GAmvicU=
 github.com/ProtonMail/go-imap-id v0.0.0-20190926060100-f94a56b9ecde h1:5koQozTDELymYOyFbQ/VSubexAEXzDR8qGM5mO8GRdw=
 github.com/ProtonMail/go-imap-id v0.0.0-20190926060100-f94a56b9ecde/go.mod h1:795VPXcRUIQ9JyMNHP4el582VokQfippgjkQP3Gk0r0=
 github.com/ProtonMail/go-mime v0.0.0-20190923161245-9b5a4261663a h1:W6RrgN/sTxg1msqzFFb+G80MFmpjMw61IU+slm+wln4=
@@ -68,8 +67,6 @@ github.com/emersion/go-imap-specialuse v0.0.0-20200722111535-598ff00e4075 h1:z8T
 github.com/emersion/go-imap-specialuse v0.0.0-20200722111535-598ff00e4075/go.mod h1:/nybxhI8kXom8Tw6BrHMl42usALvka6meORflnnYwe4=
 github.com/emersion/go-imap-unselect v0.0.0-20171113212723-b985794e5f26 h1:FiSb8+XBQQSkcX3ubr+1tAtlRJBYaFmRZqOAweZ9Wy8=
 github.com/emersion/go-imap-unselect v0.0.0-20171113212723-b985794e5f26/go.mod h1:+gnnZx3Mg3MnCzZrv0eZdp5puxXQUgGT/6N6L7ShKfM=
-github.com/emersion/go-mbox v1.0.0 h1:HN6aKbyqmgIfK9fS/gen+NRr2wXLSxZXWfdAIAnzQPc=
-github.com/emersion/go-mbox v1.0.0/go.mod h1:Yp9IVuuOYLEuMv4yjgDHvhb5mHOcYH6x92Oas3QqEZI=
 github.com/emersion/go-message v0.11.1/go.mod h1:C4jnca5HOTo4bGN9YdqNQM9sITuT3Y0K6bSUw9RklvY=
 github.com/emersion/go-message v0.12.1-0.20200903165315-e1abe21f389a h1:3C6qIGgPr1qAT0ikRD5NbyKpME/iHCDeXhpv/JJsFsE=
 github.com/emersion/go-message v0.12.1-0.20200903165315-e1abe21f389a/go.mod h1:kYIioST9GDHte9/BRWgi93rpqbDuFftMjKSMaXS8ABo=
@@ -101,8 +98,6 @@ github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/U
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI=
 github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
-github.com/jameshoulahan/go-imap v0.0.0-20200728140727-d57327f48843 h1:suxlO4AC4E4bjueAsL0m+qp8kmkxRWMGj+5bBU/KJ8g=
-github.com/jameshoulahan/go-imap v0.0.0-20200728140727-d57327f48843/go.mod h1:yKASt+C3ZiDAiCSssxg9caIckWF/JG7ZQTO7GAmvicU=
 github.com/jaytaylor/html2text v0.0.0-20200412013138-3577fbdbcff7 h1:g0fAGBisHaEQ0TRq1iBvemFRf+8AEWEmBESSiWB3Vsc=
 github.com/jaytaylor/html2text v0.0.0-20200412013138-3577fbdbcff7/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk=
 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA=

internal/frontend/qml/BridgeUI/SettingsView.qml

@@ -48,6 +48,7 @@ Item {
                     text : qsTr("Clear", "clickable link next to clear cache button in settings")
                     color: Style.main.text
                     font {
+                        family : cacheClear.font.family // use default font, not font-awesome
                         pointSize : Style.settings.fontSize * Style.pt
                         underline : true
                     }
@@ -66,6 +67,7 @@ Item {
                     text : qsTr("Clear", "clickable link next to clear keychain button in settings")
                     color: Style.main.text
                     font {
+                        family : cacheKeychain.font.family // use default font, not font-awesome
                         pointSize : Style.settings.fontSize * Style.pt
                         underline : true
                     }
@@ -125,6 +127,7 @@ Item {
                     text : qsTr("Change", "clickable link next to change ports button in settings")
                     color: Style.main.text
                     font {
+                        family : changePort.font.family // use default font, not font-awesome
                         pointSize : Style.settings.fontSize * Style.pt
                         underline : true
                     }

internal/frontend/qml/GuiIE.qml

@@ -276,6 +276,10 @@ Item {
                 winMain.dialogExport.hide()
             }
         }
+
+        onUpdateFinished : {
+            winMain.dialogUpdate.finished(hasError)
+        }
     }
 
     function folderIcon(folderName, folderType) { // translations

internal/frontend/qml/ImportExportUI/DialogExport.qml

@@ -217,7 +217,10 @@ Dialog {
                 Text {
                     anchors.centerIn: parent
                     text: {
-                        if (progressbarExport.isFinished) return qsTr("Export finished","todo")
+                        if (progressbarExport.isFinished) {
+                            if (go.progressDescription=="") return qsTr("Export finished","todo")
+                            else return qsTr("Export failed: %1").arg(go.progressDescription)
+                        }
                         if (
                             go.progressDescription == gui.enums.progressInit || 
                             (go.progress==0 && go.description=="")
@@ -450,7 +453,6 @@ Dialog {
             errorPopup.hide()
         }
         onClickedNo  : {
-            go.resumeProcess()
             errorPopup.hide()
         }
     }

internal/frontend/qml/ImportExportUI/DialogImport.qml

@@ -279,9 +279,8 @@ Dialog {
             titleTo        : root.address
         }
 
-        Rectangle {
+        Column {
             id: masterImportSettings
-            height: 150 // fixme
             anchors {
                 right  : parent.right
                 left   : parent.left
@@ -291,45 +290,47 @@ Dialog {
                 rightMargin  : Style.main.leftMargin
                 bottomMargin : Style.main.bottomMargin
             }
-            color: Style.dialog.background
 
-            Text {
+            spacing: Style.main.bottomMargin
-                id: labelMasterImportSettings
-                text: qsTr("Master import settings:")
 
-                font {
+            Row {
-                    bold: true
+                spacing: masterImportSettings.width - labelMasterImportSettings.width - resetSourceButton.width
-                    family: Style.fontawesome.name
-                    pointSize: Style.main.fontSize * Style.pt
-                }
-                color: Style.main.text
 
-                InfoToolTip {
+                Text {
-                    info: qsTr(
+                    id: labelMasterImportSettings
-                        "If master import date range is selected only emails within this range will be imported, unless it is specified differently in folder date range.",
+                    text: qsTr("Master import settings:")
-                        "Text in master import settings tooltip."
+
-                    )
+                    font {
-                    anchors {
+                        bold: true
-                        left: parent.right
+                        family: Style.fontawesome.name
-                        bottom: parent.bottom
+                        pointSize: Style.main.fontSize * Style.pt
-                        leftMargin : Style.dialog.leftMargin
+                    }
+                    color: Style.main.text
+
+                    InfoToolTip {
+                        anchors {
+                            left: parent.right
+                            bottom: parent.bottom
+                            leftMargin : Style.dialog.leftMargin
+                        }
+                        info: qsTr(
+                            "If master import date range is selected only emails within this range will be imported, unless it is specified differently in folder date range.",
+                            "Text in master import settings tooltip."
+                        )
                     }
                 }
-            }
 
-            // Reset all to default
+                // Reset all to default
-            ClickIconText {
+                ClickIconText {
-                anchors {
+                    id: resetSourceButton
-                    right: parent.right
+                    text:qsTr("Reset all settings to default")
-                    bottom: labelMasterImportSettings.bottom
+                    iconText: Style.fa.refresh
-                }
+                    textColor: Style.main.textBlue
-                text:qsTr("Reset all settings to default")
+                    onClicked: {
-                iconText: Style.fa.refresh
+                        go.resetSource()
-                textColor: Style.main.textBlue
+                        root.decrementCurrentIndex()
-                onClicked: {
+                        timer.start()
-                    go.resetSource()
+                    }
-                    root.decrementCurrentIndex()
-                    timer.start()
                 }
             }
 
@@ -348,49 +349,40 @@ Dialog {
 
             InlineDateRange {
                 id: globalDateRange
-                anchors {
-                    left      : parent.left
-                    top       : line.bottom
-                    topMargin : Style.dialog.topMargin
-                }
             }
 
             // Add global label (inline)
             InlineLabelSelect {
                 id: globalLabels
-                anchors {
+            }
-                    left      : parent.left
+        }
-                    top       : globalDateRange.bottom
+
-                    topMargin : Style.dialog.topMargin
+        // Buttons
-                }
+        Row {
-                //labelWidth : globalDateRange.labelWidth
+            spacing: Style.dialog.spacing
+            anchors {
+                right: parent.right
+                bottom: parent.bottom
+                rightMargin: Style.main.leftMargin
+                bottomMargin: Style.main.bottomMargin
             }
 
-            // Buttons
+            ButtonRounded {
-            Row {
+                id: buttonCancelThree
-                spacing: Style.dialog.spacing
+                fa_icon    : Style.fa.times
-                anchors{
+                text       : qsTr("Cancel", "todo")
-                    bottom       : parent.bottom
+                color_main : Style.dialog.textBlue
-                    right        : parent.right
+                onClicked  : root.cancel()
-                }
+            }
 
-                ButtonRounded {
+            ButtonRounded {
-                    id: buttonCancelThree
+                id: buttonNextThree
-                    fa_icon    : Style.fa.times
+                fa_icon     : Style.fa.check
-                    text       : qsTr("Cancel", "todo")
+                text        : qsTr("Import", "todo")
-                    color_main : Style.dialog.textBlue
+                color_main  : Style.dialog.background
-                    onClicked  : root.cancel()
+                color_minor : Style.dialog.textBlue
-                }
+                isOpaque    : true
-
+                onClicked   : root.okay()
-                ButtonRounded {
-                    id: buttonNextThree
-                    fa_icon     : Style.fa.check
-                    text        : qsTr("Import", "todo")
-                    color_main  : Style.dialog.background
-                    color_minor : Style.dialog.textBlue
-                    isOpaque    : true
-                    onClicked   : root.okay()
-                }
             }
         }
     }
@@ -483,18 +475,30 @@ Dialog {
                 }
             }
 
-            Text {
+            Row {
                 property int fails: go.progressFails
                 visible: fails > 0
-                color : Style.main.textRed
-                font.family: Style.fontawesome.name
-                font.pointSize: Style.main.fontSize * Style.pt
                 anchors.horizontalCenter: parent.horizontalCenter
-                text: Style.fa.exclamation_circle + " " + (
+
-                    fails == 1 ?
+                Text {
-                    qsTr("%1 message failed to be imported").arg(fails) :
+                    color: Style.main.textRed
-                    qsTr("%1 messages failed to be imported").arg(fails)
+                    font {
-                )
+                        pointSize : Style.dialog.fontSize * Style.pt
+                        family    : Style.fontawesome.name
+                    }
+                    text: Style.fa.exclamation_circle
+                }
+
+                Text {
+                    property int fails: go.progressFails
+                    color: Style.main.textRed
+                    font.pointSize: Style.main.fontSize * Style.pt
+                    text: " " + (
+                        fails == 1 ?
+                        qsTr("%1 message failed to be imported").arg(fails) :
+                        qsTr("%1 messages failed to be imported").arg(fails)
+                    )
+                }
             }
 
             Row { // buttons
@@ -575,12 +579,23 @@ Dialog {
             anchors.centerIn : finalReport
             spacing          : Style.dialog.heightSeparator
 
-            Text {
+            Row {
-                text: go.progressDescription!="" ? qsTr("Import failed: %1").arg(go.progressDescription) : Style.fa.check_circle + " " + qsTr("Import completed successfully")
                 anchors.horizontalCenter: parent.horizontalCenter
-                color: go.progressDescription!="" ? Style.main.textRed : Style.main.textGreen
+
-                font.bold : true
+                Text {
-                font.family: Style.fontawesome.name
+                    font {
+                        pointSize: Style.dialog.fontSize * Style.pt
+                        family: Style.fontawesome.name
+                    }
+                    color: Style.main.textGreen
+                    text: go.progressDescription!="" ? "" : Style.fa.check_circle
+                }
+
+                Text {
+                    text: go.progressDescription!="" ? qsTr("Import failed: %1").arg(go.progressDescription) : " " + qsTr("Import completed successfully")
+                    color: go.progressDescription!="" ? Style.main.textRed : Style.main.textGreen
+                    font.bold : true
+                }
             }
 
             Text {
@@ -773,11 +788,6 @@ Dialog {
             errorPopup.hide()
         }
         onClickedNo  : {
-            if (errorPopup.msgID == "ask_send_report") {
-                errorPopup.hide()
-                return
-            }
-            go.resumeProcess()
             errorPopup.hide()
         }
 

internal/frontend/qml/ImportExportUI/HelpView.qml

@@ -74,7 +74,7 @@ Item {
                 )
                 width: wrapper.width
                 color : Style.transparent
-                Text {
+                AccessibleText {
                     id: aboutText
                     anchors {
                         bottom: parent.bottom
@@ -82,8 +82,8 @@ Item {
                     }
                     color: Style.main.textDisabled
                     horizontalAlignment: Qt.AlignHCenter
-                    font.family : Style.fontawesome.name
+                    font.pointSize :  Style.main.fontSize * Style.pt
-                    text: "ProtonMail Import-Export app Version "+go.getBackendVersion()+"\n"+Style.fa.copyright + " 2020 Proton Technologies AG"
+                    text: "ProtonMail Import-Export app Version "+go.getBackendVersion()+"\n© 2020 Proton Technologies AG"
                 }
             }
 

internal/frontend/qml/ImportExportUI/SelectFolderMenu.qml

@@ -42,34 +42,50 @@ ComboBox {
         root.below = popup.y>0
     }
 
-    contentItem : Text {
+    contentItem : Row {
         id: boxText
-        verticalAlignment: Text.AlignVCenter
-        font {
-            family: Style.fontawesome.name
-            pointSize : Style.dialog.fontSize * Style.pt
-            bold: root.down
-        }
-        elide: Text.ElideRight
-        textFormat: Text.StyledText
 
-        text : root.displayText
+        Text {
-        color: !root.enabled ? Style.main.textDisabled : ( root.down ? Style.main.background : Style.main.text )
+            anchors.verticalCenter: parent.verticalCenter
+            font {
+                pointSize: Style.dialog.fontSize * Style.pt
+                family: Style.fontawesome.name
+            }
+            text: {
+                if (view.currentIndex >= 0) {
+                    if (!root.isFolderType) {
+                        return Style.fa.tags + " "
+                    }
+                    var tgtIcon = view.currentItem.folderIcon
+                    var tgtColor = view.currentItem.folderColor
+                    if (tgtIcon != Style.fa.folder_open) {
+                        return tgtIcon + " "
+                    }
+                    return '<font color="'+tgtColor+'">'+ tgtIcon + "</font> "
+                }
+                return ""
+            }
+            color: !root.enabled ? Style.main.textDisabled : ( root.down ? Style.main.background : Style.main.text )
+        }
+
+        Text {
+            anchors.verticalCenter: parent.verticalCenter
+            font {
+                pointSize : Style.dialog.fontSize * Style.pt
+                bold: root.down
+            }
+            elide: Text.ElideRight
+            textFormat: Text.StyledText
+
+            text : root.displayText
+            color: !root.enabled ? Style.main.textDisabled : ( root.down ? Style.main.background : Style.main.text )
+        }
     }
 
     displayText: {
         if (view.currentIndex >= 0) {
-            if (!root.isFolderType) return Style.fa.tags + " " + qsTr("Add/Remove labels")
+            if (!root.isFolderType) return qsTr("Add/Remove labels")
-
+            return view.currentItem.folderName
-            var tgtName = view.currentItem.folderName
-            var tgtIcon = view.currentItem.folderIcon
-            var tgtColor = view.currentItem.folderColor
-
-            if (tgtIcon != Style.fa.folder_open) {
-                return tgtIcon + " " + tgtName
-            }
-
-            return '<font color="'+tgtColor+'">'+ tgtIcon + "</font> " + tgtName
         }
         if (root.isFolderType) return qsTr("No folder selected")
         return qsTr("No labels selected")

internal/frontend/qml/ImportExportUI/SettingsView.qml

@@ -44,6 +44,7 @@ Item {
                     text : qsTr("Clear")
                     color: Style.main.text
                     font {
+                        family : cacheKeychain.font.family // use default font, not font-awesome
                         pointSize : Style.settings.fontSize * Style.pt
                         underline : true
                     }

internal/frontend/qml/ProtonUI/BugReportWindow.qml

@@ -84,7 +84,7 @@ Window {
             height: content.height - (
                 (clientVersion.visible ?  clientVersion.height + Style.dialog.fontSize : 0) +
                 userAddress.height  + Style.dialog.fontSize +
-                securityNote.contentHeight + Style.dialog.fontSize +
+                securityNoteText.contentHeight + Style.dialog.fontSize +
                 cancelButton.height + Style.dialog.fontSize
             )
             clip: true
@@ -215,7 +215,7 @@ Window {
         }
 
         // Note
-        AccessibleText {
+        Row {
             id: securityNote
             anchors {
                 left: parent.left
@@ -223,14 +223,32 @@ Window {
                 top: userAddress.bottom
                 topMargin: Style.dialog.fontSize
             }
-            wrapMode: Text.Wrap
+
-            color: Style.dialog.text
+            Text {
-            font.pointSize : Style.dialog.fontSize * Style.pt
+                id: securityNoteIcon
-            text:
+                font {
-            "<span style='font-family: " + Style.fontawesome.name + "'>" + Style.fa.exclamation_triangle + "</span> " +
+                    pointSize : Style.dialog.fontSize * Style.pt
-            qsTr("Bug reports are not end-to-end encrypted!", "The first part of warning in bug report form") + " " +
+                    family    : Style.fontawesome.name
-            qsTr("Please do not send any sensitive information.", "The second part of warning in bug report form") + " " +
+                }
-            qsTr("Contact us at security@protonmail.com for critical security issues.", "The third part of warning in bug report form")
+                color: Style.dialog.text
+                text  : Style.fa.exclamation_triangle
+            }
+
+            AccessibleText {
+                id: securityNoteText
+                anchors {
+                    left: securityNoteIcon.right
+                    leftMargin: 5 * Style.pt
+                    right: parent.right
+                }
+                wrapMode: Text.Wrap
+                color: Style.dialog.text
+                font.pointSize : Style.dialog.fontSize * Style.pt
+                text:
+                qsTr("Bug reports are not end-to-end encrypted!", "The first part of warning in bug report form") + " " +
+                qsTr("Please do not send any sensitive information.", "The second part of warning in bug report form") + " " +
+                qsTr("Contact us at security@protonmail.com for critical security issues.", "The third part of warning in bug report form")
+            }
         }
 
         // buttons

internal/frontend/qt-ie/error_list.go

@@ -72,6 +72,9 @@ func (e *ErrorListModel) data(index *core.QModelIndex, role int) *core.QVariant
 	case MailSubject:
 		return qtcommon.NewQVariantString(r.Subject)
 	case MailDate:
+		if r.Time.IsZero() {
+			return qtcommon.NewQVariantString("Unavailable")
+		}
 		return qtcommon.NewQVariantString(r.Time.String())
 	case MailFrom:
 		return qtcommon.NewQVariantString(r.From)

internal/imap/bridge.go

@@ -76,5 +76,9 @@ func newBridgeUserWrap(bridgeUser *users.User) *bridgeUserWrap {
 }
 
 func (u *bridgeUserWrap) GetStore() storeUserProvider {
-	return newStoreUserWrap(u.User.GetStore())
+	store := u.User.GetStore()
+	if store == nil {
+		return nil
+	}
+	return newStoreUserWrap(store)
 }

internal/imap/mailbox.go

@@ -80,7 +80,10 @@ func (im *imapMailbox) Info() (*imap.MailboxInfo, error) {
 }
 
 func (im *imapMailbox) getFlags() []string {
-	flags := []string{imap.NoInferiorsAttr} // Subfolders are not yet supported by API.
+	flags := []string{}
+	if !im.storeMailbox.IsFolder() || im.storeMailbox.IsSystem() {
+		flags = append(flags, imap.NoInferiorsAttr) // Subfolders are not supported for System or Label
+	}
 	switch im.storeMailbox.LabelID() {
 	case pmapi.SentLabel:
 		flags = append(flags, specialuse.Sent)

internal/importexport/release_notes.go

@@ -15,17 +15,18 @@
 // You should have received a copy of the GNU General Public License
 // along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
 
-// Code generated by ./release-notes.sh at 'Wed Sep 23 01:31:53 PM CEST 2020'. DO NOT EDIT.
+// Code generated by ./release-notes.sh at 'Thu Oct 29 12:57:32 PM CET 2020'. DO NOT EDIT.
 
 package importexport
 
-const ReleaseNotes = `• Speed up import by implementing parallel processing (parallel fetch, encrypt and upload of messages)
+const ReleaseNotes = `• Improvements to the import from large mbox files with multiple labels
-• Optimising the initial fetch of messages from external accounts
+• Not allow to run multiple instances of the app or transfers at the same time
-• Better handling of attachments and non-standard formatting
+• Various enhancements of the import process related to parsing
-• Improved stability of the message parser
+• Cosmetic GUI changes
-• Added persistent anonymous API cookies
+• Better error handling
 `
 
-const ReleaseFixedBugs = `• Import from mbox files with long lines
+const ReleaseFixedBugs = `• Linux font issues - Fedora specific
-• Improvements to import from Yahoo accounts
+• App response to the user pausing and canceling import or export
+• Handling errors during update
 `

internal/smtp/preferences.go

@@ -503,13 +503,6 @@ func (b *sendPreferencesBuilder) setEncryptionPreferences(mailSettings pmapi.Mai
 }
 
 func (b *sendPreferencesBuilder) setMIMEPreferences(composerMIMEType string) {
-	// If the sign flag (that we just determined above) is true we use the scheme
-	// in the encryption preferences, unless the plain text format has been
-	// selected in the composer, in which case we must enforce PGP/INLINE.
-	if !b.isInternal() && b.shouldSign() && composerMIMEType == "text/plain" {
-		b.withScheme(pgpInline)
-	}
-
 	// If the sign flag (that we just determined above) is true, then the MIME
 	// type is determined by the PGP scheme (also determined above): we should
 	// use 'text/plain' for a PGP/Inline scheme, and 'multipart/mixed' otherwise.

internal/smtp/preferences_test.go

@@ -254,6 +254,20 @@ func TestPreferencesBuilder(t *testing.T) {
 			wantMIMEType: "multipart/mixed",
 		},
 
+		{
+			name: "external with sign enabled, sending plaintext, should still send as ClearMIME",
+
+			contactMeta:  &ContactMetadata{Sign: true, SignIsSet: true},
+			receivedKeys: []pmapi.PublicKey{},
+			isInternal:   false,
+			mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, DraftMIMEType: "text/plain"},
+
+			wantEncrypt:  false,
+			wantSign:     true,
+			wantScheme:   pmapi.ClearMIMEPackage,
+			wantMIMEType: "multipart/mixed",
+		},
+
 		{
 			name: "external with pinned contact public key but no intention to encrypt/sign",
 

internal/smtp/send_recorder.go

@@ -20,6 +20,7 @@ package smtp
 import (
 	"crypto/sha256"
 	"fmt"
+	"strings"
 	"sync"
 	"time"
 
@@ -48,6 +49,15 @@ func newSendRecorder() *sendRecorder {
 }
 
 func (q *sendRecorder) getMessageHash(message *pmapi.Message) string {
+	// Outlook Calendar updates has only headers (no body) and thus have always
+	// the same hash. If the message is type of calendar, the "is sending"
+	// check to avoid potential duplicates is skipped. Duplicates should not
+	// be a problem in this case as calendar updates are small.
+	contentType := message.Header.Get("Content-Type")
+	if strings.HasPrefix(contentType, "text/calendar") {
+		return ""
+	}
+
 	h := sha256.New()
 	_, _ = h.Write([]byte(message.AddressID + message.Subject))
 	if message.Sender != nil {
@@ -101,6 +111,10 @@ func (q *sendRecorder) isSendingOrSent(client messageGetter, hash string) (isSen
 	q.lock.Lock()
 	defer q.lock.Unlock()
 
+	if hash == "" {
+		return false, false
+	}
+
 	q.deleteExpiredKeys()
 	value, ok := q.hashes[hash]
 	if !ok {

internal/smtp/send_recorder_test.go

@@ -349,6 +349,32 @@ func TestSendRecorder_getMessageHash(t *testing.T) {
 			},
 			false,
 		},
+		{ // Different content type - calendar
+			&pmapi.Message{
+				Header: mail.Header{
+					"Content-Type": []string{"text/calendar"},
+				},
+				AddressID: "address123",
+				Subject:   "Subject #1",
+				Sender: &mail.Address{
+					Address: "from@pm.me",
+				},
+				ToList: []*mail.Address{
+					{Address: "to@pm.me"},
+				},
+				CCList:  []*mail.Address{},
+				BCCList: []*mail.Address{},
+				Body:    "body",
+				Attachments: []*pmapi.Attachment{
+					{
+						Name:     "att1",
+						MIMEType: "image/png",
+						Size:     12345,
+					},
+				},
+			},
+			false,
+		},
 	}
 	for i, tc := range testCases {
 		tc := tc // bind
@@ -382,12 +408,13 @@ func TestSendRecorder_isSendingOrSent(t *testing.T) {
 		{"hash", &pmapi.Message{Type: pmapi.MessageTypeDraft, Time: time.Now().Unix()}, nil, true, false},
 		{"hash", &pmapi.Message{Type: pmapi.MessageTypeSent}, nil, false, true},
 		{"hash", &pmapi.Message{Type: pmapi.MessageTypeInboxAndSent}, nil, false, true},
+		{"", &pmapi.Message{Type: pmapi.MessageTypeInboxAndSent}, nil, false, false},
 	}
 	for i, tc := range testCases {
 		tc := tc // bind
 		t.Run(fmt.Sprintf("%d / %v / %v / %v", i, tc.hash, tc.message, tc.err), func(t *testing.T) {
 			messageGetter := &testSendRecorderGetMessageMock{message: tc.message, err: tc.err}
-			isSending, wasSent := q.isSendingOrSent(messageGetter, "hash")
+			isSending, wasSent := q.isSendingOrSent(messageGetter, tc.hash)
 			assert.Equal(t, tc.wantIsSending, isSending, "isSending does not match")
 			assert.Equal(t, tc.wantWasSent, wasSent, "wasSent does not match")
 		})

internal/transfer/message.go

@@ -29,17 +29,34 @@ type Message struct {
 	ID      string
 	Unread  bool
 	Body    []byte
-	Source  Mailbox
+	Sources []Mailbox
 	Targets []Mailbox
 }
 
+// sourceNames returns array of source mailbox names.
+func (msg Message) sourceNames() (names []string) {
+	for _, mailbox := range msg.Sources {
+		names = append(names, mailbox.Name)
+	}
+	return
+}
+
+// targetNames returns array of target mailbox names.
+func (msg Message) targetNames() (names []string) {
+	for _, mailbox := range msg.Targets {
+		names = append(names, mailbox.Name)
+	}
+	return
+}
+
 // MessageStatus holds status for message used by progress manager.
 type MessageStatus struct {
-	eventTime time.Time // Time of adding message to the process.
+	eventTime   time.Time // Time of adding message to the process.
-	rule      *Rule     // Rule with source and target mailboxes.
+	sourceNames []string  // Source mailbox names message is in.
-	SourceID  string    // Message ID at the source.
+	SourceID    string    // Message ID at the source.
-	targetID  string    // Message ID at the target (if any).
+	targetNames []string  // Target mailbox names message is in.
-	bodyHash  string    // Hash of the message body.
+	targetID    string    // Message ID at the target (if any).
+	bodyHash    string    // Hash of the message body.
 
 	exported  bool
 	imported  bool

internal/transfer/progress.go

@@ -93,7 +93,7 @@ func (p *Progress) fatal(err error) {
 	defer p.lock.Unlock()
 
 	log.WithError(err).Error("Progress finished")
-	p.isStopped = true
+	p.setStop()
 	p.fatalError = err
 	p.cleanUpdateCh()
 }
@@ -126,16 +126,17 @@ func (p *Progress) updateCount(mailbox string, count uint) {
 }
 
 // addMessage should be called as soon as there is ID of the message.
-func (p *Progress) addMessage(messageID string, rule *Rule) {
+func (p *Progress) addMessage(messageID string, sourceNames, targetNames []string) {
 	p.lock.Lock()
 	defer p.lock.Unlock()
 	defer p.update()
 
 	p.log.WithField("id", messageID).Trace("Message added")
 	p.messageStatuses[messageID] = &MessageStatus{
-		eventTime: time.Now(),
+		eventTime:   time.Now(),
-		rule:      rule,
+		sourceNames: sourceNames,
-		SourceID:  messageID,
+		SourceID:    messageID,
+		targetNames: targetNames,
 	}
 }
 
@@ -282,6 +283,15 @@ func (p *Progress) Stop() {
 	defer p.update()
 
 	p.log.Info("Progress stopped")
+	p.setStop()
+
+	// Once progress is stopped, some calls might be in progress. Results from
+	// those calls are irrelevant so we can close update channel sooner to not
+	// propagate any progress to user interface anymore.
+	p.cleanUpdateCh()
+}
+
+func (p *Progress) setStop() {
 	p.isStopped = true
 	p.pauseReason = "" // Clear pause to run paused code and stop it.
 }

internal/transfer/progress_test.go

@@ -19,6 +19,7 @@ package transfer
 
 import (
 	"testing"
+	"time"
 
 	"github.com/pkg/errors"
 	a "github.com/stretchr/testify/assert"
@@ -47,21 +48,21 @@ func TestProgressAddingMessages(t *testing.T) {
 	drainProgressUpdateChannel(&progress)
 
 	// msg1 has no problem.
-	progress.addMessage("msg1", nil)
+	progress.addMessage("msg1", []string{}, []string{})
 	progress.messageExported("msg1", []byte(""), nil)
 	progress.messageImported("msg1", "", nil)
 
 	// msg2 has an import problem.
-	progress.addMessage("msg2", nil)
+	progress.addMessage("msg2", []string{}, []string{})
 	progress.messageExported("msg2", []byte(""), nil)
 	progress.messageImported("msg2", "", errors.New("failed import"))
 
 	// msg3 has an export problem.
-	progress.addMessage("msg3", nil)
+	progress.addMessage("msg3", []string{}, []string{})
 	progress.messageExported("msg3", []byte(""), errors.New("failed export"))
 
 	// msg4 has an export problem and import is also called.
-	progress.addMessage("msg4", nil)
+	progress.addMessage("msg4", []string{}, []string{})
 	progress.messageExported("msg4", []byte(""), errors.New("failed export"))
 	progress.messageImported("msg4", "", nil)
 
@@ -91,7 +92,7 @@ func TestProgressFinish(t *testing.T) {
 	progress.finish()
 	r.Nil(t, progress.updateCh)
 
-	r.NotPanics(t, func() { progress.addMessage("msg", nil) })
+	r.NotPanics(t, func() { progress.addMessage("msg", []string{}, []string{}) })
 }
 
 func TestProgressFatalError(t *testing.T) {
@@ -101,7 +102,29 @@ func TestProgressFatalError(t *testing.T) {
 	progress.fatal(errors.New("fatal error"))
 	r.Nil(t, progress.updateCh)
 
-	r.NotPanics(t, func() { progress.addMessage("msg", nil) })
+	r.NotPanics(t, func() { progress.addMessage("msg", []string{}, []string{}) })
+}
+
+func TestFailUnpauseAndStops(t *testing.T) {
+	progress := newProgress(log, nil)
+	drainProgressUpdateChannel(&progress)
+
+	progress.Pause("pausing")
+	progress.fatal(errors.New("fatal error"))
+
+	r.Nil(t, progress.updateCh)
+	r.True(t, progress.isStopped)
+	r.False(t, progress.IsPaused())
+	r.Eventually(t, progress.shouldStop, time.Second, 10*time.Millisecond)
+}
+
+func TestStopClosesUpdates(t *testing.T) {
+	progress := newProgress(log, nil)
+	ch := progress.updateCh
+
+	progress.Stop()
+	r.Nil(t, progress.updateCh)
+	r.PanicsWithError(t, "send on closed channel", func() { ch <- struct{}{} })
 }
 
 func drainProgressUpdateChannel(progress *Progress) {

internal/transfer/provider_eml_source.go

@@ -109,7 +109,7 @@ func (p *EMLProvider) exportMessages(rule *Rule, filePaths []string, progress *P
 		// addMessage is called after time check to not report message
 		// which should not be exported but any error from reading body
 		// or parsing time is reported as an error.
-		progress.addMessage(filePath, rule)
+		progress.addMessage(filePath, msg.sourceNames(), msg.targetNames())
 		progress.messageExported(filePath, msg.Body, err)
 		if err == nil {
 			ch <- msg
@@ -134,7 +134,7 @@ func (p *EMLProvider) exportMessage(rule *Rule, filePath string) (Message, error
 		ID:      filePath,
 		Unread:  false,
 		Body:    body,
-		Source:  rule.SourceMailbox,
+		Sources: []Mailbox{rule.SourceMailbox},
 		Targets: rule.TargetMailboxes,
 	}, nil
 }

internal/transfer/provider_eml_target.go

@@ -61,7 +61,7 @@ func (p *EMLProvider) TransferFrom(rules transferRules, progress *Progress, ch <
 func (p *EMLProvider) createFolders(rules transferRules) error {
 	for rule := range rules.iterateActiveRules() {
 		for _, mailbox := range rule.TargetMailboxes {
-			path := filepath.Join(p.root, mailbox.Name)
+			path := filepath.Join(p.root, sanitizeFileName(mailbox.Name))
 			if err := os.MkdirAll(path, os.ModePerm); err != nil {
 				return err
 			}
@@ -71,7 +71,7 @@ func (p *EMLProvider) createFolders(rules transferRules) error {
 }
 
 func (p *EMLProvider) writeFile(msg Message) error {
-	fileName := filepath.Base(msg.ID)
+	fileName := sanitizeFileName(filepath.Base(msg.ID))
 	if filepath.Ext(fileName) != ".eml" {
 		fileName += ".eml"
 	}

internal/transfer/provider_imap_source.go

@@ -124,7 +124,7 @@ func (p *IMAPProvider) loadMessagesInfo(rule *Rule, progress *Progress, uidValid
 				uid:  imapMessage.Uid,
 				size: imapMessage.Size,
 			}
-			progress.addMessage(id, rule)
+			progress.addMessage(id, []string{rule.SourceMailbox.Name}, rule.TargetMailboxNames())
 		}
 
 		progress.callWrap(func() error {
@@ -231,7 +231,7 @@ func (p *IMAPProvider) exportMessage(rule *Rule, id string, imapMessage *imap.Me
 		ID:      id,
 		Unread:  unread,
 		Body:    body,
-		Source:  rule.SourceMailbox,
+		Sources: []Mailbox{rule.SourceMailbox},
 		Targets: rule.TargetMailboxes,
 	}
 }

internal/transfer/provider_mbox.go

@@ -18,8 +18,11 @@
 package transfer
 
 import (
+	"os"
 	"path/filepath"
 	"strings"
+
+	"github.com/pkg/errors"
 )
 
 // MBOXProvider implements import and export to/from MBOX structure.
@@ -44,16 +47,35 @@ func (p *MBOXProvider) ID() string {
 // In case the same folder name is used more than once (for example root/a/foo
 // and root/b/foo), it's treated as the same folder.
 func (p *MBOXProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) {
-	filePaths, err := getFilePathsWithSuffix(p.root, "mbox")
+	filePaths, err := getAllPathsWithSuffix(p.root, ".mbox")
 	if err != nil {
 		return nil, err
 	}
 
-	mailboxes := []Mailbox{}
+	mailboxNames := map[string]bool{}
 	for _, filePath := range filePaths {
 		fileName := filepath.Base(filePath)
-		mailboxName := strings.TrimSuffix(fileName, ".mbox")
+		filePath, err := p.handleAppleMailMBOXStructure(filePath)
+		if err != nil {
+			log.WithError(err).Warn("Failed to handle MBOX structure")
+			continue
+		}
 
+		mailboxName := strings.TrimSuffix(fileName, ".mbox")
+		mailboxNames[mailboxName] = true
+
+		labels, err := getGmailLabelsFromMboxFile(filepath.Join(p.root, filePath))
+		if err != nil {
+			log.WithError(err).Error("Failed to get gmail labels from mbox file")
+			continue
+		}
+		for label := range labels {
+			mailboxNames[label] = true
+		}
+	}
+
+	mailboxes := []Mailbox{}
+	for mailboxName := range mailboxNames {
 		mailboxes = append(mailboxes, Mailbox{
 			ID:          "",
 			Name:        mailboxName,
@@ -61,6 +83,20 @@ func (p *MBOXProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox,
 			IsExclusive: false,
 		})
 	}
-
 	return mailboxes, nil
 }
+
+// handleAppleMailMBOXStructure changes the path of mailbox directory to
+// the path of mbox file. Apple Mail MBOX exports has this structure:
+// `Folder.mbox` directory with `mbox` file inside.
+// Example: `Folder.mbox/mbox` (and this function converts `Folder.mbox`
+// to `Folder.mbox/mbox`).
+func (p *MBOXProvider) handleAppleMailMBOXStructure(filePath string) (string, error) {
+	if info, err := os.Stat(filepath.Join(p.root, filePath)); err == nil && info.IsDir() {
+		if _, err := os.Stat(filepath.Join(p.root, filePath, "mbox")); err != nil {
+			return "", errors.Wrap(err, "wrong mbox structure")
+		}
+		return filepath.Join(filePath, "mbox"), nil
+	}
+	return filePath, nil
+}

internal/transfer/provider_mbox_gmail_labels.go

@@ -0,0 +1,118 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"mime"
+	"os"
+	"strings"
+)
+
+type stringSet map[string]bool
+
+const xGmailLabelsHeader = "X-Gmail-Labels"
+
+// filteredOutGmailLabels is set of labels which we don't want to show to users
+// as they might be auto-generated by Gmail and unwanted.
+var filteredOutGmailLabels = []string{ //nolint[gochecknoglobals]
+	"Unread",
+	"Opened",
+	"IMAP_Junk",
+	"IMAP_NonJunk",
+	"IMAP_NotJunk",
+	"IMAP_$NotJunk",
+}
+
+func getGmailLabelsFromMboxFile(filePath string) (stringSet, error) {
+	f, err := os.Open(filePath) //nolint[gosec]
+	if err != nil {
+		return nil, err
+	}
+	return getGmailLabelsFromMboxReader(f)
+}
+
+func getGmailLabelsFromMboxReader(f io.Reader) (stringSet, error) {
+	allLabels := stringSet{}
+
+	// Scanner is not used as it does not support long lines and some mbox
+	// files contain very long lines even though that should not be happening.
+	r := bufio.NewReader(f)
+	for {
+		b, isPrefix, err := r.ReadLine()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		for isPrefix {
+			_, isPrefix, err = r.ReadLine()
+			if err != nil {
+				break
+			}
+		}
+		if bytes.HasPrefix(b, []byte(xGmailLabelsHeader)) {
+			for label := range getGmailLabelsFromValue(string(b)) {
+				allLabels[label] = true
+			}
+		}
+	}
+
+	return allLabels, nil
+}
+
+func getGmailLabelsFromMessage(body []byte) (stringSet, error) {
+	header, err := getMessageHeader(body)
+	if err != nil {
+		return nil, err
+	}
+	labels := header.Get(xGmailLabelsHeader)
+	return getGmailLabelsFromValue(labels), nil
+}
+
+func getGmailLabelsFromValue(value string) stringSet {
+	value = strings.TrimPrefix(value, xGmailLabelsHeader+":")
+	if decoded, err := new(mime.WordDecoder).DecodeHeader(value); err != nil {
+		log.WithError(err).Error("Failed to decode header")
+	} else {
+		value = decoded
+	}
+
+	labels := stringSet{}
+	for _, label := range strings.Split(value, ",") {
+		label = strings.TrimSpace(label)
+		if label == "" {
+			continue
+		}
+		skip := false
+		for _, filteredOutLabel := range filteredOutGmailLabels {
+			if label == filteredOutLabel {
+				skip = true
+				break
+			}
+		}
+		if skip {
+			continue
+		}
+		labels[label] = true
+	}
+	return labels
+}

internal/transfer/provider_mbox_gmail_labels_test.go

@@ -0,0 +1,135 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	r "github.com/stretchr/testify/require"
+)
+
+func TestGetGmailLabelsFromMboxReader(t *testing.T) {
+	mboxFile := `From - Mon May  4 16:40:31 2020
+Subject: Test 1
+X-Gmail-Labels: Foo,Bar
+
+hello
+
+From - Mon May  4 16:40:31 2020
+Subject: Test 2
+X-Gmail-Labels: Foo , Baz
+
+hello
+
+From - Mon May  4 16:40:31 2020
+Subject: Test 3
+X-Gmail-Labels: ,
+
+hello
+
+From - Mon May  4 16:40:31 2020
+Subject: Test 4
+X-Gmail-Labels:
+
+hello
+
+From - Mon May  4 16:40:31 2020
+Subject: Test 5
+
+hello
+
+`
+	mboxReader := strings.NewReader(mboxFile)
+	labels, err := getGmailLabelsFromMboxReader(mboxReader)
+	r.NoError(t, err)
+	r.Equal(t, toSet("Foo", "Bar", "Baz"), labels)
+}
+
+func TestGetGmailLabelsFromMessage(t *testing.T) {
+	tests := []struct {
+		body       string
+		wantLabels stringSet
+	}{
+		{`Subject: One
+X-Gmail-Labels: Foo,Bar
+
+Hello
+`, toSet("Foo", "Bar")},
+		{`Subject: Two
+X-Gmail-Labels: Foo , Bar ,
+
+Hello
+`, toSet("Foo", "Bar")},
+		{`Subject: Three
+X-Gmail-Labels: ,
+
+Hello
+`, toSet()},
+		{`Subject: Four
+X-Gmail-Labels:
+
+Hello
+`, toSet()},
+		{`Subject: Five
+
+Hello
+`, toSet()},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(fmt.Sprintf("%v", tc.body), func(t *testing.T) {
+			labels, err := getGmailLabelsFromMessage([]byte(tc.body))
+			r.NoError(t, err)
+			r.Equal(t, tc.wantLabels, labels)
+		})
+	}
+}
+
+func TestGetGmailLabelsFromValue(t *testing.T) {
+	tests := []struct {
+		value      string
+		wantLabels stringSet
+	}{
+		{"Foo,Bar", toSet("Foo", "Bar")},
+		{" Foo , Bar ", toSet("Foo", "Bar")},
+		{" Foo , Bar , ", toSet("Foo", "Bar")},
+		{" Foo Bar ", toSet("Foo Bar")},
+		{" , ", toSet()},
+		{" ", toSet()},
+		{"", toSet()},
+		{"=?UTF-8?Q?Archived,Category_personal,test_=F0=9F=98=80=F0=9F=99=83?=", toSet("Archived", "Category personal", "test 😀🙃")},
+		{"IMAP_NotJunk,Foo,Opened,bar,Unread", toSet("Foo", "bar")},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(fmt.Sprintf("%v", tc.value), func(t *testing.T) {
+			labels := getGmailLabelsFromValue(tc.value)
+			r.Equal(t, tc.wantLabels, labels)
+		})
+	}
+}
+
+func toSet(items ...string) stringSet {
+	set := map[string]bool{}
+	for _, item := range items {
+		set[item] = true
+	}
+	return set
+}

internal/transfer/provider_mbox_source.go

@@ -34,7 +34,7 @@ func (p *MBOXProvider) TransferTo(rules transferRules, progress *Progress, ch ch
 	log.Info("Started transfer from MBOX to channel")
 	defer log.Info("Finished transfer from MBOX to channel")
 
-	filePathsPerFolder, err := p.getFilePathsPerFolder(rules)
+	filePathsPerFolder, err := p.getFilePathsPerFolder()
 	if err != nil {
 		progress.fatal(err)
 		return
@@ -45,32 +45,29 @@ func (p *MBOXProvider) TransferTo(rules transferRules, progress *Progress, ch ch
 	}
 
 	for folderName, filePaths := range filePathsPerFolder {
-		// No error guaranteed by getFilePathsPerFolder.
+		log.WithField("folder", folderName).Debug("Estimating folder counts")
-		rule, _ := rules.getRuleBySourceMailboxName(folderName)
 		for _, filePath := range filePaths {
 			if progress.shouldStop() {
 				break
 			}
-			p.updateCount(rule, progress, filePath)
+			p.updateCount(progress, filePath)
 		}
 	}
 	progress.countsFinal()
 
 	for folderName, filePaths := range filePathsPerFolder {
-		// No error guaranteed by getFilePathsPerFolder.
+		log.WithField("folder", folderName).Debug("Processing folder")
-		rule, _ := rules.getRuleBySourceMailboxName(folderName)
-		log.WithField("rule", rule).Debug("Processing rule")
 		for _, filePath := range filePaths {
 			if progress.shouldStop() {
 				break
 			}
-			p.transferTo(rule, progress, ch, filePath)
+			p.transferTo(rules, progress, ch, folderName, filePath)
 		}
 	}
 }
 
-func (p *MBOXProvider) getFilePathsPerFolder(rules transferRules) (map[string][]string, error) {
+func (p *MBOXProvider) getFilePathsPerFolder() (map[string][]string, error) {
-	filePaths, err := getFilePathsWithSuffix(p.root, ".mbox")
+	filePaths, err := getAllPathsWithSuffix(p.root, ".mbox")
 	if err != nil {
 		return nil, err
 	}
@@ -78,19 +75,19 @@ func (p *MBOXProvider) getFilePathsPerFolder(rules transferRules) (map[string][]
 	filePathsMap := map[string][]string{}
 	for _, filePath := range filePaths {
 		fileName := filepath.Base(filePath)
-		folder := strings.TrimSuffix(fileName, ".mbox")
+		filePath, err := p.handleAppleMailMBOXStructure(filePath)
-		_, err := rules.getRuleBySourceMailboxName(folder)
+		// Skip unsupported MBOX structures. It was already filtered out in configuration step.
 		if err != nil {
-			log.WithField("msg", filePath).Trace("Mailbox skipped due to folder name")
 			continue
 		}
 
+		folder := strings.TrimSuffix(fileName, ".mbox")
 		filePathsMap[folder] = append(filePathsMap[folder], filePath)
 	}
 	return filePathsMap, nil
 }
 
-func (p *MBOXProvider) updateCount(rule *Rule, progress *Progress, filePath string) {
+func (p *MBOXProvider) updateCount(progress *Progress, filePath string) {
 	mboxReader := p.openMbox(progress, filePath)
 	if mboxReader == nil {
 		return
@@ -107,10 +104,10 @@ func (p *MBOXProvider) updateCount(rule *Rule, progress *Progress, filePath stri
 		}
 		count++
 	}
-	progress.updateCount(rule.SourceMailbox.Name, uint(count))
+	progress.updateCount(filePath, uint(count))
 }
 
-func (p *MBOXProvider) transferTo(rule *Rule, progress *Progress, ch chan<- Message, filePath string) {
+func (p *MBOXProvider) transferTo(rules transferRules, progress *Progress, ch chan<- Message, folderName, filePath string) {
 	mboxReader := p.openMbox(progress, filePath)
 	if mboxReader == nil {
 		return
@@ -134,50 +131,122 @@ func (p *MBOXProvider) transferTo(rule *Rule, progress *Progress, ch chan<- Mess
 			break
 		}
 
-		msg, err := p.exportMessage(rule, id, msgReader)
+		msg, err := p.exportMessage(rules, folderName, id, msgReader)
 
-		// Read and check time in body only if the rule specifies it
+		if err == nil && len(msg.Targets) == 0 {
-		// to not waste energy.
+			// Here should be called progress.messageSkipped(id) once we have
-		if err == nil && rule.HasTimeLimit() {
+			// this feature, and following progress.updateCount can be removed.
-			msgTime, msgTimeErr := getMessageTime(msg.Body)
+			continue
-			if msgTimeErr != nil {
-				err = msgTimeErr
-			} else if !rule.isTimeInRange(msgTime) {
-				log.WithField("msg", id).Debug("Message skipped due to time")
-				continue
-			}
 		}
 
-		// Counting only messages filtered by time to update count to correct total.
 		count++
 
 		// addMessage is called after time check to not report message
 		// which should not be exported but any error from reading body
 		// or parsing time is reported as an error.
-		progress.addMessage(id, rule)
+		progress.addMessage(id, msg.sourceNames(), msg.targetNames())
 		progress.messageExported(id, msg.Body, err)
 		if err == nil {
 			ch <- msg
 		}
 	}
-	progress.updateCount(rule.SourceMailbox.Name, uint(count))
+	progress.updateCount(filePath, uint(count))
 }
 
-func (p *MBOXProvider) exportMessage(rule *Rule, id string, msgReader io.Reader) (Message, error) {
+func (p *MBOXProvider) exportMessage(rules transferRules, folderName, id string, msgReader io.Reader) (Message, error) {
 	body, err := ioutil.ReadAll(msgReader)
 	if err != nil {
 		return Message{}, errors.Wrap(err, "failed to read message")
 	}
 
+	msgRules := p.getMessageRules(rules, folderName, id, body)
+	sources := p.getMessageSources(msgRules)
+	targets := p.getMessageTargets(msgRules, id, body)
 	return Message{
 		ID:      id,
 		Unread:  false,
 		Body:    body,
-		Source:  rule.SourceMailbox,
+		Sources: sources,
-		Targets: rule.TargetMailboxes,
+		Targets: targets,
 	}, nil
 }
 
+func (p *MBOXProvider) getMessageRules(rules transferRules, folderName, id string, body []byte) []*Rule {
+	msgRules := []*Rule{}
+
+	folderRule, err := rules.getRuleBySourceMailboxName(folderName)
+	if err != nil {
+		log.WithField("msg", id).WithField("source", folderName).Debug("Message source doesn't have a rule")
+	} else {
+		msgRules = append(msgRules, folderRule)
+	}
+
+	gmailLabels, err := getGmailLabelsFromMessage(body)
+	if err != nil {
+		log.WithError(err).Error("Failed to get gmail labels, ")
+	} else {
+		for label := range gmailLabels {
+			rule, err := rules.getRuleBySourceMailboxName(label)
+			if err != nil {
+				log.WithField("msg", id).WithField("source", label).Debug("Message source doesn't have a rule")
+				continue
+			}
+			msgRules = append(msgRules, rule)
+		}
+	}
+
+	return msgRules
+}
+
+func (p *MBOXProvider) getMessageSources(msgRules []*Rule) []Mailbox {
+	sources := []Mailbox{}
+	for _, rule := range msgRules {
+		sources = append(sources, rule.SourceMailbox)
+	}
+	return sources
+}
+
+func (p *MBOXProvider) getMessageTargets(msgRules []*Rule, id string, body []byte) []Mailbox {
+	targets := []Mailbox{}
+	haveExclusiveMailbox := false
+	for _, rule := range msgRules {
+		// Read and check time in body only if the rule specifies it
+		// to not waste energy.
+		if rule.HasTimeLimit() {
+			msgTime, err := getMessageTime(body)
+			if err != nil {
+				log.WithError(err).Error("Failed to parse time, time check skipped")
+			} else if !rule.isTimeInRange(msgTime) {
+				log.WithField("msg", id).WithField("source", rule.SourceMailbox.Name).Debug("Message skipped due to time")
+				continue
+			}
+		}
+		for _, newTarget := range rule.TargetMailboxes {
+			// msgRules is sorted. The first rule is based on the folder name,
+			// followed by the order from X-Gmail-Labels. The rule based on
+			// the folder name should have priority for exclusive target.
+			if newTarget.IsExclusive && haveExclusiveMailbox {
+				continue
+			}
+			found := false
+			for _, target := range targets {
+				if target.Hash() == newTarget.Hash() {
+					found = true
+					break
+				}
+			}
+			if found {
+				continue
+			}
+			if newTarget.IsExclusive {
+				haveExclusiveMailbox = true
+			}
+			targets = append(targets, newTarget)
+		}
+	}
+	return targets
+}
+
 func (p *MBOXProvider) openMbox(progress *Progress, mboxPath string) *mbox.Reader {
 	mboxPath = filepath.Join(p.root, mboxPath)
 	mboxFile, err := os.Open(mboxPath) //nolint[gosec]

internal/transfer/provider_mbox_target.go

@@ -57,7 +57,7 @@ func (p *MBOXProvider) TransferFrom(rules transferRules, progress *Progress, ch
 func (p *MBOXProvider) writeMessage(msg Message) error {
 	var multiErr error
 	for _, mailbox := range msg.Targets {
-		mboxName := filepath.Base(mailbox.Name)
+		mboxName := sanitizeFileName(mailbox.Name)
 		if !strings.HasSuffix(mboxName, ".mbox") {
 			mboxName += ".mbox"
 		}

internal/transfer/provider_mbox_test.go

@@ -35,27 +35,35 @@ func newTestMBOXProvider(path string) *MBOXProvider {
 }
 
 func TestMBOXProviderMailboxes(t *testing.T) {
-	provider := newTestMBOXProvider("")
-
 	tests := []struct {
+		provider      *MBOXProvider
 		includeEmpty  bool
 		wantMailboxes []Mailbox
 	}{
-		{true, []Mailbox{
+		{newTestMBOXProvider(""), true, []Mailbox{
+			{Name: "All Mail"},
 			{Name: "Foo"},
+			{Name: "Bar"},
 			{Name: "Inbox"},
 		}},
-		{false, []Mailbox{
+		{newTestMBOXProvider(""), false, []Mailbox{
+			{Name: "All Mail"},
 			{Name: "Foo"},
+			{Name: "Bar"},
 			{Name: "Inbox"},
 		}},
+		{newTestMBOXProvider("testdata/mbox-applemail"), true, []Mailbox{
+			{Name: "All Mail"},
+			{Name: "Foo"},
+			{Name: "Bar"},
+		}},
 	}
 	for _, tc := range tests {
 		tc := tc
 		t.Run(fmt.Sprintf("%v", tc.includeEmpty), func(t *testing.T) {
-			mailboxes, err := provider.Mailboxes(tc.includeEmpty, false)
+			mailboxes, err := tc.provider.Mailboxes(tc.includeEmpty, false)
 			r.NoError(t, err)
-			r.Equal(t, tc.wantMailboxes, mailboxes)
+			r.ElementsMatch(t, tc.wantMailboxes, mailboxes)
 		})
 	}
 }
@@ -67,14 +75,47 @@ func TestMBOXProviderTransferTo(t *testing.T) {
 	defer rulesClose()
 	setupMBOXRules(rules)
 
-	testTransferTo(t, rules, provider, []string{
+	msgs := testTransferTo(t, rules, provider, []string{
+		"All Mail.mbox:1",
+		"All Mail.mbox:2",
 		"Foo.mbox:1",
 		"Inbox.mbox:1",
 	})
+	got := map[string][]string{}
+	for _, msg := range msgs {
+		got[msg.ID] = msg.targetNames()
+	}
+	r.Equal(t, map[string][]string{
+		"All Mail.mbox:1": {"Archive", "Foo"}, // Bar is not in rules.
+		"All Mail.mbox:2": {"Archive", "Foo"},
+		"Foo.mbox:1":      {"Foo"},
+		"Inbox.mbox:1":    {"Inbox"},
+	}, got)
+}
+
+func TestMBOXProviderTransferToAppleMail(t *testing.T) {
+	provider := newTestMBOXProvider("testdata/mbox-applemail")
+
+	rules, rulesClose := newTestRules(t)
+	defer rulesClose()
+	setupMBOXRules(rules)
+
+	msgs := testTransferTo(t, rules, provider, []string{
+		"All Mail.mbox/mbox:1",
+		"All Mail.mbox/mbox:2",
+	})
+	got := map[string][]string{}
+	for _, msg := range msgs {
+		got[msg.ID] = msg.targetNames()
+	}
+	r.Equal(t, map[string][]string{
+		"All Mail.mbox/mbox:1": {"Archive", "Foo"}, // Bar is not in rules.
+		"All Mail.mbox/mbox:2": {"Archive", "Foo"},
+	}, got)
 }
 
 func TestMBOXProviderTransferFrom(t *testing.T) {
-	dir, err := ioutil.TempDir("", "eml")
+	dir, err := ioutil.TempDir("", "mbox")
 	r.NoError(t, err)
 	defer os.RemoveAll(dir) //nolint[errcheck]
 
@@ -94,7 +135,7 @@ func TestMBOXProviderTransferFrom(t *testing.T) {
 }
 
 func TestMBOXProviderTransferFromTo(t *testing.T) {
-	dir, err := ioutil.TempDir("", "eml")
+	dir, err := ioutil.TempDir("", "mbox")
 	r.NoError(t, err)
 	defer os.RemoveAll(dir) //nolint[errcheck]
 
@@ -103,23 +144,57 @@ func TestMBOXProviderTransferFromTo(t *testing.T) {
 
 	rules, rulesClose := newTestRules(t)
 	defer rulesClose()
-	setupEMLRules(rules)
+	setupMBOXRules(rules)
 
 	testTransferFromTo(t, rules, source, target, 5*time.Second)
 
 	checkMBOXFileStructure(t, dir, []string{
+		"Archive.mbox",
 		"Foo.mbox",
 		"Inbox.mbox",
 	})
 }
 
+func TestMBOXProviderGetMessageTargetsReturnsOnlyOneFolder(t *testing.T) {
+	provider := newTestMBOXProvider("")
+
+	folderA := Mailbox{Name: "Folder A", IsExclusive: true}
+	folderB := Mailbox{Name: "Folder B", IsExclusive: true}
+	labelA := Mailbox{Name: "Label A", IsExclusive: false}
+	labelB := Mailbox{Name: "Label B", IsExclusive: false}
+	labelC := Mailbox{Name: "Label C", IsExclusive: false}
+
+	rule1 := &Rule{TargetMailboxes: []Mailbox{folderA, labelA, labelB}}
+	rule2 := &Rule{TargetMailboxes: []Mailbox{folderB, labelC}}
+	rule3 := &Rule{TargetMailboxes: []Mailbox{folderB}}
+
+	tests := []struct {
+		rules         []*Rule
+		wantMailboxes []Mailbox
+	}{
+		{[]*Rule{}, []Mailbox{}},
+		{[]*Rule{rule1}, []Mailbox{folderA, labelA, labelB}},
+		{[]*Rule{rule1, rule2}, []Mailbox{folderA, labelA, labelB, labelC}},
+		{[]*Rule{rule1, rule3}, []Mailbox{folderA, labelA, labelB}},
+		{[]*Rule{rule3, rule1}, []Mailbox{folderB, labelA, labelB}},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(fmt.Sprintf("%v", tc.rules), func(t *testing.T) {
+			mailboxes := provider.getMessageTargets(tc.rules, "", []byte(""))
+			r.Equal(t, tc.wantMailboxes, mailboxes)
+		})
+	}
+}
+
 func setupMBOXRules(rules transferRules) {
+	_ = rules.setRule(Mailbox{Name: "All Mail"}, []Mailbox{{Name: "Archive"}}, 0, 0)
 	_ = rules.setRule(Mailbox{Name: "Inbox"}, []Mailbox{{Name: "Inbox"}}, 0, 0)
 	_ = rules.setRule(Mailbox{Name: "Foo"}, []Mailbox{{Name: "Foo"}}, 0, 0)
 }
 
 func checkMBOXFileStructure(t *testing.T, root string, expectedFiles []string) {
-	files, err := getFilePathsWithSuffix(root, ".mbox")
+	files, err := getAllPathsWithSuffix(root, ".mbox")
 	r.NoError(t, err)
 	r.Equal(t, expectedFiles, files)
 }

internal/transfer/provider_pmapi_source.go

@@ -123,7 +123,7 @@ func (p *PMAPIProvider) transferTo(rule *Rule, progress *Progress, ch chan<- Mes
 				}
 
 				msgID := fmt.Sprintf("%s_%s", rule.SourceMailbox.ID, pmapiMessage.ID)
-				progress.addMessage(msgID, rule)
+				progress.addMessage(msgID, []string{rule.SourceMailbox.Name}, rule.TargetMailboxNames())
 				msg, err := p.exportMessage(rule, progress, pmapiMessage.ID, msgID, skipEncryptedMessages)
 				progress.messageExported(msgID, msg.Body, err)
 				if err == nil {
@@ -177,7 +177,7 @@ func (p *PMAPIProvider) exportMessage(rule *Rule, progress *Progress, pmapiMsgID
 		ID:      msgID,
 		Unread:  unread,
 		Body:    body,
-		Source:  rule.SourceMailbox,
+		Sources: []Mailbox{rule.SourceMailbox},
 		Targets: rule.TargetMailboxes,
 	}, nil
 }

internal/transfer/provider_pmapi_target.go

@@ -177,6 +177,10 @@ func (p *PMAPIProvider) transferMessage(rules transferRules, progress *Progress,
 		return
 	}
 
+	if progress.shouldStop() {
+		return
+	}
+
 	importMsgReqSize := len(importMsgReq.Body)
 	if p.nextImportRequestsSize+importMsgReqSize > pmapiImportBatchMaxSize || len(p.nextImportRequests) == pmapiImportBatchMaxItems {
 		preparedImportRequestsCh <- p.nextImportRequests

internal/transfer/provider_pmapi_utils.go

@@ -72,7 +72,8 @@ func (p *PMAPIProvider) tryReconnect() error {
 
 func (p *PMAPIProvider) listMessages(filter *pmapi.MessagesFilter) (messages []*pmapi.Message, count int, err error) {
 	err = p.ensureConnection(func() error {
-		key := fmt.Sprintf("%s_%d", filter.LabelID, filter.Page)
+		// Sort is used in the key so the filter is different for estimating and real fetching.
+		key := fmt.Sprintf("%s_%s_%d", filter.LabelID, filter.Sort, filter.Page)
 		p.timeIt.start("listing", key)
 		defer p.timeIt.stop("listing", key)
 
@@ -117,8 +118,10 @@ func (p *PMAPIProvider) createDraft(msgSourceID string, message *pmapi.Message,
 
 func (p *PMAPIProvider) createAttachment(msgSourceID string, att *pmapi.Attachment, r io.Reader, sig io.Reader) (created *pmapi.Attachment, err error) {
 	err = p.ensureConnection(func() error {
-		p.timeIt.start("upload", msgSourceID)
+		// Use some attributes from attachment to have unique key for each call.
-		defer p.timeIt.stop("upload", msgSourceID)
+		key := fmt.Sprintf("%s_%s_%d", msgSourceID, att.Name, att.Size)
+		p.timeIt.start("upload", key)
+		defer p.timeIt.stop("upload", key)
 
 		created, err = p.client().CreateAttachment(att, r, sig)
 		return err

internal/transfer/provider_test.go

@@ -43,7 +43,7 @@ hello
 `, subject))
 }
 
-func testTransferTo(t *testing.T, rules transferRules, provider SourceProvider, expectedMessageIDs []string) {
+func testTransferTo(t *testing.T, rules transferRules, provider SourceProvider, expectedMessageIDs []string) []Message {
 	progress := newProgress(log, nil)
 	drainProgressUpdateChannel(&progress)
 
@@ -53,13 +53,17 @@ func testTransferTo(t *testing.T, rules transferRules, provider SourceProvider,
 		close(ch)
 	}()
 
+	msgs := []Message{}
 	gotMessageIDs := []string{}
 	for msg := range ch {
+		msgs = append(msgs, msg)
 		gotMessageIDs = append(gotMessageIDs, msg.ID)
 	}
 	r.ElementsMatch(t, expectedMessageIDs, gotMessageIDs)
 
 	r.Empty(t, progress.GetFailedMessages())
+
+	return msgs
 }
 
 func testTransferFrom(t *testing.T, rules transferRules, provider TargetProvider, messages []Message) {
@@ -69,7 +73,7 @@ func testTransferFrom(t *testing.T, rules transferRules, provider TargetProvider
 	ch := make(chan Message)
 	go func() {
 		for _, message := range messages {
-			progress.addMessage(message.ID, nil)
+			progress.addMessage(message.ID, []string{}, []string{})
 			progress.messageExported(message.ID, []byte(""), nil)
 			ch <- message
 		}

internal/transfer/report.go

@@ -114,7 +114,7 @@ type messageReport struct {
 	SourceID        string
 	TargetID        string
 	BodyHash        string
-	SourceMailbox   string
+	SourceMailboxes []string
 	TargetMailboxes []string
 	Error           string
 
@@ -130,8 +130,8 @@ func newMessageReportFromMessageStatus(messageStatus *MessageStatus, includePriv
 		SourceID:        messageStatus.SourceID,
 		TargetID:        messageStatus.targetID,
 		BodyHash:        messageStatus.bodyHash,
-		SourceMailbox:   messageStatus.rule.SourceMailbox.Name,
+		SourceMailboxes: messageStatus.sourceNames,
-		TargetMailboxes: messageStatus.rule.TargetMailboxNames(),
+		TargetMailboxes: messageStatus.targetNames,
 		Error:           messageStatus.GetErrorMessage(),
 	}
 

internal/transfer/testdata/mbox-applemail/All Mail.mbox/mbox

@@ -0,0 +1,16 @@
+From - Mon May  4 16:40:31 2020
+From: Bridge Test <bridgetest@pm.test>
+To: Bridge Test <bridgetest@protonmail.com>
+Subject: Test 1
+X-Gmail-Labels: Foo,Bar
+
+hello
+
+
+From - Mon May  4 16:40:31 2020
+From: Bridge Test <bridgetest@pm.test>
+To: Bridge Test <bridgetest@protonmail.com>
+Subject: Test 2
+X-Gmail-Labels: Foo
+
+hello

internal/transfer/testdata/mbox/All Mail.mbox

@@ -0,0 +1,16 @@
+From - Mon May  4 16:40:31 2020
+From: Bridge Test <bridgetest@pm.test>
+To: Bridge Test <bridgetest@protonmail.com>
+Subject: Test 1
+X-Gmail-Labels: Foo,Bar
+
+hello
+
+
+From - Mon May  4 16:40:31 2020
+From: Bridge Test <bridgetest@pm.test>
+To: Bridge Test <bridgetest@protonmail.com>
+Subject: Test 2
+X-Gmail-Labels: Foo
+
+hello

internal/transfer/utils.go

@@ -24,6 +24,7 @@ import (
 	"net/mail"
 	"net/textproto"
 	"path/filepath"
+	"runtime"
 	"sort"
 	"strings"
 
@@ -81,7 +82,7 @@ func getFolderNamesWithFileSuffix(root, fileSuffix string) ([]string, error) {
 // getFilePathsWithSuffix collects all file names with `suffix` under `root`.
 // File names will be with relative path based to `root`.
 func getFilePathsWithSuffix(root, suffix string) ([]string, error) {
-	fileNames, err := getFilePathsWithSuffixInner("", root, suffix)
+	fileNames, err := getFilePathsWithSuffixInner("", root, suffix, false)
 	if err != nil {
 		return nil, err
 	}
@@ -89,7 +90,18 @@ func getFilePathsWithSuffix(root, suffix string) ([]string, error) {
 	return fileNames, err
 }
 
-func getFilePathsWithSuffixInner(prefix, root, suffix string) ([]string, error) {
+// getAllPathsWithSuffix is the same as getFilePathsWithSuffix but includes
+// also directories.
+func getAllPathsWithSuffix(root, suffix string) ([]string, error) {
+	fileNames, err := getFilePathsWithSuffixInner("", root, suffix, true)
+	if err != nil {
+		return nil, err
+	}
+	sort.Strings(fileNames)
+	return fileNames, err
+}
+
+func getFilePathsWithSuffixInner(prefix, root, suffix string, includeDir bool) ([]string, error) {
 	fileNames := []string{}
 
 	files, err := ioutil.ReadDir(root)
@@ -103,10 +115,14 @@ func getFilePathsWithSuffixInner(prefix, root, suffix string) ([]string, error)
 				fileNames = append(fileNames, filepath.Join(prefix, file.Name()))
 			}
 		} else {
+			if includeDir && strings.HasSuffix(file.Name(), suffix) {
+				fileNames = append(fileNames, filepath.Join(prefix, file.Name()))
+			}
 			subfolderFileNames, err := getFilePathsWithSuffixInner(
 				filepath.Join(prefix, file.Name()),
 				filepath.Join(root, file.Name()),
 				suffix,
+				includeDir,
 			)
 			if err != nil {
 				return nil, err
@@ -139,3 +155,24 @@ func getMessageHeader(body []byte) (mail.Header, error) {
 	}
 	return mail.Header(header), nil
 }
+
+// sanitizeFileName replaces problematic special characters with underscore.
+func sanitizeFileName(fileName string) string {
+	if len(fileName) == 0 {
+		return fileName
+	}
+	if runtime.GOOS != "windows" && (fileName[0] == '-' || fileName[0] == '.') { //nolint[goconst]
+		fileName = "_" + fileName[1:]
+	}
+	return strings.Map(func(r rune) rune {
+		switch r {
+		case '\\', '/', ':', '*', '?', '"', '<', '>', '|':
+			return '_'
+		case '[', ']', '(', ')', '{', '}', '^', '#', '%', '&', '!', '@', '+', '=', '\'', '~':
+			if runtime.GOOS != "windows" {
+				return '_'
+			}
+		}
+		return r
+	}, fileName)
+}

internal/transfer/utils_test.go

@@ -21,6 +21,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"runtime"
 	"testing"
 
 	r "github.com/stretchr/testify/require"
@@ -38,6 +39,7 @@ func TestGetFolderNames(t *testing.T) {
 			"",
 			[]string{
 				"bar",
+				"bar.mbox",
 				"baz",
 				filepath.Base(root),
 				"foo",
@@ -94,6 +96,13 @@ func TestGetFilePathsWithSuffix(t *testing.T) {
 				"test/foo/msg9.eml",
 			},
 		},
+		{
+			".mbox",
+			[]string{
+				"bar.mbox",
+				"foo.mbox",
+			},
+		},
 		{
 			".txt",
 			[]string{
@@ -108,7 +117,7 @@ func TestGetFilePathsWithSuffix(t *testing.T) {
 	for _, tc := range tests {
 		tc := tc
 		t.Run(tc.suffix, func(t *testing.T) {
-			paths, err := getFilePathsWithSuffix(root, tc.suffix)
+			paths, err := getAllPathsWithSuffix(root, tc.suffix)
 			r.NoError(t, err)
 			r.Equal(t, tc.wantPaths, paths)
 		})
@@ -124,6 +133,7 @@ func createTestingFolderStructure(t *testing.T) (string, func()) {
 		"foo/baz",
 		"test/foo",
 		"qwerty",
+		"bar.mbox",
 	} {
 		err = os.MkdirAll(filepath.Join(root, path), os.ModePerm)
 		r.NoError(t, err)
@@ -141,6 +151,8 @@ func createTestingFolderStructure(t *testing.T) (string, func()) {
 		"test/foo/msg9.eml",
 		"msg10.eml",
 		"info.txt",
+		"foo.mbox",
+		"bar.mbox/mbox", // Apple Mail mbox export format.
 	} {
 		f, err := os.Create(filepath.Join(root, path))
 		r.NoError(t, err)
@@ -188,3 +200,26 @@ Body
 	r.Equal(t, header.Get("subject"), "Hello")
 	r.Equal(t, header.Get("from"), "user@example.com")
 }
+
+func TestSanitizeFileName(t *testing.T) {
+	tests := map[string]string{
+		"hello":            "hello",
+		"a\\b/c:*?d\"<>|e": "a_b_c___d____e",
+	}
+	if runtime.GOOS == "darwin" || runtime.GOOS == "linux" {
+		tests[".hello"] = "_hello"
+		tests["-hello"] = "_hello"
+	}
+	if runtime.GOOS == "windows" {
+		tests["[hello]&@=~~"] = "_hello______"
+	}
+
+	for path, wantPath := range tests {
+		path := path
+		wantPath := wantPath
+		t.Run(path, func(t *testing.T) {
+			gotPath := sanitizeFileName(path)
+			r.Equal(t, wantPath, gotPath)
+		})
+	}
+}

internal/updates/sync.go

@@ -45,7 +45,7 @@ func syncFolders(localPath, updatePath string) (err error) {
 }
 
 func removeMissing(folderToCleanPath, itemsToKeepPath string) (err error) {
-	log.Debug("remove missing")
+	log.WithField("from", folderToCleanPath).Debug("Remove missing.")
 	// Create list of files.
 	existingRelPaths := map[string]bool{}
 	err = filepath.Walk(itemsToKeepPath, func(keepThis string, _ os.FileInfo, walkErr error) error {
@@ -56,7 +56,7 @@ func removeMissing(folderToCleanPath, itemsToKeepPath string) (err error) {
 		if walkErr != nil {
 			return walkErr
 		}
-		log.Debug("path to keep ", relPath)
+		log.WithField("path", relPath).Trace("Keep the path.")
 		existingRelPaths[relPath] = true
 		return nil
 	})
@@ -95,12 +95,18 @@ func removeMissing(folderToCleanPath, itemsToKeepPath string) (err error) {
 }
 
 func restoreFromBackup(backupDir, localPath string) {
-	log.Error("recovering from ", backupDir, " to ", localPath)
+	log.WithField("from", backupDir).
-	_ = copyRecursively(backupDir, localPath)
+		WithField("to", localPath).
+		Error("recovering")
+	if err := copyRecursively(backupDir, localPath); err != nil {
+		log.WithField("from", backupDir).
+			WithField("to", localPath).
+			Error("Not able to recover.")
+	}
 }
 
 func createBackup(srcFile, dstDir string) (err error) {
-	log.Debug("backup ", srcFile, " in ", dstDir)
+	log.WithField("from", srcFile).WithField("to", dstDir).Debug("Create backup")
 	if err = mkdirAllClear(dstDir); err != nil {
 		return
 	}

internal/updates/updates.go

@@ -107,7 +107,7 @@ func NewImportExport(updateTempDir string) *Updates {
 		versionFileBaseName: "current_version_ie",
 		updateFileBaseName:  "ie/ie_upgrade",
 		linuxFileBaseName:   "ie/protonmail-import-export-app",
-		macAppBundleName:    "Import-Export app.app",
+		macAppBundleName:    "ProtonMail Import-Export app.app",
 	}
 }
 
@@ -328,10 +328,15 @@ func (u *Updates) StartUpgrade(currentStatus chan<- Progress) { // nolint[funlen
 		localPath = filepath.Dir(localPath) // .app
 
 		updatePath := filepath.Join(u.updateTempDir, u.macAppBundleName)
-		log.Warn("localPath ", localPath)
+		log.WithField("local", localPath).
-		log.Warn("updatePath ", updatePath)
+			WithField("update", updatePath).
+			Info("Syncing folders..")
 		status.Err = syncFolders(localPath, updatePath)
 		if status.Err != nil {
+			log.WithField("from", localPath).
+				WithField("to", updatePath).
+				WithError(status.Err).
+				Error("Sync failed.")
 			return
 		}
 		status.UpdateDescription(InfoRestartApp)

pkg/message/parser/parser.go

@@ -32,7 +32,7 @@ type Parser struct {
 func New(r io.Reader) (*Parser, error) {
 	p := new(Parser)
 
-	entity, err := message.Read(r)
+	entity, err := message.Read(newEndOfMailTrimmer(r))
 	if err != nil && !message.IsUnknownCharset(err) {
 		return nil, err
 	}

pkg/message/parser/trimmer.go

@@ -0,0 +1,56 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package parser
+
+import (
+	"bytes"
+	"io"
+)
+
+const endOfMail = "\r\n.\r\n"
+
+// endOfMailTrimmer wraps a reader to trim the End-Of-Mail indicator at the end
+// of the input, if present.
+//
+// During SMTP sending of a message, the DATA command indicates that you are
+// about to send the text (or body) of the message. The message text must end
+// with "\r\n.\r\n." I'm 99% sure that these 5 bytes should not be considered
+// part of the message body. However, some mail servers keep them as part of
+// the message, which our parser sometimes doesn't like. Therefore, we strip
+// them if we find them.
+type endOfMailTrimmer struct {
+	r   io.Reader
+	buf bytes.Buffer
+}
+
+func newEndOfMailTrimmer(r io.Reader) *endOfMailTrimmer {
+	return &endOfMailTrimmer{r: r}
+}
+
+func (r *endOfMailTrimmer) Read(p []byte) (int, error) {
+	_, err := io.CopyN(&r.buf, r.r, int64(len(p)+len(endOfMail)-r.buf.Len()))
+	if err != nil && err != io.EOF {
+		return 0, err
+	}
+
+	if err == io.EOF && bytes.HasSuffix(r.buf.Bytes(), []byte(endOfMail)) {
+		r.buf.Truncate(r.buf.Len() - len(endOfMail))
+	}
+
+	return r.buf.Read(p)
+}

pkg/message/parser/trimmer_test.go

@@ -0,0 +1,55 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package parser
+
+import (
+	"io"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEndOfMailTrimmer(t *testing.T) {
+	var tests = []struct {
+		in  string
+		out string
+	}{
+		{"string without eom", "string without eom"},
+		{"string with eom\r\n.\r\n", "string with eom"},
+		{"string with eom\r\n.\r\nin the middle", "string with eom\r\n.\r\nin the middle"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.in, func(t *testing.T) {
+			res := dumbRead(newEndOfMailTrimmer(strings.NewReader(tt.in)))
+			assert.Equal(t, tt.out, string(res))
+		})
+	}
+}
+
+func dumbRead(r io.Reader) []byte {
+	out := []byte{}
+
+	b := make([]byte, 1)
+	for _, err := r.Read(b); err == nil; _, err = r.Read(b) {
+		out = append(out, b...)
+	}
+
+	return out
+}

pkg/message/parser_test.go

@@ -467,6 +467,19 @@ func TestParseMultipartAlternativeLatin1(t *testing.T) {
 	assert.Equal(t, "*aoeuaoeu*\n\n", plainBody)
 }
 
+func TestParseWithTrailingEndOfMailIndicator(t *testing.T) {
+	f := getFileReader("text_html_trailing_end_of_mail.eml")
+
+	m, _, plainBody, _, err := Parse(f, "", "")
+	require.NoError(t, err)
+
+	assert.Equal(t, `"Sender" <sender@sender.com>`, m.Sender.String())
+	assert.Equal(t, `"Receiver" <receiver@receiver.com>`, m.ToList[0].String())
+
+	assert.Equal(t, "<!DOCTYPE html><html><head></head><body>boo!</body></html>", m.Body)
+	assert.Equal(t, "boo!", plainBody)
+}
+
 func getFileReader(filename string) io.Reader {
 	f, err := os.Open(filepath.Join("testdata", filename))
 	if err != nil {

pkg/message/testdata/text_html_trailing_end_of_mail.eml

@@ -0,0 +1,8 @@
+From: "Sender" <sender@sender.com>
+To: "Receiver" <receiver@receiver.com>
+Content-Type: text/html; charset="utf-8"
+Content-Transfer-Encoding: base64
+MIME-Version: 1.0
+
+PCFET0NUWVBFIEhUTUw+CjxodG1sPjxib2R5PmJvbyE8L2JvZHk+PC9odG1sPg==
+.

pkg/pmapi/mocks/mocks.go

@@ -5,11 +5,12 @@
 package mocks
 
 import (
+	io "io"
+	reflect "reflect"
+
 	crypto "github.com/ProtonMail/gopenpgp/v2/crypto"
 	pmapi "github.com/ProtonMail/proton-bridge/pkg/pmapi"
 	gomock "github.com/golang/mock/gomock"
-	io "io"
-	reflect "reflect"
 )
 
 // MockClient is a mock of Client interface

release-notes/bugs-importexport.txt

@@ -1,2 +1,3 @@
-• Import from mbox files with long lines
+• Linux font issues - Fedora specific
-• Improvements to import from Yahoo accounts
+• App response to the user pausing and canceling import or export
+• Handling errors during update

release-notes/notes-importexport.txt

@@ -1,5 +1,5 @@
-• Speed up import by implementing parallel processing (parallel fetch, encrypt and upload of messages)
+• Improvements to the import from large mbox files with multiple labels
-• Optimising the initial fetch of messages from external accounts
+• Not allow to run multiple instances of the app or transfers at the same time
-• Better handling of attachments and non-standard formatting
+• Various enhancements of the import process related to parsing
-• Improved stability of the message parser
+• Cosmetic GUI changes
-• Added persistent anonymous API cookies
+• Better error handling

test/context/context.go

@@ -73,6 +73,9 @@ type TestContext struct {
 	transferRemoteIMAPServer   *mocks.IMAPServer
 	transferProgress           *transfer.Progress
 
+	// Store releated variables.
+	bddMessageIDsToAPIIDs map[string]string
+
 	// These are the cleanup steps executed when Cleanup() is called.
 	cleanupSteps []*Cleaner
 
@@ -89,18 +92,19 @@ func New(app string) *TestContext {
 	cm := pmapi.NewClientManager(cfg.GetAPIConfig())
 
 	ctx := &TestContext{
-		t:                 &bddT{},
+		t:                     &bddT{},
-		cfg:               cfg,
+		cfg:                   cfg,
-		listener:          listener.New(),
+		listener:              listener.New(),
-		pmapiController:   newPMAPIController(cm),
+		pmapiController:       newPMAPIController(cm),
-		clientManager:     cm,
+		clientManager:         cm,
-		testAccounts:      newTestAccounts(),
+		testAccounts:          newTestAccounts(),
-		credStore:         newFakeCredStore(),
+		credStore:             newFakeCredStore(),
-		imapClients:       make(map[string]*mocks.IMAPClient),
+		imapClients:           make(map[string]*mocks.IMAPClient),
-		imapLastResponses: make(map[string]*mocks.IMAPResponse),
+		imapLastResponses:     make(map[string]*mocks.IMAPResponse),
-		smtpClients:       make(map[string]*mocks.SMTPClient),
+		smtpClients:           make(map[string]*mocks.SMTPClient),
-		smtpLastResponses: make(map[string]*mocks.SMTPResponse),
+		smtpLastResponses:     make(map[string]*mocks.SMTPResponse),
-		logger:            logrus.StandardLogger(),
+		bddMessageIDsToAPIIDs: make(map[string]string),
+		logger:                logrus.StandardLogger(),
 	}
 
 	// Ensure that the config is cleaned up after the test is over.

test/context/pmapi_controller.go

@@ -32,7 +32,6 @@ type PMAPIController interface {
 	AddUserLabel(username string, label *pmapi.Label) error
 	GetLabelIDs(username string, labelNames []string) ([]string, error)
 	AddUserMessage(username string, message *pmapi.Message) (string, error)
-	GetMessageID(username, messageIndex string) string
 	GetMessages(username, labelID string) ([]*pmapi.Message, error)
 	ReorderAddresses(user *pmapi.User, addressIDs []string) error
 	PrintCalls()

test/context/store.go

@@ -0,0 +1,39 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package context
+
+import (
+	"errors"
+)
+
+// PairMessageID sets pairing between BDD message ID and API message ID.
+func (ctx *TestContext) PairMessageID(username, bddMessageID, realMessageID string) {
+	if bddMessageID == "" {
+		return
+	}
+	ctx.bddMessageIDsToAPIIDs[username+":"+bddMessageID] = realMessageID
+}
+
+// GetAPIMessageID returns API message ID for given BDD message ID.
+func (ctx *TestContext) GetAPIMessageID(username, bddMessageID string) (string, error) {
+	msgID, ok := ctx.bddMessageIDsToAPIIDs[username+":"+bddMessageID]
+	if !ok {
+		return "", errors.New("unknown bddMessageID")
+	}
+	return msgID, nil
+}

test/fakeapi/controller_control.go

@@ -159,10 +159,6 @@ func (ctl *Controller) resetUsers() {
 	}
 }
 
-func (ctl *Controller) GetMessageID(username, messageIndex string) string {
-	return messageIndex
-}
-
 func (ctl *Controller) GetMessages(username, labelID string) ([]*pmapi.Message, error) {
 	messages := []*pmapi.Message{}
 	for _, fakeAPI := range ctl.fakeAPIs {

test/features/bridge/imap/idle/basic.feature

@@ -3,6 +3,7 @@ Feature: IMAP IDLE
     Given there is connected user "user"
     And there are 10 messages in mailbox "INBOX" for "user"
 
+  # Those tests are ignored as currently our IMAP implementation is not responding with updates to all open connections.
   @ignore
   Scenario Outline: Mark as read
     Given there is IMAP client "active" logged in as "user"
@@ -10,15 +11,15 @@ Feature: IMAP IDLE
     And there is IMAP client "idling" logged in as "user"
     And there is IMAP client "idling" selected in "INBOX"
     When IMAP client "idling" starts IDLE-ing
-    And IMAP client "active" marks message "<message>" as read
+    And IMAP client "active" marks message seq "<seq>" as read
-    Then IMAP client "idling" receives update marking message "<message>" as read within <seconds> seconds
+    Then IMAP client "idling" receives update marking message seq "<seq>" as read within <seconds> seconds
-    Then message "<message>" in "INBOX" for "user" is marked as read
+    Then message "<seq>" in "INBOX" for "user" is marked as read
 
     Examples:
-      | message | seconds |
+      | seq  | seconds |
-      | 1       | 2       |
+      | 1    | 2       |
-      | 1:5     | 2       |
+      | 1:5  | 2       |
-      | 1:10    | 5       |
+      | 1:10 | 5       |
 
   @ignore
   Scenario Outline: Mark as unread
@@ -27,15 +28,15 @@ Feature: IMAP IDLE
     And there is IMAP client "idling" logged in as "user"
     And there is IMAP client "idling" selected in "INBOX"
     When IMAP client "idling" starts IDLE-ing
-    And IMAP client "active" marks message "<message>" as unread
+    And IMAP client "active" marks message seq "<seq>" as unread
-    Then IMAP client "idling" receives update marking message "<message>" as unread within <seconds> seconds
+    Then IMAP client "idling" receives update marking message seq "<seq>" as unread within <seconds> seconds
-    And message "<message>" in "INBOX" for "user" is marked as unread
+    And message "<seq>" in "INBOX" for "user" is marked as unread
 
     Examples:
-      | message | seconds |
+      | seq  | seconds |
-      | 1       | 2       |
+      | 1    | 2       |
-      | 1:5     | 2       |
+      | 1:5  | 2       |
-      | 1:10    | 5       |
+      | 1:10 | 5       |
 
   @ignore
   Scenario Outline: Three IDLEing
@@ -50,13 +51,13 @@ Feature: IMAP IDLE
     When IMAP client "idling1" starts IDLE-ing
     And IMAP client "idling2" starts IDLE-ing
     And IMAP client "idling3" starts IDLE-ing
-    And IMAP client "active" marks message "<message>" as read
+    And IMAP client "active" marks message seq "<seq>" as read
-    Then IMAP client "idling1" receives update marking message "<message>" as read within <seconds> seconds
+    Then IMAP client "idling1" receives update marking message seq "<seq>" as read within <seconds> seconds
-    Then IMAP client "idling2" receives update marking message "<message>" as read within <seconds> seconds
+    Then IMAP client "idling2" receives update marking message seq "<seq>" as read within <seconds> seconds
-    Then IMAP client "idling3" receives update marking message "<message>" as read within <seconds> seconds
+    Then IMAP client "idling3" receives update marking message seq "<seq>" as read within <seconds> seconds
 
     Examples:
-      | message | seconds |
+      | seq  | seconds |
-      | 1       | 2       |
+      | 1    | 2       |
-      | 1:5     | 2       |
+      | 1:5  | 2       |
-      | 1:10    | 5       |
+      | 1:10 | 5       |

test/features/bridge/imap/idle/two_users.feature

@@ -8,8 +8,8 @@ Feature: IMAP IDLE with two users
     And there is IMAP client "idling" logged in as "userMoreAddresses"
     And there is IMAP client "idling" selected in "INBOX"
     When IMAP client "idling" starts IDLE-ing
-    And IMAP client "active" marks message "1" as read
+    And IMAP client "active" marks message seq "1" as read
-    Then IMAP client "idling" does not receive update for message "1" within 5 seconds
+    Then IMAP client "idling" does not receive update for message seq "1" within 5 seconds
 
   Scenario: IDLE statements are not leaked to other alias
     Given there is connected user "userMoreAddresses"
@@ -24,5 +24,5 @@ Feature: IMAP IDLE with two users
     And there is IMAP client "idling" logged in as "userMoreAddresses" with address "secondary"
     And there is IMAP client "idling" selected in "INBOX"
     When IMAP client "idling" starts IDLE-ing
-    And IMAP client "active" marks message "1" as read
+    And IMAP client "active" marks message seq "1" as read
-    Then IMAP client "idling" does not receive update for message "1" within 5 seconds
+    Then IMAP client "idling" does not receive update for message seq "1" within 5 seconds

test/features/bridge/imap/mailbox/info.feature

@@ -10,9 +10,6 @@ Feature: IMAP get mailbox info
   Scenario: Mailbox info contains mailbox name
     When IMAP client gets info of "INBOX"
     Then IMAP response contains "2 EXISTS"
-    # Messages are inserted in opposite way to keep increasing UID.
+    And IMAP response contains "UNSEEN 1"
-    # Sequence numbers are then opposite than listed above.
-    # Unseen should have first unseen message.
-    And IMAP response contains "UNSEEN 2"
     And IMAP response contains "UIDNEXT 3"
     And IMAP response contains "UIDVALIDITY"

test/features/bridge/imap/mailbox/status.feature

@@ -12,9 +12,8 @@ Feature: IMAP get mailbox status
     When IMAP client gets status of "INBOX"
     Then IMAP response contains "INBOX"
 
-  Scenario: Mailbox status contains
+  Scenario: Mailbox status contains counts and UIDs
     When IMAP client gets status of "INBOX"
-    Then IMAP response contains "INBOX"
     And IMAP response contains "MESSAGES 2"
     And IMAP response contains "UNSEEN 1"
     And IMAP response contains "UIDNEXT 3"

test/features/bridge/imap/message/copy.feature

@@ -3,8 +3,6 @@ Feature: IMAP copy messages
     Given there is connected user "user"
     And there is "user" with mailbox "Folders/mbox"
     And there is "user" with mailbox "Labels/label"
-    # Messages are inserted in opposite way to keep increasing ID.
-    # Sequence numbers are then opposite than listed above.
     And there are messages in mailbox "INBOX" for "user"
       | from              | to         | subject | body  | read  | deleted |
       | john.doe@mail.com | user@pm.me | foo     | hello | true  | false   |
@@ -13,7 +11,7 @@ Feature: IMAP copy messages
     And there is IMAP client selected in "INBOX"
 
   Scenario: Copy message to label
-    When IMAP client copies messages "2" to "Labels/label"
+    When IMAP client copies message seq "1" to "Labels/label"
     Then IMAP response is "OK"
     And mailbox "INBOX" for "user" has messages
       | from              | to         | subject | body  | read  | deleted |
@@ -24,7 +22,7 @@ Feature: IMAP copy messages
       | john.doe@mail.com | user@pm.me | foo     | hello | true | false   |
 
   Scenario: Copy all messages to label
-    When IMAP client copies messages "1:*" to "Labels/label"
+    When IMAP client copies message seq "1:*" to "Labels/label"
     Then IMAP response is "OK"
     And mailbox "INBOX" for "user" has messages
       | from              | to         | subject | body  | read  | deleted |
@@ -36,7 +34,7 @@ Feature: IMAP copy messages
       | jane.doe@mail.com | name@pm.me | bar     | world | false | true    |
 
   Scenario: Copy message to folder does move
-    When IMAP client copies messages "2" to "Folders/mbox"
+    When IMAP client copies message seq "1" to "Folders/mbox"
     Then IMAP response is "OK"
     And mailbox "INBOX" for "user" has messages
       | from              | to         | subject | body  | read  | deleted |
@@ -46,7 +44,7 @@ Feature: IMAP copy messages
       | john.doe@mail.com | user@pm.me | foo     | hello | true | false   |
 
   Scenario: Copy all messages to folder does move
-    When IMAP client copies messages "1:*" to "Folders/mbox"
+    When IMAP client copies message seq "1:*" to "Folders/mbox"
     Then IMAP response is "OK"
     And mailbox "INBOX" for "user" has 0 messages
     And mailbox "Folders/mbox" for "user" has messages

test/features/bridge/imap/message/create.feature

@@ -34,7 +34,7 @@ Feature: IMAP create messages
     Then IMAP response is "OK"
     When the event loop of "userMoreAddresses" loops once
     Then mailbox "Sent" for "userMoreAddresses" has messages
-      | from      | to                 | subject | read |
+      | from        | to                 | subject | read |
       | [secondary] | john.doe@email.com | foo     | true |
     And mailbox "INBOX" for "userMoreAddresses" has no messages
 
@@ -57,7 +57,7 @@ Feature: IMAP create messages
     And mailbox "INBOX" for "userMoreAddresses" has no messages
 
   # Importing duplicate messages when messageID cannot be found in Sent already.
-  # 
+  #
   # Previously, we discarded messages for which sender matches account address to
   # avoid duplicates, but this led to discarding messages imported through mail client.
   Scenario: Imports a similar (duplicate) message to sent
@@ -67,4 +67,4 @@ Feature: IMAP create messages
     And there is IMAP client selected in "Sent"
     When IMAP client creates message "Meet the Twins" from address "primary" of "userMoreAddresses" to "chosen@one.com" with body "Hello, Mr. Anderson" in "Sent"
     Then IMAP response is "OK"
-    And mailbox "Sent" for "userMoreAddresses" has 2 messages
+    And mailbox "Sent" for "userMoreAddresses" has 2 messages

test/features/bridge/imap/message/delete.feature

@@ -8,10 +8,10 @@ Feature: IMAP remove messages from mailbox
     Given there are 10 messages in mailbox "<mailbox>" for "user"
     And there is IMAP client logged in as "user"
     And there is IMAP client selected in "<mailbox>"
-    When IMAP client marks message "2" as deleted
+    When IMAP client marks message seq "2" as deleted
     Then IMAP response is "OK"
     And mailbox "<mailbox>" for "user" has 10 messages
-    And message "9" in "<mailbox>" for "user" is marked as deleted
+    And message "2" in "<mailbox>" for "user" is marked as deleted
     And IMAP response contains "\* 2 FETCH[ (]*FLAGS \([^)]*\\Deleted"
     When IMAP client sends expunge
     Then IMAP response is "OK"
@@ -30,7 +30,7 @@ Feature: IMAP remove messages from mailbox
     Given there are 5 messages in mailbox "<mailbox>" for "user"
     And there is IMAP client logged in as "user"
     And there is IMAP client selected in "<mailbox>"
-    When IMAP client marks message "1:*" as deleted
+    When IMAP client marks message seq "1:*" as deleted
     Then IMAP response is "OK"
     When IMAP client sends expunge
     Then IMAP response is "OK"
@@ -53,9 +53,9 @@ Feature: IMAP remove messages from mailbox
     Given there are 5 messages in mailbox "<mailbox>" for "user"
     And there is IMAP client logged in as "user"
     And there is IMAP client selected in "<mailbox>"
-    When IMAP client marks message "1:*" as deleted
+    When IMAP client marks message seq "1:*" as deleted
     Then IMAP response is "OK"
-    When IMAP client marks message "1:3" as undeleted
+    When IMAP client marks message seq "1:3" as undeleted
     Then IMAP response is "OK"
     When IMAP client sends expunge
     Then IMAP response is "OK"
@@ -75,10 +75,10 @@ Feature: IMAP remove messages from mailbox
     Given there are 10 messages in mailbox "INBOX" for "user"
     And there is IMAP client logged in as "user"
     And there is IMAP client selected in "INBOX"
-    When IMAP client marks message "2" as deleted
+    When IMAP client marks message seq "2" as deleted
     Then IMAP response is "OK"
     And mailbox "INBOX" for "user" has 10 messages
-    And message "9" in "INBOX" for "user" is marked as deleted
+    And message "2" in "INBOX" for "user" is marked as deleted
     When IMAP client sends command "<leave>"
     Then IMAP response is "OK"
     And mailbox "INBOX" for "user" has <n> messages
@@ -97,5 +97,5 @@ Feature: IMAP remove messages from mailbox
     Given there are 1 messages in mailbox "INBOX" for "user"
     And there is IMAP client logged in as "user"
     And there is IMAP client selected in "All Mail"
-    When IMAP client marks message "1" as deleted
+    When IMAP client marks message seq "1" as deleted
     Then IMAP response is "IMAP error: NO operation not allowed for 'All Mail' folder"

test/features/bridge/imap/message/delete_from_trash.feature

@@ -6,14 +6,14 @@ Feature: IMAP remove messages from Trash
 
   Scenario Outline: Message in Trash/Spam and some other label is not permanently deleted
     Given there are messages in mailbox "<mailbox>" for "user"
-      | from              | to         | subject | body  |
+      | id | from              | to         | subject | body  |
-      | john.doe@mail.com | user@pm.me | foo     | hello |
+      | 1  | john.doe@mail.com | user@pm.me | foo     | hello |
-      | jane.doe@mail.com | name@pm.me | bar     | world |
+      | 2  | jane.doe@mail.com | name@pm.me | bar     | world |
     And there is IMAP client logged in as "user"
     And there is IMAP client selected in "<mailbox>"
-    When IMAP client copies messages "2" to "Labels/label"
+    When IMAP client copies message seq "2" to "Labels/label"
     Then IMAP response is "OK"
-    When IMAP client marks message "2" as deleted
+    When IMAP client marks message seq "2" as deleted
     Then IMAP response is "OK"
     And mailbox "<mailbox>" for "user" has 2 messages
     And mailbox "All Mail" for "user" has 2 messages
@@ -31,12 +31,12 @@ Feature: IMAP remove messages from Trash
 
   Scenario Outline: Message in Trash/Spam only is permanently deleted
     Given there are messages in mailbox "<mailbox>" for "user"
-      | from              | to         | subject | body  |
+      | id | from              | to         | subject | body  |
-      | john.doe@mail.com | user@pm.me | foo     | hello |
+      | 1  | john.doe@mail.com | user@pm.me | foo     | hello |
-      | jane.doe@mail.com | name@pm.me | bar     | world |
+      | 2  | jane.doe@mail.com | name@pm.me | bar     | world |
     And there is IMAP client logged in as "user"
     And there is IMAP client selected in "<mailbox>"
-    When IMAP client marks message "2" as deleted
+    When IMAP client marks message seq "2" as deleted
     Then IMAP response is "OK"
     And mailbox "<mailbox>" for "user" has 2 messages
     And mailbox "All Mail" for "user" has 2 messages

test/features/bridge/imap/message/move.feature

@@ -2,8 +2,6 @@ Feature: IMAP move messages
   Background:
     Given there is connected user "user"
     And there is "user" with mailbox "Folders/mbox"
-    # Messages are inserted in opposite way to keep increasing ID.
-    # Sequence numbers are then opposite than listed above.
     And there are messages in mailbox "INBOX" for "user"
       | from              | to         | subject | body  |
       | john.doe@mail.com | user@pm.me | foo     | hello |
@@ -12,7 +10,7 @@ Feature: IMAP move messages
     And there is IMAP client selected in "INBOX"
 
   Scenario: Move message
-    When IMAP client moves messages "2" to "Folders/mbox"
+    When IMAP client moves message seq "1" to "Folders/mbox"
     Then IMAP response is "OK"
     And mailbox "INBOX" for "user" has messages
       | from              | to         | subject |
@@ -22,7 +20,7 @@ Feature: IMAP move messages
       | john.doe@mail.com | user@pm.me | foo     |
 
   Scenario: Move all messages
-    When IMAP client moves messages "1:*" to "Folders/mbox"
+    When IMAP client moves message seq "1:*" to "Folders/mbox"
     Then IMAP response is "OK"
     And mailbox "INBOX" for "user" has 0 messages
     And mailbox "Folders/mbox" for "user" has messages
@@ -31,7 +29,7 @@ Feature: IMAP move messages
       | jane.doe@mail.com | name@pm.me | bar     |
 
   Scenario: Move message from All Mail is not possible
-    When IMAP client moves messages "2" to "Folders/mbox"
+    When IMAP client moves message seq "1" to "Folders/mbox"
     Then IMAP response is "OK"
     And mailbox "All Mail" for "user" has messages
       | from              | to         | subject |

test/features/bridge/imap/message/move_without_support.feature

@@ -6,15 +6,13 @@ Feature: IMAP move messages by append and delete (without MOVE support, e.g., Ou
     And there is IMAP client "target" logged in as "user"
 
   Scenario Outline: Move message from INBOX to mailbox by append and delete
-    # Messages are inserted in opposite way to keep increasing ID.
-    # Sequence numbers are then opposite than listed below.
     Given there are messages in mailbox "INBOX" for "user"
-      | from              | to         | subject | body  |
+      | id | from              | to         | subject | body  |
-      | john.doe@mail.com | user@pm.me | foo     | hello |
+      | 1  | john.doe@mail.com | user@pm.me | foo     | hello |
-      | jane.doe@mail.com | name@pm.me | bar     | world |
+      | 2  | jane.doe@mail.com | name@pm.me | bar     | world |
     And there is IMAP client "source" selected in "INBOX"
     And there is IMAP client "target" selected in "<mailbox>"
-    When IMAP clients "source" and "target" move message "1" of "user" from "INBOX" to "<mailbox>" by append and delete
+    When IMAP clients "source" and "target" move message seq "2" of "user" from "INBOX" to "<mailbox>" by append and delete
     Then IMAP response to "source" is "OK"
     Then IMAP response to "target" is "OK"
     When IMAP client "source" sends expunge
@@ -34,15 +32,13 @@ Feature: IMAP move messages by append and delete (without MOVE support, e.g., Ou
       | Trash        |
 
   Scenario Outline: Move message from Trash/Spam to INBOX by append and delete
-    # Messages are inserted in opposite way to keep increasing ID.
-    # Sequence numbers are then opposite than listed below.
     Given there are messages in mailbox "<mailbox>" for "user"
-      | from              | to         | subject | body  |
+      | id | from              | to         | subject | body  |
-      | john.doe@mail.com | user@pm.me | foo     | hello |
+      | 1  | john.doe@mail.com | user@pm.me | foo     | hello |
-      | jane.doe@mail.com | name@pm.me | bar     | world |
+      | 2  | jane.doe@mail.com | name@pm.me | bar     | world |
     And there is IMAP client "source" selected in "<mailbox>"
     And there is IMAP client "target" selected in "INBOX"
-    When IMAP clients "source" and "target" move message "1" of "user" from "<mailbox>" to "INBOX" by append and delete
+    When IMAP clients "source" and "target" move message seq "2" of "user" from "<mailbox>" to "INBOX" by append and delete
     Then IMAP response to "source" is "OK"
     Then IMAP response to "target" is "OK"
     When IMAP client "source" sends expunge

test/features/bridge/imap/message/search.feature

@@ -1,8 +1,6 @@
 Feature: IMAP search messages
   Background:
     Given there is connected user "user"
-    # Messages are inserted in opposite way to keep increasing ID.
-    # Sequence numbers are then opposite than listed above.
     Given there are messages in mailbox "INBOX" for "user"
       | from               | to         | cc         | subject | read  | starred | deleted | body  |
       | john.doe@email.com | user@pm.me |            | foo     | false | false   | false   | hello |
@@ -34,17 +32,17 @@ Feature: IMAP search messages
   Scenario: Search by Subject
     When IMAP client searches for "SUBJECT foo"
     Then IMAP response is "OK"
-    And IMAP response contains "SEARCH 3[^0-9]*$"
+    And IMAP response contains "SEARCH 1[^0-9]*$"
 
   Scenario: Search by From
     When IMAP client searches for "FROM jane.doe@email.com"
     Then IMAP response is "OK"
-    And IMAP response contains "SEARCH 1 2[^0-9]*$"
+    And IMAP response contains "SEARCH 2 3[^0-9]*$"
 
   Scenario: Search by To
     When IMAP client searches for "TO user@pm.me"
     Then IMAP response is "OK"
-    And IMAP response contains "SEARCH 2 3[^0-9]*$"
+    And IMAP response contains "SEARCH 1 2[^0-9]*$"
 
   Scenario: Search by CC
     When IMAP client searches for "CC name@pm.me"
@@ -64,22 +62,22 @@ Feature: IMAP search messages
   Scenario: Search seen messages
     When IMAP client searches for "SEEN"
     Then IMAP response is "OK"
-    And IMAP response contains "SEARCH 1 2[^0-9]*$"
+    And IMAP response contains "SEARCH 2 3[^0-9]*$"
 
   Scenario: Search unseen messages
     When IMAP client searches for "UNSEEN"
     Then IMAP response is "OK"
-    And IMAP response contains "SEARCH 3[^0-9]*$"
+    And IMAP response contains "SEARCH 1[^0-9]*$"
 
   Scenario: Search deleted messages
     When IMAP client searches for "DELETED"
     Then IMAP response is "OK"
-    And IMAP response contains "SEARCH 1[^0-9]*$"
+    And IMAP response contains "SEARCH 3[^0-9]*$"
 
   Scenario: Search undeleted messages
     When IMAP client searches for "UNDELETED"
     Then IMAP response is "OK"
-    And IMAP response contains "SEARCH 2 3[^0-9]*$"
+    And IMAP response contains "SEARCH 1 2[^0-9]*$"
 
   Scenario: Search recent messages
     When IMAP client searches for "RECENT"
@@ -89,4 +87,4 @@ Feature: IMAP search messages
   Scenario: Search by more criterias
     When IMAP client searches for "SUBJECT baz TO name@pm.me SEEN UNFLAGGED"
     Then IMAP response is "OK"
-    And IMAP response contains "SEARCH 1[^0-9]*$"
+    And IMAP response contains "SEARCH 3[^0-9]*$"

test/features/bridge/imap/message/update.feature

@@ -1,23 +1,21 @@
 Feature: IMAP update messages
   Background:
     Given there is connected user "user"
-    # Messages are inserted in opposite way to keep increasing ID.
-    # Sequence numbers are then opposite than listed above.
     And there are messages in mailbox "INBOX" for "user"
-      | from              | to         | subject | body  | read  | starred | deleted |
+      | id | from              | to         | subject | body  | read  | starred | deleted |
-      | john.doe@mail.com | user@pm.me | foo     | hello | false | false   | false   |
+      | 1  | john.doe@mail.com | user@pm.me | foo     | hello | false | false   | false   |
-      | jane.doe@mail.com | name@pm.me | bar     | world | true  | true    | false   |
+      | 2  | jane.doe@mail.com | name@pm.me | bar     | world | true  | true    | false   |
     And there is IMAP client logged in as "user"
     And there is IMAP client selected in "INBOX"
 
   Scenario: Mark message as read
-    When IMAP client marks message "2" as read
+    When IMAP client marks message seq "1" as read
     Then IMAP response is "OK"
     And message "1" in "INBOX" for "user" is marked as read
     And message "1" in "INBOX" for "user" is marked as unstarred
 
   Scenario: Mark message as unread
-    When IMAP client marks message "1" as unread
+    When IMAP client marks message seq "2" as unread
     Then IMAP response is "OK"
     And message "2" in "INBOX" for "user" is marked as unread
     And message "2" in "INBOX" for "user" is marked as starred
@@ -25,32 +23,32 @@ Feature: IMAP update messages
   Scenario: Mark message as starred
     Then message "1" in "INBOX" for "user" is marked as unread
     And message "1" in "INBOX" for "user" is marked as unstarred
-    When IMAP client marks message "2" as starred
+    When IMAP client marks message seq "1" as starred
     Then IMAP response is "OK"
     And message "1" in "INBOX" for "user" is marked as unread
     And message "1" in "INBOX" for "user" is marked as starred
 
   Scenario: Mark message as unstarred
-    When IMAP client marks message "1" as unstarred
+    When IMAP client marks message seq "2" as unstarred
     Then IMAP response is "OK"
     And message "2" in "INBOX" for "user" is marked as read
     And message "2" in "INBOX" for "user" is marked as unstarred
 
   Scenario: Mark message as read and starred
-    When IMAP client marks message "2" with "\Seen \Flagged"
+    When IMAP client marks message seq "1" with "\Seen \Flagged"
     Then IMAP response is "OK"
     And message "1" in "INBOX" for "user" is marked as read
     And message "1" in "INBOX" for "user" is marked as starred
 
   Scenario: Mark message as read only
-    When IMAP client marks message "1" with "\Seen"
+    When IMAP client marks message seq "2" with "\Seen"
     Then IMAP response is "OK"
     And message "2" in "INBOX" for "user" is marked as read
     # Unstarred because we set flags without \Starred.
     And message "2" in "INBOX" for "user" is marked as unstarred
 
   Scenario: Mark message as spam only
-    When IMAP client marks message "1" with "Junk"
+    When IMAP client marks message seq "2" with "Junk"
     Then IMAP response is "OK"
     # Unread and unstarred because we set flags without \Seen and \Starred.
     And message "1" in "Spam" for "user" is marked as unread
@@ -59,23 +57,23 @@ Feature: IMAP update messages
   Scenario: Mark message as deleted
     # Mark message as Starred so we can check that mark as Deleted is not
     # tempering with Starred flag
-    When IMAP client marks message "1" as starred
+    When IMAP client marks message seq "2" as starred
     Then IMAP response is "OK"
-    When IMAP client marks message "1" as deleted
+    When IMAP client marks message seq "2" as deleted
     Then IMAP response is "OK"
     And message "2" in "INBOX" for "user" is marked as read
     And message "2" in "INBOX" for "user" is marked as starred
     And message "2" in "INBOX" for "user" is marked as deleted
 
   Scenario: Mark message as undeleted
-    When IMAP client marks message "1" as undeleted
+    When IMAP client marks message seq "2" as undeleted
     Then IMAP response is "OK"
     And message "2" in "INBOX" for "user" is marked as read
     And message "2" in "INBOX" for "user" is marked as starred
     And message "2" in "INBOX" for "user" is marked as undeleted
 
   Scenario: Mark message as deleted only
-    When IMAP client marks message "1" with "\Deleted"
+    When IMAP client marks message seq "2" with "\Deleted"
     Then IMAP response is "OK"
     And message "2" in "INBOX" for "user" is marked as unread
     And message "2" in "INBOX" for "user" is marked as unstarred

test/features/bridge/smtp/send/html.feature

@@ -13,7 +13,6 @@ Feature: SMTP sending of HTML messages
       Content-Transfer-Encoding: quoted-printable
       Content-Type: text/html; charset=utf-8
       In-Reply-To: <base64hashOfSomeMessage@protonmail.internalid>
-      References: <base64hashOfSomeConversation@protonmail.internalid> <base64hashOfSomeConversation@protonmail.conversationid>
 
       <html><body>This is body of <b>HTML mail</b> without attachment<body></html>
 

test/imap_actions_messages_test.go

@@ -32,28 +32,28 @@ func IMAPActionsMessagesFeatureContext(s *godog.Suite) {
 	s.Step(`^IMAP client fetches "([^"]*)"$`, imapClientFetches)
 	s.Step(`^IMAP client fetches by UID "([^"]*)"$`, imapClientFetchesByUID)
 	s.Step(`^IMAP client searches for "([^"]*)"$`, imapClientSearchesFor)
-	s.Step(`^IMAP client copies messages "([^"]*)" to "([^"]*)"$`, imapClientCopiesMessagesTo)
+	s.Step(`^IMAP client copies message seq "([^"]*)" to "([^"]*)"$`, imapClientCopiesMessagesTo)
-	s.Step(`^IMAP client moves messages "([^"]*)" to "([^"]*)"$`, imapClientMovesMessagesTo)
+	s.Step(`^IMAP client moves message seq "([^"]*)" to "([^"]*)"$`, imapClientMovesMessagesTo)
-	s.Step(`^IMAP clients "([^"]*)" and "([^"]*)" move message "([^"]*)" of "([^"]*)" from "([^"]*)" to "([^"]*)" by append and delete$`, imapClientsMoveMessageOfUserFromToByAppendAndDelete)
+	s.Step(`^IMAP clients "([^"]*)" and "([^"]*)" move message seq "([^"]*)" of "([^"]*)" from "([^"]*)" to "([^"]*)" by append and delete$`, imapClientsMoveMessageSeqOfUserFromToByAppendAndDelete)
 	s.Step(`^IMAP client imports message to "([^"]*)"$`, imapClientCreatesMessage)
 	s.Step(`^IMAP client imports message to "([^"]*)" with encoding "([^"]*)"$`, imapClientCreatesMessageWithEncoding)
 	s.Step(`^IMAP client creates message "([^"]*)" from "([^"]*)" to "([^"]*)" with body "([^"]*)" in "([^"]*)"$`, imapClientCreatesMessageFromToWithBody)
 	s.Step(`^IMAP client creates message "([^"]*)" from "([^"]*)" to address "([^"]*)" of "([^"]*)" with body "([^"]*)" in "([^"]*)"$`, imapClientCreatesMessageFromToAddressOfUserWithBody)
 	s.Step(`^IMAP client creates message "([^"]*)" from address "([^"]*)" of "([^"]*)" to "([^"]*)" with body "([^"]*)" in "([^"]*)"$`, imapClientCreatesMessageFromAddressOfUserToWithBody)
-	s.Step(`^IMAP client marks message "([^"]*)" with "([^"]*)"$`, imapClientMarksMessageWithFlags)
+	s.Step(`^IMAP client marks message seq "([^"]*)" with "([^"]*)"$`, imapClientMarksMessageSeqWithFlags)
-	s.Step(`^IMAP client "([^"]*)" marks message "([^"]*)" with "([^"]*)"$`, imapClientNamedMarksMessageWithFlags)
+	s.Step(`^IMAP client "([^"]*)" marks message seq "([^"]*)" with "([^"]*)"$`, imapClientNamedMarksMessageSeqWithFlags)
-	s.Step(`^IMAP client marks message "([^"]*)" as read$`, imapClientMarksMessageAsRead)
+	s.Step(`^IMAP client marks message seq "([^"]*)" as read$`, imapClientMarksMessageSeqAsRead)
-	s.Step(`^IMAP client "([^"]*)" marks message "([^"]*)" as read$`, imapClientNamedMarksMessageAsRead)
+	s.Step(`^IMAP client "([^"]*)" marks message seq "([^"]*)" as read$`, imapClientNamedMarksMessageSeqAsRead)
-	s.Step(`^IMAP client marks message "([^"]*)" as unread$`, imapClientMarksMessageAsUnread)
+	s.Step(`^IMAP client marks message seq "([^"]*)" as unread$`, imapClientMarksMessageSeqAsUnread)
-	s.Step(`^IMAP client "([^"]*)" marks message "([^"]*)" as unread$`, imapClientNamedMarksMessageAsUnread)
+	s.Step(`^IMAP client "([^"]*)" marks message seq "([^"]*)" as unread$`, imapClientNamedMarksMessageSeqAsUnread)
-	s.Step(`^IMAP client marks message "([^"]*)" as starred$`, imapClientMarksMessageAsStarred)
+	s.Step(`^IMAP client marks message seq "([^"]*)" as starred$`, imapClientMarksMessageSeqAsStarred)
-	s.Step(`^IMAP client "([^"]*)" marks message "([^"]*)" as starred$`, imapClientNamedMarksMessageAsStarred)
+	s.Step(`^IMAP client "([^"]*)" marks message seq "([^"]*)" as starred$`, imapClientNamedMarksMessageSeqAsStarred)
-	s.Step(`^IMAP client marks message "([^"]*)" as unstarred$`, imapClientMarksMessageAsUnstarred)
+	s.Step(`^IMAP client marks message seq "([^"]*)" as unstarred$`, imapClientMarksMessageSeqAsUnstarred)
-	s.Step(`^IMAP client "([^"]*)" marks message "([^"]*)" as unstarred$`, imapClientNamedMarksMessageAsUnstarred)
+	s.Step(`^IMAP client "([^"]*)" marks message seq "([^"]*)" as unstarred$`, imapClientNamedMarksMessageSeqAsUnstarred)
-	s.Step(`^IMAP client marks message "([^"]*)" as deleted$`, imapClientMarksMessageAsDeleted)
+	s.Step(`^IMAP client marks message seq "([^"]*)" as deleted$`, imapClientMarksMessageSeqAsDeleted)
-	s.Step(`^IMAP client "([^"]*)" marks message "([^"]*)" as deleted$`, imapClientNamedMarksMessageAsDeleted)
+	s.Step(`^IMAP client "([^"]*)" marks message seq "([^"]*)" as deleted$`, imapClientNamedMarksMessageSeqAsDeleted)
-	s.Step(`^IMAP client marks message "([^"]*)" as undeleted$`, imapClientMarksMessageAsUndeleted)
+	s.Step(`^IMAP client marks message seq "([^"]*)" as undeleted$`, imapClientMarksMessageSeqAsUndeleted)
-	s.Step(`^IMAP client "([^"]*)" marks message "([^"]*)" as undeleted$`, imapClientNamedMarksMessageAsUndeleted)
+	s.Step(`^IMAP client "([^"]*)" marks message seq "([^"]*)" as undeleted$`, imapClientNamedMarksMessageSeqAsUndeleted)
 	s.Step(`^IMAP client starts IDLE-ing$`, imapClientStartsIDLEing)
 	s.Step(`^IMAP client "([^"]*)" starts IDLE-ing$`, imapClientNamedStartsIDLEing)
 	s.Step(`^IMAP client sends expunge$`, imapClientExpunge)
@@ -84,19 +84,19 @@ func imapClientSearchesFor(query string) error {
 	return nil
 }
 
-func imapClientCopiesMessagesTo(messageRange, newMailboxName string) error {
+func imapClientCopiesMessagesTo(messageSeq, newMailboxName string) error {
-	res := ctx.GetIMAPClient("imap").Copy(messageRange, newMailboxName)
+	res := ctx.GetIMAPClient("imap").Copy(messageSeq, newMailboxName)
 	ctx.SetIMAPLastResponse("imap", res)
 	return nil
 }
 
-func imapClientMovesMessagesTo(messageRange, newMailboxName string) error {
+func imapClientMovesMessagesTo(messageSeq, newMailboxName string) error {
-	res := ctx.GetIMAPClient("imap").Move(messageRange, newMailboxName)
+	res := ctx.GetIMAPClient("imap").Move(messageSeq, newMailboxName)
 	ctx.SetIMAPLastResponse("imap", res)
 	return nil
 }
 
-func imapClientsMoveMessageOfUserFromToByAppendAndDelete(sourceIMAPClient, targetIMAPClient, messageUID, bddUserID, sourceMailboxName, targetMailboxName string) error {
+func imapClientsMoveMessageSeqOfUserFromToByAppendAndDelete(sourceIMAPClient, targetIMAPClient, messageSeq, bddUserID, sourceMailboxName, targetMailboxName string) error {
 	account := ctx.GetTestAccount(bddUserID)
 	if account == nil {
 		return godog.ErrPending
@@ -105,7 +105,7 @@ func imapClientsMoveMessageOfUserFromToByAppendAndDelete(sourceIMAPClient, targe
 	if err != nil {
 		return internalError(err, "getting store mailbox")
 	}
-	uid, err := strconv.ParseUint(messageUID, 10, 32)
+	uid, err := strconv.ParseUint(messageSeq, 10, 32)
 	if err != nil {
 		return internalError(err, "parsing message UID")
 	}
@@ -136,7 +136,7 @@ func imapClientsMoveMessageOfUserFromToByAppendAndDelete(sourceIMAPClient, targe
 
 	go func() {
 		defer wg.Done()
-		_ = imapClientNamedMarksMessageAsDeleted(sourceIMAPClient, messageUID)
+		_ = imapClientNamedMarksMessageSeqAsDeleted(sourceIMAPClient, messageSeq)
 	}()
 
 	wg.Wait()
@@ -195,72 +195,72 @@ func imapClientCreatesMessageFromAddressOfUserToWithBody(subject, bddAddressID,
 	return imapClientCreatesMessageFromToWithBody(subject, account.Address(), to, body, mailboxName)
 }
 
-func imapClientMarksMessageWithFlags(messageRange, flags string) error {
+func imapClientMarksMessageSeqWithFlags(messageSeq, flags string) error {
-	return imapClientNamedMarksMessageWithFlags("imap", messageRange, flags)
+	return imapClientNamedMarksMessageSeqWithFlags("imap", messageSeq, flags)
 }
 
-func imapClientNamedMarksMessageWithFlags(imapClient, messageRange, flags string) error {
+func imapClientNamedMarksMessageSeqWithFlags(imapClient, messageSeq, flags string) error {
-	res := ctx.GetIMAPClient(imapClient).SetFlags(messageRange, flags)
+	res := ctx.GetIMAPClient(imapClient).SetFlags(messageSeq, flags)
 	ctx.SetIMAPLastResponse(imapClient, res)
 	return nil
 }
 
-func imapClientMarksMessageAsRead(messageRange string) error {
+func imapClientMarksMessageSeqAsRead(messageSeq string) error {
-	return imapClientNamedMarksMessageAsRead("imap", messageRange)
+	return imapClientNamedMarksMessageSeqAsRead("imap", messageSeq)
 }
 
-func imapClientNamedMarksMessageAsRead(imapClient, messageRange string) error {
+func imapClientNamedMarksMessageSeqAsRead(imapClient, messageSeq string) error {
-	res := ctx.GetIMAPClient(imapClient).MarkAsRead(messageRange)
+	res := ctx.GetIMAPClient(imapClient).MarkAsRead(messageSeq)
 	ctx.SetIMAPLastResponse(imapClient, res)
 	return nil
 }
 
-func imapClientMarksMessageAsUnread(messageRange string) error {
+func imapClientMarksMessageSeqAsUnread(messageSeq string) error {
-	return imapClientNamedMarksMessageAsUnread("imap", messageRange)
+	return imapClientNamedMarksMessageSeqAsUnread("imap", messageSeq)
 }
 
-func imapClientNamedMarksMessageAsUnread(imapClient, messageRange string) error {
+func imapClientNamedMarksMessageSeqAsUnread(imapClient, messageSeq string) error {
-	res := ctx.GetIMAPClient(imapClient).MarkAsUnread(messageRange)
+	res := ctx.GetIMAPClient(imapClient).MarkAsUnread(messageSeq)
 	ctx.SetIMAPLastResponse(imapClient, res)
 	return nil
 }
 
-func imapClientMarksMessageAsStarred(messageRange string) error {
+func imapClientMarksMessageSeqAsStarred(messageSeq string) error {
-	return imapClientNamedMarksMessageAsStarred("imap", messageRange)
+	return imapClientNamedMarksMessageSeqAsStarred("imap", messageSeq)
 }
 
-func imapClientNamedMarksMessageAsStarred(imapClient, messageRange string) error {
+func imapClientNamedMarksMessageSeqAsStarred(imapClient, messageSeq string) error {
-	res := ctx.GetIMAPClient(imapClient).MarkAsStarred(messageRange)
+	res := ctx.GetIMAPClient(imapClient).MarkAsStarred(messageSeq)
 	ctx.SetIMAPLastResponse(imapClient, res)
 	return nil
 }
 
-func imapClientMarksMessageAsUnstarred(messageRange string) error {
+func imapClientMarksMessageSeqAsUnstarred(messageSeq string) error {
-	return imapClientNamedMarksMessageAsUnstarred("imap", messageRange)
+	return imapClientNamedMarksMessageSeqAsUnstarred("imap", messageSeq)
 }
 
-func imapClientNamedMarksMessageAsUnstarred(imapClient, messageRange string) error {
+func imapClientNamedMarksMessageSeqAsUnstarred(imapClient, messageSeq string) error {
-	res := ctx.GetIMAPClient(imapClient).MarkAsUnstarred(messageRange)
+	res := ctx.GetIMAPClient(imapClient).MarkAsUnstarred(messageSeq)
 	ctx.SetIMAPLastResponse(imapClient, res)
 	return nil
 }
 
-func imapClientMarksMessageAsDeleted(messageRange string) error {
+func imapClientMarksMessageSeqAsDeleted(messageSeq string) error {
-	return imapClientNamedMarksMessageAsDeleted("imap", messageRange)
+	return imapClientNamedMarksMessageSeqAsDeleted("imap", messageSeq)
 }
 
-func imapClientNamedMarksMessageAsDeleted(imapClient, messageRange string) error {
+func imapClientNamedMarksMessageSeqAsDeleted(imapClient, messageSeq string) error {
-	res := ctx.GetIMAPClient(imapClient).MarkAsDeleted(messageRange)
+	res := ctx.GetIMAPClient(imapClient).MarkAsDeleted(messageSeq)
 	ctx.SetIMAPLastResponse(imapClient, res)
 	return nil
 }
 
-func imapClientMarksMessageAsUndeleted(messageRange string) error {
+func imapClientMarksMessageSeqAsUndeleted(messageSeq string) error {
-	return imapClientNamedMarksMessageAsUndeleted("imap", messageRange)
+	return imapClientNamedMarksMessageSeqAsUndeleted("imap", messageSeq)
 }
 
-func imapClientNamedMarksMessageAsUndeleted(imapClient, messageRange string) error {
+func imapClientNamedMarksMessageSeqAsUndeleted(imapClient, messageSeq string) error {
-	res := ctx.GetIMAPClient(imapClient).MarkAsUndeleted(messageRange)
+	res := ctx.GetIMAPClient(imapClient).MarkAsUndeleted(messageSeq)
 	ctx.SetIMAPLastResponse(imapClient, res)
 	return nil
 }

test/imap_checks_test.go

@@ -32,11 +32,11 @@ func IMAPChecksFeatureContext(s *godog.Suite) {
 	s.Step(`^IMAP response to "([^"]*)" contains "([^"]*)"$`, imapResponseNamedContains)
 	s.Step(`^IMAP response has (\d+) message(?:s)?$`, imapResponseHasNumberOfMessages)
 	s.Step(`^IMAP response to "([^"]*)" has (\d+) message(?:s)?$`, imapResponseNamedHasNumberOfMessages)
-	s.Step(`^IMAP client receives update marking message "([^"]*)" as read within (\d+) seconds$`, imapClientReceivesUpdateMarkingMessagesAsReadWithin)
+	s.Step(`^IMAP client receives update marking message seq "([^"]*)" as read within (\d+) seconds$`, imapClientReceivesUpdateMarkingMessageSeqAsReadWithin)
-	s.Step(`^IMAP client "([^"]*)" receives update marking message "([^"]*)" as read within (\d+) seconds$`, imapClientNamedReceivesUpdateMarkingMessagesAsReadWithin)
+	s.Step(`^IMAP client "([^"]*)" receives update marking message seq "([^"]*)" as read within (\d+) seconds$`, imapClientNamedReceivesUpdateMarkingMessageSeqAsReadWithin)
-	s.Step(`^IMAP client receives update marking message "([^"]*)" as unread within (\d+) seconds$`, imapClientReceivesUpdateMarkingMessagesAsUnreadWithin)
+	s.Step(`^IMAP client receives update marking message seq "([^"]*)" as unread within (\d+) seconds$`, imapClientReceivesUpdateMarkingMessageSeqAsUnreadWithin)
-	s.Step(`^IMAP client "([^"]*)" receives update marking message "([^"]*)" as unread within (\d+) seconds$`, imapClientNamedReceivesUpdateMarkingMessagesAsUnreadWithin)
+	s.Step(`^IMAP client "([^"]*)" receives update marking message seq "([^"]*)" as unread within (\d+) seconds$`, imapClientNamedReceivesUpdateMarkingMessageSeqAsUnreadWithin)
-	s.Step(`^IMAP client "([^"]*)" does not receive update for message "([^"]*)" within (\d+) seconds$`, imapClientDoesNotReceiveUpdateForMessageWithin)
+	s.Step(`^IMAP client "([^"]*)" does not receive update for message seq "([^"]*)" within (\d+) seconds$`, imapClientDoesNotReceiveUpdateForMessageSeqWithin)
 }
 
 func imapResponseIs(expectedResponse string) error {
@@ -73,26 +73,26 @@ func imapResponseNamedHasNumberOfMessages(clientID string, expectedCount int) er
 	return ctx.GetTestingError()
 }
 
-func imapClientReceivesUpdateMarkingMessagesAsReadWithin(messageUIDs string, seconds int) error {
+func imapClientReceivesUpdateMarkingMessageSeqAsReadWithin(messageSeq string, seconds int) error {
-	return imapClientNamedReceivesUpdateMarkingMessagesAsReadWithin("imap", messageUIDs, seconds)
+	return imapClientNamedReceivesUpdateMarkingMessageSeqAsReadWithin("imap", messageSeq, seconds)
 }
 
-func imapClientNamedReceivesUpdateMarkingMessagesAsReadWithin(clientID, messageUIDs string, seconds int) error {
+func imapClientNamedReceivesUpdateMarkingMessageSeqAsReadWithin(clientID, messageSeq string, seconds int) error {
 	regexps := []string{}
-	iterateOverSeqSet(messageUIDs, func(messageUID string) {
+	iterateOverSeqSet(messageSeq, func(messageUID string) {
 		regexps = append(regexps, `FETCH \(FLAGS \(.*\\Seen.*\) UID `+messageUID)
 	})
 	ctx.GetIMAPLastResponse(clientID).WaitForSections(time.Duration(seconds)*time.Second, regexps...)
 	return ctx.GetTestingError()
 }
 
-func imapClientReceivesUpdateMarkingMessagesAsUnreadWithin(messageUIDs string, seconds int) error {
+func imapClientReceivesUpdateMarkingMessageSeqAsUnreadWithin(messageSeq string, seconds int) error {
-	return imapClientNamedReceivesUpdateMarkingMessagesAsUnreadWithin("imap", messageUIDs, seconds)
+	return imapClientNamedReceivesUpdateMarkingMessageSeqAsUnreadWithin("imap", messageSeq, seconds)
 }
 
-func imapClientNamedReceivesUpdateMarkingMessagesAsUnreadWithin(clientID, messageUIDs string, seconds int) error {
+func imapClientNamedReceivesUpdateMarkingMessageSeqAsUnreadWithin(clientID, messageSeq string, seconds int) error {
 	regexps := []string{}
-	iterateOverSeqSet(messageUIDs, func(messageUID string) {
+	iterateOverSeqSet(messageSeq, func(messageUID string) {
 		// Golang does not support negative look ahead. Following complex regexp checks \Seen is not there.
 		regexps = append(regexps, `FETCH \(FLAGS \(([^S]|S[^e]|Se[^e]|See[^n])*\) UID `+messageUID)
 	})
@@ -100,9 +100,9 @@ func imapClientNamedReceivesUpdateMarkingMessagesAsUnreadWithin(clientID, messag
 	return ctx.GetTestingError()
 }
 
-func imapClientDoesNotReceiveUpdateForMessageWithin(clientID, messageUIDs string, seconds int) error {
+func imapClientDoesNotReceiveUpdateForMessageSeqWithin(clientID, messageSeq string, seconds int) error {
 	regexps := []string{}
-	iterateOverSeqSet(messageUIDs, func(messageUID string) {
+	iterateOverSeqSet(messageSeq, func(messageUID string) {
 		regexps = append(regexps, `FETCH.*UID `+messageUID)
 	})
 	ctx.GetIMAPLastResponse(clientID).WaitForNotSections(time.Duration(seconds)*time.Second, regexps...)

test/liveapi/messages.go

@@ -23,7 +23,6 @@ import (
 	"io"
 	"mime/multipart"
 	"net/http"
-	"strconv"
 
 	messageUtils "github.com/ProtonMail/proton-bridge/pkg/message"
 	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
@@ -127,14 +126,6 @@ func buildMessageBody(message *pmapi.Message, body *bytes.Buffer) error {
 	return nil
 }
 
-func (ctl *Controller) GetMessageID(username, messageIndex string) string {
-	idx, err := strconv.Atoi(messageIndex)
-	if err != nil {
-		panic(fmt.Sprintf("message index %s not found", messageIndex))
-	}
-	return ctl.messageIDsByUsername[username][idx-1]
-}
-
 func (ctl *Controller) GetMessages(username, labelID string) ([]*pmapi.Message, error) {
 	client, ok := ctl.pmapiByUsername[username]
 	if !ok {

test/store_checks_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/ProtonMail/proton-bridge/test/accounts"
 	"github.com/cucumber/godog"
 	"github.com/cucumber/godog/gherkin"
+	"github.com/hashicorp/go-multierror"
 )
 
 func StoreChecksFeatureContext(s *godog.Suite) {
@@ -202,6 +203,14 @@ func messagesContainsMessageRow(account *accounts.TestAccount, allMessages []int
 		matches := true
 		for n, cell := range row.Cells {
 			switch head[n].Value {
+			case "id":
+				id, err := ctx.GetAPIMessageID(account.Username(), cell.Value)
+				if err != nil {
+					return false, fmt.Errorf("unknown BDD message ID: %s", cell.Value)
+				}
+				if message.ID != id {
+					matches = false
+				}
 			case "from": //nolint[goconst]
 				address := ctx.EnsureAddress(account.Username(), cell.Value)
 				if !areAddressesSame(message.Sender.Address, address) {
@@ -278,8 +287,8 @@ func areAddressesSame(first, second string) bool {
 	return firstAddress.Address == secondAddress.Address
 }
 
-func messagesInMailboxForUserIsMarkedAsRead(messageIDs, mailboxName, bddUserID string) error {
+func messagesInMailboxForUserIsMarkedAsRead(bddMessageIDs, mailboxName, bddUserID string) error {
-	return checkMessages(bddUserID, mailboxName, messageIDs, func(message *store.Message) error {
+	return checkMessages(bddUserID, mailboxName, bddMessageIDs, func(message *store.Message) error {
 		if message.Message().Unread == 0 {
 			return nil
 		}
@@ -287,8 +296,8 @@ func messagesInMailboxForUserIsMarkedAsRead(messageIDs, mailboxName, bddUserID s
 	})
 }
 
-func messagesInMailboxForUserIsMarkedAsUnread(messageIDs, mailboxName, bddUserID string) error {
+func messagesInMailboxForUserIsMarkedAsUnread(bddMessageIDs, mailboxName, bddUserID string) error {
-	return checkMessages(bddUserID, mailboxName, messageIDs, func(message *store.Message) error {
+	return checkMessages(bddUserID, mailboxName, bddMessageIDs, func(message *store.Message) error {
 		if message.Message().Unread == 1 {
 			return nil
 		}
@@ -296,8 +305,8 @@ func messagesInMailboxForUserIsMarkedAsUnread(messageIDs, mailboxName, bddUserID
 	})
 }
 
-func messagesInMailboxForUserIsMarkedAsStarred(messageIDs, mailboxName, bddUserID string) error {
+func messagesInMailboxForUserIsMarkedAsStarred(bddMessageIDs, mailboxName, bddUserID string) error {
-	return checkMessages(bddUserID, mailboxName, messageIDs, func(message *store.Message) error {
+	return checkMessages(bddUserID, mailboxName, bddMessageIDs, func(message *store.Message) error {
 		if hasItem(message.Message().LabelIDs, "10") {
 			return nil
 		}
@@ -305,8 +314,8 @@ func messagesInMailboxForUserIsMarkedAsStarred(messageIDs, mailboxName, bddUserI
 	})
 }
 
-func messagesInMailboxForUserIsMarkedAsUnstarred(messageIDs, mailboxName, bddUserID string) error {
+func messagesInMailboxForUserIsMarkedAsUnstarred(bddMessageIDs, mailboxName, bddUserID string) error {
-	return checkMessages(bddUserID, mailboxName, messageIDs, func(message *store.Message) error {
+	return checkMessages(bddUserID, mailboxName, bddMessageIDs, func(message *store.Message) error {
 		if !hasItem(message.Message().LabelIDs, "10") {
 			return nil
 		}
@@ -314,8 +323,8 @@ func messagesInMailboxForUserIsMarkedAsUnstarred(messageIDs, mailboxName, bddUse
 	})
 }
 
-func messagesInMailboxForUserIsMarkedAsDeleted(messageIDs, mailboxName, bddUserID string) error {
+func messagesInMailboxForUserIsMarkedAsDeleted(bddMessageIDs, mailboxName, bddUserID string) error {
-	return checkMessages(bddUserID, mailboxName, messageIDs, func(message *store.Message) error {
+	return checkMessages(bddUserID, mailboxName, bddMessageIDs, func(message *store.Message) error {
 		if message.IsMarkedDeleted() {
 			return nil
 		}
@@ -323,8 +332,8 @@ func messagesInMailboxForUserIsMarkedAsDeleted(messageIDs, mailboxName, bddUserI
 	})
 }
 
-func messagesInMailboxForUserIsMarkedAsUndeleted(messageIDs, mailboxName, bddUserID string) error {
+func messagesInMailboxForUserIsMarkedAsUndeleted(bddMessageIDs, mailboxName, bddUserID string) error {
-	return checkMessages(bddUserID, mailboxName, messageIDs, func(message *store.Message) error {
+	return checkMessages(bddUserID, mailboxName, bddMessageIDs, func(message *store.Message) error {
 		if !message.IsMarkedDeleted() {
 			return nil
 		}
@@ -332,14 +341,14 @@ func messagesInMailboxForUserIsMarkedAsUndeleted(messageIDs, mailboxName, bddUse
 	})
 }
 
-func checkMessages(bddUserID, mailboxName, messageIDs string, callback func(*store.Message) error) error {
+func checkMessages(bddUserID, mailboxName, bddMessageIDs string, callback func(*store.Message) error) error {
 	account := ctx.GetTestAccount(bddUserID)
 	if account == nil {
 		return godog.ErrPending
 	}
-	messages, err := getMessages(account.Username(), account.AddressID(), mailboxName, messageIDs)
+	messages, err := getMessages(account.Username(), account.AddressID(), mailboxName, bddMessageIDs)
 	if err != nil {
-		return internalError(err, "getting messages %s", messageIDs)
+		return internalError(err, "getting messages %s", bddMessageIDs)
 	}
 	for _, message := range messages {
 		if err := callback(message); err != nil {
@@ -349,18 +358,23 @@ func checkMessages(bddUserID, mailboxName, messageIDs string, callback func(*sto
 	return nil
 }
 
-func getMessages(username, addressID, mailboxName, messageIDs string) ([]*store.Message, error) {
+func getMessages(username, addressID, mailboxName, bddMessageIDs string) ([]*store.Message, error) {
 	msgs := []*store.Message{}
-	var msg *store.Message
+	var allErrs *multierror.Error
-	var err error
+	iterateOverSeqSet(bddMessageIDs, func(bddMessageID string) {
-	iterateOverSeqSet(messageIDs, func(messageID string) {
+		messageID, err := ctx.GetAPIMessageID(username, bddMessageID)
-		messageID = ctx.GetPMAPIController().GetMessageID(username, messageID)
+		if err != nil {
-		msg, err = getMessage(username, addressID, mailboxName, messageID)
+			allErrs = multierror.Append(allErrs, err)
-		if err == nil {
+			return
-			msgs = append(msgs, msg)
 		}
+		msg, err := getMessage(username, addressID, mailboxName, messageID)
+		if err != nil {
+			allErrs = multierror.Append(allErrs, err)
+			return
+		}
+		msgs = append(msgs, msg)
 	})
-	return msgs, err
+	return msgs, allErrs.ErrorOrNil()
 }
 
 func getMessage(username, addressID, mailboxName, messageID string) (*store.Message, error) {

test/store_setup_test.go

@@ -87,8 +87,11 @@ func thereAreMessagesInMailboxesForAddressOfUser(mailboxNames, bddAddressID, bdd
 
 	var markMessageIDsDeleted []string
 
+	// Inserting in the opposite order becase sync is done from newest to oldest.
+	// The goal is to have simply predictable IMAP sequence numbers if possible.
 	head := messages.Rows[0].Cells
-	for _, row := range messages.Rows[1:] {
+	for i := len(messages.Rows) - 1; i > 0; i-- {
+		row := messages.Rows[i]
 		message := &pmapi.Message{
 			MIMEType:  "text/plain",
 			LabelIDs:  labelIDs,
@@ -99,10 +102,13 @@ func thereAreMessagesInMailboxesForAddressOfUser(mailboxNames, bddAddressID, bdd
 			message.Flags |= pmapi.FlagSent
 		}
 
+		bddMessageID := ""
 		hasDeletedFlag := false
 
 		for n, cell := range row.Cells {
 			switch head[n].Value {
+			case "id":
+				bddMessageID = cell.Value
 			case "from":
 				message.Sender = &mail.Address{
 					Address: ctx.EnsureAddress(account.Username(), cell.Value),
@@ -147,6 +153,7 @@ func thereAreMessagesInMailboxesForAddressOfUser(mailboxNames, bddAddressID, bdd
 		if err != nil {
 			return internalError(err, "adding message")
 		}
+		ctx.PairMessageID(account.Username(), bddMessageID, lastMessageID)
 
 		if hasDeletedFlag {
 			markMessageIDsDeleted = append(markMessageIDsDeleted, lastMessageID)
@@ -223,7 +230,7 @@ func thereAreSomeMessagesInMailboxesForAddressOfUser(numberOfMessages int, mailb
 		if err != nil {
 			return internalError(err, "getting labels %s for %s", mailboxNames, account.Username())
 		}
-		_, err = ctx.GetPMAPIController().AddUserMessage(account.Username(), &pmapi.Message{
+		lastMessageID, err := ctx.GetPMAPIController().AddUserMessage(account.Username(), &pmapi.Message{
 			MIMEType:  "text/plain",
 			LabelIDs:  labelIDs,
 			AddressID: account.AddressID(),
@@ -234,6 +241,11 @@ func thereAreSomeMessagesInMailboxesForAddressOfUser(numberOfMessages int, mailb
 		if err != nil {
 			return internalError(err, "adding message")
 		}
+
+		// Generating IDs in the opposite order becase sync is done from newest to oldest.
+		// The goal is to have simply predictable IMAP sequence numbers if possible.
+		bddMessageID := fmt.Sprintf("%d", numberOfMessages-i+1)
+		ctx.PairMessageID(account.Username(), bddMessageID, lastMessageID)
 	}
 	return internalError(ctx.WaitForSync(account.Username()), "waiting for sync")
 }

test/transfer_checks_test.go

@@ -48,7 +48,10 @@ func TransferChecksFeatureContext(s *godog.Suite) {
 func progressFinishedWith(wantResponse string) error {
 	progress := ctx.GetTransferProgress()
 	// Wait till transport is finished.
-	for range progress.GetUpdateChannel() {
+	updateCh := progress.GetUpdateChannel()
+	if updateCh != nil {
+		for range updateCh {
+		}
 	}
 
 	err := progress.GetFatalError()