Import-Export Elbe 1.2.1

• Further improvements to address and date parsing
• Better handling and displaying of skipped messages
• Improved error reporting

Changelog.md

@@ -1,8 +1,30 @@
-# ProtonMail Bridge Changelog
+# ProtonMail Bridge and Import-Export app Changelog
 
 Changelog [format](http://keepachangelog.com/en/1.0.0/)
 
-## Unreleased
+## [IE 1.2.1] Elbe
+
+### Added
+* GODT-799 Skipped messages do not change total counts but shows as separate number.
+
+## Fixed
+* GODT-799 Fix skipping unwanted folders importing from mbox files.
+* GODT-769 Close connection before deleting labels to prevent panics accessing deleted bucket.
+
+### Removed
+* GODT-766 Remove GUI popup for IMAP TLS error.
+
+
+## [Bridge 1.5.0] Golden Gate
+### Changed
+* Updated go-mbox dependency back to upstream.
+
+### Fixed
+* GODT-847 Waiting for unilateral update during deleting the message.
+* GODT-849 Show in error counts in the end also lost messages.
+* GODT-835 Do not include conversation ID in references to show properly conversation threads in clients.
+* GODT-685 Improve deb packaging regarding dejavu font
+
 
 ## [IE 1.2.0] Elbe
 
@@ -26,6 +48,11 @@ Changelog [format](http://keepachangelog.com/en/1.0.0/)
 * GODT-827 Do not spam sentry with bad ID by integration test.
 * GODT-700 Fix UTF-7 incompatibility.
 * GODT-837 Fix flaky TestFailUnpauseAndStops.
+* GODT-782 Don't use TLS pinning when checking connectivity status.
+
+### Changed
+* TLS pins conform to official list.
+
 
 ## [Bridge 1.4.5] Forth
 

Makefile

@@ -10,8 +10,8 @@ TARGET_OS?=${GOOS}
 .PHONY: build build-ie build-nogui build-ie-nogui check-has-go
 
 # Keep version hardcoded so app build works also without Git repository.
-BRIDGE_APP_VERSION?=1.4.5-git
+BRIDGE_APP_VERSION?=1.5.0-git
-IE_APP_VERSION?=1.2.0-git
+IE_APP_VERSION?=1.2.1-git
 APP_VERSION:=${BRIDGE_APP_VERSION}
 SRC_ICO:=logo.ico
 SRC_ICNS:=Bridge.icns
@@ -57,7 +57,6 @@ ifeq "${TARGET_CMD}" "Import-Export"
     TGZ_TARGET:=ie_${TARGET_OS}_${REVISION}.tgz
 endif
 
-
 build: ${TGZ_TARGET}
 build-ie:
 	TARGET_CMD=Import-Export $(MAKE) build
@@ -265,7 +264,6 @@ run-ie-qt:
 run-ie-nogui:
 	TARGET_CMD=Import-Export $(MAKE) run-nogui
 
-
 clean-frontend-qt:
 	$(MAKE) -C internal/frontend/qt -f Makefile.local clean
 clean-frontend-qt-ie:

go.mod

@@ -18,6 +18,7 @@ require (
 	github.com/ProtonMail/go-apple-mobileconfig v0.0.0-20160701194735-7ea9927a11f6
 	github.com/ProtonMail/go-autostart v0.0.0-20181114175602-c5272053443a
 	github.com/ProtonMail/go-imap-id v0.0.0-20190926060100-f94a56b9ecde
+	github.com/ProtonMail/go-rfc5322 v0.2.0
 	github.com/ProtonMail/go-vcard v0.0.0-20180326232728-33aaa0a0c8a5
 	github.com/ProtonMail/gopenpgp/v2 v2.0.1
 	github.com/PuerkitoBio/goquery v1.5.1
@@ -35,7 +36,7 @@ require (
 	github.com/emersion/go-imap-quota v0.0.0-20200423100218-dcfd1b7d2b41
 	github.com/emersion/go-imap-specialuse v0.0.0-20200722111535-598ff00e4075
 	github.com/emersion/go-imap-unselect v0.0.0-20171113212723-b985794e5f26
-	github.com/emersion/go-mbox v1.0.0
+	github.com/emersion/go-mbox v1.0.2
 	github.com/emersion/go-message v0.12.1-0.20200903165315-e1abe21f389a
 	github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21
 	github.com/emersion/go-textwrapper v0.0.0-20160606182133-d0e65e56babe
@@ -59,7 +60,7 @@ require (
 	github.com/nsf/jsondiff v0.0.0-20200515183724-f29ed568f4ce
 	github.com/olekukonko/tablewriter v0.0.4 // indirect
 	github.com/pkg/errors v0.9.1
-	github.com/sirupsen/logrus v1.6.0
+	github.com/sirupsen/logrus v1.7.0
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
 	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
 	github.com/stretchr/testify v1.6.1
@@ -74,8 +75,7 @@ require (
 
 replace (
 	github.com/docker/docker-credential-helpers => github.com/ProtonMail/docker-credential-helpers v1.1.0
-	github.com/emersion/go-imap => github.com/ProtonMail/go-imap v0.0.0-20201016095853-a7520cc904d3
+	github.com/emersion/go-imap => github.com/ProtonMail/go-imap v0.0.0-20201102134601-418cd74e9474
-	github.com/emersion/go-mbox => github.com/ProtonMail/mbox v0.0.0-20200918064939-909a18c9af45
 	github.com/emersion/go-smtp => github.com/ProtonMail/go-smtp v0.0.0-20181206232543-8261df20d309
 	github.com/jameskeane/bcrypt => github.com/ProtonMail/bcrypt v0.0.0-20170924085257-7509ea014998
 	golang.org/x/crypto => github.com/ProtonMail/crypto v0.0.0-20200818122824-ed5d25e28db8

go.sum

@@ -15,20 +15,20 @@ github.com/ProtonMail/go-apple-mobileconfig v0.0.0-20160701194735-7ea9927a11f6 h
 github.com/ProtonMail/go-apple-mobileconfig v0.0.0-20160701194735-7ea9927a11f6/go.mod h1:EtDfBMIDWmVe4viZCuBTEfe3OIIo0ghbpOaAZVO+hVg=
 github.com/ProtonMail/go-autostart v0.0.0-20181114175602-c5272053443a h1:fXK2KsfnkBV9Nh+9SKzHchYjuE9s0vI20JG1mbtEAcc=
 github.com/ProtonMail/go-autostart v0.0.0-20181114175602-c5272053443a/go.mod h1:oTGdE7/DlWIr23G0IKW3OXK9wZ5Hw1GGiaJFccTvZi4=
-github.com/ProtonMail/go-imap v0.0.0-20201016095853-a7520cc904d3 h1:Jvv9t3rSg/ID3Fh+uYsxgmvNI9fYnlab4vtBsbPtmq8=
+github.com/ProtonMail/go-imap v0.0.0-20201102134601-418cd74e9474 h1:D0RwDtkBw0Gt7hmbb1ivdEulplJAwu1i2jzh4HM45fo=
-github.com/ProtonMail/go-imap v0.0.0-20201016095853-a7520cc904d3/go.mod h1:yKASt+C3ZiDAiCSssxg9caIckWF/JG7ZQTO7GAmvicU=
+github.com/ProtonMail/go-imap v0.0.0-20201102134601-418cd74e9474/go.mod h1:yKASt+C3ZiDAiCSssxg9caIckWF/JG7ZQTO7GAmvicU=
 github.com/ProtonMail/go-imap-id v0.0.0-20190926060100-f94a56b9ecde h1:5koQozTDELymYOyFbQ/VSubexAEXzDR8qGM5mO8GRdw=
 github.com/ProtonMail/go-imap-id v0.0.0-20190926060100-f94a56b9ecde/go.mod h1:795VPXcRUIQ9JyMNHP4el582VokQfippgjkQP3Gk0r0=
 github.com/ProtonMail/go-mime v0.0.0-20190923161245-9b5a4261663a h1:W6RrgN/sTxg1msqzFFb+G80MFmpjMw61IU+slm+wln4=
 github.com/ProtonMail/go-mime v0.0.0-20190923161245-9b5a4261663a/go.mod h1:NYt+V3/4rEeDuaev/zw1zCq8uqVEuPHzDPo3OZrlGJ4=
+github.com/ProtonMail/go-rfc5322 v0.2.0 h1:tndoDGFtiCvESta9KLUeMksojz8qf76PefnkoQ+fqeg=
+github.com/ProtonMail/go-rfc5322 v0.2.0/go.mod h1:mzZWlMWnQJuYLL7JpzuPF5+FimV2lZ9f0jeq24kJjpU=
 github.com/ProtonMail/go-smtp v0.0.0-20181206232543-8261df20d309 h1:2pzfKjhBjSnw3BgmfTYRFQr1rFGxhfhUY0KKkg+RYxE=
 github.com/ProtonMail/go-smtp v0.0.0-20181206232543-8261df20d309/go.mod h1:6UoBvDAMA/cTBwS3Y7tGpKnY5RH1F1uYHschT6eqAkI=
 github.com/ProtonMail/go-vcard v0.0.0-20180326232728-33aaa0a0c8a5 h1:Uga1DHFN4GUxuDQr0F71tpi8I9HqPIlZodZAI1lR6VQ=
 github.com/ProtonMail/go-vcard v0.0.0-20180326232728-33aaa0a0c8a5/go.mod h1:oeP9CMN+ajWp5jKp1kue5daJNwMMxLF+ujPaUIoJWlA=
 github.com/ProtonMail/gopenpgp/v2 v2.0.1 h1:x0uvDhry5WzoHeJO4J3dgMLhG4Z9PeBJ2O+sDOY0LcU=
 github.com/ProtonMail/gopenpgp/v2 v2.0.1/go.mod h1:wQQCJo7DURO6S9VwH+kSDEYs/B63yZnAEfGlOg8YNBY=
-github.com/ProtonMail/mbox v0.0.0-20200918064939-909a18c9af45 h1:GDh55hDI2sNiirDqEWV8b6EB729u78Qxu3nKF970n6g=
-github.com/ProtonMail/mbox v0.0.0-20200918064939-909a18c9af45/go.mod h1:Yp9IVuuOYLEuMv4yjgDHvhb5mHOcYH6x92Oas3QqEZI=
 github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
 github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
 github.com/abiosoft/ishell v2.0.0+incompatible h1:zpwIuEHc37EzrsIYah3cpevrIc8Oma7oZPxr03tlmmw=
@@ -39,6 +39,8 @@ github.com/allan-simon/go-singleinstance v0.0.0-20160830203053-79edcfdc2dfc h1:m
 github.com/allan-simon/go-singleinstance v0.0.0-20160830203053-79edcfdc2dfc/go.mod h1:qqsTQiwdyqxU05iDCsi0oN3P4nrVxAmn8xCtODDSf/U=
 github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
 github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/antlr/antlr4 v0.0.0-20201029161626-9a95f0cc3d7c h1:j/C2kxPfyE0d87/ggAjIsCV5Cdkqmjb+O0W8W+1J+IY=
+github.com/antlr/antlr4 v0.0.0-20201029161626-9a95f0cc3d7c/go.mod h1:T7PbCXFs94rrTttyxjbyT5+/1V8T2TYDejxUfHJjw1Y=
 github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 h1:JLaf/iINcLyjwbtTsCJjc6rtlASgHeIJPrB6QmwURnA=
 github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
 github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE=
@@ -67,6 +69,8 @@ github.com/emersion/go-imap-specialuse v0.0.0-20200722111535-598ff00e4075 h1:z8T
 github.com/emersion/go-imap-specialuse v0.0.0-20200722111535-598ff00e4075/go.mod h1:/nybxhI8kXom8Tw6BrHMl42usALvka6meORflnnYwe4=
 github.com/emersion/go-imap-unselect v0.0.0-20171113212723-b985794e5f26 h1:FiSb8+XBQQSkcX3ubr+1tAtlRJBYaFmRZqOAweZ9Wy8=
 github.com/emersion/go-imap-unselect v0.0.0-20171113212723-b985794e5f26/go.mod h1:+gnnZx3Mg3MnCzZrv0eZdp5puxXQUgGT/6N6L7ShKfM=
+github.com/emersion/go-mbox v1.0.2 h1:tE/rT+lEugK9y0myEymCCHnwlZN04hlXPrbKkxRBA5I=
+github.com/emersion/go-mbox v1.0.2/go.mod h1:Yp9IVuuOYLEuMv4yjgDHvhb5mHOcYH6x92Oas3QqEZI=
 github.com/emersion/go-message v0.11.1/go.mod h1:C4jnca5HOTo4bGN9YdqNQM9sITuT3Y0K6bSUw9RklvY=
 github.com/emersion/go-message v0.12.1-0.20200903165315-e1abe21f389a h1:3C6qIGgPr1qAT0ikRD5NbyKpME/iHCDeXhpv/JJsFsE=
 github.com/emersion/go-message v0.12.1-0.20200903165315-e1abe21f389a/go.mod h1:kYIioST9GDHte9/BRWgi93rpqbDuFftMjKSMaXS8ABo=
@@ -107,8 +111,6 @@ github.com/keybase/go-keychain v0.0.0-20200502122510-cda31fe0c86d/go.mod h1:W6Eb
 github.com/keybase/go.dbus v0.0.0-20200324223359-a94be52c0b03/go.mod h1:a8clEhrrGV/d76/f9r2I41BwANMihfZYV9C223vaxqE=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
-github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -146,8 +148,8 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5I
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
+github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
-github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA=
 github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog=
 github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo=

internal/bridge/credits.go

@@ -15,8 +15,8 @@
 // You should have received a copy of the GNU General Public License
 // along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
 
-// Code generated by ./credits.sh at Tue Sep 29 14:56:25 CEST 2020. DO NOT EDIT.
+// Code generated by ./credits.sh at Wed Nov 4 13:57:47 CET 2020. DO NOT EDIT.
 
 package bridge
 
-const Credits = "github.com/0xAX/notificator;github.com/Masterminds/semver/v3;github.com/ProtonMail/bcrypt;github.com/ProtonMail/crypto;github.com/ProtonMail/docker-credential-helpers;github.com/ProtonMail/go-appdir;github.com/ProtonMail/go-apple-mobileconfig;github.com/ProtonMail/go-autostart;github.com/ProtonMail/go-imap;github.com/ProtonMail/go-imap-id;github.com/ProtonMail/go-smtp;github.com/ProtonMail/go-vcard;github.com/ProtonMail/gopenpgp/v2;github.com/ProtonMail/mbox;github.com/PuerkitoBio/goquery;github.com/abiosoft/ishell;github.com/abiosoft/readline;github.com/allan-simon/go-singleinstance;github.com/certifi/gocertifi;github.com/chzyer/logex;github.com/chzyer/test;github.com/cucumber/godog;github.com/docker/docker-credential-helpers;github.com/emersion/go-imap;github.com/emersion/go-imap-appendlimit;github.com/emersion/go-imap-idle;github.com/emersion/go-imap-move;github.com/emersion/go-imap-quota;github.com/emersion/go-imap-specialuse;github.com/emersion/go-imap-unselect;github.com/emersion/go-mbox;github.com/emersion/go-message;github.com/emersion/go-sasl;github.com/emersion/go-smtp;github.com/emersion/go-textwrapper;github.com/emersion/go-vcard;github.com/fatih/color;github.com/flynn-archive/go-shlex;github.com/getsentry/raven-go;github.com/go-resty/resty/v2;github.com/golang/mock;github.com/google/go-cmp;github.com/google/uuid;github.com/gopherjs/gopherjs;github.com/hashicorp/go-multierror;github.com/jameskeane/bcrypt;github.com/jaytaylor/html2text;github.com/kardianos/osext;github.com/keybase/go-keychain;github.com/logrusorgru/aurora;github.com/mattn/go-runewidth;github.com/miekg/dns;github.com/myesui/uuid;github.com/nsf/jsondiff;github.com/olekukonko/tablewriter;github.com/pkg/errors;github.com/sirupsen/logrus;github.com/skratchdot/open-golang;github.com/ssor/bom;github.com/stretchr/testify;github.com/therecipe/qt;github.com/twinj/uuid;github.com/urfave/cli;go.etcd.io/bbolt;golang.org/x/crypto;golang.org/x/net;golang.org/x/text;gopkg.in/stretchr/testify.v1;;Font Awesome 4.7.0;;Qt 5.13 by Qt group;"
+const Credits = "github.com/0xAX/notificator;github.com/Masterminds/semver/v3;github.com/ProtonMail/bcrypt;github.com/ProtonMail/crypto;github.com/ProtonMail/docker-credential-helpers;github.com/ProtonMail/go-appdir;github.com/ProtonMail/go-apple-mobileconfig;github.com/ProtonMail/go-autostart;github.com/ProtonMail/go-imap;github.com/ProtonMail/go-imap-id;github.com/ProtonMail/go-smtp;github.com/ProtonMail/go-vcard;github.com/ProtonMail/gopenpgp/v2;github.com/PuerkitoBio/goquery;github.com/abiosoft/ishell;github.com/abiosoft/readline;github.com/allan-simon/go-singleinstance;github.com/antlr/antlr4;github.com/certifi/gocertifi;github.com/chzyer/logex;github.com/chzyer/test;github.com/cucumber/godog;github.com/docker/docker-credential-helpers;github.com/emersion/go-imap;github.com/emersion/go-imap-appendlimit;github.com/emersion/go-imap-idle;github.com/emersion/go-imap-move;github.com/emersion/go-imap-quota;github.com/emersion/go-imap-specialuse;github.com/emersion/go-imap-unselect;github.com/emersion/go-mbox;github.com/emersion/go-message;github.com/emersion/go-sasl;github.com/emersion/go-smtp;github.com/emersion/go-textwrapper;github.com/emersion/go-vcard;github.com/fatih/color;github.com/flynn-archive/go-shlex;github.com/getsentry/raven-go;github.com/go-resty/resty/v2;github.com/golang/mock;github.com/google/go-cmp;github.com/google/uuid;github.com/gopherjs/gopherjs;github.com/hashicorp/go-multierror;github.com/jameskeane/bcrypt;github.com/jaytaylor/html2text;github.com/kardianos/osext;github.com/keybase/go-keychain;github.com/logrusorgru/aurora;github.com/mattn/go-runewidth;github.com/miekg/dns;github.com/myesui/uuid;github.com/nsf/jsondiff;github.com/olekukonko/tablewriter;github.com/pkg/errors;github.com/sirupsen/logrus;github.com/skratchdot/open-golang;github.com/ssor/bom;github.com/stretchr/testify;github.com/therecipe/qt;github.com/twinj/uuid;github.com/urfave/cli;go.etcd.io/bbolt;golang.org/x/crypto;golang.org/x/net;golang.org/x/text;gopkg.in/stretchr/testify.v1;;Font Awesome 4.7.0;;Qt 5.13 by Qt group;"

internal/bridge/release_notes.go

@@ -15,21 +15,17 @@
 // You should have received a copy of the GNU General Public License
 // along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
 
-// Code generated by ./release-notes.sh at 'Mon Sep 21 01:29:10 PM CEST 2020'. DO NOT EDIT.
+// Code generated by ./release-notes.sh at 'Wed Nov  4 12:24:35 PM CET 2020'. DO NOT EDIT.
 
 package bridge
 
-const ReleaseNotes = `• Bulletproofing against any potential data loss and/or duplication
+const ReleaseNotes = `• Ensured better message flow by refactoring both address and date parsing
-• Performance improvements for handling attachments and non-standard formatting
+• Improved secure connectivity checks
-• Better stability of the message parser
+• Better deb packaging
-• Additional foreign encoding support for outgoing messages
+• More robust error handling
-• Complete refactor of the way messages are parsed to simplify code maintenance
-• Improved User-Agent detection
-• Added MacOS Big Sur compatibility
-• Added persistent anonymous API cookies
 `
 
-const ReleaseFixedBugs = `• Fixed rare mail loss when moving from Spam folder
+const ReleaseFixedBugs = `• Ensured that conversations are properly threaded
-• Limited log size
+• Fixed Linux font issues (Fedora)
-• Fixed Linux font issues (mouse hover).
+• Better handling of Mime encrypted messages
 `

internal/events/events.go

@@ -40,7 +40,6 @@ const (
 	NoActiveKeyForRecipientEvent = "noActiveKeyForRecipient"
 	UpgradeApplicationEvent      = "upgradeApplication"
 	TLSCertIssue                 = "tlsCertPinningIssue"
-	IMAPTLSBadCert               = "imapTLSBadCert"
 
 	// LogoutEventTimeout is the minimum time to permit between logout events being sent.
 	LogoutEventTimeout = 3 * time.Minute

internal/frontend/cli-ie/importexport.go

@@ -184,9 +184,17 @@ func (f *frontendCLI) setTransferRules(t *transfer.Transfer) bool {
 }
 
 func (f *frontendCLI) printTransferProgress(progress *transfer.Progress) {
-	failed, imported, exported, added, total := progress.GetCounts()
+	counts := progress.GetCounts()
-	if total != 0 {
+	if counts.Total != 0 {
-		f.Println(fmt.Sprintf("Progress update: %d (%d / %d) / %d, failed: %d", imported, exported, added, total, failed))
+		f.Println(fmt.Sprintf(
+			"Progress update: %d (%d / %d) / %d, skipped: %d, failed: %d",
+			counts.Imported,
+			counts.Exported,
+			counts.Added,
+			counts.Total,
+			counts.Skipped,
+			counts.Failed,
+		))
 	}
 
 	if progress.IsPaused() {

internal/frontend/qml/Gui.qml

@@ -237,13 +237,6 @@ Item {
             winMain.tlsBarState="notOK"
         }
 
-        onShowIMAPCertTroubleshoot : {
-            go.notifyBubble(1, qsTr(
-                "Bridge was unable to establish a connection with your Email client. <br> <a href=\"https://protonmail.com/support/knowledge-base/bridge-ssl-connection-issue\">Learn more</a> <br>",
-                "notification message"
-            ))
-        }
-
 
     }
 

internal/frontend/qml/ImportExportUI/DialogImport.qml

@@ -599,7 +599,7 @@ Dialog {
             }
 
             Text {
-                text: qsTr("<b>Import summary:</b><br>Total number of emails: %1<br>Imported emails: %2<br>Errors: %3").arg(go.total).arg(finalReport.imported).arg(go.progressFails)
+                text: qsTr("<b>Import summary:</b><br>Total number of emails: %1<br>Imported emails: %2<br>Filtered out emails: %3<br>Errors: %4").arg(go.total).arg(go.progressImported).arg(go.progressSkipped).arg(go.progressFails)
                 anchors.horizontalCenter: parent.horizontalCenter
                 textFormat: Text.RichText
                 horizontalAlignment: Text.AlignHCenter

internal/frontend/qml/tst_Gui.qml

@@ -122,7 +122,6 @@ Window {
         ListElement { title: "Minimize this"  }
         ListElement { title: "SendAlertPopup" }
         ListElement { title: "TLSCertError"   }
-        ListElement { title: "IMAPCertError"  }
     }
 
     ListView {
@@ -209,9 +208,6 @@ Window {
                     case "TLSCertError" :
                     go.showCertIssue()
                     break;
-                    case "IMAPCertError" :
-                    go.showIMAPCertTroubleshoot()
-                    break;
                     default :
                     console.log("Not implemented " + data)
                 }
@@ -314,7 +310,6 @@ Window {
         signal failedAutostartCode(string code)
 
         signal showCertIssue()
-        signal showIMAPCertTroubleshoot()
 
         signal updateFinished(bool hasError)
 

internal/frontend/qml/tst_GuiIE.qml

@@ -840,6 +840,8 @@ Window {
 
         property real progress: 0.0
         property int progressFails: 0
+        property int progressImported: 0
+        property int progressSkipped: 0
         property string progressDescription: "nothing"
         property string progressInit: "init"
         property int total: 42
@@ -1011,6 +1013,8 @@ Window {
         property SequentialAnimation animateProgressBar : SequentialAnimation {
             id: apb
             property real speedup : 1.0;
+            PropertyAnimation{ target: go; properties: "progressSkipped"; to: 0; duration: 1; }
+            PropertyAnimation{ target: go; properties: "progressImported"; to: 0; duration: 1; }
             PropertyAnimation{ target: go; properties: "importLogFileName"; to: ""; duration: 1; }
             PropertyAnimation{ target: go; properties: "progressDescription"; to: go.progressInit; duration: 1; }
             PropertyAnimation{ duration: 2000/apb.speedup; }
@@ -1024,6 +1028,8 @@ Window {
             PropertyAnimation{ target: go; properties: "progress"; to: 0.01; duration: 1; }
             PropertyAnimation{ duration: 1000/apb.speedup; }
             PropertyAnimation{ target: go; properties: "progress"; to: 0.1; duration: 1; }
+            PropertyAnimation{ target: go; properties: "progressSkipped"; to: 12; duration: 1; }
+            PropertyAnimation{ target: go; properties: "progressImported"; to: 13.1; duration: 1; }
             PropertyAnimation{ duration: 1000/apb.speedup; }
             PropertyAnimation{ target: go; properties: "progress"; to: 0.3; duration: 1; }
             PropertyAnimation{ target: go; properties: "progressFails"; to: 1; duration: 1; }

internal/frontend/qt-common/account_model.go

@@ -21,6 +21,7 @@ package qtcommon
 
 import (
 	"fmt"
+
 	"github.com/therecipe/qt/core"
 )
 

internal/frontend/qt-ie/frontend.go

@@ -337,18 +337,25 @@ func (f *FrontendQt) setProgressManager(progress *transfer.Progress) {
 			if progress.IsStopped() {
 				break
 			}
-			failed, imported, _, _, total := progress.GetCounts()
+			counts := progress.GetCounts()
-			f.Qml.SetTotal(int(total))
+			f.Qml.SetTotal(int(counts.Total))
-			f.Qml.SetProgressFails(int(failed))
+			f.Qml.SetProgressImported(int(counts.Imported))
+			f.Qml.SetProgressSkipped(int(counts.Skipped))
+			f.Qml.SetProgressFails(int(counts.Failed))
 			f.Qml.SetProgressDescription(progress.PauseReason())
-			if total > 0 {
+			if counts.Total > 0 {
-				newProgress := float32(imported+failed) / float32(total)
+				newProgress := counts.Progress()
 				if newProgress >= 0 && newProgress != f.Qml.Progress() {
 					f.Qml.SetProgress(newProgress)
 					f.Qml.ProgressChanged(newProgress)
 				}
 			}
 		}
+		// Counts will add lost messages only once the progress is completeled.
+		counts := progress.GetCounts()
+		f.Qml.SetProgressImported(int(counts.Imported))
+		f.Qml.SetProgressSkipped(int(counts.Skipped))
+		f.Qml.SetProgressFails(int(counts.Failed))
 
 		if err := progress.GetFatalError(); err != nil {
 			f.Qml.SetProgressDescription(err.Error())

internal/frontend/qt-ie/import.go

@@ -77,6 +77,8 @@ func (f *FrontendQt) StartImport(email string) { // TODO email not needed
 	log.Trace("Starting import")
 
 	f.Qml.SetProgressDescription("init") // TODO use const
+	f.Qml.SetProgressImported(0)
+	f.Qml.SetProgressSkipped(0)
 	f.Qml.SetProgressFails(0)
 	f.Qml.SetProgress(0.0)
 	f.Qml.SetTotal(1)

internal/frontend/qt-ie/ui.go

@@ -43,6 +43,8 @@ type GoQMLInterface struct {
 	_ string  `property:lastError`
 	_ float32 `property:progress`
 	_ string  `property:progressDescription`
+	_ int     `property:progressImported`
+	_ int     `property:progressSkipped`
 	_ int     `property:progressFails`
 	_ int     `property:total`
 	_ string  `property:importLogFileName`

internal/frontend/qt/frontend.go

@@ -40,17 +40,17 @@ import (
 	"github.com/ProtonMail/proton-bridge/internal/bridge"
 	"github.com/ProtonMail/proton-bridge/internal/events"
 	"github.com/ProtonMail/proton-bridge/internal/frontend/autoconfig"
-	"github.com/ProtonMail/proton-bridge/internal/frontend/qt-common"
+	qtcommon "github.com/ProtonMail/proton-bridge/internal/frontend/qt-common"
 	"github.com/ProtonMail/proton-bridge/internal/frontend/types"
 	"github.com/ProtonMail/proton-bridge/internal/preferences"
 	"github.com/ProtonMail/proton-bridge/internal/updates"
 	"github.com/ProtonMail/proton-bridge/pkg/config"
-	"github.com/ProtonMail/proton-bridge/pkg/ports"
-	"github.com/ProtonMail/proton-bridge/pkg/useragent"
 	"github.com/ProtonMail/proton-bridge/pkg/listener"
 	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
-	"github.com/sirupsen/logrus"
+	"github.com/ProtonMail/proton-bridge/pkg/ports"
+	"github.com/ProtonMail/proton-bridge/pkg/useragent"
 	"github.com/kardianos/osext"
+	"github.com/sirupsen/logrus"
 	"github.com/skratchdot/open-golang/open"
 	"github.com/therecipe/qt/core"
 	"github.com/therecipe/qt/gui"
@@ -187,7 +187,6 @@ func (s *FrontendQt) watchEvents() {
 	updateApplicationCh := s.getEventChannel(events.UpgradeApplicationEvent)
 	newUserCh := s.getEventChannel(events.UserRefreshEvent)
 	certIssue := s.getEventChannel(events.TLSCertIssue)
-	imapCertIssue := s.getEventChannel(events.IMAPTLSBadCert)
 	for {
 		select {
 		case errorDetails := <-errorCh:
@@ -227,8 +226,6 @@ func (s *FrontendQt) watchEvents() {
 			s.Qml.LoadAccounts()
 		case <-certIssue:
 			s.Qml.ShowCertIssue()
-		case <-imapCertIssue:
-			s.Qml.ShowIMAPCertTroubleshoot()
 		}
 	}
 }

internal/frontend/qt/ui.go

@@ -135,7 +135,6 @@ type GoQMLInterface struct {
 	_ func(x, y float32)                      `slot:"saveOutgoingNoEncPopupCoord"`
 	_ func(recipient string)                  `signal:"showNoActiveKeyForRecipient"`
 	_ func()                                  `signal:"showCertIssue"`
-	_ func()                                  `signal:"ShowIMAPCertTroubleshoot"`
 
 	_ func()              `slot:"startUpdate"`
 	_ func(hasError bool) `signal:"updateFinished"`

internal/imap/backend.go

@@ -28,7 +28,6 @@ import (
 	"github.com/ProtonMail/proton-bridge/pkg/listener"
 	"github.com/emersion/go-imap"
 	goIMAPBackend "github.com/emersion/go-imap/backend"
-	"github.com/sirupsen/logrus"
 )
 
 type panicHandler interface {
@@ -198,11 +197,3 @@ func (ib *imapBackend) monitorDisconnectedUsers() {
 		ib.deleteUser(address)
 	}
 }
-
-func (ib *imapBackend) upgradeError(err error) {
-	logrus.WithError(err).Error("IMAP connection couldn't be upgraded to TLS during STARTTLS")
-
-	if strings.Contains(err.Error(), "remote error: tls: bad certificate") {
-		ib.eventListener.Emit(events.IMAPTLSBadCert, err.Error())
-	}
-}

internal/imap/server.go

@@ -58,7 +58,6 @@ func NewIMAPServer(debugClient, debugServer bool, port int, tls *tls.Config, ima
 	s.AllowInsecureAuth = true
 	s.ErrorLog = newServerErrorLogger("server-imap")
 	s.AutoLogout = 30 * time.Minute
-	s.UpgradeError = imapBackend.upgradeError
 
 	serverID := imapid.ID{
 		imapid.FieldName:       "ProtonMail Bridge",

internal/importexport/credits.go

@@ -15,8 +15,8 @@
 // You should have received a copy of the GNU General Public License
 // along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
 
-// Code generated by ./credits.sh at Tue Sep 29 14:56:25 CEST 2020. DO NOT EDIT.
+// Code generated by ./credits.sh at Wed Nov 4 13:57:47 CET 2020. DO NOT EDIT.
 
 package importexport
 
-const Credits = "github.com/0xAX/notificator;github.com/Masterminds/semver/v3;github.com/ProtonMail/bcrypt;github.com/ProtonMail/crypto;github.com/ProtonMail/docker-credential-helpers;github.com/ProtonMail/go-appdir;github.com/ProtonMail/go-apple-mobileconfig;github.com/ProtonMail/go-autostart;github.com/ProtonMail/go-imap;github.com/ProtonMail/go-imap-id;github.com/ProtonMail/go-smtp;github.com/ProtonMail/go-vcard;github.com/ProtonMail/gopenpgp/v2;github.com/ProtonMail/mbox;github.com/PuerkitoBio/goquery;github.com/abiosoft/ishell;github.com/abiosoft/readline;github.com/allan-simon/go-singleinstance;github.com/certifi/gocertifi;github.com/chzyer/logex;github.com/chzyer/test;github.com/cucumber/godog;github.com/docker/docker-credential-helpers;github.com/emersion/go-imap;github.com/emersion/go-imap-appendlimit;github.com/emersion/go-imap-idle;github.com/emersion/go-imap-move;github.com/emersion/go-imap-quota;github.com/emersion/go-imap-specialuse;github.com/emersion/go-imap-unselect;github.com/emersion/go-mbox;github.com/emersion/go-message;github.com/emersion/go-sasl;github.com/emersion/go-smtp;github.com/emersion/go-textwrapper;github.com/emersion/go-vcard;github.com/fatih/color;github.com/flynn-archive/go-shlex;github.com/getsentry/raven-go;github.com/go-resty/resty/v2;github.com/golang/mock;github.com/google/go-cmp;github.com/google/uuid;github.com/gopherjs/gopherjs;github.com/hashicorp/go-multierror;github.com/jameskeane/bcrypt;github.com/jaytaylor/html2text;github.com/kardianos/osext;github.com/keybase/go-keychain;github.com/logrusorgru/aurora;github.com/mattn/go-runewidth;github.com/miekg/dns;github.com/myesui/uuid;github.com/nsf/jsondiff;github.com/olekukonko/tablewriter;github.com/pkg/errors;github.com/sirupsen/logrus;github.com/skratchdot/open-golang;github.com/ssor/bom;github.com/stretchr/testify;github.com/therecipe/qt;github.com/twinj/uuid;github.com/urfave/cli;go.etcd.io/bbolt;golang.org/x/crypto;golang.org/x/net;golang.org/x/text;gopkg.in/stretchr/testify.v1;;Font Awesome 4.7.0;;Qt 5.13 by Qt group;"
+const Credits = "github.com/0xAX/notificator;github.com/Masterminds/semver/v3;github.com/ProtonMail/bcrypt;github.com/ProtonMail/crypto;github.com/ProtonMail/docker-credential-helpers;github.com/ProtonMail/go-appdir;github.com/ProtonMail/go-apple-mobileconfig;github.com/ProtonMail/go-autostart;github.com/ProtonMail/go-imap;github.com/ProtonMail/go-imap-id;github.com/ProtonMail/go-smtp;github.com/ProtonMail/go-vcard;github.com/ProtonMail/gopenpgp/v2;github.com/PuerkitoBio/goquery;github.com/abiosoft/ishell;github.com/abiosoft/readline;github.com/allan-simon/go-singleinstance;github.com/antlr/antlr4;github.com/certifi/gocertifi;github.com/chzyer/logex;github.com/chzyer/test;github.com/cucumber/godog;github.com/docker/docker-credential-helpers;github.com/emersion/go-imap;github.com/emersion/go-imap-appendlimit;github.com/emersion/go-imap-idle;github.com/emersion/go-imap-move;github.com/emersion/go-imap-quota;github.com/emersion/go-imap-specialuse;github.com/emersion/go-imap-unselect;github.com/emersion/go-mbox;github.com/emersion/go-message;github.com/emersion/go-sasl;github.com/emersion/go-smtp;github.com/emersion/go-textwrapper;github.com/emersion/go-vcard;github.com/fatih/color;github.com/flynn-archive/go-shlex;github.com/getsentry/raven-go;github.com/go-resty/resty/v2;github.com/golang/mock;github.com/google/go-cmp;github.com/google/uuid;github.com/gopherjs/gopherjs;github.com/hashicorp/go-multierror;github.com/jameskeane/bcrypt;github.com/jaytaylor/html2text;github.com/kardianos/osext;github.com/keybase/go-keychain;github.com/logrusorgru/aurora;github.com/mattn/go-runewidth;github.com/miekg/dns;github.com/myesui/uuid;github.com/nsf/jsondiff;github.com/olekukonko/tablewriter;github.com/pkg/errors;github.com/sirupsen/logrus;github.com/skratchdot/open-golang;github.com/ssor/bom;github.com/stretchr/testify;github.com/therecipe/qt;github.com/twinj/uuid;github.com/urfave/cli;go.etcd.io/bbolt;golang.org/x/crypto;golang.org/x/net;golang.org/x/text;gopkg.in/stretchr/testify.v1;;Font Awesome 4.7.0;;Qt 5.13 by Qt group;"

internal/importexport/release_notes.go

@@ -15,18 +15,14 @@
 // You should have received a copy of the GNU General Public License
 // along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
 
-// Code generated by ./release-notes.sh at 'Thu Oct 29 12:57:32 PM CET 2020'. DO NOT EDIT.
+// Code generated by ./release-notes.sh at 'Wed Nov 11 01:57:14 PM CET 2020'. DO NOT EDIT.
 
 package importexport
 
-const ReleaseNotes = `• Improvements to the import from large mbox files with multiple labels
+const ReleaseNotes = `• Further improvements to address and date parsing
-• Not allow to run multiple instances of the app or transfers at the same time
+• Better handling and displaying of skipped messages
-• Various enhancements of the import process related to parsing
+• Improved error reporting
-• Cosmetic GUI changes
-• Better error handling
 `
 
-const ReleaseFixedBugs = `• Linux font issues - Fedora specific
+const ReleaseFixedBugs = `
-• App response to the user pausing and canceling import or export
-• Handling errors during update
 `

internal/store/change.go

@@ -37,7 +37,7 @@ func (store *Store) SetIMAPUpdateChannel(updates chan imapBackend.Update) {
 	}
 }
 
-func (store *Store) imapNotice(address, notice string) {
+func (store *Store) imapNotice(address, notice string) *imapBackend.StatusUpdate {
 	update := new(imapBackend.StatusUpdate)
 	update.Update = imapBackend.NewUpdate(address, "")
 	update.StatusResp = &imap.StatusResp{
@@ -46,13 +46,14 @@ func (store *Store) imapNotice(address, notice string) {
 		Info: notice,
 	}
 	store.imapSendUpdate(update)
+	return update
 }
 
 func (store *Store) imapUpdateMessage(
 	address, mailboxName string,
 	uid, sequenceNumber uint32,
 	msg *pmapi.Message, hasDeletedFlag bool,
-) {
+) *imapBackend.MessageUpdate {
 	store.log.WithFields(logrus.Fields{
 		"address": address,
 		"mailbox": mailboxName,
@@ -70,9 +71,10 @@ func (store *Store) imapUpdateMessage(
 	}
 	update.Message.Uid = uid
 	store.imapSendUpdate(update)
+	return update
 }
 
-func (store *Store) imapDeleteMessage(address, mailboxName string, sequenceNumber uint32) {
+func (store *Store) imapDeleteMessage(address, mailboxName string, sequenceNumber uint32) *imapBackend.ExpungeUpdate {
 	store.log.WithFields(logrus.Fields{
 		"address": address,
 		"mailbox": mailboxName,
@@ -82,9 +84,10 @@ func (store *Store) imapDeleteMessage(address, mailboxName string, sequenceNumbe
 	update.Update = imapBackend.NewUpdate(address, mailboxName)
 	update.SeqNum = sequenceNumber
 	store.imapSendUpdate(update)
+	return update
 }
 
-func (store *Store) imapMailboxCreated(address, mailboxName string) {
+func (store *Store) imapMailboxCreated(address, mailboxName string) *imapBackend.MailboxInfoUpdate {
 	store.log.WithFields(logrus.Fields{
 		"address": address,
 		"mailbox": mailboxName,
@@ -97,9 +100,10 @@ func (store *Store) imapMailboxCreated(address, mailboxName string) {
 		Name:       mailboxName,
 	}
 	store.imapSendUpdate(update)
+	return update
 }
 
-func (store *Store) imapMailboxStatus(address, mailboxName string, total, unread, unreadSeqNum uint) {
+func (store *Store) imapMailboxStatus(address, mailboxName string, total, unread, unreadSeqNum uint) *imapBackend.MailboxUpdate {
 	store.log.WithFields(logrus.Fields{
 		"address":      address,
 		"mailbox":      mailboxName,
@@ -114,6 +118,7 @@ func (store *Store) imapMailboxStatus(address, mailboxName string, total, unread
 	update.MailboxStatus.Unseen = uint32(unread)
 	update.MailboxStatus.UnseenSeqNum = uint32(unreadSeqNum)
 	store.imapSendUpdate(update)
+	return update
 }
 
 func (store *Store) imapSendUpdate(update imapBackend.Update) {

internal/store/mailbox.go

@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"strings"
+	"sync/atomic"
 
 	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
 	"github.com/sirupsen/logrus"
@@ -38,6 +39,8 @@ type Mailbox struct {
 	color       string
 
 	log *logrus.Entry
+
+	isDeleting atomic.Value
 }
 
 func newMailbox(storeAddress *Address, labelID, labelPrefix, labelName, color string) (mb *Mailbox, err error) {
@@ -59,6 +62,7 @@ func txNewMailbox(tx *bolt.Tx, storeAddress *Address, labelID, labelPrefix, labe
 		color:        color,
 		log:          l,
 	}
+	mb.isDeleting.Store(false)
 
 	err := initMailboxBucket(tx, mb.getBucketName())
 	if err != nil {
@@ -215,6 +219,7 @@ func (storeMailbox *Mailbox) Rename(newName string) error {
 // Deletion has to be propagated to all the same mailboxes in all addresses.
 // The propagation is processed by the event loop.
 func (storeMailbox *Mailbox) Delete() error {
+	storeMailbox.isDeleting.Store(true)
 	return storeMailbox.storeAddress.deleteMailbox(storeMailbox.labelID)
 }
 
@@ -226,6 +231,14 @@ func (storeMailbox *Mailbox) GetDelimiter() string {
 // deleteMailboxEvent deletes the mailbox bucket.
 // This is called from the event loop.
 func (storeMailbox *Mailbox) deleteMailboxEvent() error {
+	if !storeMailbox.isDeleting.Load().(bool) {
+		// Deleting label removes bucket. Any ongoing connection selected
+		// in such mailbox then might panic because of non-existing bucket.
+		// Closing connetions prevents that panic but if the connection
+		// asked for deletion, it should not be closed so it can receive
+		// successful response.
+		storeMailbox.store.user.CloseAllConnections()
+	}
 	return storeMailbox.db().Update(func(tx *bolt.Tx) error {
 		return tx.Bucket(mailboxesBucket).DeleteBucket(storeMailbox.getBucketName())
 	})

internal/store/mailbox_message.go

@@ -18,6 +18,8 @@
 package store
 
 import (
+	"time"
+
 	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -501,7 +503,7 @@ func (storeMailbox *Mailbox) txMarkMessagesAsDeleted(tx *bolt.Tx, apiIDs []strin
 
 		// In order to send flags in format
 		// S: * 2 FETCH (FLAGS (\Deleted \Seen))
-		storeMailbox.store.imapUpdateMessage(
+		update := storeMailbox.store.imapUpdateMessage(
 			storeMailbox.storeAddress.address,
 			storeMailbox.labelName,
 			uid,
@@ -509,6 +511,14 @@ func (storeMailbox *Mailbox) txMarkMessagesAsDeleted(tx *bolt.Tx, apiIDs []strin
 			msg,
 			markAsDeleted,
 		)
+
+		// txMarkMessagesAsDeleted is called only during processing request
+		// from IMAP call (i.e., not from event loop) and in such cases we
+		// have to wait to propagate update back before closing the response.
+		select {
+		case <-time.After(1 * time.Second):
+		case <-update.Done():
+		}
 	}
 
 	return nil

internal/store/mocks/mocks.go

@@ -5,9 +5,10 @@
 package mocks
 
 import (
+	reflect "reflect"
+
 	pmapi "github.com/ProtonMail/proton-bridge/pkg/pmapi"
 	gomock "github.com/golang/mock/gomock"
-	reflect "reflect"
 )
 
 // MockPanicHandler is a mock of PanicHandler interface
@@ -105,6 +106,18 @@ func (m *MockBridgeUser) EXPECT() *MockBridgeUserMockRecorder {
 	return m.recorder
 }
 
+// CloseAllConnections mocks base method
+func (m *MockBridgeUser) CloseAllConnections() {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "CloseAllConnections")
+}
+
+// CloseAllConnections indicates an expected call of CloseAllConnections
+func (mr *MockBridgeUserMockRecorder) CloseAllConnections() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CloseAllConnections", reflect.TypeOf((*MockBridgeUser)(nil).CloseAllConnections))
+}
+
 // CloseConnection mocks base method
 func (m *MockBridgeUser) CloseConnection(arg0 string) {
 	m.ctrl.T.Helper()

internal/store/mocks/utils_mocks.go

@@ -5,9 +5,10 @@
 package mocks
 
 import (
-	gomock "github.com/golang/mock/gomock"
 	reflect "reflect"
 	time "time"
+
+	gomock "github.com/golang/mock/gomock"
 )
 
 // MockListener is a mock of Listener interface

internal/store/types.go

@@ -36,6 +36,7 @@ type BridgeUser interface {
 	GetPrimaryAddress() string
 	GetStoreAddresses() []string
 	UpdateUser() error
+	CloseAllConnections()
 	CloseConnection(string)
 	Logout() error
 }

internal/transfer/message.go

@@ -58,6 +58,7 @@ type MessageStatus struct {
 	targetID    string    // Message ID at the target (if any).
 	bodyHash    string    // Hash of the message body.
 
+	skipped   bool
 	exported  bool
 	imported  bool
 	exportErr error
@@ -96,7 +97,7 @@ func (status *MessageStatus) setDetailsFromHeader(header mail.Header) {
 }
 
 func (status *MessageStatus) hasError(includeMissing bool) bool {
-	return status.exportErr != nil || status.importErr != nil || (includeMissing && !status.imported)
+	return status.exportErr != nil || status.importErr != nil || (includeMissing && !status.skipped && !status.imported)
 }
 
 // GetErrorMessage returns error message.
@@ -105,6 +106,9 @@ func (status *MessageStatus) GetErrorMessage() string {
 }
 
 func (status *MessageStatus) getErrorMessage(includeMissing bool) string {
+	if status.skipped {
+		return ""
+	}
 	if status.exportErr != nil {
 		return fmt.Sprintf("failed to export: %s", status.exportErr)
 	}

internal/transfer/mocks/mocks.go

@@ -5,9 +5,10 @@
 package mocks
 
 import (
+	reflect "reflect"
+
 	pmapi "github.com/ProtonMail/proton-bridge/pkg/pmapi"
 	gomock "github.com/golang/mock/gomock"
-	reflect "reflect"
 )
 
 // MockPanicHandler is a mock of PanicHandler interface

internal/transfer/progress.go

@@ -140,6 +140,19 @@ func (p *Progress) addMessage(messageID string, sourceNames, targetNames []strin
 	}
 }
 
+// messageSkipped should be called once the message is skipped due to some
+// filter such as time or folder and so on.
+func (p *Progress) messageSkipped(messageID string) {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+	defer p.update()
+
+	p.log.WithField("id", messageID).Debug("Message skipped")
+
+	p.messageStatuses[messageID].skipped = true
+	p.logMessage(messageID)
+}
+
 // messageExported should be called right before message is exported.
 func (p *Progress) messageExported(messageID string, body []byte, err error) {
 	p.lock.Lock()
@@ -330,35 +343,40 @@ func (p *Progress) GetFailedMessages() []*MessageStatus {
 }
 
 // GetCounts returns counts of exported and imported messages.
-func (p *Progress) GetCounts() (failed, imported, exported, added, total uint) {
+func (p *Progress) GetCounts() ProgressCounts {
 	p.lock.Lock()
 	defer p.lock.Unlock()
 
+	counts := ProgressCounts{}
+
 	// Return counts only once total is estimated or the process already
 	// ended (for a case when it ended quickly to report it correctly).
 	if p.updateCh != nil && !p.messageCounted {
-		return
+		return counts
 	}
 
 	// Include lost messages in the process only when transfer is done.
 	includeMissing := p.updateCh == nil
 
 	for _, mailboxCount := range p.messageCounts {
-		total += mailboxCount
+		counts.Total += mailboxCount
 	}
 	for _, status := range p.messageStatuses {
-		added++
+		counts.Added++
+		if status.skipped {
+			counts.Skipped++
+		}
 		if status.exported {
-			exported++
+			counts.Exported++
 		}
 		if status.imported {
-			imported++
+			counts.Imported++
 		}
 		if status.hasError(includeMissing) {
-			failed++
+			counts.Failed++
 		}
 	}
-	return
+	return counts
 }
 
 // GenerateBugReport generates similar file to import log except private information.

internal/transfer/progress_counts.go

@@ -0,0 +1,35 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package transfer
+
+// ProgressCounts holds counts counted by Progress.
+type ProgressCounts struct {
+	Failed,
+	Skipped,
+	Imported,
+	Exported,
+	Added,
+	Total uint
+}
+
+// Progress returns ratio between processed messages (fully imported, skipped
+// and failed ones) and total number of messages as percentage (0 - 1).
+func (c *ProgressCounts) Progress() float32 {
+	progressed := c.Imported + c.Skipped + c.Failed
+	return float32(progressed) / float32(c.Total)
+}

internal/transfer/progress_test.go

@@ -39,8 +39,8 @@ func TestProgressUpdateCount(t *testing.T) {
 
 	progress.finish()
 
-	_, _, _, _, total := progress.GetCounts() //nolint[dogsled]
+	counts := progress.GetCounts()
-	r.Equal(t, uint(42), total)
+	r.Equal(t, uint(42), counts.Total)
 }
 
 func TestProgressAddingMessages(t *testing.T) {
@@ -66,13 +66,18 @@ func TestProgressAddingMessages(t *testing.T) {
 	progress.messageExported("msg4", []byte(""), errors.New("failed export"))
 	progress.messageImported("msg4", "", nil)
 
+	// msg5 is skipped.
+	progress.addMessage("msg5", []string{}, []string{})
+	progress.messageSkipped("msg5")
+
 	progress.finish()
 
-	failed, imported, exported, added, _ := progress.GetCounts()
+	counts := progress.GetCounts()
-	a.Equal(t, uint(4), added)
+	a.Equal(t, uint(5), counts.Added)
-	a.Equal(t, uint(2), exported)
+	a.Equal(t, uint(2), counts.Exported)
-	a.Equal(t, uint(2), imported)
+	a.Equal(t, uint(2), counts.Imported)
-	a.Equal(t, uint(3), failed)
+	a.Equal(t, uint(1), counts.Skipped)
+	a.Equal(t, uint(3), counts.Failed)
 
 	errorsMap := map[string]string{}
 	for _, status := range progress.GetFailedMessages() {

internal/transfer/provider_eml_source.go

@@ -82,8 +82,6 @@ func (p *EMLProvider) getFilePathsPerFolder(rules transferRules) (map[string][]s
 }
 
 func (p *EMLProvider) exportMessages(rule *Rule, filePaths []string, progress *Progress, ch chan<- Message) {
-	count := uint(len(filePaths))
-
 	for _, filePath := range filePaths {
 		if progress.shouldStop() {
 			break
@@ -91,6 +89,8 @@ func (p *EMLProvider) exportMessages(rule *Rule, filePaths []string, progress *P
 
 		msg, err := p.exportMessage(rule, filePath)
 
+		progress.addMessage(filePath, msg.sourceNames(), msg.targetNames())
+
 		// Read and check time in body only if the rule specifies it
 		// to not waste energy.
 		if err == nil && rule.HasTimeLimit() {
@@ -99,17 +99,11 @@ func (p *EMLProvider) exportMessages(rule *Rule, filePaths []string, progress *P
 				err = msgTimeErr
 			} else if !rule.isTimeInRange(msgTime) {
 				log.WithField("msg", filePath).Debug("Message skipped due to time")
-
+				progress.messageSkipped(filePath)
-				count--
-				progress.updateCount(rule.SourceMailbox.Name, count)
 				continue
 			}
 		}
 
-		// addMessage is called after time check to not report message
-		// which should not be exported but any error from reading body
-		// or parsing time is reported as an error.
-		progress.addMessage(filePath, msg.sourceNames(), msg.targetNames())
 		progress.messageExported(filePath, msg.Body, err)
 		if err == nil {
 			ch <- msg

internal/transfer/provider_mbox_source.go

@@ -114,7 +114,6 @@ func (p *MBOXProvider) transferTo(rules transferRules, progress *Progress, ch ch
 	}
 
 	index := 0
-	count := 0
 	for {
 		if progress.shouldStop() {
 			break
@@ -133,24 +132,18 @@ func (p *MBOXProvider) transferTo(rules transferRules, progress *Progress, ch ch
 
 		msg, err := p.exportMessage(rules, folderName, id, msgReader)
 
+		progress.addMessage(id, msg.sourceNames(), msg.targetNames())
+
 		if err == nil && len(msg.Targets) == 0 {
-			// Here should be called progress.messageSkipped(id) once we have
+			progress.messageSkipped(id)
-			// this feature, and following progress.updateCount can be removed.
 			continue
 		}
 
-		count++
-
-		// addMessage is called after time check to not report message
-		// which should not be exported but any error from reading body
-		// or parsing time is reported as an error.
-		progress.addMessage(id, msg.sourceNames(), msg.targetNames())
 		progress.messageExported(id, msg.Body, err)
 		if err == nil {
 			ch <- msg
 		}
 	}
-	progress.updateCount(filePath, uint(count))
 }
 
 func (p *MBOXProvider) exportMessage(rules transferRules, folderName, id string, msgReader io.Reader) (Message, error) {
@@ -177,7 +170,7 @@ func (p *MBOXProvider) getMessageRules(rules transferRules, folderName, id strin
 	folderRule, err := rules.getRuleBySourceMailboxName(folderName)
 	if err != nil {
 		log.WithField("msg", id).WithField("source", folderName).Debug("Message source doesn't have a rule")
-	} else {
+	} else if folderRule.Active {
 		msgRules = append(msgRules, folderRule)
 	}
 
@@ -191,7 +184,9 @@ func (p *MBOXProvider) getMessageRules(rules transferRules, folderName, id strin
 				log.WithField("msg", id).WithField("source", label).Debug("Message source doesn't have a rule")
 				continue
 			}
-			msgRules = append(msgRules, rule)
+			if rule.Active {
+				msgRules = append(msgRules, rule)
+			}
 		}
 	}
 

internal/transfer/provider_mbox_test.go

@@ -155,6 +155,29 @@ func TestMBOXProviderTransferFromTo(t *testing.T) {
 	})
 }
 
+func TestMBOXProviderGetMessageRules(t *testing.T) {
+	provider := newTestMBOXProvider("")
+
+	body := []byte(`Subject: Test
+X-Gmail-Labels: foo,bar
+
+`)
+	rules := transferRules{
+		rules: map[string]*Rule{
+			"1": {Active: true, SourceMailbox: Mailbox{Name: "folder"}},
+			"2": {Active: false, SourceMailbox: Mailbox{Name: "foo"}},
+			"3": {Active: true, SourceMailbox: Mailbox{Name: "bar"}},
+			"4": {Active: false, SourceMailbox: Mailbox{Name: "baz"}},
+			"5": {Active: true, SourceMailbox: Mailbox{Name: "other"}},
+		},
+	}
+
+	gotRules := provider.getMessageRules(rules, "folder", "id", body)
+	r.Equal(t, 2, len(gotRules))
+	r.Equal(t, "folder", gotRules[0].SourceMailbox.Name)
+	r.Equal(t, "bar", gotRules[1].SourceMailbox.Name)
+}
+
 func TestMBOXProviderGetMessageTargetsReturnsOnlyOneFolder(t *testing.T) {
 	provider := newTestMBOXProvider("")
 

internal/transfer/utils.go

@@ -28,6 +28,7 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/ProtonMail/go-rfc5322"
 	"github.com/pkg/errors"
 )
 
@@ -136,14 +137,21 @@ func getFilePathsWithSuffixInner(prefix, root, suffix string, includeDir bool) (
 
 // getMessageTime returns time of the message specified in the message header.
 func getMessageTime(body []byte) (int64, error) {
-	mailHeader, err := getMessageHeader(body)
+	hdr, err := getMessageHeader(body)
 	if err != nil {
 		return 0, err
 	}
-	if t, err := mailHeader.Date(); err == nil && !t.IsZero() {
+
-		return t.Unix(), nil
+	t, err := rfc5322.ParseDateTime(hdr.Get("Date"))
+	if err != nil {
+		return 0, err
 	}
-	return 0, nil
+
+	if t.IsZero() {
+		return 0, nil
+	}
+
+	return t.Unix(), nil
 }
 
 // getMessageHeader returns headers of the message body.

internal/users/mocks/mocks.go

@@ -5,11 +5,12 @@
 package mocks
 
 import (
+	reflect "reflect"
+
 	store "github.com/ProtonMail/proton-bridge/internal/store"
 	credentials "github.com/ProtonMail/proton-bridge/internal/users/credentials"
 	pmapi "github.com/ProtonMail/proton-bridge/pkg/pmapi"
 	gomock "github.com/golang/mock/gomock"
-	reflect "reflect"
 )
 
 // MockConfiger is a mock of Configer interface

internal/users/user.go

@@ -437,7 +437,7 @@ func (u *User) SwitchAddressMode() (err error) {
 
 	u.lock.Lock()
 	defer u.lock.Unlock()
-	u.closeAllConnections()
+	u.CloseAllConnections()
 
 	if u.store == nil {
 		err = errors.New("store is not initialised")
@@ -509,7 +509,7 @@ func (u *User) Logout() (err error) {
 	// Do not close whole store, just event loop. Some information might be needed offline (e.g. addressID)
 	u.closeEventLoop()
 
-	u.closeAllConnections()
+	u.CloseAllConnections()
 
 	runtime.GC()
 
@@ -532,8 +532,8 @@ func (u *User) closeEventLoop() {
 	u.store.CloseEventLoop()
 }
 
-// closeAllConnections calls CloseConnection for all users addresses.
+// CloseAllConnections calls CloseConnection for all users addresses.
-func (u *User) closeAllConnections() {
+func (u *User) CloseAllConnections() {
 	for _, address := range u.creds.EmailList() {
 		u.CloseConnection(address)
 	}

internal/users/users.go

@@ -186,7 +186,7 @@ func (u *Users) watchAPIAuths() {
 
 func (u *Users) closeAllConnections() {
 	for _, user := range u.users {
-		user.closeAllConnections()
+		user.CloseAllConnections()
 	}
 }
 

pkg/message/header.go

@@ -19,9 +19,7 @@ package message
 
 import (
 	"mime"
-	"net/mail"
 	"net/textproto"
-	"regexp"
 	"strings"
 	"time"
 
@@ -86,10 +84,6 @@ func GetHeader(msg *pmapi.Message) textproto.MIMEHeader { //nolint[funlen]
 	}
 	if msg.ConversationID != "" {
 		h.Set("X-Pm-ConversationID-Id", msg.ConversationID)
-		if references := h.Get("References"); !strings.Contains(references, msg.ConversationID) {
-			references += " <" + msg.ConversationID + "@" + pmapi.ConversationIDDomain + ">"
-			h.Set("References", references)
-		}
 	}
 
 	return h
@@ -141,46 +135,3 @@ func GetAttachmentHeader(att *pmapi.Attachment) textproto.MIMEHeader {
 
 	return h
 }
-
-var reEmailComment = regexp.MustCompile("[(][^)]*[)]") // nolint[gochecknoglobals]
-
-// parseAddressComment removes the comments completely even though they should be allowed
-// http://tools.wordtothewise.com/rfc/822
-// NOTE: This should be supported in go>1.10 but it seems it's not ¯\_(ツ)_/¯
-func parseAddressComment(raw string) string {
-	return reEmailComment.ReplaceAllString(raw, "")
-}
-
-func parseAddressList(val string) (addrs []*mail.Address, err error) {
-	if val == "" || val == "<>" {
-		return
-	}
-
-	addrs, err = mail.ParseAddressList(parseAddressComment(val))
-	if err == nil {
-		if addrs == nil {
-			addrs = []*mail.Address{}
-		}
-		return
-	}
-
-	// Probably missing encoding error -- try to at least parse addresses in brackets.
-	first := strings.Index(val, "<")
-	last := strings.LastIndex(val, ">")
-	if first < 0 || last < 0 || first >= last {
-		return
-	}
-	var addrList []string
-	open := first
-	for open < last && 0 <= open {
-		val = val[open:]
-		close := strings.Index(val, ">")
-		addrList = append(addrList, val[:close+1])
-		val = val[close:]
-		open = strings.Index(val, "<")
-		last = strings.LastIndex(val, ">")
-	}
-	val = strings.Join(addrList, ", ")
-
-	return mail.ParseAddressList(val)
-}

pkg/message/init.go

@@ -0,0 +1,27 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package message
+
+import (
+	"github.com/ProtonMail/go-rfc5322"
+	pmmime "github.com/ProtonMail/proton-bridge/pkg/mime"
+)
+
+func init() { // nolint[noinit]
+	rfc5322.CharsetReader = pmmime.CharsetReader
+}

pkg/message/parser.go

@@ -26,6 +26,7 @@ import (
 	"net/textproto"
 	"strings"
 
+	"github.com/ProtonMail/go-rfc5322"
 	"github.com/ProtonMail/proton-bridge/pkg/message/parser"
 	pmmime "github.com/ProtonMail/proton-bridge/pkg/mime"
 	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
@@ -365,7 +366,6 @@ func attachPublicKey(p *parser.Part, key, keyName string) {
 	})
 }
 
-// NOTE: We should use our own ParseAddressList here.
 func parseMessageHeader(m *pmapi.Message, h message.Header) error { // nolint[funlen]
 	mimeHeader, err := toMailHeader(h)
 	if err != nil {
@@ -373,59 +373,64 @@ func parseMessageHeader(m *pmapi.Message, h message.Header) error { // nolint[fu
 	}
 	m.Header = mimeHeader
 
-	if err := forEachDecodedHeaderField(h, func(key, val string) error {
+	fields := h.Fields()
-		switch strings.ToLower(key) {
+
+	for fields.Next() {
+		switch strings.ToLower(fields.Key()) {
 		case "subject":
-			m.Subject = val
+			s, err := fields.Text()
+			if err != nil {
+				if s, err = pmmime.DecodeHeader(fields.Value()); err != nil {
+					return errors.Wrap(err, "failed to parse subject")
+				}
+			}
+
+			m.Subject = s
 
 		case "from":
-			sender, err := parseAddressList(val)
+			sender, err := rfc5322.ParseAddressList(fields.Value())
 			if err != nil {
-				return err
+				return errors.Wrap(err, "failed to parse from")
 			}
 			if len(sender) > 0 {
 				m.Sender = sender[0]
 			}
 
 		case "to":
-			toList, err := parseAddressList(val)
+			toList, err := rfc5322.ParseAddressList(fields.Value())
 			if err != nil {
-				return err
+				return errors.Wrap(err, "failed to parse to")
 			}
 			m.ToList = toList
 
 		case "reply-to":
-			replyTos, err := parseAddressList(val)
+			replyTos, err := rfc5322.ParseAddressList(fields.Value())
 			if err != nil {
-				return err
+				return errors.Wrap(err, "failed to parse reply-to")
 			}
 			m.ReplyTos = replyTos
 
 		case "cc":
-			ccList, err := parseAddressList(val)
+			ccList, err := rfc5322.ParseAddressList(fields.Value())
 			if err != nil {
-				return err
+				return errors.Wrap(err, "failed to parse cc")
 			}
 			m.CCList = ccList
 
 		case "bcc":
-			bccList, err := parseAddressList(val)
+			bccList, err := rfc5322.ParseAddressList(fields.Value())
 			if err != nil {
-				return err
+				return errors.Wrap(err, "failed to parse bcc")
 			}
 			m.BCCList = bccList
 
 		case "date":
-			date, err := mail.ParseDate(val)
+			date, err := rfc5322.ParseDateTime(fields.Value())
 			if err != nil {
-				return err
+				return errors.Wrap(err, "failed to parse date")
 			}
 			m.Time = date.Unix()
 		}
-
-		return nil
-	}); err != nil {
-		return err
 	}
 
 	return nil
@@ -469,29 +474,6 @@ func parseAttachment(h message.Header) (*pmapi.Attachment, error) {
 	return att, nil
 }
 
-func forEachDecodedHeaderField(h message.Header, fn func(string, string) error) error {
-	fields := h.Fields()
-
-	for fields.Next() {
-		text, err := fields.Text()
-		if err != nil {
-			if !message.IsUnknownCharset(err) {
-				return err
-			}
-
-			if text, err = pmmime.DecodeHeader(fields.Value()); err != nil {
-				return err
-			}
-		}
-
-		if err := fn(fields.Key(), text); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
 func toMailHeader(h message.Header) (mail.Header, error) {
 	mimeHeader := make(mail.Header)
 
@@ -517,3 +499,26 @@ func toMIMEHeader(h message.Header) (textproto.MIMEHeader, error) {
 
 	return mimeHeader, nil
 }
+
+func forEachDecodedHeaderField(h message.Header, fn func(string, string) error) error {
+	fields := h.Fields()
+
+	for fields.Next() {
+		text, err := fields.Text()
+		if err != nil {
+			if !message.IsUnknownCharset(err) {
+				return err
+			}
+
+			if text, err = pmmime.DecodeHeader(fields.Value()); err != nil {
+				return err
+			}
+		}
+
+		if err := fn(fields.Key(), text); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

pkg/message/parser_test.go

@@ -498,80 +498,3 @@ func readerToString(r io.Reader) string {
 
 	return string(b)
 }
-
-func TestRFC822AddressFormat(t *testing.T) { //nolint[funlen]
-	tests := []struct {
-		address  string
-		expected []string
-	}{
-		{
-			" normal name  <username@server.com>",
-			[]string{
-				"\"normal name\" <username@server.com>",
-			},
-		},
-		{
-			" \"comma, name\"  <username@server.com>",
-			[]string{
-				"\"comma, name\" <username@server.com>",
-			},
-		},
-		{
-			" name  <username@server.com> (ignore comment)",
-			[]string{
-				"\"name\" <username@server.com>",
-			},
-		},
-		{
-			" name (ignore comment)  <username@server.com>,  (Comment as name) username2@server.com",
-			[]string{
-				"\"name\" <username@server.com>",
-				"<username2@server.com>",
-			},
-		},
-		{
-			" normal name  <username@server.com>, (comment)All.(around)address@(the)server.com",
-			[]string{
-				"\"normal name\" <username@server.com>",
-				"<All.address@server.com>",
-			},
-		},
-		{
-			" normal name  <username@server.com>, All.(\"comma, in comment\")address@(the)server.com",
-			[]string{
-				"\"normal name\" <username@server.com>",
-				"<All.address@server.com>",
-			},
-		},
-		{
-			" \"normal name\"  <username@server.com>, \"comma, name\" <address@server.com>",
-			[]string{
-				"\"normal name\" <username@server.com>",
-				"\"comma, name\" <address@server.com>",
-			},
-		},
-		{
-			" \"comma, one\"  <username@server.com>, \"comma, two\" <address@server.com>",
-			[]string{
-				"\"comma, one\" <username@server.com>",
-				"\"comma, two\" <address@server.com>",
-			},
-		},
-		{
-			" \"comma, name\"  <username@server.com>, another, name <address@server.com>",
-			[]string{
-				"\"comma, name\" <username@server.com>",
-				"\"another, name\" <address@server.com>",
-			},
-		},
-	}
-
-	for _, data := range tests {
-		result, err := parseAddressList(data.address)
-		assert.NoError(t, err)
-		assert.Len(t, result, len(data.expected))
-		for i, result := range result {
-			assert.Equal(t, data.expected[i], result.String())
-		}
-	}
-}

pkg/mime/encoding.go

@@ -32,17 +32,19 @@ import (
 	"golang.org/x/text/encoding/htmlindex"
 )
 
-var wordDec = &mime.WordDecoder{
+func CharsetReader(charset string, input io.Reader) (io.Reader, error) {
-	CharsetReader: func(charset string, input io.Reader) (io.Reader, error) {
+	dec, err := SelectDecoder(charset)
-		dec, err := SelectDecoder(charset)
+	if err != nil {
-		if err != nil {
+		return nil, err
-			return nil, err
+	}
-		}
+	if dec == nil { // utf-8
-		if dec == nil { // utf-8
+		return input, nil
-			return input, nil
+	}
-		}
+	return dec.Reader(input), nil
-		return dec.Reader(input), nil
+}
-	},
+
+var WordDec = &mime.WordDecoder{
+	CharsetReader: CharsetReader,
 }
 
 // Expects trimmed lowercase.
@@ -180,7 +182,7 @@ func SelectDecoder(charset string) (decoder *encoding.Decoder, err error) {
 
 // DecodeHeader if needed. Returns error if raw contains non-utf8 characters.
 func DecodeHeader(raw string) (decoded string, err error) {
-	if decoded, err = wordDec.DecodeHeader(raw); err != nil {
+	if decoded, err = WordDec.DecodeHeader(raw); err != nil {
 		decoded = raw
 	}
 	if !utf8.ValidString(decoded) {

pkg/pmapi/check_connection.go

@@ -0,0 +1,94 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"time"
+)
+
+const protonStatusURL = "http://protonstatus.com/vpn_status"
+
+// ErrNoInternetConnection indicates that both protonstatus and the API are unreachable.
+var ErrNoInternetConnection = errors.New("no internet connection")
+
+// CheckConnection returns an error if there is no internet connection.
+// This should be moved to the ConnectionManager when it is implemented.
+func (cm *ClientManager) CheckConnection() error {
+	// We use a normal dialer here which doesn't check tls fingerprints.
+	client := &http.Client{Timeout: time.Second * 10}
+
+	// Do not cumulate timeouts, use goroutines.
+	retStatus := make(chan error)
+	retAPI := make(chan error)
+
+	// vpn_status endpoint is fast and returns only OK. We check the connection only.
+	go checkConnection(client, protonStatusURL, retStatus)
+
+	// Check of API reachability also uses a fast endpoint.
+	go checkConnection(client, cm.GetRootURL()+"/tests/ping", retAPI)
+
+	errStatus := <-retStatus
+	errAPI := <-retAPI
+
+	switch {
+	case errStatus == nil && errAPI == nil:
+		return nil
+
+	case errStatus == nil && errAPI != nil:
+		cm.log.Error("ProtonStatus is reachable but API is not")
+		return ErrAPINotReachable
+
+	case errStatus != nil && errAPI == nil:
+		cm.log.Warn("API is reachable but protonstatus is not")
+		return nil
+
+	case errStatus != nil && errAPI != nil:
+		cm.log.Error("Both ProtonStatus and API are unreachable")
+		return ErrNoInternetConnection
+	}
+
+	return nil
+}
+
+// CheckConnection returns an error if there is no internet connection.
+func CheckConnection() error {
+	client := &http.Client{Timeout: time.Second * 10}
+	retStatus := make(chan error)
+	go checkConnection(client, protonStatusURL, retStatus)
+	return <-retStatus
+}
+
+func checkConnection(client *http.Client, url string, errorChannel chan error) {
+	resp, err := client.Get(url)
+	if err != nil {
+		errorChannel <- err
+		return
+	}
+
+	_ = resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		errorChannel <- fmt.Errorf("HTTP status code %d", resp.StatusCode)
+		return
+	}
+
+	errorChannel <- nil
+}

pkg/pmapi/clientmanager.go

@@ -305,72 +305,6 @@ func (cm *ClientManager) GetAuthUpdateChannel() chan ClientAuth {
 	return cm.authUpdates
 }
 
-// ErrNoInternetConnection indicates that both protonstatus and the API are unreachable.
-var ErrNoInternetConnection = errors.New("no internet connection")
-
-// CheckConnection returns an error if there is no internet connection.
-// This should be moved to the ConnectionManager when it is implemented.
-func (cm *ClientManager) CheckConnection() error {
-	client := getHTTPClient(cm.config, cm.roundTripper, cm.cookieJar)
-
-	// Do not cumulate timeouts, use goroutines.
-	retStatus := make(chan error)
-	retAPI := make(chan error)
-
-	// vpn_status endpoint is fast and returns only OK. We check the connection only.
-	go checkConnection(client, "https://protonstatus.com/vpn_status", retStatus)
-
-	// Check of API reachability also uses a fast endpoint.
-	go checkConnection(client, cm.GetRootURL()+"/tests/ping", retAPI)
-
-	errStatus := <-retStatus
-	errAPI := <-retAPI
-
-	switch {
-	case errStatus == nil && errAPI == nil:
-		return nil
-
-	case errStatus == nil && errAPI != nil:
-		cm.log.Error("ProtonStatus is reachable but API is not")
-		return ErrAPINotReachable
-
-	case errStatus != nil && errAPI == nil:
-		cm.log.Warn("API is reachable but protonstatus is not")
-		return nil
-
-	case errStatus != nil && errAPI != nil:
-		cm.log.Error("Both ProtonStatus and API are unreachable")
-		return ErrNoInternetConnection
-	}
-
-	return nil
-}
-
-// CheckConnection returns an error if there is no internet connection.
-func CheckConnection() error {
-	client := &http.Client{Timeout: time.Second * 10}
-	retStatus := make(chan error)
-	go checkConnection(client, "https://protonstatus.com/vpn_status", retStatus)
-	return <-retStatus
-}
-
-func checkConnection(client *http.Client, url string, errorChannel chan error) {
-	resp, err := client.Get(url)
-	if err != nil {
-		errorChannel <- err
-		return
-	}
-
-	_ = resp.Body.Close()
-
-	if resp.StatusCode != 200 {
-		errorChannel <- fmt.Errorf("HTTP status code %d", resp.StatusCode)
-		return
-	}
-
-	errorChannel <- nil
-}
-
 // setTokenIfUnset sets the token for the given userID if it wasn't already set.
 // The set token does not expire.
 func (cm *ClientManager) setTokenIfUnset(userID, token string) {

pkg/pmapi/config_qa.go

@@ -15,7 +15,7 @@
 // You should have received a copy of the GNU General Public License
 // along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
 
-// +build pmapi_env
+// +build pmapi_qa
 
 package pmapi
 

pkg/pmapi/mocks/mocks.go

@@ -5,12 +5,11 @@
 package mocks
 
 import (
-	io "io"
-	reflect "reflect"
-
 	crypto "github.com/ProtonMail/gopenpgp/v2/crypto"
 	pmapi "github.com/ProtonMail/proton-bridge/pkg/pmapi"
 	gomock "github.com/golang/mock/gomock"
+	io "io"
+	reflect "reflect"
 )
 
 // MockClient is a mock of Client interface

pkg/pmapi/proxy_test.go

@@ -382,7 +382,7 @@ func TestProxyProvider_UseProxy_RevertIfProxyStopsWorkingAndOriginalAPIIsReachab
 	// The error should be ErrAPINotReachable because the connection dropped intermittently but
 	// the original API is now reachable (see Alternative-Routing-v2 spec for details).
 	url, err = cm.switchToReachableServer()
-	require.EqualError(t, err, ErrAPINotReachable.Error())
+	require.Error(t, err)
 	require.Equal(t, rootURL, url)
 	require.Equal(t, rootURL, cm.getHost())
 }

pkg/pmapi/tlsreport.go

@@ -35,13 +35,21 @@ var ErrTLSMismatch = errors.New("no TLS fingerprint match found")
 // TrustedAPIPins contains trusted public keys of the protonmail API and proxies.
 // NOTE: the proxy pins are the same for all proxy servers, guaranteed by infra team ;)
 var TrustedAPIPins = []string{ // nolint[gochecknoglobals]
+	// api.protonmail.ch
 	`pin-sha256="drtmcR2kFkM8qJClsuWgUzxgBkePfRCkRpqUesyDmeE="`, // current
-	`pin-sha256="YRGlaY0jyJ4Jw2/4M8FIftwbDIQfh8Sdro96CeEel54="`, // hot
+	`pin-sha256="YRGlaY0jyJ4Jw2/4M8FIftwbDIQfh8Sdro96CeEel54="`, // hot backup
-	`pin-sha256="AfMENBVvOS8MnISprtvyPsjKlPooqh8nMB/pvCrpJpw="`, // cold
+	`pin-sha256="AfMENBVvOS8MnISprtvyPsjKlPooqh8nMB/pvCrpJpw="`, // cold backup
-	`pin-sha256="EU6TS9MO0L/GsDHvVc9D5fChYLNy5JdGYpJw0ccgetM="`, // proxy main
+
-	`pin-sha256="iKPIHPnDNqdkvOnTClQ8zQAIKG0XavaPkcEo0LBAABA="`, // proxy backup 1
+	// protonmail.com
-	`pin-sha256="MSlVrBCdL0hKyczvgYVSRNm88RicyY04Q2y5qrBt0xA="`, // proxy backup 2
+	`pin-sha256="8joiNBdqaYiQpKskgtkJsqRxF7zN0C0aqfi8DacknnI="`, // current
-	`pin-sha256="C2UxW0T1Ckl9s+8cXfjXxlEqwAfPM4HiW2y3UdtBeCw="`, // proxy backup 3
+	`pin-sha256="JMI8yrbc6jB1FYGyyWRLFTmDNgIszrNEMGlgy972e7w="`, // hot backup
+	`pin-sha256="Iu44zU84EOCZ9vx/vz67/MRVrxF1IO4i4NIa8ETwiIY="`, // cold backup
+
+	// proxies
+	`pin-sha256="EU6TS9MO0L/GsDHvVc9D5fChYLNy5JdGYpJw0ccgetM="`, // main
+	`pin-sha256="iKPIHPnDNqdkvOnTClQ8zQAIKG0XavaPkcEo0LBAABA="`, // backup 1
+	`pin-sha256="MSlVrBCdL0hKyczvgYVSRNm88RicyY04Q2y5qrBt0xA="`, // backup 2
+	`pin-sha256="C2UxW0T1Ckl9s+8cXfjXxlEqwAfPM4HiW2y3UdtBeCw="`, // backup 3
 }
 
 // TLSReportURI is the address where TLS reports should be sent.

release-notes/bugs-bridge.txt

@@ -1,3 +1,3 @@
-• Fixed rare mail loss when moving from Spam folder
+• Ensured that conversations are properly threaded
-• Limited log size
+• Fixed Linux font issues (Fedora)
-• Fixed Linux font issues (mouse hover).
+• Better handling of Mime encrypted messages

release-notes/bugs-importexport.txt

@@ -1,3 +0,0 @@
-• Linux font issues - Fedora specific
-• App response to the user pausing and canceling import or export
-• Handling errors during update

release-notes/notes-bridge.txt

@@ -1,8 +1,4 @@
-• Bulletproofing against any potential data loss and/or duplication
+• Ensured better message flow by refactoring both address and date parsing
-• Performance improvements for handling attachments and non-standard formatting
+• Improved secure connectivity checks
-• Better stability of the message parser
+• Better deb packaging
-• Additional foreign encoding support for outgoing messages
+• More robust error handling
-• Complete refactor of the way messages are parsed to simplify code maintenance
-• Improved User-Agent detection
-• Added MacOS Big Sur compatibility
-• Added persistent anonymous API cookies

release-notes/notes-importexport.txt

@@ -1,5 +1,3 @@
-• Improvements to the import from large mbox files with multiple labels
+• Further improvements to address and date parsing
-• Not allow to run multiple instances of the app or transfers at the same time
+• Better handling and displaying of skipped messages
-• Various enhancements of the import process related to parsing
+• Improved error reporting
-• Cosmetic GUI changes
-• Better error handling

test/features/bridge/imap/message/update_spam.feature

@@ -4,14 +4,14 @@ Feature: IMAP update messages in Spam folder
     # Messages are inserted in opposite way to keep increasing ID.
     # Sequence numbers are then opposite than listed above.
     And there are messages in mailbox "Spam" for "user"
-      | from              | to         | subject | body  | read  | starred | deleted |
+      | id | from              | to         | subject | body  | read  | starred | deleted |
-      | john.doe@mail.com | user@pm.me | foo     | hello | false | false   | false   |
+      | 1  | john.doe@mail.com | user@pm.me | foo     | hello | false | false   | false   |
-      | jane.doe@mail.com | name@pm.me | bar     | world | true  | true    | false   |
+      | 2  | jane.doe@mail.com | name@pm.me | bar     | world | true  | true    | false   |
     And there is IMAP client logged in as "user"
     And there is IMAP client selected in "Spam"
 
   Scenario: Mark message as read only
-    When IMAP client marks message "2" with "\Seen"
+    When IMAP client marks message seq "1" with "\Seen"
     Then IMAP response is "OK"
     And message "1" in "Spam" for "user" is marked as read
     And message "1" in "Spam" for "user" is marked as unstarred

test/mocks/imap_response.go

@@ -155,7 +155,7 @@ func (ir *IMAPResponse) AssertSectionsInOrder(wantRegexps ...string) *IMAPRespon
 func (ir *IMAPResponse) AssertSections(wantRegexps ...string) *IMAPResponse {
 	ir.wait()
 	for _, wantRegexp := range wantRegexps {
-		a.NoError(ir.t, ir.hasSectionRegexp(wantRegexp), "regexp %v not found", wantRegexp)
+		a.NoError(ir.t, ir.hasSectionRegexp(wantRegexp), "regexp %v not found\nSections: %v", wantRegexp, ir.sections)
 	}
 	return ir
 }

test/mocks/smtp.go

@@ -23,12 +23,12 @@ import (
 	"fmt"
 	"io"
 	"net"
-	"net/mail"
 	"os"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/ProtonMail/go-rfc5322"
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -148,12 +148,12 @@ func (c *SMTPClient) SendMail(r io.Reader, bcc string) *SMTPResponse {
 
 		from = c.address
 		if from == "" && strings.HasPrefix(line, "From: ") {
-			if addr, err := mail.ParseAddress(line[6:]); err == nil {
+			if addrs, err := rfc5322.ParseAddressList(line[6:]); err == nil {
-				from = addr.Address
+				from = addrs[0].Address
 			}
 		}
 		if strings.HasPrefix(line, "To: ") || strings.HasPrefix(line, "CC: ") {
-			if addrs, err := mail.ParseAddressList(line[4:]); err == nil {
+			if addrs, err := rfc5322.ParseAddressList(line[4:]); err == nil {
 				for _, addr := range addrs {
 					tos = append(tos, addr.Address)
 				}

test/store_checks_test.go

@@ -19,10 +19,10 @@ package tests
 
 import (
 	"fmt"
-	"net/mail"
 	"strings"
 	"time"
 
+	"github.com/ProtonMail/go-rfc5322"
 	"github.com/ProtonMail/proton-bridge/internal/store"
 	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
 	"github.com/ProtonMail/proton-bridge/test/accounts"
@@ -276,15 +276,15 @@ func messagesContainsMessageRow(account *accounts.TestAccount, allMessages []int
 }
 
 func areAddressesSame(first, second string) bool {
-	firstAddress, err := mail.ParseAddress(first)
+	firstAddress, err := rfc5322.ParseAddressList(first)
 	if err != nil {
 		return false
 	}
-	secondAddress, err := mail.ParseAddress(second)
+	secondAddress, err := rfc5322.ParseAddressList(second)
 	if err != nil {
 		return false
 	}
-	return firstAddress.Address == secondAddress.Address
+	return firstAddress[0].Address == secondAddress[0].Address
 }
 
 func messagesInMailboxForUserIsMarkedAsRead(bddMessageIDs, mailboxName, bddUserID string) error {

test/transfer_checks_test.go

@@ -21,13 +21,13 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"net/mail"
 	"os"
 	"path/filepath"
 	"sort"
 	"strings"
 	"time"
 
+	"github.com/ProtonMail/go-rfc5322"
 	"github.com/cucumber/godog"
 	"github.com/cucumber/godog/gherkin"
 	"github.com/emersion/go-mbox"
@@ -65,15 +65,15 @@ func progressFinishedWith(wantResponse string) error {
 
 func transferExportedNumberOfMessages(wantCount int) error {
 	progress := ctx.GetTransferProgress()
-	_, _, exported, _, _ := progress.GetCounts() //nolint[dogsled]
+	counts := progress.GetCounts()
-	a.Equal(ctx.GetTestingT(), uint(wantCount), exported)
+	a.Equal(ctx.GetTestingT(), uint(wantCount), counts.Exported)
 	return ctx.GetTestingError()
 }
 
 func transferImportedNumberOfMessages(wantCount int) error {
 	progress := ctx.GetTransferProgress()
-	_, imported, _, _, _ := progress.GetCounts() //nolint[dogsled]
+	counts := progress.GetCounts()
-	a.Equal(ctx.GetTestingT(), uint(wantCount), imported)
+	a.Equal(ctx.GetTestingT(), uint(wantCount), counts.Imported)
 	return ctx.GetTestingError()
 }
 
@@ -243,7 +243,7 @@ func parseTime(input string) (time.Time, error) {
 }
 
 func parseAddresses(input string) ([]string, error) {
-	addresses, err := mail.ParseAddressList(input)
+	addresses, err := rfc5322.ParseAddressList(input)
 	if err != nil {
 		return nil, err
 	}
@@ -255,11 +255,11 @@ func parseAddresses(input string) ([]string, error) {
 }
 
 func parseAddress(input string) (string, error) {
-	address, err := mail.ParseAddress(input)
+	address, err := rfc5322.ParseAddressList(input)
 	if err != nil {
 		return "", err
 	}
-	return address.Address, nil
+	return address[0].Address, nil
 }
 
 // BySubject implements sort.Interface based on the subject field.

unreleased.md

@@ -0,0 +1,5 @@
+# ProtonMail Bridge and Import-Export app Changelog
+
+Changelog [format](http://keepachangelog.com/en/1.0.0/)
+
+## Unreleased