We build too many walls and not enough bridges

pkg/algo/algo.go

@@ -0,0 +1,19 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+// Package algo provides some algorithm utils.
+package algo

pkg/algo/sets.go

@@ -0,0 +1,47 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package algo
+
+import "reflect"
+
+// SetIntersection complexity: O(n^2), could be better but this is simple enough
+func SetIntersection(a, b interface{}, eq func(a, b interface{}) bool) []interface{} {
+	set := make([]interface{}, 0)
+	av := reflect.ValueOf(a)
+
+	for i := 0; i < av.Len(); i++ {
+		el := av.Index(i).Interface()
+		if contains(b, el, eq) {
+			set = append(set, el)
+		}
+	}
+
+	return set
+}
+
+func contains(a, e interface{}, eq func(a, b interface{}) bool) bool {
+	v := reflect.ValueOf(a)
+
+	for i := 0; i < v.Len(); i++ {
+		if eq(v.Index(i).Interface(), e) {
+			return true
+		}
+	}
+
+	return false
+}

pkg/algo/sets_test.go

@@ -0,0 +1,71 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package algo
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+type T struct {
+	k, v int
+}
+
+func TestSetIntersection(t *testing.T) {
+	keysAreEqual := func(a, b interface{}) bool {
+		return a.(T).k == b.(T).k
+	}
+
+	type args struct {
+		a  interface{}
+		b  interface{}
+		eq func(a, b interface{}) bool
+	}
+
+	tests := []struct {
+		name string
+		args args
+		want interface{}
+	}{
+		{
+			name: "integer sets",
+			args: args{a: []int{1, 2, 3}, b: []int{3, 4, 5}, eq: func(a, b interface{}) bool { return a == b }},
+			want: []int{3},
+		},
+		{
+			name: "string sets",
+			args: args{a: []string{"1", "2", "3"}, b: []string{"3", "4", "5"}, eq: func(a, b interface{}) bool { return a == b }},
+			want: []string{"3"},
+		},
+		{
+			name: "custom comp, only compare on keys, prefer first set if keys are the same",
+			args: args{a: []T{{k: 1, v: 1}, {k: 2, v: 2}}, b: []T{{k: 2, v: 1234}, {k: 3, v: 3}}, eq: keysAreEqual},
+			want: []T{{k: 2, v: 2}},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// using cmp.Equal because it handles the interfaces correctly; testify/assert doesn't
+			// treat these as equal because their types are different ([]interface vs []int)
+			if got := SetIntersection(tt.args.a, tt.args.b, tt.args.eq); cmp.Equal(got, tt.want) {
+				t.Errorf("SetIntersection() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/args/args.go

@@ -0,0 +1,35 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package args
+
+import (
+	"os"
+	"strings"
+)
+
+// FilterProcessSerialNumberFromArgs removes additional flag from MacOS. More info ProcessSerialNumber
+// http://mirror.informatimago.com/next/developer.apple.com/documentation/Carbon/Reference/Process_Manager/prmref_main/data_type_5.html#//apple_ref/doc/uid/TP30000208/C001951
+func FilterProcessSerialNumberFromArgs() {
+	tmp := os.Args[:0]
+	for _, arg := range os.Args {
+		if !strings.Contains(arg, "-psn_") {
+			tmp = append(tmp, arg)
+		}
+	}
+	os.Args = tmp
+}

pkg/config/config.go

@@ -0,0 +1,259 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package config
+
+import (
+	"io/ioutil"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/ProtonMail/go-appdir"
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/hashicorp/go-multierror"
+)
+
+var (
+	log = GetLogEntry("config") //nolint[gochecknoglobals]
+)
+
+type appDirProvider interface {
+	UserConfig() string
+	UserCache() string
+	UserLogs() string
+}
+
+type Config struct {
+	appName        string
+	version        string
+	revision       string
+	cacheVersion   string
+	appDirs        appDirProvider
+	appDirsVersion appDirProvider
+	apiConfig      *pmapi.ClientConfig
+}
+
+// New returns fully initialized config struct.
+// `appName` should be in camelCase format for folder or file names. It's also used in API
+// as `AppVersion` which is converted to CamelCase.
+// `version` is the version of the app (e.g. v1.2.3).
+// `cacheVersion` is the version of the cache files (setting a different number will remove the old ones).
+func New(appName, version, revision, cacheVersion string) *Config {
+	appDirs := appdir.New(filepath.Join("protonmail", appName))
+	appDirsVersion := appdir.New(filepath.Join("protonmail", appName, cacheVersion))
+	return newConfig(appName, version, revision, cacheVersion, appDirs, appDirsVersion)
+}
+
+func newConfig(appName, version, revision, cacheVersion string, appDirs, appDirsVersion appDirProvider) *Config {
+	return &Config{
+		appName:        appName,
+		version:        version,
+		revision:       revision,
+		cacheVersion:   cacheVersion,
+		appDirs:        appDirs,
+		appDirsVersion: appDirsVersion,
+		apiConfig: &pmapi.ClientConfig{
+			AppVersion: strings.Title(appName) + "_" + version,
+			ClientID:   appName,
+			Transport: &http.Transport{
+				DialContext:           (&net.Dialer{Timeout: 3 * time.Second}).DialContext,
+				TLSHandshakeTimeout:   10 * time.Second,
+				ResponseHeaderTimeout: 10 * time.Second,
+			},
+			// TokenManager should not be required, but PMAPI still doesn't handle not-set cases everywhere.
+			TokenManager: pmapi.NewTokenManager(),
+		},
+	}
+}
+
+// CreateDirs creates all folders that are necessary for bridge to properly function.
+func (c *Config) CreateDirs() error {
+	// Log files.
+	if err := os.MkdirAll(c.appDirs.UserLogs(), 0700); err != nil {
+		return err
+	}
+	// TLS files.
+	if err := os.MkdirAll(c.appDirs.UserConfig(), 0750); err != nil {
+		return err
+	}
+	// Lock, events, preferences, user_info, db files.
+	if err := os.MkdirAll(c.appDirsVersion.UserCache(), 0750); err != nil {
+		return err
+	}
+	return nil
+}
+
+// ClearData removes all files except the lock file.
+// The lock file will be removed when the Bridge stops.
+func (c *Config) ClearData() error {
+	dirs := []string{
+		c.appDirs.UserLogs(),
+		c.appDirs.UserConfig(),
+		c.appDirs.UserCache(),
+	}
+	shouldRemove := func(filePath string) bool {
+		return filePath != c.GetLockPath()
+	}
+	return c.removeAllExcept(dirs, shouldRemove)
+}
+
+// ClearOldData removes all old files, such as old log files or old versions of cache and so on.
+func (c *Config) ClearOldData() error {
+	// `appDirs` is parent for `appDirsVersion`.
+	// `dir` then contains all subfolders and only `cacheVersion` should stay.
+	// But on Windows all files (dirs) are in the same one - we cannot remove log, lock or tls files.
+	dir := c.appDirs.UserCache()
+
+	return c.removeExcept(dir, func(filePath string) bool {
+		fileName := filepath.Base(filePath)
+		return (fileName != c.cacheVersion &&
+			!logFileRgx.MatchString(fileName) &&
+			filePath != c.GetTLSCertPath() &&
+			filePath != c.GetTLSKeyPath() &&
+			filePath != c.GetEventsPath() &&
+			filePath != c.GetIMAPCachePath() &&
+			filePath != c.GetLockPath() &&
+			filePath != c.GetPreferencesPath())
+	})
+}
+
+func (c *Config) removeAllExcept(dirs []string, shouldRemove func(string) bool) error {
+	var result *multierror.Error
+	for _, dir := range dirs {
+		if err := c.removeExcept(dir, shouldRemove); err != nil {
+			result = multierror.Append(result, err)
+		}
+	}
+	return result.ErrorOrNil()
+}
+
+func (c *Config) removeExcept(dir string, shouldRemove func(string) bool) error {
+	files, err := ioutil.ReadDir(dir)
+	if err != nil {
+		return err
+	}
+
+	var result *multierror.Error
+	for _, file := range files {
+		filePath := filepath.Join(dir, file.Name())
+		if !shouldRemove(filePath) {
+			continue
+		}
+
+		if !file.IsDir() {
+			if err := os.RemoveAll(filePath); err != nil {
+				result = multierror.Append(result, err)
+			}
+			continue
+		}
+
+		subDir := filepath.Join(dir, file.Name())
+		if err := c.removeExcept(subDir, shouldRemove); err != nil {
+			result = multierror.Append(result, err)
+		} else {
+			// Remove dir itself only if it's empty.
+			subFiles, err := ioutil.ReadDir(subDir)
+			if err != nil {
+				result = multierror.Append(result, err)
+			} else if len(subFiles) == 0 {
+				if err := os.RemoveAll(subDir); err != nil {
+					result = multierror.Append(result, err)
+				}
+			}
+		}
+	}
+	return result.ErrorOrNil()
+}
+
+// IsDevMode should be used for development conditions such us whether to send sentry reports.
+func (c *Config) IsDevMode() bool {
+	return os.Getenv("PROTONMAIL_ENV") == "dev"
+}
+
+// GetLogDir returns folder for log files.
+func (c *Config) GetLogDir() string {
+	return c.appDirs.UserLogs()
+}
+
+// GetLogPrefix returns prefix for log files. Bridge uses format vVERSION.
+func (c *Config) GetLogPrefix() string {
+	return "v" + c.version + "_" + c.revision
+}
+
+// GetTLSCertPath returns path to certificate; used for TLS servers (IMAP, SMTP and API).
+func (c *Config) GetTLSCertPath() string {
+	return filepath.Join(c.appDirs.UserConfig(), "cert.pem")
+}
+
+// GetTLSKeyPath returns path to private key; used for TLS servers (IMAP, SMTP and API).
+func (c *Config) GetTLSKeyPath() string {
+	return filepath.Join(c.appDirs.UserConfig(), "key.pem")
+}
+
+// GetDBDir returns folder for db files.
+func (c *Config) GetDBDir() string {
+	return filepath.Join(c.appDirsVersion.UserCache())
+}
+
+// GetEventsPath returns path to events file containing the last processed event IDs.
+func (c *Config) GetEventsPath() string {
+	return filepath.Join(c.appDirsVersion.UserCache(), "events.json")
+}
+
+// GetIMAPCachePath returns path to file with IMAP status.
+func (c *Config) GetIMAPCachePath() string {
+	return filepath.Join(c.appDirsVersion.UserCache(), "user_info.json")
+}
+
+// GetLockPath returns path to lock file to check if bridge is already running.
+func (c *Config) GetLockPath() string {
+	return filepath.Join(c.appDirsVersion.UserCache(), c.appName+".lock")
+}
+
+// GetUpdateDir returns folder for update files; such as new binary.
+func (c *Config) GetUpdateDir() string {
+	return filepath.Join(c.appDirsVersion.UserCache(), "updates")
+}
+
+// GetPreferencesPath returns path to preference file.
+func (c *Config) GetPreferencesPath() string {
+	return filepath.Join(c.appDirsVersion.UserCache(), "prefs.json")
+}
+
+// GetAPIConfig returns config for ProtonMail API.
+func (c *Config) GetAPIConfig() *pmapi.ClientConfig {
+	return c.apiConfig
+}
+
+// GetDefaultAPIPort returns default Bridge local API port.
+func (c *Config) GetDefaultAPIPort() int {
+	return 1042
+}
+
+// GetDefaultIMAPPort returns default Bridge IMAP port.
+func (c *Config) GetDefaultIMAPPort() int {
+	return 1143
+}
+
+// GetDefaultSMTPPort returns default Bridge SMTP port.
+func (c *Config) GetDefaultSMTPPort() int {
+	return 1025
+}

pkg/config/config_test.go

@@ -0,0 +1,238 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package config
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	gomock "github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/require"
+)
+
+const testAppName = "bridge-test"
+
+var testConfigDir string //nolint[gochecknoglobals]
+
+func TestMain(m *testing.M) {
+	setupTestConfig()
+	setupTestLogs()
+	code := m.Run()
+	shutdownTestConfig()
+	shutdownTestLogs()
+	shutdownTestPreferences()
+	os.Exit(code)
+}
+
+func setupTestConfig() {
+	var err error
+	testConfigDir, err = ioutil.TempDir("", "config")
+	if err != nil {
+		panic(err)
+	}
+}
+
+func shutdownTestConfig() {
+	_ = os.RemoveAll(testConfigDir)
+}
+
+type mocks struct {
+	t *testing.T
+
+	ctrl          *gomock.Controller
+	appDir        *MockappDirer
+	appDirVersion *MockappDirer
+}
+
+func initMocks(t *testing.T) mocks {
+	mockCtrl := gomock.NewController(t)
+	return mocks{
+		t: t,
+
+		ctrl:          mockCtrl,
+		appDir:        NewMockappDirer(mockCtrl),
+		appDirVersion: NewMockappDirer(mockCtrl),
+	}
+}
+
+func TestClearDataLinux(t *testing.T) {
+	m := initMocks(t)
+	defer m.ctrl.Finish()
+
+	createTestStructureLinux(m, testConfigDir)
+	cfg := newConfig(testAppName, "v1", "rev123", "c2", m.appDir, m.appDirVersion)
+	require.NoError(t, cfg.ClearData())
+	checkFileNames(t, testConfigDir, []string{
+		"cache",
+		"cache/c2",
+		"cache/c2/bridge-test.lock",
+		"config",
+		"logs",
+	})
+}
+
+func TestClearDataWindows(t *testing.T) {
+	m := initMocks(t)
+	defer m.ctrl.Finish()
+
+	createTestStructureWindows(m, testConfigDir)
+	cfg := newConfig(testAppName, "v1", "rev123", "c2", m.appDir, m.appDirVersion)
+	require.NoError(t, cfg.ClearData())
+	checkFileNames(t, testConfigDir, []string{
+		"cache",
+		"cache/c2",
+		"cache/c2/bridge-test.lock",
+		"config",
+	})
+}
+
+// OldData touches only cache folder.
+// Removes only c1 folder as nothing else is part of cache folder on Linux/Mac.
+func TestClearOldDataLinux(t *testing.T) {
+	m := initMocks(t)
+	defer m.ctrl.Finish()
+
+	createTestStructureLinux(m, testConfigDir)
+	cfg := newConfig(testAppName, "v1", "rev123", "c2", m.appDir, m.appDirVersion)
+	require.NoError(t, cfg.ClearOldData())
+	checkFileNames(t, testConfigDir, []string{
+		"cache",
+		"cache/c2",
+		"cache/c2/bridge-test.lock",
+		"cache/c2/events.json",
+		"cache/c2/mailbox-user@pm.me.db",
+		"cache/c2/prefs.json",
+		"cache/c2/updates",
+		"cache/c2/user_info.json",
+		"config",
+		"config/cert.pem",
+		"config/key.pem",
+		"logs",
+		"logs/other.log",
+		"logs/v1_10.log",
+		"logs/v1_11.log",
+		"logs/v2_12.log",
+		"logs/v2_13.log",
+	})
+}
+
+// OldData touches only cache folder. Removes everything except c2 folder
+// and bridge log files which are part of cache folder on Windows.
+func TestClearOldDataWindows(t *testing.T) {
+	m := initMocks(t)
+	defer m.ctrl.Finish()
+
+	createTestStructureWindows(m, testConfigDir)
+	cfg := newConfig(testAppName, "v1", "rev123", "c2", m.appDir, m.appDirVersion)
+	require.NoError(t, cfg.ClearOldData())
+	checkFileNames(t, testConfigDir, []string{
+		"cache",
+		"cache/c2",
+		"cache/c2/bridge-test.lock",
+		"cache/c2/events.json",
+		"cache/c2/mailbox-user@pm.me.db",
+		"cache/c2/prefs.json",
+		"cache/c2/updates",
+		"cache/c2/user_info.json",
+		"cache/v1_10.log",
+		"cache/v1_11.log",
+		"cache/v2_12.log",
+		"cache/v2_13.log",
+		"config",
+		"config/cert.pem",
+		"config/key.pem",
+	})
+}
+
+func createTestStructureLinux(m mocks, baseDir string) {
+	logsDir := filepath.Join(baseDir, "logs")
+	configDir := filepath.Join(baseDir, "config")
+	cacheDir := filepath.Join(baseDir, "cache")
+	versionedOldCacheDir := filepath.Join(baseDir, "cache", "c1")
+	versionedCacheDir := filepath.Join(baseDir, "cache", "c2")
+	createTestStructure(m, baseDir, logsDir, configDir, cacheDir, versionedOldCacheDir, versionedCacheDir)
+}
+
+func createTestStructureWindows(m mocks, baseDir string) {
+	logsDir := filepath.Join(baseDir, "cache")
+	configDir := filepath.Join(baseDir, "config")
+	cacheDir := filepath.Join(baseDir, "cache")
+	versionedOldCacheDir := filepath.Join(baseDir, "cache", "c1")
+	versionedCacheDir := filepath.Join(baseDir, "cache", "c2")
+	createTestStructure(m, baseDir, logsDir, configDir, cacheDir, versionedOldCacheDir, versionedCacheDir)
+}
+
+func createTestStructure(m mocks, baseDir, logsDir, configDir, cacheDir, versionedOldCacheDir, versionedCacheDir string) {
+	m.appDir.EXPECT().UserLogs().Return(logsDir).AnyTimes()
+	m.appDir.EXPECT().UserConfig().Return(configDir).AnyTimes()
+	m.appDir.EXPECT().UserCache().Return(cacheDir).AnyTimes()
+	m.appDirVersion.EXPECT().UserCache().Return(versionedCacheDir).AnyTimes()
+
+	require.NoError(m.t, os.RemoveAll(baseDir))
+	require.NoError(m.t, os.MkdirAll(baseDir, 0700))
+	require.NoError(m.t, os.MkdirAll(logsDir, 0700))
+	require.NoError(m.t, os.MkdirAll(configDir, 0700))
+	require.NoError(m.t, os.MkdirAll(cacheDir, 0700))
+	require.NoError(m.t, os.MkdirAll(versionedOldCacheDir, 0700))
+	require.NoError(m.t, os.MkdirAll(versionedCacheDir, 0700))
+	require.NoError(m.t, os.MkdirAll(filepath.Join(versionedCacheDir, "updates"), 0700))
+
+	require.NoError(m.t, ioutil.WriteFile(filepath.Join(logsDir, "other.log"), []byte("Hello"), 0755))
+	require.NoError(m.t, ioutil.WriteFile(filepath.Join(logsDir, "v1_10.log"), []byte("Hello"), 0755))
+	require.NoError(m.t, ioutil.WriteFile(filepath.Join(logsDir, "v1_11.log"), []byte("Hello"), 0755))
+	require.NoError(m.t, ioutil.WriteFile(filepath.Join(logsDir, "v2_12.log"), []byte("Hello"), 0755))
+	require.NoError(m.t, ioutil.WriteFile(filepath.Join(logsDir, "v2_13.log"), []byte("Hello"), 0755))
+
+	require.NoError(m.t, ioutil.WriteFile(filepath.Join(configDir, "cert.pem"), []byte("Hello"), 0755))
+	require.NoError(m.t, ioutil.WriteFile(filepath.Join(configDir, "key.pem"), []byte("Hello"), 0755))
+
+	require.NoError(m.t, ioutil.WriteFile(filepath.Join(versionedOldCacheDir, "prefs.json"), []byte("Hello"), 0755))
+	require.NoError(m.t, ioutil.WriteFile(filepath.Join(versionedOldCacheDir, "events.json"), []byte("Hello"), 0755))
+	require.NoError(m.t, ioutil.WriteFile(filepath.Join(versionedOldCacheDir, "user_info.json"), []byte("Hello"), 0755))
+	require.NoError(m.t, ioutil.WriteFile(filepath.Join(versionedOldCacheDir, "mailbox-user@pm.me.db"), []byte("Hello"), 0755))
+	require.NoError(m.t, ioutil.WriteFile(filepath.Join(versionedCacheDir, "prefs.json"), []byte("Hello"), 0755))
+	require.NoError(m.t, ioutil.WriteFile(filepath.Join(versionedCacheDir, "events.json"), []byte("Hello"), 0755))
+	require.NoError(m.t, ioutil.WriteFile(filepath.Join(versionedCacheDir, "user_info.json"), []byte("Hello"), 0755))
+	require.NoError(m.t, ioutil.WriteFile(filepath.Join(versionedCacheDir, testAppName+".lock"), []byte("Hello"), 0755))
+	require.NoError(m.t, ioutil.WriteFile(filepath.Join(versionedCacheDir, "mailbox-user@pm.me.db"), []byte("Hello"), 0755))
+}
+
+func checkFileNames(t *testing.T, dir string, expectedFileNames []string) {
+	fileNames := getFileNames(t, dir)
+	require.Equal(t, expectedFileNames, fileNames)
+}
+
+func getFileNames(t *testing.T, dir string) []string {
+	files, err := ioutil.ReadDir(dir)
+	require.NoError(t, err)
+
+	fileNames := []string{}
+	for _, file := range files {
+		fileNames = append(fileNames, file.Name())
+		if file.IsDir() {
+			subDir := filepath.Join(dir, file.Name())
+			subFileNames := getFileNames(t, subDir)
+			for _, subFileName := range subFileNames {
+				fileNames = append(fileNames, file.Name()+"/"+subFileName)
+			}
+		}
+	}
+	return fileNames
+}

pkg/config/logs.go

@@ -0,0 +1,252 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package config
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"runtime/pprof"
+	"sort"
+	"strconv"
+	"time"
+
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/sirupsen/logrus"
+)
+
+type logConfiger interface {
+	GetLogDir() string
+	GetLogPrefix() string
+}
+
+const (
+	// Zendesk now has a file size limit of 20MB. When the last N log files
+	// are zipped, it should fit under 20MB. Value in MB (average file has
+	// few hundreds kB).
+	maxLogFileSize = 10 * 1024 * 1024 //nolint[gochecknoglobals]
+	// Including the current logfile.
+	maxNumberLogFiles = 3 //nolint[gochecknoglobals]
+)
+
+// logFile is pointer to currently open file used by logrus.
+var logFile *os.File //nolint[gochecknoglobals]
+
+var logFileRgx = regexp.MustCompile("^v.*\\.log$")           //nolint[gochecknoglobals]
+var logCrashRgx = regexp.MustCompile("^v.*_crash_.*\\.log$") //nolint[gochecknoglobals]
+
+// GetLogEntry returns logrus.Entry with PID and `packageName`.
+func GetLogEntry(packageName string) *logrus.Entry {
+	return logrus.WithFields(logrus.Fields{
+		"pkg": packageName,
+	})
+}
+
+// HandlePanic reports the crash to sentry or local file when sentry fails.
+func HandlePanic(cfg *Config, output string) {
+	if !cfg.IsDevMode() {
+		c := pmapi.NewClient(cfg.GetAPIConfig(), "no-user-id")
+		err := c.ReportSentryCrash(fmt.Errorf(output))
+		if err != nil {
+			log.Error("Sentry crash report failed: ", err)
+		}
+	}
+
+	filename := getLogFilename(cfg.GetLogPrefix() + "_crash_")
+	filepath := filepath.Join(cfg.GetLogDir(), filename)
+	f, err := os.OpenFile(filepath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600)
+	if err != nil {
+		log.Error("Cannot open file to write crash report: ", err)
+		return
+	}
+
+	_, _ = f.WriteString(output)
+	_ = pprof.Lookup("goroutine").WriteTo(f, 2)
+
+	log.Warn("Crash report saved to ", filepath)
+}
+
+// GetGID returns goroutine number which can be used to distiguish logs from
+// the concurent processes. Keep in mind that it returns the number of routine
+// which executes the function.
+func GetGID() uint64 {
+	b := make([]byte, 64)
+	b = b[:runtime.Stack(b, false)]
+	b = bytes.TrimPrefix(b, []byte("goroutine "))
+	b = b[:bytes.IndexByte(b, ' ')]
+	n, _ := strconv.ParseUint(string(b), 10, 64)
+	return n
+}
+
+// SetupLog set up log level, formatter and output (file or stdout).
+// Returns whether should be used debug for IMAP and SMTP servers.
+func SetupLog(cfg logConfiger, levelFlag string) (debugClient, debugServer bool) {
+	level, useFile := getLogLevelAndFile(levelFlag)
+
+	logrus.SetLevel(level)
+
+	if useFile {
+		logrus.SetFormatter(&logrus.JSONFormatter{})
+		setLogFile(cfg.GetLogDir(), cfg.GetLogPrefix())
+		watchLogFileSize(cfg.GetLogDir(), cfg.GetLogPrefix())
+	} else {
+		logrus.SetFormatter(&logrus.TextFormatter{
+			ForceColors:     true,
+			FullTimestamp:   true,
+			TimestampFormat: time.StampMilli,
+		})
+		logrus.SetOutput(os.Stdout)
+	}
+
+	switch levelFlag {
+	case "debug-client", "debug-client-json":
+		debugClient = true
+	case "debug-server", "debug-server-json", "trace":
+		fmt.Println("THE LOG WILL CONTAIN **DECRYPTED** MESSAGE DATA")
+		log.Warning("================================================")
+		log.Warning("THIS LOG WILL CONTAIN **DECRYPTED** MESSAGE DATA")
+		log.Warning("================================================")
+		debugClient = true
+		debugServer = true
+	}
+
+	return debugClient, debugServer
+}
+
+func setLogFile(logDir, logPrefix string) {
+	if logFile != nil {
+		return
+	}
+
+	filename := getLogFilename(logPrefix)
+	var err error
+	logFile, err = os.OpenFile(filepath.Join(logDir, filename), os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600)
+	if err != nil {
+		panic(err)
+	}
+
+	logrus.SetOutput(logFile)
+
+	// Users sometimes change the name of the log file. We want to always log
+	// information about bridge version (included in log prefix) and OS.
+	log.Warn("Bridge version: ", logPrefix, " ", runtime.GOOS)
+}
+
+func getLogFilename(logPrefix string) string {
+	currentTime := strconv.Itoa(int(time.Now().Unix()))
+	return logPrefix + "_" + currentTime + ".log"
+}
+
+func watchLogFileSize(logDir, logPrefix string) {
+	go func() {
+		for {
+			time.Sleep(60 * time.Second)
+			checkLogFileSize(logDir, logPrefix)
+		}
+	}()
+}
+
+func checkLogFileSize(logDir, logPrefix string) {
+	if logFile == nil {
+		return
+	}
+
+	stat, err := logFile.Stat()
+	if err != nil {
+		log.Error("Log file size check failed: ", err)
+		return
+	}
+
+	if stat.Size() >= maxLogFileSize {
+		log.Warn("Current log file ", logFile.Name(), " is too big, opening new file")
+		closeLogFile()
+		setLogFile(logDir, logPrefix)
+	}
+
+	if err := clearLogs(logDir); err != nil {
+		log.Error("Cannot clear logs ", err)
+	}
+}
+
+func closeLogFile() {
+	if logFile != nil {
+		_ = logFile.Close()
+		logFile = nil
+	}
+}
+
+func clearLogs(logDir string) error {
+	files, err := ioutil.ReadDir(logDir)
+	if err != nil {
+		return err
+	}
+
+	var logsWithPrefix []string
+	var crashesWithPrefix []string
+
+	for _, file := range files {
+		if logFileRgx.MatchString(file.Name()) {
+			if logCrashRgx.MatchString(file.Name()) {
+				crashesWithPrefix = append(crashesWithPrefix, file.Name())
+			} else {
+				logsWithPrefix = append(logsWithPrefix, file.Name())
+			}
+		} else {
+			// Older versions of Bridge stored logs in subfolders for each version.
+			// That also has to be cleared and the functionality can be removed after some time.
+			if file.IsDir() {
+				if err := clearLogs(filepath.Join(logDir, file.Name())); err != nil {
+					return err
+				}
+			} else {
+				removeLog(logDir, file.Name())
+			}
+		}
+	}
+
+	removeOldLogs(logDir, logsWithPrefix)
+	removeOldLogs(logDir, crashesWithPrefix)
+	return nil
+}
+
+func removeOldLogs(logDir string, filenames []string) {
+	count := len(filenames)
+	if count <= maxNumberLogFiles {
+		return
+	}
+
+	sort.Strings(filenames) // Sorted by timestamp: oldest first.
+	for _, filename := range filenames[:count-maxNumberLogFiles] {
+		removeLog(logDir, filename)
+	}
+}
+
+func removeLog(logDir, filename string) {
+	// We need to be sure to delete only log files.
+	// Directory with logs can also contain other files.
+	if !logFileRgx.MatchString(filename) {
+		return
+	}
+	if err := os.RemoveAll(filepath.Join(logDir, filename)); err != nil {
+		log.Error("Cannot remove old logs ", err)
+	}
+}

pkg/config/logs_all.go

@@ -0,0 +1,49 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+// +build !build_qa
+
+package config
+
+import (
+	"github.com/sirupsen/logrus"
+)
+
+func getLogLevelAndFile(levelFlag string) (level logrus.Level, useFile bool) {
+	useFile = true
+	switch levelFlag {
+	case "panic":
+		level = logrus.PanicLevel
+	case "fatal":
+		level = logrus.FatalLevel
+	case "error":
+		level = logrus.ErrorLevel
+	case "warn":
+		level = logrus.WarnLevel
+	case "info":
+		level = logrus.InfoLevel
+	case "debug", "debug-client", "debug-server", "debug-client-json", "debug-server-json":
+		level = logrus.DebugLevel
+		useFile = false
+	case "trace":
+		level = logrus.TraceLevel
+		useFile = false
+	default:
+		level = logrus.InfoLevel
+	}
+	return
+}

pkg/config/logs_qa.go

@@ -0,0 +1,50 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+// +build build_qa
+
+package config
+
+import (
+	"github.com/sirupsen/logrus"
+)
+
+// getLogLevelAndFile for QA build is altered in a way even decrypted data are stored
+// in the log file when forced with `debug-client-json` or `debug-server-json`.
+func getLogLevelAndFile(levelFlag string) (level logrus.Level, useFile bool) {
+	useFile = true
+	switch levelFlag {
+	case "panic":
+		level = logrus.PanicLevel
+	case "fatal":
+		level = logrus.FatalLevel
+	case "error":
+		level = logrus.ErrorLevel
+	case "warn":
+		level = logrus.WarnLevel
+	case "info":
+		level = logrus.InfoLevel
+	case "debug-client-json", "debug-server-json":
+		level = logrus.DebugLevel
+	case "debug", "debug-client", "debug-server":
+		level = logrus.DebugLevel
+		useFile = false
+	default:
+		level = logrus.InfoLevel
+	}
+	return
+}

pkg/config/logs_test.go

@@ -0,0 +1,225 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package config
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/require"
+)
+
+type testLogConfig struct{ logDir, logPrefix string }
+
+func (c *testLogConfig) GetLogDir() string    { return c.logDir }
+func (c *testLogConfig) GetLogPrefix() string { return c.logPrefix }
+
+var testLogDir string //nolint[gochecknoglobals]
+
+func setupTestLogs() {
+	var err error
+	testLogDir, err = ioutil.TempDir("", "log")
+	if err != nil {
+		panic(err)
+	}
+}
+
+func shutdownTestLogs() {
+	_ = os.RemoveAll(testLogDir)
+}
+
+func TestLogNameLength(t *testing.T) {
+	cfg := New("bridge-test", "longVersion123", "longRevision1234567890", "c2")
+	name := getLogFilename(cfg.GetLogPrefix())
+	if len(name) > 128 {
+		t.Fatal("Name of the log is too long - limit for encrypted linux is 128 characters")
+	}
+}
+
+// Info and higher levels writes to the file.
+func TestSetupLogInfo(t *testing.T) {
+	dir := beforeEachCreateTestDir(t, "setupInfo")
+
+	SetupLog(&testLogConfig{dir, "v"}, "info")
+	require.Equal(t, "info", logrus.GetLevel().String())
+
+	logrus.Info("test message")
+	files := checkLogFiles(t, dir, 1)
+	checkLogContains(t, dir, files[0].Name(), "test message")
+}
+
+// Debug levels writes to stdout.
+func TestSetupLogDebug(t *testing.T) {
+	dir := beforeEachCreateTestDir(t, "setupDebug")
+
+	SetupLog(&testLogConfig{dir, "v"}, "debug")
+	require.Equal(t, "debug", logrus.GetLevel().String())
+
+	logrus.Info("test message")
+	checkLogFiles(t, dir, 0)
+}
+
+func TestReopenLogFile(t *testing.T) {
+	dir := beforeEachCreateTestDir(t, "reopenLogFile")
+
+	setLogFile(dir, "v1")
+
+	done := make(chan interface{})
+
+	log.Info("first message")
+
+	go func() {
+		<-done // Wait for closing file and opening new one.
+		log.Info("second message")
+		done <- nil
+	}()
+
+	closeLogFile()
+	setLogFile(dir, "v2")
+
+	done <- nil
+	<-done // Wait for second log message.
+
+	files := checkLogFiles(t, dir, 2)
+	checkLogContains(t, dir, files[0].Name(), "first message")
+	checkLogContains(t, dir, files[1].Name(), "second message")
+}
+
+func TestCheckLogFileSizeSmall(t *testing.T) {
+	dir := beforeEachCreateTestDir(t, "logFileSizeSmall")
+
+	setLogFile(dir, "v1")
+	originalFileName := logFile.Name()
+
+	_, _ = logFile.WriteString("small file")
+	checkLogFileSize(dir, "v2")
+
+	require.Equal(t, originalFileName, logFile.Name())
+}
+
+func TestCheckLogFileSizeBig(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	dir := beforeEachCreateTestDir(t, "logFileSizeBig")
+
+	setLogFile(dir, "v1")
+	originalFileName := logFile.Name()
+
+	// The limit for big file is 10*1024*1024 - keep the string 10 letters long.
+	for i := 0; i < 1024*1024; i++ {
+		_, _ = logFile.WriteString("big file!\n")
+	}
+	checkLogFileSize(dir, "v2")
+
+	require.NotEqual(t, originalFileName, logFile.Name())
+}
+
+// ClearLogs removes only bridge old log files keeping last three of them.
+func TestClearLogsLinux(t *testing.T) {
+	m := initMocks(t)
+	defer m.ctrl.Finish()
+
+	dir := beforeEachCreateTestDir(t, "clearLogs")
+
+	createTestStructureLinux(m, dir)
+	require.NoError(t, clearLogs(dir))
+	checkFileNames(t, dir, []string{
+		"cache",
+		"cache/c1",
+		"cache/c1/events.json",
+		"cache/c1/mailbox-user@pm.me.db",
+		"cache/c1/prefs.json",
+		"cache/c1/user_info.json",
+		"cache/c2",
+		"cache/c2/bridge-test.lock",
+		"cache/c2/events.json",
+		"cache/c2/mailbox-user@pm.me.db",
+		"cache/c2/prefs.json",
+		"cache/c2/updates",
+		"cache/c2/user_info.json",
+		"config",
+		"config/cert.pem",
+		"config/key.pem",
+		"logs",
+		"logs/other.log",
+		"logs/v1_11.log",
+		"logs/v2_12.log",
+		"logs/v2_13.log",
+	})
+}
+
+// ClearLogs removes only bridge old log files even when log folder
+// is shared with other files on Windows.
+func TestClearLogsWindows(t *testing.T) {
+	m := initMocks(t)
+	defer m.ctrl.Finish()
+
+	dir := beforeEachCreateTestDir(t, "clearLogs")
+
+	createTestStructureWindows(m, dir)
+	require.NoError(t, clearLogs(dir))
+	checkFileNames(t, dir, []string{
+		"cache",
+		"cache/c1",
+		"cache/c1/events.json",
+		"cache/c1/mailbox-user@pm.me.db",
+		"cache/c1/prefs.json",
+		"cache/c1/user_info.json",
+		"cache/c2",
+		"cache/c2/bridge-test.lock",
+		"cache/c2/events.json",
+		"cache/c2/mailbox-user@pm.me.db",
+		"cache/c2/prefs.json",
+		"cache/c2/updates",
+		"cache/c2/user_info.json",
+		"cache/other.log",
+		"cache/v1_11.log",
+		"cache/v2_12.log",
+		"cache/v2_13.log",
+		"config",
+		"config/cert.pem",
+		"config/key.pem",
+	})
+}
+
+func beforeEachCreateTestDir(t *testing.T, dir string) string {
+	// Make sure opened file (from the previous test) is cleared.
+	closeLogFile()
+
+	dir = filepath.Join(testLogDir, dir)
+	require.NoError(t, os.MkdirAll(dir, 0700))
+	return dir
+}
+
+func checkLogFiles(t *testing.T, dir string, expectedCount int) []os.FileInfo {
+	files, err := ioutil.ReadDir(dir)
+	require.NoError(t, err)
+	require.Equal(t, expectedCount, len(files))
+	return files
+}
+
+func checkLogContains(t *testing.T, dir, fileName, expectedSubstr string) {
+	data, err := ioutil.ReadFile(filepath.Join(dir, fileName)) //nolint[gosec]
+	require.NoError(t, err)
+	require.Contains(t, string(data), expectedSubstr)
+}

pkg/config/mock_config.go

@@ -0,0 +1,76 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: config/config.go
+
+// Package config is a generated GoMock package.
+package config
+
+import (
+	reflect "reflect"
+
+	gomock "github.com/golang/mock/gomock"
+)
+
+// MockappDirer is a mock of appDirer interface
+type MockappDirer struct {
+	ctrl     *gomock.Controller
+	recorder *MockappDirerMockRecorder
+}
+
+// MockappDirerMockRecorder is the mock recorder for MockappDirer
+type MockappDirerMockRecorder struct {
+	mock *MockappDirer
+}
+
+// NewMockappDirer creates a new mock instance
+func NewMockappDirer(ctrl *gomock.Controller) *MockappDirer {
+	mock := &MockappDirer{ctrl: ctrl}
+	mock.recorder = &MockappDirerMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockappDirer) EXPECT() *MockappDirerMockRecorder {
+	return m.recorder
+}
+
+// UserConfig mocks base method
+func (m *MockappDirer) UserConfig() string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UserConfig")
+	ret0, _ := ret[0].(string)
+	return ret0
+}
+
+// UserConfig indicates an expected call of UserConfig
+func (mr *MockappDirerMockRecorder) UserConfig() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UserConfig", reflect.TypeOf((*MockappDirer)(nil).UserConfig))
+}
+
+// UserCache mocks base method
+func (m *MockappDirer) UserCache() string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UserCache")
+	ret0, _ := ret[0].(string)
+	return ret0
+}
+
+// UserCache indicates an expected call of UserCache
+func (mr *MockappDirerMockRecorder) UserCache() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UserCache", reflect.TypeOf((*MockappDirer)(nil).UserCache))
+}
+
+// UserLogs mocks base method
+func (m *MockappDirer) UserLogs() string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UserLogs")
+	ret0, _ := ret[0].(string)
+	return ret0
+}
+
+// UserLogs indicates an expected call of UserLogs
+func (mr *MockappDirerMockRecorder) UserLogs() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UserLogs", reflect.TypeOf((*MockappDirer)(nil).UserLogs))
+}

pkg/config/preferences.go

@@ -0,0 +1,127 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package config
+
+import (
+	"encoding/json"
+	"errors"
+	"os"
+	"strconv"
+	"sync"
+)
+
+type Preferences struct {
+	cache map[string]string
+	path  string
+	lock  *sync.RWMutex
+}
+
+// NewPreferences returns loaded preferences.
+func NewPreferences(preferencesPath string) *Preferences {
+	p := &Preferences{
+		path: preferencesPath,
+		lock: &sync.RWMutex{},
+	}
+	if err := p.load(); err != nil {
+		log.Warn("Cannot load preferences: ", err)
+	}
+	return p
+}
+
+func (p *Preferences) load() error {
+	if p.cache != nil {
+		return nil
+	}
+
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	p.cache = map[string]string{}
+
+	f, err := os.Open(p.path)
+	if err != nil {
+		return err
+	}
+	defer f.Close() //nolint[errcheck]
+
+	return json.NewDecoder(f).Decode(&p.cache)
+}
+
+func (p *Preferences) save() error {
+	if p.cache == nil {
+		return errors.New("cannot save preferences: cache is nil")
+	}
+
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	f, err := os.Create(p.path)
+	if err != nil {
+		return err
+	}
+	defer f.Close() //nolint[errcheck]
+
+	return json.NewEncoder(f).Encode(p.cache)
+}
+
+func (p *Preferences) SetDefault(key, value string) {
+	if p.Get(key) == "" {
+		p.Set(key, value)
+	}
+}
+
+func (p *Preferences) Get(key string) string {
+	p.lock.RLock()
+	defer p.lock.RUnlock()
+
+	return p.cache[key]
+}
+
+func (p *Preferences) GetBool(key string) bool {
+	return p.Get(key) == "true"
+}
+
+func (p *Preferences) GetInt(key string) int {
+	value, err := strconv.Atoi(p.Get(key))
+	if err != nil {
+		log.Error("Cannot parse int: ", err)
+	}
+	return value
+}
+
+func (p *Preferences) Set(key, value string) {
+	p.lock.Lock()
+	p.cache[key] = value
+	p.lock.Unlock()
+
+	if err := p.save(); err != nil {
+		log.Warn("Cannot save preferences: ", err)
+	}
+}
+
+func (p *Preferences) SetBool(key string, value bool) {
+	if value {
+		p.Set(key, "true")
+	} else {
+		p.Set(key, "false")
+	}
+}
+
+func (p *Preferences) SetInt(key string, value int) {
+	p.Set(key, strconv.Itoa(value))
+}

pkg/config/preferences_test.go

@@ -0,0 +1,109 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package config
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+const testPrefFilePath = "/tmp/pref.json"
+
+func shutdownTestPreferences() {
+	_ = os.RemoveAll(testPrefFilePath)
+}
+
+func TestLoadNoPreferences(t *testing.T) {
+	pref := newTestEmptyPreferences(t)
+	require.Equal(t, "", pref.Get("key"))
+}
+
+func TestLoadBadPreferences(t *testing.T) {
+	require.NoError(t, ioutil.WriteFile(testPrefFilePath, []byte("{\"key\":\"value"), 0700))
+	pref := NewPreferences(testPrefFilePath)
+	require.Equal(t, "", pref.Get("key"))
+}
+
+func TestPreferencesGet(t *testing.T) {
+	pref := newTestPreferences(t)
+	require.Equal(t, "value", pref.Get("str"))
+	require.Equal(t, "42", pref.Get("int"))
+	require.Equal(t, "true", pref.Get("bool"))
+	require.Equal(t, "t", pref.Get("falseBool"))
+}
+
+func TestPreferencesGetInt(t *testing.T) {
+	pref := newTestPreferences(t)
+	require.Equal(t, 0, pref.GetInt("str"))
+	require.Equal(t, 42, pref.GetInt("int"))
+	require.Equal(t, 0, pref.GetInt("bool"))
+	require.Equal(t, 0, pref.GetInt("falseBool"))
+}
+
+func TestPreferencesGetBool(t *testing.T) {
+	pref := newTestPreferences(t)
+	require.Equal(t, false, pref.GetBool("str"))
+	require.Equal(t, false, pref.GetBool("int"))
+	require.Equal(t, true, pref.GetBool("bool"))
+	require.Equal(t, false, pref.GetBool("falseBool"))
+}
+
+func TestPreferencesSetDefault(t *testing.T) {
+	pref := newTestEmptyPreferences(t)
+	pref.SetDefault("key", "value")
+	pref.SetDefault("key", "othervalue")
+	require.Equal(t, "value", pref.Get("key"))
+}
+
+func TestPreferencesSet(t *testing.T) {
+	pref := newTestEmptyPreferences(t)
+	pref.Set("str", "value")
+	checkSavedPreferences(t, "{\"str\":\"value\"}")
+}
+
+func TestPreferencesSetInt(t *testing.T) {
+	pref := newTestEmptyPreferences(t)
+	pref.SetInt("int", 42)
+	checkSavedPreferences(t, "{\"int\":\"42\"}")
+}
+
+func TestPreferencesSetBool(t *testing.T) {
+	pref := newTestEmptyPreferences(t)
+	pref.SetBool("trueBool", true)
+	pref.SetBool("falseBool", false)
+	checkSavedPreferences(t, "{\"falseBool\":\"false\",\"trueBool\":\"true\"}")
+}
+
+func newTestEmptyPreferences(t *testing.T) *Preferences {
+	require.NoError(t, os.RemoveAll(testPrefFilePath))
+	return NewPreferences(testPrefFilePath)
+}
+
+func newTestPreferences(t *testing.T) *Preferences {
+	require.NoError(t, ioutil.WriteFile(testPrefFilePath, []byte("{\"str\":\"value\",\"int\":\"42\",\"bool\":\"true\",\"falseBool\":\"t\"}"), 0700))
+	return NewPreferences(testPrefFilePath)
+}
+
+func checkSavedPreferences(t *testing.T, expected string) {
+	data, err := ioutil.ReadFile(testPrefFilePath)
+	require.NoError(t, err)
+	require.Equal(t, expected+"\n", string(data))
+}

pkg/config/tls.go

@@ -0,0 +1,170 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package config
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"fmt"
+	"math/big"
+	"net"
+	"os"
+	"os/exec"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/kardianos/osext"
+)
+
+type tlsConfiger interface {
+	GetTLSCertPath() string
+	GetTLSKeyPath() string
+}
+
+var tlsTemplate = x509.Certificate{ //nolint[gochecknoglobals]
+	SerialNumber: big.NewInt(-1),
+	Subject: pkix.Name{
+		Country:            []string{"CH"},
+		Organization:       []string{"Proton Technologies AG"},
+		OrganizationalUnit: []string{"ProtonMail"},
+		CommonName:         "127.0.0.1",
+	},
+	KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
+	ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
+	BasicConstraintsValid: true,
+	IsCA:                  true,
+	IPAddresses:           []net.IP{net.ParseIP("127.0.0.1")},
+	NotBefore:             time.Now(),
+	NotAfter:              time.Now().Add(20 * 365 * 24 * time.Hour),
+}
+
+var ErrTLSCertExpireSoon = fmt.Errorf("TLS certificate will expire soon")
+
+// GetTLSConfig tries to load TLS config or generate new one which is then returned.
+func GetTLSConfig(cfg tlsConfiger) (tlsConfig *tls.Config, err error) {
+	certPath := cfg.GetTLSCertPath()
+	keyPath := cfg.GetTLSKeyPath()
+	tlsConfig, err = loadTLSConfig(certPath, keyPath)
+	if err != nil {
+		log.WithError(err).Warn("Cannot load cert, generating a new one")
+		tlsConfig, err = generateTLSConfig(certPath, keyPath)
+		if err != nil {
+			return
+		}
+
+		if runtime.GOOS == "darwin" {
+			// If this fails, log the error but continue to load.
+			if p, err := osext.Executable(); err == nil {
+				p = strings.TrimSuffix(p, "MacOS/Desktop-Bridge") // This needs to match the executable name.
+				p += "Resources/addcert.scpt"
+				if err := exec.Command("/usr/bin/osascript", p).Run(); err != nil { // nolint[gosec]
+					log.WithError(err).Error("Failed to add cert to system keychain")
+				}
+			}
+		}
+	}
+
+	tlsConfig.ServerName = "127.0.0.1"
+	tlsConfig.ClientAuth = tls.VerifyClientCertIfGiven
+
+	caCertPool := x509.NewCertPool()
+	caCertPool.AddCert(tlsConfig.Certificates[0].Leaf)
+	tlsConfig.RootCAs = caCertPool
+	tlsConfig.ClientCAs = caCertPool
+
+	/* This is deprecated:
+	 * SA1019: tlsConfig.BuildNameToCertificate is deprecated:
+	 * NameToCertificate only allows associating a single certificate with a given name.
+	 * Leave that field nil to let the library select the first compatible chain from Certificates.
+	 */
+	tlsConfig.BuildNameToCertificate() // nolint[staticcheck]
+
+	return tlsConfig, err
+}
+
+func loadTLSConfig(certPath, keyPath string) (tlsConfig *tls.Config, err error) {
+	c, err := tls.LoadX509KeyPair(certPath, keyPath)
+	if err != nil {
+		return
+	}
+
+	c.Leaf, err = x509.ParseCertificate(c.Certificate[0])
+	if err != nil {
+		return
+	}
+
+	tlsConfig = &tls.Config{
+		Certificates: []tls.Certificate{c},
+	}
+
+	if time.Now().Add(31 * 24 * time.Hour).After(c.Leaf.NotAfter) {
+		err = ErrTLSCertExpireSoon
+		return
+	}
+	return
+}
+
+// See https://golang.org/src/crypto/tls/generate_cert.go
+func generateTLSConfig(certPath, keyPath string) (tlsConfig *tls.Config, err error) {
+	priv, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		err = fmt.Errorf("failed to generate private key: %s", err)
+		return
+	}
+
+	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
+	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+	if err != nil {
+		err = fmt.Errorf("failed to generate serial number: %s", err)
+		return
+	}
+
+	tlsTemplate.SerialNumber = serialNumber
+	derBytes, err := x509.CreateCertificate(rand.Reader, &tlsTemplate, &tlsTemplate, &priv.PublicKey, priv)
+	if err != nil {
+		err = fmt.Errorf("failed to create certificate: %s", err)
+		return
+	}
+
+	certOut, err := os.Create(certPath)
+	if err != nil {
+		return
+	}
+	defer certOut.Close() //nolint[errcheck]
+	err = pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
+	if err != nil {
+		return
+	}
+
+	keyOut, err := os.OpenFile(keyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return
+	}
+	defer keyOut.Close() //nolint[errcheck]
+	err = pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
+	if err != nil {
+		return
+	}
+
+	return loadTLSConfig(certPath, keyPath)
+}

pkg/config/tls_test.go

@@ -0,0 +1,63 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package config
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+type testTLSConfig struct{ certPath, keyPath string }
+
+func (c *testTLSConfig) GetTLSCertPath() string { return c.certPath }
+func (c *testTLSConfig) GetTLSKeyPath() string  { return c.keyPath }
+
+func TestTLSKeyRenewal(t *testing.T) {
+	// Remove keys.
+	configPath := "/tmp"
+	certPath := filepath.Join(configPath, "cert.pem")
+	keyPath := filepath.Join(configPath, "key.pem")
+	_ = os.Remove(certPath)
+	_ = os.Remove(keyPath)
+
+	// Put old key there.
+	tlsTemplate.NotBefore = time.Now().Add(-365 * 24 * time.Hour)
+	tlsTemplate.NotAfter = time.Now()
+	cert, err := generateTLSConfig(certPath, keyPath)
+	require.Equal(t, err, ErrTLSCertExpireSoon)
+	require.Equal(t, len(cert.Certificates), 1)
+	time.Sleep(time.Second)
+	now, notValidAfter := time.Now(), cert.Certificates[0].Leaf.NotAfter
+	require.True(t, now.After(notValidAfter), "old certificate expected to not be valid at %v but have valid until %v", now, notValidAfter)
+
+	// Renew key.
+	tlsTemplate.NotBefore = time.Now()
+	tlsTemplate.NotAfter = time.Now().Add(2 * 365 * 24 * time.Hour)
+	cert, err = GetTLSConfig(&testTLSConfig{certPath, keyPath})
+	if runtime.GOOS != "darwin" { // Darwin is not supported.
+		require.NoError(t, err)
+	}
+	require.Equal(t, len(cert.Certificates), 1)
+	now, notValidAfter = time.Now(), cert.Certificates[0].Leaf.NotAfter
+	require.False(t, now.After(notValidAfter), "new certificate expected to be valid at %v but have valid until %v", now, notValidAfter)
+}

pkg/connection/check_connection.go

@@ -0,0 +1,88 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package connection
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/ProtonMail/proton-bridge/pkg/config"
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+)
+
+// Errors for possible connection issues
+var (
+	ErrNoInternetConnection = errors.New("no internet connection")
+	ErrCanNotReachAPI       = errors.New("can not reach PM API")
+	log                     = config.GetLogEntry("connection") //nolint[gochecknoglobals]
+)
+
+// CheckInternetConnection does a check of API connection. It checks two of our endpoints in parallel.
+// One endpoint is part of the protonmail API, while the other is not.
+// This allows us to determine whether there is a problem with the connection itself or only a problem with our API.
+// Two errors can be returned, ErrNoInternetConnection or ErrCanNotReachAPI.
+func CheckInternetConnection() error {
+	client := &http.Client{
+		Transport: pmapi.NewPMAPIPinning(pmapi.CurrentUserAgent).TransportWithPinning(),
+	}
+
+	// Do not cumulate timeouts, use goroutines.
+	retStatus := make(chan error)
+	retAPI := make(chan error)
+
+	// Check protonstatus.com without SSL for performance reasons. vpn_status endpoint is fast and
+	// returns only OK; this endpoint is not known by the public. We check the connection only.
+	go checkConnection(client, "http://protonstatus.com/vpn_status", retStatus)
+
+	// Check of API reachability also uses a fast endpoint.
+	go checkConnection(client, pmapi.GlobalGetRootURL()+"/tests/ping", retAPI)
+
+	errStatus := <-retStatus
+	errAPI := <-retAPI
+
+	if errStatus != nil {
+		if errAPI != nil {
+			log.Error("Checking internet connection failed with ", errStatus, " and ", errAPI)
+			return ErrNoInternetConnection
+		}
+		log.Warning("API OK, but status: ", errStatus)
+		return nil
+	}
+
+	if errAPI != nil {
+		log.Error("Status OK, but API: ", errAPI)
+		return ErrCanNotReachAPI
+	}
+
+	return nil
+}
+
+func checkConnection(client *http.Client, url string, errorChannel chan error) {
+	resp, err := client.Get(url)
+	if err != nil {
+		errorChannel <- err
+		return
+	}
+	_ = resp.Body.Close()
+	if resp.StatusCode != 200 {
+		errorChannel <- fmt.Errorf("HTTP status code %d", resp.StatusCode)
+		return
+	}
+	errorChannel <- nil
+}

pkg/connection/check_connection_test.go

@@ -0,0 +1,91 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package connection
+
+import (
+	"net/http"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/ProtonMail/proton-bridge/pkg/dialer"
+	"github.com/stretchr/testify/require"
+)
+
+const testServerPort = "18000"
+const testRequestTimeout = 10 * time.Second
+
+func TestMain(m *testing.M) {
+	go startServer()
+	time.Sleep(100 * time.Millisecond) // We need to wait till server is fully running.
+	code := m.Run()
+	os.Exit(code)
+}
+
+func startServer() {
+	http.HandleFunc("/ok", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("OK"))
+	})
+	http.HandleFunc("/timeout", func(w http.ResponseWriter, r *http.Request) {
+		time.Sleep(10 * time.Second)
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("OK"))
+	})
+	http.HandleFunc("/serverError", func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "error", http.StatusInternalServerError)
+	})
+	panic(http.ListenAndServe(":"+testServerPort, nil))
+}
+
+func TestCheckConnection(t *testing.T) {
+	checkCheckConnection(t, "ok", "")
+}
+
+func TestCheckConnectionTimeout(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	checkCheckConnection(t, "timeout", "Client.Timeout exceeded while awaiting headers")
+}
+
+func TestCheckConnectionServerError(t *testing.T) {
+	checkCheckConnection(t, "serverError", "HTTP status code 500")
+}
+
+func checkCheckConnection(t *testing.T, path string, expectedErrMessage string) {
+	client := dialer.DialTimeoutClient()
+	client.Timeout = testRequestTimeout
+
+	ch := make(chan error)
+
+	go checkConnection(client, "http://localhost:"+testServerPort+"/"+path, ch)
+
+	timeout := time.After(testRequestTimeout + time.Second)
+	select {
+	case err := <-ch:
+		if expectedErrMessage == "" {
+			require.NoError(t, err)
+		} else {
+			require.Error(t, err, expectedErrMessage)
+		}
+	case <-timeout:
+		t.Error("checkConnection timeout failed")
+	}
+}

pkg/dialer/dial_client.go

@@ -0,0 +1,46 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package dialer
+
+import (
+	"net"
+	"net/http"
+	"time"
+)
+
+const (
+	// ClientTimeout is the timeout for the whole request (from dial to
+	// receiving the response body). It should be large enough to download
+	// even the largest attachments or the new binary of the Bridge, but
+	// should be hit if the server hangs (default is infinite which is bad).
+	clientTimeout = 30 * time.Minute
+	dialTimeout   = 3 * time.Second
+)
+
+// DialTimeoutClient creates client with overridden dialTimeout.
+func DialTimeoutClient() *http.Client {
+	transport := &http.Transport{
+		Dial: func(network, addr string) (net.Conn, error) {
+			return net.DialTimeout(network, addr, dialTimeout)
+		},
+	}
+	return &http.Client{
+		Timeout:   clientTimeout,
+		Transport: transport,
+	}
+}

pkg/keychain/keychain.go

@@ -0,0 +1,131 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+// Package keychain implements a native secure password store for each platform.
+package keychain
+
+import (
+	"errors"
+	"strings"
+	"sync"
+
+	"github.com/ProtonMail/proton-bridge/pkg/config"
+	"github.com/docker/docker-credential-helpers/credentials"
+)
+
+const (
+	KeychainVersion = "k11" //nolint[golint]
+)
+
+var (
+	log = config.GetLogEntry("bridgeUtils/keychain") //nolint[gochecknoglobals]
+
+	ErrWrongKeychainURL    = errors.New("wrong keychain base URL")
+	ErrMacKeychainRebuild  = errors.New("keychain error -25293")
+	ErrMacKeychainList     = errors.New("function `osxkeychain.List()` is not valid function for mac keychain. Use `Access.ListKeychain()` instead")
+	ErrNoKeychainInstalled = errors.New("no keychain management installed on this system")
+	accessLocker           = &sync.Mutex{} //nolint[gochecknoglobals]
+)
+
+// NewAccess creates a new native keychain.
+func NewAccess(appName string) (*Access, error) {
+	newHelper, err := newKeychain()
+	if err != nil {
+		return nil, err
+	}
+	return &Access{
+		helper:            newHelper,
+		KeychainURL:       "protonmail/" + appName + "/users",
+		KeychainOldURL:    "protonmail/users",
+		KeychainMacURL:    "ProtonMail" + strings.Title(appName) + "Service",
+		KeychainOldMacURL: "ProtonMailService",
+	}, nil
+}
+
+type Access struct {
+	helper credentials.Helper
+	KeychainURL,
+	KeychainOldURL,
+	KeychainMacURL,
+	KeychainOldMacURL string
+}
+
+func (s *Access) List() (userIDs []string, err error) {
+	accessLocker.Lock()
+	defer accessLocker.Unlock()
+
+	var userIDByURL map[string]string
+	userIDByURL, err = s.ListKeychain()
+
+	if err != nil {
+		return
+	}
+
+	for itemURL, userID := range userIDByURL {
+		if itemURL == s.KeychainName(userID) {
+			userIDs = append(userIDs, userID)
+		}
+
+		// Clean up old keychain name.
+		if itemURL == s.KeychainOldName(userID) {
+			_ = s.helper.Delete(s.KeychainOldName(userID))
+		}
+	}
+
+	return
+}
+
+func (s *Access) Delete(userID string) (err error) {
+	accessLocker.Lock()
+	defer accessLocker.Unlock()
+	return s.helper.Delete(s.KeychainName(userID))
+}
+
+func (s *Access) Get(userID string) (secret string, err error) {
+	accessLocker.Lock()
+	defer accessLocker.Unlock()
+	_, secret, err = s.helper.Get(s.KeychainName(userID))
+	return
+}
+
+func (s *Access) Put(userID, secret string) error {
+	accessLocker.Lock()
+	defer accessLocker.Unlock()
+
+	// On macOS, adding a credential that already exists does not update it and returns an error.
+	// So let's remove it first.
+	_ = s.helper.Delete(s.KeychainName(userID))
+
+	cred := &credentials.Credentials{
+		ServerURL: s.KeychainName(userID),
+		Username:  userID,
+		Secret:    secret,
+	}
+
+	return s.helper.Add(cred)
+}
+
+func splitServiceAndID(keychainName string) (serviceName string, userID string, err error) { //nolint[unused]
+	splitted := strings.FieldsFunc(keychainName, func(c rune) bool { return c == '/' })
+	n := len(splitted)
+	if n <= 1 {
+		return "", "", ErrWrongKeychainURL
+	}
+	userID = splitted[len(splitted)-1]
+	serviceName = strings.Join(splitted[:len(splitted)-1], "/")
+	return
+}

pkg/keychain/keychain_darwin.go

@@ -0,0 +1,140 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package keychain
+
+import (
+	"strings"
+
+	"github.com/docker/docker-credential-helpers/credentials"
+	mackeychain "github.com/keybase/go-keychain"
+)
+
+func (s *Access) KeychainName(userID string) string {
+	return s.KeychainMacURL + "/" + userID
+}
+
+func (s *Access) KeychainOldName(userID string) string {
+	return s.KeychainOldMacURL + "/" + userID
+}
+
+type osxkeychain struct {
+}
+
+func newKeychain() (credentials.Helper, error) {
+	log.Debug("creating osckeychain")
+	return &osxkeychain{}, nil
+}
+
+func newQuery(serviceName, username string) mackeychain.Item {
+	query := mackeychain.NewItem()
+	query.SetSecClass(mackeychain.SecClassGenericPassword)
+	query.SetService(serviceName)
+	query.SetAccount(username)
+	return query
+}
+
+func parseError(original error) error {
+	if original != nil && strings.Contains(original.Error(), "25293") {
+		return ErrMacKeychainRebuild
+	}
+	return original
+}
+
+// Add appends credentials to the store (assuming old record with same ID is already deleted).
+func (s *osxkeychain) Add(cred *credentials.Credentials) error {
+	serviceName, userID, err := splitServiceAndID(cred.ServerURL)
+	if err != nil {
+		return err
+	}
+
+	query := newQuery(serviceName, userID)
+	query.SetData([]byte(cred.Secret))
+	err = mackeychain.AddItem(query)
+	return parseError(err)
+}
+
+// Delete removes credentials from the store.
+func (s *osxkeychain) Delete(serverURL string) error {
+	serviceName, userID, err := splitServiceAndID(serverURL)
+	if err != nil {
+		return err
+	}
+
+	query := newQuery(serviceName, userID)
+	err = mackeychain.DeleteItem(query)
+	if err != nil && !strings.Contains(err.Error(), "25300") { // Missing item is not error.
+		return err
+	}
+	return nil
+}
+
+// Get retrieves credentials from the store.
+// It returns username and secret as strings.
+func (s *osxkeychain) Get(serverURL string) (userID string, secret string, err error) {
+	serviceName, userID, err := splitServiceAndID(serverURL)
+	if err != nil {
+		return
+	}
+
+	query := newQuery(serviceName, userID)
+	query.SetMatchLimit(mackeychain.MatchLimitOne)
+	query.SetReturnData(true)
+	results, err := mackeychain.QueryItem(query)
+	if err != nil {
+		return "", "", parseError(err)
+	}
+
+	if len(results) == 1 {
+		secret = string(results[0].Data)
+	}
+
+	return
+}
+
+// ListKeychain lists items in our services.
+func (s *Access) ListKeychain() (userIDByURL map[string]string, err error) {
+	// Pick up correct service name and trim '/'.
+	serviceName, _, err := splitServiceAndID(s.KeychainOldName("not-id"))
+	if err != nil {
+		return
+	}
+
+	userIDByURL = make(map[string]string)
+
+	if oldIDs, err := mackeychain.GetGenericPasswordAccounts(serviceName); err == nil {
+		for _, userIDold := range oldIDs {
+			userIDByURL[s.KeychainOldName(userIDold)] = userIDold
+		}
+	}
+
+	serviceName, _, _ = splitServiceAndID(s.KeychainName("not-id"))
+	if userIDs, err := mackeychain.GetGenericPasswordAccounts(serviceName); err == nil {
+		for _, userID := range userIDs {
+			userIDByURL[s.KeychainName(userID)] = userID
+		}
+	}
+
+	return
+}
+
+// List returns the stored serverURLs and their associated usernames.
+// NOTE: This is not valid for go-keychain. Use ListKeychain instead.
+func (s *osxkeychain) List() (userIDByURL map[string]string, err error) {
+	err = ErrMacKeychainList
+	return
+}

pkg/keychain/keychain_linux.go

@@ -0,0 +1,73 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package keychain
+
+import (
+	"github.com/docker/docker-credential-helpers/credentials"
+	"github.com/docker/docker-credential-helpers/pass"
+	"github.com/docker/docker-credential-helpers/secretservice"
+)
+
+func newKeychain() (credentials.Helper, error) {
+	log.Debug("creating pass")
+	passHelper := &pass.Pass{}
+	passErr := checkPassIsUsable(passHelper)
+	if passErr == nil {
+		return passHelper, nil
+	}
+
+	log.Debug("creating secretservice")
+	sserviceHelper := &secretservice.Secretservice{}
+	_, sserviceErr := sserviceHelper.List()
+	if sserviceErr == nil {
+		return sserviceHelper, nil
+	}
+
+	log.Error("No keychain! Pass: ", passErr, ", secretService: ", sserviceErr)
+	return nil, ErrNoKeychainInstalled
+}
+
+func checkPassIsUsable(passHelper *pass.Pass) (err error) {
+	creds := &credentials.Credentials{
+		ServerURL: "initCheck/pass",
+		Username:  "pass",
+		Secret:    "pass",
+	}
+
+	if err = passHelper.Add(creds); err != nil {
+		return
+	}
+	// Pass is not asked about unlock until you try to decrypt.
+	if _, _, err = passHelper.Get(creds.ServerURL); err != nil {
+		return
+	}
+	_ = passHelper.Delete(creds.ServerURL) // Doesn't matter if you are able to clear.
+	return
+}
+
+func (s *Access) KeychainName(userID string) string {
+	return s.KeychainURL + "/" + userID
+}
+
+func (s *Access) KeychainOldName(userID string) string {
+	return s.KeychainOldURL + "/" + userID
+}
+
+func (s *Access) ListKeychain() (map[string]string, error) {
+	return s.helper.List()
+}

pkg/keychain/keychain_test.go

@@ -0,0 +1,152 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package keychain
+
+import (
+	"encoding/base64"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+var suffix = []byte("\x00avoidFix\x00\x00\x00\x00\x00\x00\x00") //nolint[gochecknoglobals]
+
+var testData = map[string]string{ //nolint[gochecknoglobals]
+	"user1": base64.StdEncoding.EncodeToString(append([]byte("data1"), suffix...)),
+	"user2": base64.StdEncoding.EncodeToString(append([]byte("data2"), suffix...)),
+}
+
+func TestSplitServiceAndID(t *testing.T) {
+	acc, err := NewAccess("bridge")
+	require.NoError(t, err)
+	expectedUserID := "user"
+
+	acc.KeychainURL = "Something/With/Several/Slashes/"
+	acc.KeychainMacURL = acc.KeychainURL
+	expectedServiceName := acc.KeychainURL
+	serviceName, userID, err := splitServiceAndID(acc.KeychainName(expectedUserID))
+	require.NoError(t, err)
+	require.Equal(t, expectedUserID, userID)
+	require.Equal(t, expectedServiceName, serviceName+"/")
+
+	acc.KeychainURL = "SomethingWithoutSlash"
+	acc.KeychainMacURL = acc.KeychainURL
+	expectedServiceName = acc.KeychainURL
+	serviceName, userID, err = splitServiceAndID(acc.KeychainName(expectedUserID))
+	require.NoError(t, err)
+	require.Equal(t, expectedUserID, userID)
+	require.Equal(t, expectedServiceName, serviceName)
+}
+
+func TestInsertReadRemove(t *testing.T) { // nolint[funlen]
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	access, err := NewAccess("bridge")
+	require.NoError(t, err)
+	access.KeychainURL = "protonmail/testchain/users"
+	access.KeychainMacURL = "ProtonMailTestChainService"
+
+	// Clear before test.
+	for id := range testData {
+		// Keychain can be empty.
+		_ = access.Delete(id)
+	}
+
+	for id, secret := range testData {
+		expectedList, _ := access.List()
+		// Add expected secrets.
+		expectedSecret := secret
+		require.NoError(t, access.Put(id, expectedSecret))
+
+		// Check list.
+		actualList, err := access.List()
+		require.NoError(t, err)
+		expectedList = append(expectedList, id)
+		require.ElementsMatch(t, expectedList, actualList)
+
+		// Get and check what was inserted.
+		actualSecret, err := access.Get(id)
+		require.NoError(t, err)
+		require.Equal(t, expectedSecret, actualSecret)
+
+		// Put what changed.
+
+		expectedSecret = "edited_" + id
+		expectedSecret = base64.StdEncoding.EncodeToString(append([]byte(expectedSecret), suffix...))
+
+		nJobs := 100
+		nWorkers := 3
+		jobs := make(chan interface{}, nJobs)
+		done := make(chan interface{})
+		for i := 0; i < nWorkers; i++ {
+			go func() {
+				for {
+					_, more := <-jobs
+					if more {
+						require.NoError(t, access.Put(id, expectedSecret))
+					} else {
+						done <- nil
+						return
+					}
+				}
+			}()
+		}
+
+		for i := 0; i < nJobs; i++ {
+			jobs <- nil
+		}
+		close(jobs)
+		for i := 0; i < nWorkers; i++ {
+			<-done
+		}
+
+		// Check list.
+		actualList, err = access.List()
+		require.NoError(t, err)
+		require.ElementsMatch(t, expectedList, actualList)
+
+		// Get and check what changed.
+		actualSecret, err = access.Get(id)
+		require.NoError(t, err)
+		require.Equal(t, expectedSecret, actualSecret)
+
+		if id != "user1" {
+			// Remove.
+			err = access.Delete(id)
+			require.NoError(t, err)
+
+			// Check removed.
+			actualList, err = access.List()
+			require.NoError(t, err)
+			expectedList = expectedList[:len(expectedList)-1]
+			require.ElementsMatch(t, expectedList, actualList)
+		}
+	}
+
+	// Clear first.
+	err = access.Delete("user1")
+	require.NoError(t, err)
+
+	actualList, err := access.List()
+	require.NoError(t, err)
+	for id := range testData {
+		require.NotContains(t, actualList, id)
+	}
+}

pkg/keychain/keychain_windows.go

@@ -0,0 +1,40 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package keychain
+
+import (
+	"github.com/docker/docker-credential-helpers/credentials"
+	"github.com/docker/docker-credential-helpers/wincred"
+)
+
+func newKeychain() (credentials.Helper, error) {
+	log.Debug("creating wincred")
+	return &wincred.Wincred{}, nil
+}
+
+func (s *Access) KeychainName(userID string) string {
+	return s.KeychainURL + "/" + userID
+}
+
+func (s *Access) KeychainOldName(userID string) string {
+	return s.KeychainOldURL + "/" + userID
+}
+
+func (s *Access) ListKeychain() (map[string]string, error) {
+	return s.helper.List()
+}

pkg/listener/listener.go

@@ -0,0 +1,180 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package listener
+
+import (
+	"sync"
+	"time"
+
+	"github.com/ProtonMail/proton-bridge/pkg/config"
+)
+
+var log = config.GetLogEntry("bridgeUtils/listener") //nolint[gochecknoglobals]
+
+// Listener has a list of channels watching for updates.
+type Listener interface {
+	SetLimit(eventName string, limit time.Duration)
+	Add(eventName string, channel chan<- string)
+	Remove(eventName string, channel chan<- string)
+	Emit(eventName string, data string)
+	SetBuffer(eventName string)
+	RetryEmit(eventName string)
+}
+
+type listener struct {
+	channels  map[string][]chan<- string
+	limits    map[string]time.Duration
+	lastEmits map[string]map[string]time.Time
+	buffered  map[string][]string
+	lock      *sync.RWMutex
+}
+
+// New returns a new Listener which initially has no topics.
+func New() Listener {
+	return &listener{
+		channels:  nil,
+		limits:    make(map[string]time.Duration),
+		lastEmits: make(map[string]map[string]time.Time),
+		buffered:  make(map[string][]string),
+		lock:      &sync.RWMutex{},
+	}
+}
+
+// SetLimit sets the limit for the `eventName`. When the same event (name and data)
+// is emitted within last time duration (`limit`), event is dropped. Zero limit clears
+// the limit for the specific `eventName`.
+func (l *listener) SetLimit(eventName string, limit time.Duration) {
+	if limit == 0 {
+		delete(l.limits, eventName)
+		return
+	}
+	l.limits[eventName] = limit
+}
+
+// Add adds an event listener.
+func (l *listener) Add(eventName string, channel chan<- string) {
+	l.lock.Lock()
+	defer l.lock.Unlock()
+
+	if l.channels == nil {
+		l.channels = make(map[string][]chan<- string)
+	}
+	if _, ok := l.channels[eventName]; ok {
+		l.channels[eventName] = append(l.channels[eventName], channel)
+	} else {
+		l.channels[eventName] = []chan<- string{channel}
+	}
+}
+
+// Remove removes an event listener.
+func (l *listener) Remove(eventName string, channel chan<- string) {
+	l.lock.Lock()
+	defer l.lock.Unlock()
+
+	if _, ok := l.channels[eventName]; ok {
+		for i := range l.channels[eventName] {
+			if l.channels[eventName][i] == channel {
+				l.channels[eventName] = append(l.channels[eventName][:i], l.channels[eventName][i+1:]...)
+				break
+			}
+		}
+	}
+}
+
+// Emit emits an event in parallel to all listeners (channels).
+func (l *listener) Emit(eventName string, data string) {
+	l.emit(eventName, data, false)
+}
+
+func (l *listener) emit(eventName, data string, isReEmit bool) {
+	l.lock.RLock()
+	defer l.lock.RUnlock()
+
+	if !l.shouldEmit(eventName, data) {
+		log.Warn("Emit of ", eventName, " with data ", data, " skipped")
+		return
+	}
+
+	if _, ok := l.channels[eventName]; ok {
+		for i, handler := range l.channels[eventName] {
+			go func(handler chan<- string, i int) {
+				handler <- data
+				log.Debugf("emitted %s data %s -> %d", eventName, data, i)
+			}(handler, i)
+		}
+	} else if !isReEmit {
+		if bufferedData, ok := l.buffered[eventName]; ok {
+			l.buffered[eventName] = append(bufferedData, data)
+			log.Debugf("Buffering event %s data %s", eventName, data)
+		} else {
+			log.Warnf("No channel is listening to %s data %s", eventName, data)
+		}
+	}
+}
+
+func (l *listener) shouldEmit(eventName, data string) bool {
+	if _, ok := l.limits[eventName]; !ok {
+		return true
+	}
+
+	l.clearLastEmits()
+
+	if eventLastEmits, ok := l.lastEmits[eventName]; ok {
+		if _, ok := eventLastEmits[data]; ok {
+			return false
+		}
+	} else {
+		l.lastEmits[eventName] = make(map[string]time.Time)
+	}
+
+	l.lastEmits[eventName][data] = time.Now()
+	return true
+}
+
+func (l *listener) clearLastEmits() {
+	for eventName, lastEmits := range l.lastEmits {
+		limit, ok := l.limits[eventName]
+		if !ok { // Limits were disabled.
+			delete(l.lastEmits, eventName)
+			continue
+		}
+		for key, lastEmit := range lastEmits {
+			if time.Since(lastEmit) > limit {
+				delete(lastEmits, key)
+			}
+		}
+	}
+}
+
+func (l *listener) SetBuffer(eventName string) {
+	if _, ok := l.buffered[eventName]; !ok {
+		l.buffered[eventName] = []string{}
+	}
+}
+
+func (l *listener) RetryEmit(eventName string) {
+	if _, ok := l.channels[eventName]; !ok || len(l.channels[eventName]) == 0 {
+		return
+	}
+	if bufferedData, ok := l.buffered[eventName]; ok {
+		for _, data := range bufferedData {
+			l.emit(eventName, data, true)
+		}
+		l.buffered[eventName] = []string{}
+	}
+}

pkg/listener/listener_test.go

@@ -0,0 +1,172 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package listener
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/require"
+)
+
+func Example() {
+	eventListener := New()
+
+	ch := make(chan string)
+	eventListener.Add("eventname", ch)
+	for eventdata := range ch {
+		fmt.Println(eventdata + " world")
+	}
+
+	eventListener.Emit("eventname", "hello")
+}
+
+func TestAddAndEmitSameEvent(t *testing.T) {
+	listener, channel := newListener()
+
+	listener.Emit("event", "hello!")
+	checkChannelEmitted(t, channel, "hello!")
+}
+
+func TestAddAndEmitDifferentEvent(t *testing.T) {
+	listener, channel := newListener()
+
+	listener.Emit("other", "hello!")
+	checkChannelNotEmitted(t, channel)
+}
+
+func TestAddAndRemove(t *testing.T) {
+	listener := New()
+
+	channel := make(chan string)
+	listener.Add("event", channel)
+	listener.Remove("event", channel)
+	listener.Emit("event", "hello!")
+
+	checkChannelNotEmitted(t, channel)
+}
+
+func TestNoLimit(t *testing.T) {
+	listener, channel := newListener()
+
+	listener.Emit("event", "hello!")
+	checkChannelEmitted(t, channel, "hello!")
+
+	listener.Emit("event", "hello!")
+	checkChannelEmitted(t, channel, "hello!")
+}
+
+func TestLimit(t *testing.T) {
+	listener, channel := newListener()
+	listener.SetLimit("event", 1*time.Second)
+
+	channel2 := make(chan string)
+	listener.Add("event", channel2)
+
+	listener.Emit("event", "hello!")
+	checkChannelEmitted(t, channel, "hello!")
+	checkChannelEmitted(t, channel2, "hello!")
+
+	listener.Emit("event", "hello!")
+	checkChannelNotEmitted(t, channel)
+	checkChannelNotEmitted(t, channel2)
+
+	time.Sleep(1 * time.Second)
+
+	listener.Emit("event", "hello!")
+	checkChannelEmitted(t, channel, "hello!")
+	checkChannelEmitted(t, channel2, "hello!")
+}
+
+func TestLimitDifferentData(t *testing.T) {
+	listener, channel := newListener()
+	listener.SetLimit("event", 1*time.Second)
+
+	listener.Emit("event", "hello!")
+	checkChannelEmitted(t, channel, "hello!")
+
+	listener.Emit("event", "hello?")
+	checkChannelEmitted(t, channel, "hello?")
+}
+
+func TestReEmit(t *testing.T) {
+	logrus.SetLevel(logrus.DebugLevel)
+	listener := New()
+	listener.Emit("event", "hello?")
+
+	listener.SetBuffer("event")
+	listener.SetBuffer("other")
+
+	listener.Emit("event", "hello1")
+	listener.Emit("event", "hello2")
+	listener.Emit("other", "hello!")
+	listener.Emit("event", "hello3")
+	listener.Emit("other", "hello!")
+
+	eventCH := make(chan string, 3)
+	listener.Add("event", eventCH)
+
+	otherCH := make(chan string)
+	listener.Add("other", otherCH)
+
+	listener.RetryEmit("event")
+	listener.RetryEmit("other")
+	time.Sleep(time.Millisecond)
+
+	receivedEvents := map[string]int{}
+	for i := 0; i < 5; i++ {
+		select {
+		case res := <-eventCH:
+			receivedEvents[res]++
+		case res := <-otherCH:
+			receivedEvents[res+":other"]++
+		case <-time.After(10 * time.Millisecond):
+			t.Fatalf("Channel not emitted %d times", i+1)
+		}
+	}
+	expectedEvents := map[string]int{"hello1": 1, "hello2": 1, "hello3": 1, "hello!:other": 2}
+	require.Equal(t, expectedEvents, receivedEvents)
+}
+
+func newListener() (Listener, chan string) {
+	listener := New()
+
+	channel := make(chan string)
+	listener.Add("event", channel)
+
+	return listener, channel
+}
+
+func checkChannelEmitted(t testing.TB, channel chan string, expectedData string) {
+	select {
+	case res := <-channel:
+		require.Equal(t, expectedData, res)
+	case <-time.After(10 * time.Millisecond):
+		t.Fatalf("Channel not emitted with expected data: %s", expectedData)
+	}
+}
+
+func checkChannelNotEmitted(t testing.TB, channel chan string) {
+	select {
+	case res := <-channel:
+		t.Fatalf("Channel emitted with a unexpected response: %s", res)
+	case <-time.After(10 * time.Millisecond):
+	}
+}

pkg/message/address.go

@@ -0,0 +1,56 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package message
+
+import (
+	"net/mail"
+	"strings"
+
+	"github.com/emersion/go-imap"
+)
+
+func getAddresses(addrs []*mail.Address) (imapAddrs []*imap.Address) {
+	for _, a := range addrs {
+		if a == nil {
+			continue
+		}
+
+		parts := strings.SplitN(a.Address, "@", 2)
+		if len(parts) != 2 {
+			continue
+		}
+
+		imapAddrs = append(imapAddrs, &imap.Address{
+			PersonalName: a.Name,
+			MailboxName:  parts[0],
+			HostName:     parts[1],
+		})
+	}
+
+	return
+}
+
+func formatAddressList(addrs []*mail.Address) (s string) {
+	for i, addr := range addrs {
+		if i > 0 {
+			s += ", "
+		}
+		s += addr.String()
+	}
+	return
+}

pkg/message/body.go

@@ -0,0 +1,75 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package message
+
+import (
+	"encoding/base64"
+	"fmt"
+	"io"
+	"mime/quotedprintable"
+
+	pmcrypto "github.com/ProtonMail/gopenpgp/crypto"
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/emersion/go-textwrapper"
+	openpgperrors "golang.org/x/crypto/openpgp/errors"
+)
+
+func WriteBody(w io.Writer, kr *pmcrypto.KeyRing, m *pmapi.Message) error {
+	// Decrypt body.
+	if err := m.Decrypt(kr); err != nil && err != openpgperrors.ErrSignatureExpired {
+		return err
+	}
+	if m.MIMEType != pmapi.ContentTypeMultipartMixed {
+		// Encode it.
+		qp := quotedprintable.NewWriter(w)
+		if _, err := io.WriteString(qp, m.Body); err != nil {
+			return err
+		}
+		return qp.Close()
+	}
+	_, err := io.WriteString(w, m.Body)
+	return err
+}
+
+func WriteAttachmentBody(w io.Writer, kr *pmcrypto.KeyRing, m *pmapi.Message, att *pmapi.Attachment, r io.Reader) (err error) {
+	// Decrypt it
+	var dr io.Reader
+	dr, err = att.Decrypt(r, kr)
+	if err == openpgperrors.ErrKeyIncorrect {
+		// Do not fail if attachment is encrypted with a different key.
+		dr = r
+		err = nil
+		att.Name += ".gpg"
+		att.MIMEType = "application/pgp-encrypted"
+	} else if err != nil && err != openpgperrors.ErrSignatureExpired {
+		err = fmt.Errorf("cannot decrypt attachment: %v", err)
+		return
+	}
+
+	// Encode it.
+	ww := textwrapper.NewRFC822(w)
+	bw := base64.NewEncoder(base64.StdEncoding, ww)
+
+	var n int64
+	if n, err = io.Copy(bw, dr); err != nil {
+		err = fmt.Errorf("cannot write attachment: %v (wrote %v bytes)", err, n)
+	}
+
+	_ = bw.Close()
+	return
+}

pkg/message/envelope.go

@@ -0,0 +1,48 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package message
+
+import (
+	"net/mail"
+	"time"
+
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/emersion/go-imap"
+)
+
+func GetEnvelope(m *pmapi.Message) *imap.Envelope {
+	messageID := m.ExternalID
+	if messageID == "" {
+		messageID = m.Header.Get("Message-Id")
+	} else {
+		messageID = "<" + messageID + ">"
+	}
+
+	return &imap.Envelope{
+		Date:      time.Unix(m.Time, 0),
+		Subject:   m.Subject,
+		From:      getAddresses([]*mail.Address{m.Sender}),
+		Sender:    getAddresses([]*mail.Address{m.Sender}),
+		ReplyTo:   getAddresses(m.ReplyTos),
+		To:        getAddresses(m.ToList),
+		Cc:        getAddresses(m.CCList),
+		Bcc:       getAddresses(m.BCCList),
+		InReplyTo: m.Header.Get("In-Reply-To"),
+		MessageId: messageID,
+	}
+}

pkg/message/flags.go

@@ -0,0 +1,83 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package message
+
+import (
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/emersion/go-imap"
+)
+
+//nolint[gochecknoglobals]
+var (
+	AppleMailJunkFlag      = imap.CanonicalFlag("$Junk")
+	ThunderbirdJunkFlag    = imap.CanonicalFlag("Junk")
+	ThunderbirdNonJunkFlag = imap.CanonicalFlag("NonJunk")
+)
+
+func GetFlags(m *pmapi.Message) (flags []string) {
+	if m.Unread == 0 {
+		flags = append(flags, imap.SeenFlag)
+	}
+	if !m.Has(pmapi.FlagSent) && !m.Has(pmapi.FlagReceived) {
+		flags = append(flags, imap.DraftFlag)
+	}
+	if m.Has(pmapi.FlagReplied) || m.Has(pmapi.FlagRepliedAll) {
+		flags = append(flags, imap.AnsweredFlag)
+	}
+
+	hasSpam := false
+
+	for _, l := range m.LabelIDs {
+		if l == pmapi.StarredLabel {
+			flags = append(flags, imap.FlaggedFlag)
+		}
+		if l == pmapi.SpamLabel {
+			flags = append(flags, AppleMailJunkFlag, ThunderbirdJunkFlag)
+			hasSpam = true
+		}
+	}
+
+	if !hasSpam {
+		flags = append(flags, ThunderbirdNonJunkFlag)
+	}
+
+	return
+}
+
+func ParseFlags(m *pmapi.Message, flags []string) {
+	if (m.Flags & pmapi.FlagSent) == 0 {
+		m.Flags |= pmapi.FlagReceived
+	}
+	m.Unread = 1
+	for _, f := range flags {
+		switch f {
+		case imap.SeenFlag:
+			m.Unread = 0
+		case imap.DraftFlag:
+			m.Flags &= ^pmapi.FlagSent
+			m.Flags &= ^pmapi.FlagReceived
+			m.LabelIDs = append(m.LabelIDs, pmapi.DraftLabel)
+		case imap.FlaggedFlag:
+			m.LabelIDs = append(m.LabelIDs, pmapi.StarredLabel)
+		case imap.AnsweredFlag:
+			m.Flags |= pmapi.FlagReplied
+		case AppleMailJunkFlag, ThunderbirdJunkFlag:
+			m.LabelIDs = append(m.LabelIDs, pmapi.SpamLabel)
+		}
+	}
+}

pkg/message/header.go

@@ -0,0 +1,214 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package message
+
+import (
+	"mime"
+	"net/mail"
+	"net/textproto"
+	"strings"
+	"time"
+
+	pmmime "github.com/ProtonMail/proton-bridge/pkg/mime"
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+)
+
+// GetHeader builds the header for the message.
+func GetHeader(msg *pmapi.Message) textproto.MIMEHeader { //nolint[funlen]
+	h := make(textproto.MIMEHeader)
+
+	// Copy the custom header fields if there are some.
+	if msg.Header != nil {
+		h = textproto.MIMEHeader(msg.Header)
+	}
+
+	// Add or rewrite fields.
+	h.Set("Subject", pmmime.EncodeHeader(msg.Subject))
+	if msg.Sender != nil {
+		h.Set("From", pmmime.EncodeHeader(msg.Sender.String()))
+	}
+	if len(msg.ReplyTos) > 0 {
+		h.Set("Reply-To", pmmime.EncodeHeader(formatAddressList(msg.ReplyTos)))
+	}
+	if len(msg.ToList) > 0 {
+		h.Set("To", pmmime.EncodeHeader(formatAddressList(msg.ToList)))
+	}
+	if len(msg.CCList) > 0 {
+		h.Set("Cc", pmmime.EncodeHeader(formatAddressList(msg.CCList)))
+	}
+	if len(msg.BCCList) > 0 {
+		h.Set("Bcc", pmmime.EncodeHeader(formatAddressList(msg.BCCList)))
+	}
+
+	// Add or rewrite date related fields.
+	if msg.Time > 0 {
+		h.Set("X-Pm-Date", time.Unix(msg.Time, 0).Format(time.RFC1123Z))
+		if d, err := msg.Header.Date(); err != nil || d.IsZero() { // Fix date if needed.
+			h.Set("Date", time.Unix(msg.Time, 0).Format(time.RFC1123Z))
+		}
+	}
+
+	// Use External-Id if available to ensure email clients:
+	// * build the conversations threads correctly (Thunderbird, Mac Outlook, Apple Mail)
+	// * do not think the message is lost (Apple Mail)
+	if msg.ExternalID != "" {
+		h.Set("X-Pm-External-Id", "<"+msg.ExternalID+">")
+		if h.Get("Message-Id") == "" {
+			h.Set("Message-Id", "<"+msg.ExternalID+">")
+		}
+	}
+	if msg.ID != "" {
+		if h.Get("Message-Id") == "" {
+			h.Set("Message-Id", "<"+msg.ID+"@protonmail.internalid>")
+		}
+		h.Set("X-Pm-Internal-Id", msg.ID)
+		// Forward References, and include the message ID here (to improve outlook support).
+		if references := h.Get("References"); !strings.Contains(references, msg.ID) {
+			references += " <" + msg.ID + "@protonmail.internalid>"
+			h.Set("References", references)
+		}
+	}
+	if msg.ConversationID != "" {
+		h.Set("X-Pm-ConversationID-Id", msg.ConversationID)
+		if references := h.Get("References"); !strings.Contains(references, msg.ConversationID) {
+			references += " <" + msg.ConversationID + "@protonmail.conversationid>"
+			h.Set("References", references)
+		}
+	}
+
+	return h
+}
+
+func SetBodyContentFields(h *textproto.MIMEHeader, m *pmapi.Message) {
+	h.Set("Content-Type", m.MIMEType+"; charset=utf-8")
+	h.Set("Content-Disposition", "inline")
+	h.Set("Content-Transfer-Encoding", "quoted-printable")
+}
+
+func GetBodyHeader(m *pmapi.Message) textproto.MIMEHeader {
+	h := make(textproto.MIMEHeader)
+	SetBodyContentFields(&h, m)
+	return h
+}
+
+func GetRelatedHeader(m *pmapi.Message) textproto.MIMEHeader {
+	h := make(textproto.MIMEHeader)
+	h.Set("Content-Type", "multipart/related; boundary="+GetRelatedBoundary(m))
+	return h
+}
+
+func GetAttachmentHeader(att *pmapi.Attachment) textproto.MIMEHeader {
+	mediaType := att.MIMEType
+	if mediaType == "application/pgp-encrypted" {
+		mediaType = "application/octet-stream"
+	}
+
+	encodedName := pmmime.EncodeHeader(att.Name)
+	disposition := "attachment" //nolint[goconst]
+	if strings.Contains(att.Header.Get("Content-Disposition"), "inline") {
+		disposition = "inline"
+	}
+
+	h := make(textproto.MIMEHeader)
+	h.Set("Content-Type", mime.FormatMediaType(mediaType, map[string]string{"name": encodedName}))
+	h.Set("Content-Transfer-Encoding", "base64")
+	h.Set("Content-Disposition", mime.FormatMediaType(disposition, map[string]string{"filename": encodedName}))
+
+	// Forward some original header lines.
+	forward := []string{"Content-Id", "Content-Description", "Content-Location"}
+	for _, k := range forward {
+		v := att.Header.Get(k)
+		if v != "" {
+			h.Set(k, v)
+		}
+	}
+
+	return h
+}
+
+// ========= Header parsing and sanitizing functions =========
+
+func parseHeader(h mail.Header) (m *pmapi.Message, err error) { //nolint[unparam]
+	m = pmapi.NewMessage()
+
+	if subject, err := pmmime.DecodeHeader(h.Get("Subject")); err == nil {
+		m.Subject = subject
+	}
+	if addrs, err := sanitizeAddressList(h, "From"); err == nil && len(addrs) > 0 {
+		m.Sender = addrs[0]
+	}
+	if addrs, err := sanitizeAddressList(h, "Reply-To"); err == nil && len(addrs) > 0 {
+		m.ReplyTos = addrs
+	}
+	if addrs, err := sanitizeAddressList(h, "To"); err == nil {
+		m.ToList = addrs
+	}
+	if addrs, err := sanitizeAddressList(h, "Cc"); err == nil {
+		m.CCList = addrs
+	}
+	if addrs, err := sanitizeAddressList(h, "Bcc"); err == nil {
+		m.BCCList = addrs
+	}
+	m.Time = 0
+	if t, err := h.Date(); err == nil && !t.IsZero() {
+		m.Time = t.Unix()
+	}
+
+	m.Header = h
+	return
+}
+
+func sanitizeAddressList(h mail.Header, field string) (addrs []*mail.Address, err error) {
+	raw := h.Get(field)
+	if raw == "" {
+		err = mail.ErrHeaderNotPresent
+		return
+	}
+	var decoded string
+	decoded, err = pmmime.DecodeHeader(raw)
+	if err != nil {
+		return
+	}
+	addrs, err = mail.ParseAddressList(parseAddressComment(decoded))
+	if err == nil {
+		if addrs == nil {
+			addrs = []*mail.Address{}
+		}
+		return
+	}
+	// Probably missing encoding error -- try to at least parse addresses in brackets.
+	addrStr := h.Get(field)
+	first := strings.Index(addrStr, "<")
+	last := strings.LastIndex(addrStr, ">")
+	if first < 0 || last < 0 || first >= last {
+		return
+	}
+	var addrList []string
+	open := first
+	for open < last && 0 <= open {
+		addrStr = addrStr[open:]
+		close := strings.Index(addrStr, ">")
+		addrList = append(addrList, addrStr[:close+1])
+		addrStr = addrStr[close:]
+		open = strings.Index(addrStr, "<")
+		last = strings.LastIndex(addrStr, ">")
+	}
+	addrStr = strings.Join(addrList, ", ")
+	//
+	return mail.ParseAddressList(addrStr)
+}

pkg/message/html.go

@@ -0,0 +1,71 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package message
+
+import (
+	"bytes"
+	escape "html"
+	"strings"
+
+	"github.com/andybalholm/cascadia"
+	"golang.org/x/net/html"
+)
+
+func plaintextToHTML(text string) (output string) {
+	text = escape.EscapeString(text)
+	text = strings.Replace(text, "\n\r", "<br>", -1)
+	text = strings.Replace(text, "\r\n", "<br>", -1)
+	text = strings.Replace(text, "\n", "<br>", -1)
+	text = strings.Replace(text, "\r", "<br>", -1)
+
+	return "<div>" + text + "</div>"
+}
+
+func stripHTML(input string) (stripped string, err error) {
+	reader := strings.NewReader(input)
+	doc, _ := html.Parse(reader)
+	body := cascadia.MustCompile("body").MatchFirst(doc)
+	var buf1 bytes.Buffer
+	if err = html.Render(&buf1, body); err != nil {
+		stripped = input
+		return
+	}
+	stripped = buf1.String()
+	// Handle double body tags edge case.
+	if strings.Index(stripped, "<body") == 0 {
+		startIndex := strings.Index(stripped, ">")
+		if startIndex < 5 {
+			return
+		}
+		stripped = stripped[startIndex+1:]
+		// Closing body tag is optional.
+		closingIndex := strings.Index(stripped, "</body>")
+		if closingIndex > -1 {
+			stripped = stripped[:closingIndex]
+		}
+	}
+	return
+}
+
+func addOuterHTMLTags(input string) (output string) {
+	return "<html><head></head><body>" + input + "</body></html>"
+}
+
+func makeEmbeddedImageHTML(cid, name string) (output string) {
+	return "<img class=\"proton-embedded\" alt=\"" + name + "\" src=\"cid:" + cid + "\">"
+}

pkg/message/message.go

@@ -0,0 +1,188 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package message
+
+import (
+	"crypto/sha512"
+	"fmt"
+	"strings"
+
+	pmmime "github.com/ProtonMail/proton-bridge/pkg/mime"
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/emersion/go-imap"
+	"github.com/jhillyerd/enmime"
+	log "github.com/sirupsen/logrus"
+)
+
+const textPlain = "text/plain"
+
+func GetBoundary(m *pmapi.Message) string {
+	// The boundary needs to be deterministic because messages are not supposed to
+	// change.
+	return fmt.Sprintf("%x", sha512.Sum512_256([]byte(m.ID)))
+}
+
+func GetRelatedBoundary(m *pmapi.Message) string {
+	// The boundary needs to be deterministic because messages are not supposed to
+	// change.
+	return fmt.Sprintf("%x", sha512.Sum512_256([]byte(m.ID+m.ID)))
+}
+
+func GetBodyStructure(m *pmapi.Message) (bs *imap.BodyStructure) { //nolint[funlen]
+	bs = &imap.BodyStructure{
+		MimeType:    "multipart",
+		MimeSubType: "mixed",
+		Params:      map[string]string{"boundary": GetBoundary(m)},
+	}
+	var inlineParts []*imap.BodyStructure
+	var attParts []*imap.BodyStructure
+
+	for _, att := range m.Attachments {
+		typeParts := strings.SplitN(att.MIMEType, "/", 2)
+		if len(typeParts) != 2 {
+			continue
+		}
+
+		if typeParts[0] == "application" && typeParts[1] == "pgp-encrypted" {
+			typeParts[1] = "octet-stream"
+		}
+
+		part := &imap.BodyStructure{
+			MimeType:    typeParts[0],
+			MimeSubType: typeParts[1],
+			Params:      map[string]string{"name": att.Name},
+			Encoding:    "base64",
+		}
+
+		if strings.Contains(att.Header.Get("Content-Disposition"), "inline") {
+			part.Disposition = "inline"
+			inlineParts = append(inlineParts, part)
+		} else {
+			part.Disposition = "attachment"
+			attParts = append(attParts, part)
+		}
+	}
+
+	if len(inlineParts) > 0 {
+		// Set to multipart-related for inline attachments.
+		relatedPart := &imap.BodyStructure{
+			MimeType:    "multipart",
+			MimeSubType: "related",
+			Params:      map[string]string{"boundary": GetRelatedBoundary(m)},
+		}
+
+		subType := "html"
+
+		if m.MIMEType == textPlain {
+			subType = "plain"
+		}
+
+		relatedPart.Parts = append(relatedPart.Parts, &imap.BodyStructure{
+			MimeType:    "text",
+			MimeSubType: subType,
+			Params:      map[string]string{"charset": "utf-8"},
+			Encoding:    "quoted-printable",
+			Disposition: "inline",
+		})
+
+		bs.Parts = append(bs.Parts, relatedPart)
+	} else {
+		subType := "html"
+
+		if m.MIMEType == textPlain {
+			subType = "plain"
+		}
+
+		bs.Parts = append(bs.Parts, &imap.BodyStructure{
+			MimeType:    "text",
+			MimeSubType: subType,
+			Params:      map[string]string{"charset": "utf-8"},
+			Encoding:    "quoted-printable",
+			Disposition: "inline",
+		})
+	}
+
+	bs.Parts = append(bs.Parts, attParts...)
+
+	return bs
+}
+
+func SeparateInlineAttachments(m *pmapi.Message) (atts, inlines []*pmapi.Attachment) {
+	for _, att := range m.Attachments {
+		if strings.Contains(att.Header.Get("Content-Disposition"), "inline") {
+			inlines = append(inlines, att)
+		} else {
+			atts = append(atts, att)
+		}
+	}
+	return
+}
+
+func GetMIMEBodyStructure(m *pmapi.Message, parsedMsg *enmime.Envelope) (bs *imap.BodyStructure, err error) {
+	// We recursively look through the MIME structure.
+	root := parsedMsg.Root
+	if root == nil {
+		return GetBodyStructure(m), nil
+	}
+
+	mediaType, params, err := pmmime.ParseMediaType(root.ContentType)
+	if err != nil {
+		log.Warnf("Cannot parse Content-Type '%v': %v", root.ContentType, err)
+		err = nil
+		mediaType = textPlain
+	}
+
+	typeParts := strings.SplitN(mediaType, "/", 2)
+
+	bs = &imap.BodyStructure{
+		MimeType: typeParts[0],
+		Params:   params,
+	}
+
+	if len(typeParts) > 1 {
+		bs.MimeSubType = typeParts[1]
+	}
+
+	bs.Parts = getChildrenParts(root)
+
+	return
+}
+
+func getChildrenParts(root *enmime.Part) (parts []*imap.BodyStructure) {
+	for child := root.FirstChild; child != nil; child = child.NextSibling {
+		mediaType, params, err := pmmime.ParseMediaType(child.ContentType)
+		if err != nil {
+			log.Warnf("Cannot parse Content-Type '%v': %v", child.ContentType, err)
+			mediaType = textPlain
+		}
+		typeParts := strings.SplitN(mediaType, "/", 2)
+		childrenParts := getChildrenParts(child)
+		part := &imap.BodyStructure{
+			MimeType:    typeParts[0],
+			Params:      params,
+			Encoding:    child.Charset,
+			Disposition: child.Disposition,
+			Parts:       childrenParts,
+		}
+		if len(typeParts) > 1 {
+			part.MimeSubType = typeParts[1]
+		}
+		parts = append(parts, part)
+	}
+	return
+}

pkg/message/parser.go

@@ -0,0 +1,468 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package message
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"mime"
+	"mime/quotedprintable"
+	"net/mail"
+	"net/textproto"
+	"regexp"
+	"strconv"
+	"strings"
+
+	pmmime "github.com/ProtonMail/proton-bridge/pkg/mime"
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/jaytaylor/html2text"
+	log "github.com/sirupsen/logrus"
+)
+
+func parseAttachment(filename string, mediaType string, h textproto.MIMEHeader) (att *pmapi.Attachment) {
+	if decoded, err := pmmime.DecodeHeader(filename); err == nil {
+		filename = decoded
+	}
+	if filename == "" {
+		ext, err := mime.ExtensionsByType(mediaType)
+		if err == nil && len(ext) > 0 {
+			filename = "attachment" + ext[0]
+		}
+	}
+
+	att = &pmapi.Attachment{
+		Name:     filename,
+		MIMEType: mediaType,
+		Header:   h,
+	}
+
+	headerContentID := strings.Trim(h.Get("Content-Id"), " <>")
+
+	if headerContentID != "" {
+		att.ContentID = headerContentID
+	}
+
+	return
+}
+
+var reEmailComment = regexp.MustCompile("[(][^)]*[)]") //nolint[gochecknoglobals]
+
+// parseAddressComment removes the comments completely even though they should be allowed
+// http://tools.wordtothewise.com/rfc/822
+// NOTE: This should be supported in go>1.10 but it seems it's not ¯\_(ツ)_/¯
+func parseAddressComment(raw string) string {
+	return reEmailComment.ReplaceAllString(raw, "")
+}
+
+// Some clients incorrectly format messages with embedded attachments to have a format like
+// I. text/plain II. attachment III. text/plain
+// which we need to convert to a single HTML part with an embedded attachment.
+func combineParts(m *pmapi.Message, parts []io.Reader, headers []textproto.MIMEHeader, convertPlainToHTML bool, atts *[]io.Reader) (isHTML bool, err error) { //nolint[funlen]
+	isHTML = true
+	foundText := false
+
+	for i := len(parts) - 1; i >= 0; i-- {
+		part := parts[i]
+		h := headers[i]
+
+		disp, dispParams, _ := pmmime.ParseMediaType(h.Get("Content-Disposition"))
+
+		d := pmmime.DecodeContentEncoding(part, h.Get("Content-Transfer-Encoding"))
+		if d == nil {
+			log.Warnf("Unsupported Content-Transfer-Encoding '%v'", h.Get("Content-Transfer-Encoding"))
+			d = part
+		}
+
+		contentType := h.Get("Content-Type")
+		if contentType == "" {
+			contentType = "text/plain"
+		}
+		mediaType, params, _ := pmmime.ParseMediaType(contentType)
+
+		if strings.HasPrefix(mediaType, "text/") && mediaType != "text/calendar" && disp != "attachment" {
+			// This is text.
+			var b []byte
+			if b, err = ioutil.ReadAll(d); err != nil {
+				continue
+			}
+			b, err = pmmime.DecodeCharset(b, params)
+			if err != nil {
+				log.Warn("Decode charset error: ", err)
+				return false, err
+			}
+			contents := string(b)
+			if strings.Contains(mediaType, "text/plain") && len(contents) > 0 {
+				if !convertPlainToHTML {
+					isHTML = false
+				} else {
+					contents = plaintextToHTML(contents)
+				}
+			} else if strings.Contains(mediaType, "text/html") && len(contents) > 0 {
+				contents, err = stripHTML(contents)
+				if err != nil {
+					return isHTML, err
+				}
+			}
+			m.Body = contents + m.Body
+			foundText = true
+		} else {
+			// This is an attachment.
+			filename := dispParams["filename"]
+			if filename == "" {
+				// Using "name" in Content-Type is discouraged.
+				filename = params["name"]
+			}
+			if filename == "" && mediaType == "text/calendar" {
+				filename = "event.ics"
+			}
+
+			att := parseAttachment(filename, mediaType, h)
+
+			b := &bytes.Buffer{}
+			if _, err = io.Copy(b, d); err != nil {
+				continue
+			}
+			if foundText && att.ContentID == "" && strings.Contains(mediaType, "image") {
+				// Treat this as an inline attachment even though it is not marked as one.
+				hasher := sha256.New()
+				_, _ = hasher.Write([]byte(att.Name + strconv.Itoa(b.Len())))
+				bytes := hasher.Sum(nil)
+				cid := hex.EncodeToString(bytes) + "@protonmail.com"
+
+				att.ContentID = cid
+				embeddedHTML := makeEmbeddedImageHTML(cid, att.Name)
+				m.Body = embeddedHTML + m.Body
+			}
+
+			m.Attachments = append(m.Attachments, att)
+			*atts = append(*atts, b)
+		}
+	}
+	if isHTML {
+		m.Body = addOuterHTMLTags(m.Body)
+	}
+	return isHTML, nil
+}
+
+func checkHeaders(headers []textproto.MIMEHeader) bool {
+	foundAttachment := false
+
+	for i := 0; i < len(headers); i++ {
+		h := headers[i]
+
+		mediaType, _, _ := pmmime.ParseMediaType(h.Get("Content-Type"))
+
+		if !strings.HasPrefix(mediaType, "text/") {
+			foundAttachment = true
+		} else if foundAttachment {
+			// This means that there is a text part after the first attachment,
+			// so we will have to convert the body from plain->HTML.
+			return true
+		}
+	}
+	return false
+}
+
+// ============================== 7bit Filter ==========================
+// For every MIME part in the tree that has "8bit" or "binary" content
+// transfer encoding: transcode it to "quoted-printable".
+
+type SevenBitFilter struct {
+	target pmmime.VisitAcceptor
+}
+
+func NewSevenBitFilter(targetAccepter pmmime.VisitAcceptor) *SevenBitFilter {
+	return &SevenBitFilter{
+		target: targetAccepter,
+	}
+}
+
+func decodePart(partReader io.Reader, header textproto.MIMEHeader) (decodedPart io.Reader) {
+	decodedPart = pmmime.DecodeContentEncoding(partReader, header.Get("Content-Transfer-Encoding"))
+	if decodedPart == nil {
+		log.Warnf("Unsupported Content-Transfer-Encoding '%v'", header.Get("Content-Transfer-Encoding"))
+		decodedPart = partReader
+	}
+	return
+}
+
+func (sd SevenBitFilter) Accept(partReader io.Reader, header textproto.MIMEHeader, hasPlainSibling bool, isFirst, isLast bool) error {
+	cte := strings.ToLower(header.Get("Content-Transfer-Encoding"))
+	if isFirst && pmmime.IsLeaf(header) && cte != "quoted-printable" && cte != "base64" && cte != "7bit" {
+		decodedPart := decodePart(partReader, header)
+
+		filteredHeader := textproto.MIMEHeader{}
+		for k, v := range header {
+			filteredHeader[k] = v
+		}
+		filteredHeader.Set("Content-Transfer-Encoding", "quoted-printable")
+		//filteredHeader.Set("Content-Transfer-Encoding", "base64")
+
+		filteredBuffer := &bytes.Buffer{}
+		decodedSlice, _ := ioutil.ReadAll(decodedPart)
+		w := quotedprintable.NewWriter(filteredBuffer)
+		//w := base64.NewEncoder(base64.StdEncoding, filteredBuffer)
+		if _, err := w.Write(decodedSlice); err != nil {
+			log.Errorf("cannot write quotedprintable from %q: %v", cte, err)
+		}
+		if err := w.Close(); err != nil {
+			log.Errorf("cannot close quotedprintable from %q: %v", cte, err)
+		}
+
+		_ = sd.target.Accept(filteredBuffer, filteredHeader, hasPlainSibling, true, isLast)
+	} else {
+		_ = sd.target.Accept(partReader, header, hasPlainSibling, isFirst, isLast)
+	}
+	return nil
+}
+
+// =================== HTML Only convertor ==================================
+// In any part of MIME tree structure, replace standalone text/html with
+// multipart/alternative containing both text/html and text/plain.
+
+type HTMLOnlyConvertor struct {
+	target pmmime.VisitAcceptor
+}
+
+func NewHTMLOnlyConvertor(targetAccepter pmmime.VisitAcceptor) *HTMLOnlyConvertor {
+	return &HTMLOnlyConvertor{
+		target: targetAccepter,
+	}
+}
+
+func randomBoundary() string {
+	var buf [30]byte
+	_, err := io.ReadFull(rand.Reader, buf[:])
+	if err != nil {
+		panic(err)
+	}
+
+	return fmt.Sprintf("%x", buf[:])
+}
+
+func (hoc HTMLOnlyConvertor) Accept(partReader io.Reader, header textproto.MIMEHeader, hasPlainSiblings bool, isFirst, isLast bool) error {
+	mediaType, _, err := pmmime.ParseMediaType(header.Get("Content-Type"))
+	if isFirst && err == nil && mediaType == "text/html" && !hasPlainSiblings {
+		multiPartHeaders := make(textproto.MIMEHeader)
+		for k, v := range header {
+			multiPartHeaders[k] = v
+		}
+		boundary := randomBoundary()
+		multiPartHeaders.Set("Content-Type", "multipart/alternative; boundary=\""+boundary+"\"")
+		childCte := header.Get("Content-Transfer-Encoding")
+
+		_ = hoc.target.Accept(partReader, multiPartHeaders, false, true, false)
+
+		partData, _ := ioutil.ReadAll(partReader)
+
+		htmlChildHeaders := make(textproto.MIMEHeader)
+		htmlChildHeaders.Set("Content-Transfer-Encoding", childCte)
+		htmlChildHeaders.Set("Content-Type", "text/html")
+		htmlReader := bytes.NewReader(partData)
+		_ = hoc.target.Accept(htmlReader, htmlChildHeaders, false, true, false)
+
+		_ = hoc.target.Accept(partReader, multiPartHeaders, hasPlainSiblings, false, false)
+
+		plainChildHeaders := make(textproto.MIMEHeader)
+		plainChildHeaders.Set("Content-Transfer-Encoding", childCte)
+		plainChildHeaders.Set("Content-Type", "text/plain")
+		unHtmlized, err := html2text.FromReader(bytes.NewReader(partData))
+		if err != nil {
+			unHtmlized = string(partData)
+		}
+		plainReader := strings.NewReader(unHtmlized)
+		_ = hoc.target.Accept(plainReader, plainChildHeaders, false, true, true)
+
+		_ = hoc.target.Accept(partReader, multiPartHeaders, hasPlainSiblings, false, true)
+	} else {
+		_ = hoc.target.Accept(partReader, header, hasPlainSiblings, isFirst, isLast)
+	}
+	return nil
+}
+
+// ======= Public Key Attacher ========
+
+type PublicKeyAttacher struct {
+	target                pmmime.VisitAcceptor
+	attachedPublicKey     string
+	attachedPublicKeyName string
+	appendToMultipart     bool
+	depth                 int
+}
+
+func NewPublicKeyAttacher(targetAccepter pmmime.VisitAcceptor, attachedPublicKey, attachedPublicKeyName string) *PublicKeyAttacher {
+	return &PublicKeyAttacher{
+		target:                targetAccepter,
+		attachedPublicKey:     attachedPublicKey,
+		attachedPublicKeyName: attachedPublicKeyName,
+		appendToMultipart:     false,
+		depth:                 0,
+	}
+}
+
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
+
+func split(input string, sliceLength int) string {
+	processed := input
+	result := ""
+	for len(processed) > 0 {
+		cutPoint := min(sliceLength, len(processed))
+		part := processed[0:cutPoint]
+		result = result + part + "\n"
+		processed = processed[cutPoint:]
+	}
+	return result
+}
+
+func createKeyAttachment(publicKey, publicKeyName string) (headers textproto.MIMEHeader, contents io.Reader) {
+	attachmentHeaders := make(textproto.MIMEHeader)
+	attachmentHeaders.Set("Content-Type", "application/pgp-key; name=\""+publicKeyName+"\"")
+	attachmentHeaders.Set("Content-Transfer-Encoding", "base64")
+	attachmentHeaders.Set("Content-Disposition", "attachment; filename=\""+publicKeyName+".asc.pgp\"")
+
+	buffer := &bytes.Buffer{}
+	w := base64.NewEncoder(base64.StdEncoding, buffer)
+	_, _ = w.Write([]byte(publicKey))
+	_ = w.Close()
+
+	return attachmentHeaders, strings.NewReader(split(buffer.String(), 73))
+}
+
+func (pka *PublicKeyAttacher) Accept(partReader io.Reader, header textproto.MIMEHeader, hasPlainSiblings bool, isFirst, isLast bool) error {
+	if isFirst && !pmmime.IsLeaf(header) {
+		pka.depth++
+	}
+	if isLast && !pmmime.IsLeaf(header) {
+		defer func() {
+			pka.depth--
+		}()
+	}
+	isRoot := (header.Get("From") != "")
+	mediaType, _, err := pmmime.ParseMediaType(header.Get("Content-Type"))
+	if isRoot && isFirst && err == nil && pka.attachedPublicKey != "" { //nolint[gocritic]
+		if strings.HasPrefix(mediaType, "multipart/mixed") {
+			pka.appendToMultipart = true
+			_ = pka.target.Accept(partReader, header, hasPlainSiblings, isFirst, isLast)
+		} else {
+			// Create two siblings with attachment in the case toplevel is not multipart/mixed.
+			multiPartHeaders := make(textproto.MIMEHeader)
+			for k, v := range header {
+				multiPartHeaders[k] = v
+			}
+			boundary := randomBoundary()
+			multiPartHeaders.Set("Content-Type", "multipart/mixed; boundary=\""+boundary+"\"")
+			multiPartHeaders.Del("Content-Transfer-Encoding")
+
+			_ = pka.target.Accept(partReader, multiPartHeaders, false, true, false)
+
+			originalHeader := make(textproto.MIMEHeader)
+			originalHeader.Set("Content-Type", header.Get("Content-Type"))
+			if header.Get("Content-Transfer-Encoding") != "" {
+				originalHeader.Set("Content-Transfer-Encoding", header.Get("Content-Transfer-Encoding"))
+			}
+
+			_ = pka.target.Accept(partReader, originalHeader, false, true, false)
+			_ = pka.target.Accept(partReader, multiPartHeaders, hasPlainSiblings, false, false)
+
+			attachmentHeaders, attachmentReader := createKeyAttachment(pka.attachedPublicKey, pka.attachedPublicKeyName)
+
+			_ = pka.target.Accept(attachmentReader, attachmentHeaders, false, true, true)
+			_ = pka.target.Accept(partReader, multiPartHeaders, hasPlainSiblings, false, true)
+		}
+	} else if isLast && pka.depth == 1 && pka.attachedPublicKey != "" {
+		_ = pka.target.Accept(partReader, header, hasPlainSiblings, isFirst, false)
+		attachmentHeaders, attachmentReader := createKeyAttachment(pka.attachedPublicKey, pka.attachedPublicKeyName)
+		_ = pka.target.Accept(attachmentReader, attachmentHeaders, hasPlainSiblings, true, true)
+		_ = pka.target.Accept(partReader, header, hasPlainSiblings, isFirst, true)
+	} else {
+		_ = pka.target.Accept(partReader, header, hasPlainSiblings, isFirst, isLast)
+	}
+	return nil
+}
+
+// ======= Parser ==========
+
+func Parse(r io.Reader, attachedPublicKey, attachedPublicKeyName string) (m *pmapi.Message, mimeBody string, plainContents string, atts []io.Reader, err error) {
+	secondReader := new(bytes.Buffer)
+	_, _ = secondReader.ReadFrom(r)
+
+	mimeBody = secondReader.String()
+
+	mm, err := mail.ReadMessage(secondReader)
+	if err != nil {
+		return
+	}
+
+	if m, err = parseHeader(mm.Header); err != nil {
+		return
+	}
+
+	h := textproto.MIMEHeader(m.Header)
+	mmBodyData, err := ioutil.ReadAll(mm.Body)
+	if err != nil {
+		return
+	}
+
+	printAccepter := pmmime.NewMIMEPrinter()
+
+	publicKeyAttacher := NewPublicKeyAttacher(printAccepter, attachedPublicKey, attachedPublicKeyName)
+	sevenBitFilter := NewSevenBitFilter(publicKeyAttacher)
+
+	plainTextCollector := pmmime.NewPlainTextCollector(sevenBitFilter)
+	htmlOnlyConvertor := NewHTMLOnlyConvertor(plainTextCollector)
+
+	visitor := pmmime.NewMimeVisitor(htmlOnlyConvertor)
+	err = pmmime.VisitAll(bytes.NewReader(mmBodyData), h, visitor)
+	/*
+		err = visitor.VisitAll(h, bytes.NewReader(mmBodyData))
+	*/
+	if err != nil {
+		return
+	}
+
+	mimeBody = printAccepter.String()
+	plainContents = plainTextCollector.GetPlainText()
+
+	parts, headers, err := pmmime.GetAllChildParts(bytes.NewReader(mmBodyData), h)
+
+	if err != nil {
+		return
+	}
+
+	convertPlainToHTML := checkHeaders(headers)
+	isHTML, err := combineParts(m, parts, headers, convertPlainToHTML, &atts)
+
+	if isHTML {
+		m.MIMEType = "text/html"
+	} else {
+		m.MIMEType = "text/plain"
+	}
+
+	return m, mimeBody, plainContents, atts, err
+}

pkg/message/parser_test.go

@@ -0,0 +1,107 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package message
+
+import (
+	"net/mail"
+	"testing"
+)
+
+func TestRFC822AddressFormat(t *testing.T) { //nolint[funlen]
+	tests := []struct {
+		address  string
+		expected []string
+	}{
+		{
+			" normal name  <username@server.com>",
+			[]string{
+				"\"normal name\" <username@server.com>",
+			},
+		},
+		{
+			" \"comma, name\"  <username@server.com>",
+			[]string{
+				"\"comma, name\" <username@server.com>",
+			},
+		},
+		{
+			" name  <username@server.com> (ignore comment)",
+			[]string{
+				"\"name\" <username@server.com>",
+			},
+		},
+		{
+			" name (ignore comment)  <username@server.com>,  (Comment as name) username2@server.com",
+			[]string{
+				"\"name\" <username@server.com>",
+				"<username2@server.com>",
+			},
+		},
+		{
+			" normal name  <username@server.com>, (comment)All.(around)address@(the)server.com",
+			[]string{
+				"\"normal name\" <username@server.com>",
+				"<All.address@server.com>",
+			},
+		},
+		{
+			" normal name  <username@server.com>, All.(\"comma, in comment\")address@(the)server.com",
+			[]string{
+				"\"normal name\" <username@server.com>",
+				"<All.address@server.com>",
+			},
+		},
+		{
+			" \"normal name\"  <username@server.com>, \"comma, name\" <address@server.com>",
+			[]string{
+				"\"normal name\" <username@server.com>",
+				"\"comma, name\" <address@server.com>",
+			},
+		},
+		{
+			" \"comma, one\"  <username@server.com>, \"comma, two\" <address@server.com>",
+			[]string{
+				"\"comma, one\" <username@server.com>",
+				"\"comma, two\" <address@server.com>",
+			},
+		},
+		{
+			" \"comma, name\"  <username@server.com>, another, name <address@server.com>",
+			[]string{
+				"\"comma, name\" <username@server.com>",
+				"\"another, name\" <address@server.com>",
+			},
+		},
+	}
+
+	for _, data := range tests {
+		uncommented := parseAddressComment(data.address)
+		result, err := mail.ParseAddressList(uncommented)
+		if err != nil {
+			t.Errorf("Can not parse '%s' created from '%s': %v", uncommented, data.address, err)
+		}
+		if len(result) != len(data.expected) {
+			t.Errorf("Wrong parsing of '%s' created from '%s': expected '%s' but have '%+v'", uncommented, data.address, data.expected, result)
+		}
+		for i, result := range result {
+			if data.expected[i] != result.String() {
+				t.Errorf("Wrong parsing\nof %q\ncreated from %q:\nexpected %q\nbut have %q", uncommented, data.address, data.expected, result.String())
+			}
+		}
+	}
+}

pkg/message/section.go

@@ -0,0 +1,413 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package message
+
+import (
+	"bufio"
+	"bytes"
+	"errors"
+	"io"
+	"io/ioutil"
+	"net/textproto"
+	"strconv"
+	"strings"
+
+	pmmime "github.com/ProtonMail/proton-bridge/pkg/mime"
+	"github.com/emersion/go-imap"
+)
+
+type sectionInfo struct {
+	header                    textproto.MIMEHeader
+	start, bsize, size, lines int
+	reader                    io.Reader
+}
+
+// Count and read.
+func (si *sectionInfo) Read(p []byte) (n int, err error) {
+	n, err = si.reader.Read(p)
+	si.size += n
+	si.lines += bytes.Count(p, []byte("\n"))
+	return
+}
+
+type boundaryReader struct {
+	reader *bufio.Reader
+
+	closed, first bool
+	skipped       int
+
+	nl               []byte // "\r\n" or "\n" (set after seeing first boundary line)
+	nlDashBoundary   []byte // nl + "--boundary"
+	dashBoundaryDash []byte // "--boundary--"
+	dashBoundary     []byte // "--boundary"
+}
+
+func newBoundaryReader(r *bufio.Reader, boundary string) (br *boundaryReader, err error) {
+	b := []byte("\r\n--" + boundary + "--")
+	br = &boundaryReader{
+		reader:           r,
+		closed:           false,
+		first:            true,
+		nl:               b[:2],
+		nlDashBoundary:   b[:len(b)-2],
+		dashBoundaryDash: b[2:],
+		dashBoundary:     b[2 : len(b)-2],
+	}
+	err = br.WriteNextPartTo(nil)
+	return
+}
+
+func skipLWSPChar(b []byte) []byte {
+	for len(b) > 0 && (b[0] == ' ' || b[0] == '\t') {
+		b = b[1:]
+	}
+	return b
+}
+
+func (br *boundaryReader) isFinalBoundary(line []byte) bool {
+	if !bytes.HasPrefix(line, br.dashBoundaryDash) {
+		return false
+	}
+	rest := line[len(br.dashBoundaryDash):]
+	rest = skipLWSPChar(rest)
+	return len(rest) == 0 || bytes.Equal(rest, br.nl)
+}
+
+func (br *boundaryReader) isBoundaryDelimiterLine(line []byte) (ret bool) {
+	if !bytes.HasPrefix(line, br.dashBoundary) {
+		return false
+	}
+	rest := line[len(br.dashBoundary):]
+	rest = skipLWSPChar(rest)
+
+	if br.first && len(rest) == 1 && rest[0] == '\n' {
+		br.nl = br.nl[1:]
+		br.nlDashBoundary = br.nlDashBoundary[1:]
+	}
+	return bytes.Equal(rest, br.nl)
+}
+
+func (br *boundaryReader) WriteNextPartTo(part io.Writer) (err error) {
+	if br.closed {
+		return io.EOF
+	}
+
+	var line, slice []byte
+	br.skipped = 0
+
+	for {
+		slice, err = br.reader.ReadSlice('\n')
+		line = append(line, slice...)
+		if err == bufio.ErrBufferFull {
+			continue
+		}
+
+		br.skipped += len(line)
+
+		if err == io.EOF && br.isFinalBoundary(line) {
+			err = nil
+			br.closed = true
+			return
+		}
+
+		if err != nil {
+			return
+		}
+
+		if br.isBoundaryDelimiterLine(line) {
+			br.first = false
+			return
+		}
+
+		if br.isFinalBoundary(line) {
+			br.closed = true
+			return
+		}
+
+		if part != nil {
+			if _, err = part.Write(line); err != nil {
+				return
+			}
+		}
+
+		line = []byte{}
+	}
+}
+
+type BodyStructure map[string]*sectionInfo
+
+func NewBodyStructure(reader io.Reader) (structure *BodyStructure, err error) {
+	structure = &BodyStructure{}
+	err = structure.Parse(reader)
+	return
+}
+
+func (bs *BodyStructure) Parse(r io.Reader) error {
+	return bs.parseAllChildSections(r, []int{}, 0)
+}
+
+func (bs *BodyStructure) parseAllChildSections(r io.Reader, currentPath []int, start int) (err error) { //nolint[funlen]
+	info := &sectionInfo{
+		start:  start,
+		size:   0,
+		bsize:  0,
+		lines:  0,
+		reader: r,
+	}
+
+	bufInfo := bufio.NewReader(info)
+	tp := textproto.NewReader(bufInfo)
+
+	if info.header, err = tp.ReadMIMEHeader(); err != nil {
+		return
+	}
+
+	bodyInfo := &sectionInfo{reader: tp.R}
+	bodyReader := bufio.NewReader(bodyInfo)
+
+	mediaType, params, _ := pmmime.ParseMediaType(info.header.Get("Content-Type"))
+
+	// If multipart, call getAllParts, else read to count lines.
+	if (strings.HasPrefix(mediaType, "multipart/") || mediaType == "message/rfc822") && params["boundary"] != "" {
+		newPath := append(currentPath, 1)
+
+		var br *boundaryReader
+		br, err = newBoundaryReader(bodyReader, params["boundary"])
+		// New reader seeks first boundary.
+		if err != nil {
+			// Return also EOF.
+			return
+		}
+
+		for err == nil {
+			start += br.skipped
+			part := &bytes.Buffer{}
+			err = br.WriteNextPartTo(part)
+			if err != nil {
+				break
+			}
+			err = bs.parseAllChildSections(part, newPath, start)
+			part.Reset()
+			newPath[len(newPath)-1]++
+		}
+		br.reader = nil
+
+		if err == io.EOF {
+			err = nil
+		}
+		if err != nil {
+			return
+		}
+	} else {
+		// Count length.
+		_, _ = bodyReader.WriteTo(ioutil.Discard)
+	}
+
+	// Clear all buffers.
+	bodyReader = nil
+	bodyInfo.reader = nil
+	tp.R = nil
+	tp = nil
+	bufInfo = nil // nolint
+	info.reader = nil
+
+	// Store boundaries.
+	info.bsize = bodyInfo.size
+	path := stringPathFromInts(currentPath)
+	(*bs)[path] = info
+
+	// Fix start of subsections.
+	newPath := append(currentPath, 1)
+	shift := info.size - info.bsize
+	subInfo, err := bs.getInfo(newPath)
+
+	// If it has subparts.
+	for err == nil {
+		subInfo.start += shift
+
+		// Level down.
+		subInfo, err = bs.getInfo(append(newPath, 1))
+		if err == nil {
+			newPath = append(newPath, 1)
+			continue
+		}
+
+		// Next.
+		newPath[len(newPath)-1]++
+		subInfo, err = bs.getInfo(newPath)
+		if err == nil {
+			continue
+		}
+
+		// Level up.
+		for {
+			newPath = newPath[:len(newPath)-1]
+			if len(newPath) > 0 {
+				newPath[len(newPath)-1]++
+				subInfo, err = bs.getInfo(newPath)
+				if err != nil {
+					err = nil
+					continue
+				}
+			}
+			break
+		}
+
+		// The end.
+		if len(newPath) == 0 {
+			break
+		}
+	}
+
+	return nil
+}
+
+func stringPathFromInts(ints []int) (ret string) {
+	for i, n := range ints {
+		if i != 0 {
+			ret += "."
+		}
+		ret += strconv.Itoa(n)
+	}
+	return
+}
+
+func (bs *BodyStructure) getInfo(sectionPath []int) (sectionInfo *sectionInfo, err error) {
+	path := stringPathFromInts(sectionPath)
+	sectionInfo, ok := (*bs)[path]
+	if !ok {
+		err = errors.New("wrong section " + path)
+	}
+	return
+}
+
+func (bs *BodyStructure) GetSection(wholeMail io.ReadSeeker, sectionPath []int) (section []byte, err error) {
+	info, err := bs.getInfo(sectionPath)
+	if err != nil {
+		return
+	}
+	if _, err = wholeMail.Seek(int64(info.start), io.SeekStart); err != nil {
+		return
+	}
+	section = make([]byte, info.size)
+	_, err = wholeMail.Read(section)
+	return
+}
+
+func (bs *BodyStructure) GetSectionContent(wholeMail io.ReadSeeker, sectionPath []int) (section []byte, err error) {
+	info, err := bs.getInfo(sectionPath)
+	if err != nil {
+		return
+	}
+	if _, err = wholeMail.Seek(int64(info.start+info.size-info.bsize), io.SeekStart); err != nil {
+		return
+	}
+	section = make([]byte, info.bsize)
+	_, err = wholeMail.Read(section)
+	return
+
+	/* This is slow:
+	sectionBuf, err := bs.GetSection(wholeMail, sectionPath)
+	if err != nil {
+		return
+	}
+
+	tp := textproto.NewReader(bufio.NewReader(buf))
+	if _, err = tp.ReadMIMEHeader(); err != nil {
+		return err
+	}
+
+	sectionBuf = &bytes.Buffer{}
+	_, err = io.Copy(sectionBuf, tp.R)
+	return
+	*/
+}
+
+func (bs *BodyStructure) GetSectionHeader(sectionPath []int) (header textproto.MIMEHeader, err error) {
+	info, err := bs.getInfo(sectionPath)
+	if err != nil {
+		return
+	}
+	header = info.header
+	return
+}
+
+func (bs *BodyStructure) Size() uint32 {
+	info, err := bs.getInfo([]int{})
+	if err != nil {
+		return uint32(0)
+	}
+	return uint32(info.size)
+}
+
+func (bs *BodyStructure) IMAPBodyStructure(currentPart []int) (imapBS *imap.BodyStructure, err error) {
+	var info *sectionInfo
+	if info, err = bs.getInfo(currentPart); err != nil {
+		return
+	}
+
+	mediaType, params, _ := pmmime.ParseMediaType(info.header.Get("Content-Type"))
+
+	mediaTypeSep := strings.Split(mediaType, "/")
+
+	// If it is empty or missing it will not crash.
+	mediaTypeSep = append(mediaTypeSep, "")
+
+	imapBS = &imap.BodyStructure{
+		MimeType:    mediaTypeSep[0],
+		MimeSubType: mediaTypeSep[1],
+		Params:      params,
+		Size:        uint32(info.bsize),
+		Lines:       uint32(info.lines),
+	}
+
+	if val := info.header.Get("Content-ID"); val != "" {
+		imapBS.Id = val
+	}
+
+	if val := info.header.Get("Content-Transfer-Encoding"); val != "" {
+		imapBS.Encoding = val
+	}
+
+	if val := info.header.Get("Content-Description"); val != "" {
+		imapBS.Description = val
+	}
+
+	if val := info.header.Get("Content-Disposition"); val != "" {
+		imapBS.Disposition = val
+	}
+
+	nextPart := append(currentPart, 1)
+	for {
+		if _, err := bs.getInfo(nextPart); err != nil {
+			break
+		}
+		var subStruct *imap.BodyStructure
+		subStruct, err = bs.IMAPBodyStructure(nextPart)
+		if err != nil {
+			return
+		}
+		if imapBS.Parts == nil {
+			imapBS.Parts = []*imap.BodyStructure{}
+		}
+		imapBS.Parts = append(imapBS.Parts, subStruct)
+		nextPart[len(nextPart)-1]++
+	}
+
+	return imapBS, nil
+}

pkg/message/section_test.go

@@ -0,0 +1,414 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package message
+
+import (
+	"fmt"
+	"path/filepath"
+	"runtime"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func debug(msg string, v ...interface{}) {
+	_, file, line, _ := runtime.Caller(1)
+	fmt.Printf("%s:%d: \033[2;33m"+msg+"\033[0;39m\n", append([]interface{}{filepath.Base(file), line}, v...)...)
+}
+
+func TestParseBodyStructure(t *testing.T) {
+	expectedStructure := map[string]string{
+		"":        "multipart/mixed; boundary=\"0000MAIN\"",
+		"1":       "text/plain",
+		"2":       "application/octet-stream",
+		"3":       "message/rfc822; boundary=\"0003MSG\"",
+		"3.1":     "text/plain",
+		"3.2":     "application/octet-stream",
+		"4":       "multipart/mixed; boundary=\"0004ATTACH\"",
+		"4.1":     "image/gif",
+		"4.2":     "message/rfc822; boundary=\"0042MSG\"",
+		"4.2.1":   "text/plain",
+		"4.2.2":   "multipart/alternative; boundary=\"0422ALTER\"",
+		"4.2.2.1": "text/plain",
+		"4.2.2.2": "text/html",
+	}
+	mailReader := strings.NewReader(sampleMail)
+	bs, err := NewBodyStructure(mailReader)
+	require.NoError(t, err)
+
+	paths := []string{}
+	for path := range *bs {
+		paths = append(paths, path)
+	}
+	sort.Strings(paths)
+
+	debug("%10s: %-50s %5s %5s %5s %5s", "section", "type", "start", "size", "bsize", "lines")
+	for _, path := range paths {
+		sec := (*bs)[path]
+		contentType := sec.header.Get("Content-Type")
+		debug("%10s: %-50s %5d %5d %5d %5d", path, contentType, sec.start, sec.size, sec.bsize, sec.lines)
+		require.Equal(t, expectedStructure[path], contentType)
+	}
+
+	require.True(t, len(*bs) == len(expectedStructure), "Wrong number of sections expected %d but have %d", len(expectedStructure), len(*bs))
+}
+
+func TestGetSection(t *testing.T) {
+	structReader := strings.NewReader(sampleMail)
+	bs, err := NewBodyStructure(structReader)
+	require.NoError(t, err)
+	// Whole section.
+	for _, try := range testPaths {
+		mailReader := strings.NewReader(sampleMail)
+		info, err := bs.getInfo(try.path)
+		require.NoError(t, err)
+		section, err := bs.GetSection(mailReader, try.path)
+		require.NoError(t, err)
+
+		debug("section %v: %d %d\n___\n%s\n‾‾‾\n", try.path, info.start, info.size, string(section))
+
+		require.True(t, string(section) == try.expectedSection, "not same as expected:\n___\n%s\n‾‾‾", try.expectedSection)
+	}
+	// Body content.
+	for _, try := range testPaths {
+		mailReader := strings.NewReader(sampleMail)
+		info, err := bs.getInfo(try.path)
+		require.NoError(t, err)
+		section, err := bs.GetSectionContent(mailReader, try.path)
+		require.NoError(t, err)
+
+		debug("content %v: %d %d\n___\n%s\n‾‾‾\n", try.path, info.start+info.size-info.bsize, info.bsize, string(section))
+
+		require.True(t, string(section) == try.expectedBody, "not same as expected:\n___\n%s\n‾‾‾", try.expectedBody)
+	}
+}
+
+/* Structure example:
+HEADER     ([RFC-2822] header of the message)
+TEXT       ([RFC-2822] text body of the message) MULTIPART/MIXED
+1          TEXT/PLAIN
+2          APPLICATION/OCTET-STREAM
+3          MESSAGE/RFC822
+3.HEADER   ([RFC-2822] header of the message)
+3.TEXT     ([RFC-2822] text body of the message) MULTIPART/MIXED
+3.1        TEXT/PLAIN
+3.2        APPLICATION/OCTET-STREAM
+4          MULTIPART/MIXED
+4.1        IMAGE/GIF
+4.1.MIME   ([MIME-IMB] header for the IMAGE/GIF)
+4.2        MESSAGE/RFC822
+4.2.HEADER ([RFC-2822] header of the message)
+4.2.TEXT   ([RFC-2822] text body of the message) MULTIPART/MIXED
+4.2.1      TEXT/PLAIN
+4.2.2      MULTIPART/ALTERNATIVE
+4.2.2.1    TEXT/PLAIN
+4.2.2.2    TEXT/RICHTEXT
+*/
+
+var sampleMail = `Subject: Sample mail
+From: John Doe <jdoe@machine.example>
+To: Mary Smith <mary@example.net>
+Date: Fri, 21 Nov 1997 09:55:06 -0600
+Content-Type: multipart/mixed; boundary="0000MAIN"
+
+main summary
+
+--0000MAIN
+Content-Type: text/plain
+
+1. main message
+
+
+--0000MAIN
+Content-Type: application/octet-stream
+Content-Disposition: inline; filename="main_signature.sig"
+Content-Transfer-Encoding: base64
+
+2/MainOctetStream
+
+--0000MAIN
+Subject: Inside mail 3
+From: Mary Smith <mary@example.net>
+To: John Doe <jdoe@machine.example>
+Date: Fri, 20 Nov 1997 09:55:06 -0600
+Content-Type: message/rfc822; boundary="0003MSG"
+
+3. message summary
+
+--0003MSG
+Content-Type: text/plain
+
+3.1 message text
+
+--0003MSG
+Content-Type: application/octet-stream
+Content-Disposition: attachment; filename="msg_3_signature.sig"
+Content-Transfer-Encoding: base64
+
+3/2/MessageOctestStream/==
+
+--0003MSG--
+
+--0000MAIN
+Content-Type: multipart/mixed; boundary="0004ATTACH"
+
+4 attach summary
+
+--0004ATTACH
+Content-Type: image/gif
+Content-Disposition: attachment; filename="att4.1_gif.sig"
+Content-Transfer-Encoding: base64
+
+4/1/Gif=
+
+--0004ATTACH
+Subject: Inside mail 4.2
+From: Mary Smith <mary@example.net>
+To: John Doe <jdoe@machine.example>
+Date: Fri, 10 Nov 1997 09:55:06 -0600
+Content-Type: message/rfc822; boundary="0042MSG"
+
+4.2 message summary
+
+--0042MSG
+Content-Type: text/plain
+
+4.2.1 message text
+
+--0042MSG
+Content-Type: multipart/alternative; boundary="0422ALTER"
+
+4.2.2 alternative summary
+
+--0422ALTER
+Content-Type: text/plain
+
+4.2.2.1 plain text
+
+--0422ALTER
+Content-Type: text/html
+
+<h1>4.2.2.2 html text</h1>
+
+--0422ALTER--
+
+--0042MSG--
+
+--0004ATTACH--
+
+--0000MAIN--
+
+
+`
+
+var testPaths = []struct {
+	path                          []int
+	expectedSection, expectedBody string
+}{
+	{[]int{},
+		sampleMail,
+		`main summary
+
+--0000MAIN
+Content-Type: text/plain
+
+1. main message
+
+
+--0000MAIN
+Content-Type: application/octet-stream
+Content-Disposition: inline; filename="main_signature.sig"
+Content-Transfer-Encoding: base64
+
+2/MainOctetStream
+
+--0000MAIN
+Subject: Inside mail 3
+From: Mary Smith <mary@example.net>
+To: John Doe <jdoe@machine.example>
+Date: Fri, 20 Nov 1997 09:55:06 -0600
+Content-Type: message/rfc822; boundary="0003MSG"
+
+3. message summary
+
+--0003MSG
+Content-Type: text/plain
+
+3.1 message text
+
+--0003MSG
+Content-Type: application/octet-stream
+Content-Disposition: attachment; filename="msg_3_signature.sig"
+Content-Transfer-Encoding: base64
+
+3/2/MessageOctestStream/==
+
+--0003MSG--
+
+--0000MAIN
+Content-Type: multipart/mixed; boundary="0004ATTACH"
+
+4 attach summary
+
+--0004ATTACH
+Content-Type: image/gif
+Content-Disposition: attachment; filename="att4.1_gif.sig"
+Content-Transfer-Encoding: base64
+
+4/1/Gif=
+
+--0004ATTACH
+Subject: Inside mail 4.2
+From: Mary Smith <mary@example.net>
+To: John Doe <jdoe@machine.example>
+Date: Fri, 10 Nov 1997 09:55:06 -0600
+Content-Type: message/rfc822; boundary="0042MSG"
+
+4.2 message summary
+
+--0042MSG
+Content-Type: text/plain
+
+4.2.1 message text
+
+--0042MSG
+Content-Type: multipart/alternative; boundary="0422ALTER"
+
+4.2.2 alternative summary
+
+--0422ALTER
+Content-Type: text/plain
+
+4.2.2.1 plain text
+
+--0422ALTER
+Content-Type: text/html
+
+<h1>4.2.2.2 html text</h1>
+
+--0422ALTER--
+
+--0042MSG--
+
+--0004ATTACH--
+
+--0000MAIN--
+
+
+`,
+	},
+
+	{[]int{1},
+		`Content-Type: text/plain
+
+1. main message
+
+
+`,
+		`1. main message
+
+
+`,
+	},
+	{[]int{3},
+		`Subject: Inside mail 3
+From: Mary Smith <mary@example.net>
+To: John Doe <jdoe@machine.example>
+Date: Fri, 20 Nov 1997 09:55:06 -0600
+Content-Type: message/rfc822; boundary="0003MSG"
+
+3. message summary
+
+--0003MSG
+Content-Type: text/plain
+
+3.1 message text
+
+--0003MSG
+Content-Type: application/octet-stream
+Content-Disposition: attachment; filename="msg_3_signature.sig"
+Content-Transfer-Encoding: base64
+
+3/2/MessageOctestStream/==
+
+--0003MSG--
+
+`,
+		`3. message summary
+
+--0003MSG
+Content-Type: text/plain
+
+3.1 message text
+
+--0003MSG
+Content-Type: application/octet-stream
+Content-Disposition: attachment; filename="msg_3_signature.sig"
+Content-Transfer-Encoding: base64
+
+3/2/MessageOctestStream/==
+
+--0003MSG--
+
+`,
+	},
+	{[]int{3, 1},
+		`Content-Type: text/plain
+
+3.1 message text
+
+`,
+		`3.1 message text
+
+`,
+	},
+	{[]int{3, 2},
+		`Content-Type: application/octet-stream
+Content-Disposition: attachment; filename="msg_3_signature.sig"
+Content-Transfer-Encoding: base64
+
+3/2/MessageOctestStream/==
+
+`,
+		`3/2/MessageOctestStream/==
+
+`,
+	},
+	{[]int{4, 2, 2, 1},
+		`Content-Type: text/plain
+
+4.2.2.1 plain text
+
+`,
+		`4.2.2.1 plain text
+
+`,
+	},
+	{[]int{4, 2, 2, 2},
+		`Content-Type: text/html
+
+<h1>4.2.2.2 html text</h1>
+
+`,
+		`<h1>4.2.2.2 html text</h1>
+
+`,
+	},
+}

pkg/mime/Changelog.md

@@ -0,0 +1,24 @@
+# Do not modify this file!
+It is here for historical reasons only. All changes should be documented in the
+Changelog at the root of this repository.
+
+
+# Changelog
+
+## [2019-12-10] v1.0.2
+
+### Added
+* support for shift_JIS (cp932) encoding
+
+## [2019-09-30] v1.0.1
+
+### Changed
+* fix divide by zero
+
+## [2019-09-26] v1.0.0
+
+### Changed
+* Import-Export#192: filter header parameters
+    * ignore twice the same parameter (take the latest)
+    * convert non utf8 RFC2231 parameters to a single line utf8 RFC2231
+

pkg/mime/encoding.go

@@ -0,0 +1,254 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmmime
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"mime"
+	"mime/quotedprintable"
+	"regexp"
+	"strings"
+	"unicode/utf8"
+
+	"encoding/base64"
+
+	"golang.org/x/text/encoding"
+	"golang.org/x/text/encoding/htmlindex"
+	"golang.org/x/text/transform"
+)
+
+var wordDec = &mime.WordDecoder{
+	CharsetReader: func(charset string, input io.Reader) (io.Reader, error) {
+		dec, err := selectDecoder(charset)
+		if err != nil {
+			return nil, err
+		}
+		if dec == nil { // utf-8
+			return input, nil
+		}
+		return dec.Reader(input), nil
+	},
+}
+
+// Expects trimmed lowercase.
+func getEncoding(charset string) (enc encoding.Encoding, err error) {
+	preparsed := strings.Trim(strings.ToLower(charset), " \t\r\n")
+
+	// koi
+	re := regexp.MustCompile("(cs)?koi[-_ ]?8?[-_ ]?(r|ru|u|uk)?$")
+	matches := re.FindAllStringSubmatch(preparsed, -1)
+	if len(matches) == 1 && len(matches[0]) == 3 {
+		preparsed = "koi8-"
+		switch matches[0][2] {
+		case "u", "uk":
+			preparsed += "u"
+		default:
+			preparsed += "r"
+		}
+	}
+
+	// windows-XXXX
+	re = regexp.MustCompile("(cp|(cs)?win(dows)?)[-_ ]?([0-9]{3,4})$")
+	matches = re.FindAllStringSubmatch(preparsed, -1)
+	if len(matches) == 1 && len(matches[0]) == 5 {
+		switch matches[0][4] {
+		case "874", "1250", "1251", "1252", "1253", "1254", "1255", "1256", "1257", "1258":
+			preparsed = "windows-" + matches[0][4]
+		}
+	}
+
+	// iso
+	re = regexp.MustCompile("iso[-_ ]?([0-9]{4})[-_ ]?([0-9]+|jp)?[-_ ]?(i|e)?")
+	matches = re.FindAllStringSubmatch(preparsed, -1)
+	if len(matches) == 1 && len(matches[0]) == 4 {
+		if matches[0][1] == "2022" && matches[0][2] == "jp" {
+			preparsed = "iso-2022-jp"
+		}
+		if matches[0][1] == "8859" {
+			switch matches[0][2] {
+			case "1", "2", "3", "4", "5", "7", "8", "9", "10", "11", "13", "14", "15", "16":
+				preparsed = "iso-8859-" + matches[0][2]
+				if matches[0][3] == "i" {
+					preparsed += "-" + matches[0][3]
+				}
+			case "":
+				preparsed = "iso-8859-1"
+			}
+		}
+	}
+
+	// Latin is tricky.
+	re = regexp.MustCompile("^(cs|csiso)?l(atin)?[-_ ]?([0-9]{1,2})$")
+	matches = re.FindAllStringSubmatch(preparsed, -1)
+	if len(matches) == 1 && len(matches[0]) == 4 {
+		switch matches[0][3] {
+		case "1":
+			preparsed = "windows-1252"
+		case "2", "3", "4", "5":
+			preparsed = "iso-8859-" + matches[0][3]
+		case "6":
+			preparsed = "iso-8859-10"
+		case "8":
+			preparsed = "iso-8859-14"
+		case "9":
+			preparsed = "iso-8859-15"
+		case "10":
+			preparsed = "iso-8859-16"
+		}
+	}
+
+	// Missing substitutions.
+	switch preparsed {
+	case "csutf8", "iso-utf-8", "utf8mb4":
+		preparsed = "utf-8"
+
+	case "cp932", "windows-932", "windows-31J", "ibm-943", "cp943":
+		preparsed = "shift_jis"
+	case "eucjp", "ibm-eucjp":
+		preparsed = "euc-jp"
+	case "euckr", "ibm-euckr", "cp949":
+		preparsed = "euc-kr"
+	case "euccn", "ibm-euccn":
+		preparsed = "gbk"
+	case "zht16mswin950", "cp950":
+		preparsed = "big5"
+
+	case "csascii",
+		"ansi_x3.4-1968",
+		"ansi_x3.4-1986",
+		"ansi_x3.110-1983",
+		"cp850",
+		"cp858",
+		"us",
+		"iso646",
+		"iso-646",
+		"iso646-us",
+		"iso_646.irv:1991",
+		"cp367",
+		"ibm367",
+		"ibm-367",
+		"iso-ir-6":
+		preparsed = "ascii"
+
+	case "ibm852":
+		preparsed = "iso-8859-2"
+	case "iso-ir-199", "iso-celtic":
+		preparsed = "iso-8859-14"
+	case "iso-ir-226":
+		preparsed = "iso-8859-16"
+
+	case "macroman":
+		preparsed = "macintosh"
+	}
+
+	enc, _ = htmlindex.Get(preparsed)
+	if enc == nil {
+		err = fmt.Errorf("can not get encodig for '%s' (or '%s')", charset, preparsed)
+	}
+	return
+}
+
+func selectDecoder(charset string) (decoder *encoding.Decoder, err error) {
+	var enc encoding.Encoding
+	lcharset := strings.Trim(strings.ToLower(charset), " \t\r\n")
+	switch lcharset {
+	case "utf7", "utf-7", "unicode-1-1-utf-7":
+		return NewUtf7Decoder(), nil
+	default:
+		enc, err = getEncoding(lcharset)
+	}
+	if err == nil {
+		decoder = enc.NewDecoder()
+	}
+	return
+}
+
+// DecodeHeader if needed. Returns error if raw contains non-utf8 characters.
+func DecodeHeader(raw string) (decoded string, err error) {
+	if decoded, err = wordDec.DecodeHeader(raw); err != nil {
+		decoded = raw
+	}
+	if !utf8.ValidString(decoded) {
+		err = fmt.Errorf("header contains non utf8 chars: %v", err)
+	}
+	return
+}
+
+// EncodeHeader using quoted printable and utf8
+func EncodeHeader(s string) string {
+	return mime.QEncoding.Encode("utf-8", s)
+}
+
+// DecodeCharset decodes the orginal using content type parameters.
+// When charset is missing it checks thaht the content is valid utf8.
+func DecodeCharset(original []byte, contentTypeParams map[string]string) ([]byte, error) {
+	var decoder *encoding.Decoder
+	var err error
+	if charset, ok := contentTypeParams["charset"]; ok {
+		decoder, err = selectDecoder(charset)
+	} else {
+		if utf8.Valid(original) {
+			return original, nil
+		}
+		err = fmt.Errorf("non-utf8 content without charset specification")
+	}
+
+	if err != nil {
+		return original, err
+	}
+
+	utf8 := make([]byte, len(original))
+	nDst, nSrc, err := decoder.Transform(utf8, original, false)
+	for err == transform.ErrShortDst {
+		if nDst < 1 {
+			nDst = 1
+		}
+		if nSrc < 1 {
+			nSrc = 1
+		}
+		utf8 = make([]byte, (nDst/nSrc+1)*len(original))
+		nDst, nSrc, err = decoder.Transform(utf8, original, false)
+	}
+	if err != nil {
+		return original, err
+	}
+	utf8 = bytes.Trim(utf8, "\x00")
+
+	return utf8, nil
+}
+
+// DecodeContentEncoding wraps the reader with decoder based on content encoding.
+func DecodeContentEncoding(r io.Reader, contentEncoding string) (d io.Reader) {
+	switch strings.ToLower(contentEncoding) {
+	case "quoted-printable":
+		d = quotedprintable.NewReader(r)
+	case "base64":
+		d = base64.NewDecoder(base64.StdEncoding, r)
+	case "7bit", "8bit", "binary", "": // Nothing to do
+		d = r
+	}
+	return
+}
+
+// ParseMediaType from MIME doesn't support RFC2231 for non asci / utf8 encodings so we have to pre-parse it.
+func ParseMediaType(v string) (mediatype string, params map[string]string, err error) {
+	v, _ = changeEncodingAndKeepLastParamDefinition(v)
+	return mime.ParseMediaType(v)
+}

pkg/mime/encoding_test.go

@@ -0,0 +1,445 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmmime
+
+import (
+	"bytes"
+	//"fmt"
+	"strings"
+	"testing"
+
+	"golang.org/x/text/encoding/htmlindex"
+
+	a "github.com/stretchr/testify/assert"
+)
+
+func TestDecodeHeader(t *testing.T) {
+	testData := []struct{ raw, expected string }{
+		{
+			"",
+			"",
+		},
+		{
+			"=?iso-2022-jp?Q?=1B$B!Z=1B(BTimes_Car_PLUS=1B$B![JV5Q>Z=1B(B?=",
+			"【Times Car PLUS】返却証",
+		},
+		{
+			`=?iso-2022-jp?Q?iTunes_Movie_=1B$B%K%e!<%j%j!<%9$HCmL\:nIJ=1B(B?=`,
+			"iTunes Movie ニューリリースと注目作品",
+		},
+		{
+			"=?UTF-8?B?w4TDi8OPw5bDnA==?= =?UTF-8?B?IMOkw6vDr8O2w7w=?=",
+			"ÄËÏÖÜ äëïöü",
+		},
+		{
+			"=?ISO-8859-2?B?xMtJ1tw=?= =?ISO-8859-2?B?IOTrafb8?=",
+			"ÄËIÖÜ äëiöü",
+		},
+		{
+			"=?uknown?B?xMtJ1tw=?= =?ISO-8859-2?B?IOTrafb8?=",
+			"=?uknown?B?xMtJ1tw=?= =?ISO-8859-2?B?IOTrafb8?=",
+		},
+	}
+
+	for _, val := range testData {
+		if decoded, err := DecodeHeader(val.raw); strings.Compare(val.expected, decoded) != 0 {
+			t.Errorf("Incorrect decoding of header %q expected %q but have %q; Error %v", val.raw, val.expected, decoded, err)
+		} else {
+			// fmt.Println("Header", val.raw, "successfully decoded", decoded, ". Error", err)
+		}
+	}
+}
+
+type testParseMediaTypeData struct {
+	arg, wantMediaType string
+	wantParams         map[string]string
+}
+
+func (d *testParseMediaTypeData) run(t *testing.T) {
+	gotMediaType, params, err := ParseMediaType(d.arg)
+	a.Nil(t, err)
+	a.Equal(t, d.wantMediaType, gotMediaType)
+	a.Equal(t, d.wantParams, params)
+}
+
+func TestParseMediaType(t *testing.T) {
+	testTable := map[string]testParseMediaTypeData{
+		"TwiceTheSameParameter": {
+			arg:           "attachment; filename=joy.txt; filename=JOY.TXT; title=hi;",
+			wantMediaType: "attachment",
+			wantParams:    map[string]string{"filename": "JOY.TXT", "title": "hi"},
+		},
+		"SingleLineUTF8": {
+			arg:           "attachment;\nfilename*=utf-8''%F0%9F%98%81%F0%9F%98%82.txt;\n title=smile",
+			wantMediaType: "attachment",
+			wantParams:    map[string]string{"filename": "😁😂.txt", "title": "smile"},
+		},
+		"MultiLineUTF8": {
+			arg:           "attachment;\nfilename*0*=utf-8''%F0%9F%98%81;   title=smile;\nfilename*1*=%F0%9F%98%82;\nfilename*2=.txt",
+			wantMediaType: "attachment",
+			wantParams:    map[string]string{"filename": "😁😂.txt", "title": "smile"},
+		},
+		"MultiLineFirstNoEncNextUTF8": {
+			arg:           "attachment;\nfilename*0*=utf-8''joy  ;\n title*=utf-8''smile;  \nfilename*1*=%F0%9F%98%82;\nfilename*2=.txt",
+			wantMediaType: "attachment",
+			wantParams:    map[string]string{"filename": "joy😂.txt", "title": "smile"},
+		},
+		"SingleLineBig5": {
+			arg:           "attachment;\nfilename*=big5''%B3%C6%A7%D1%BF%FD.m4a; title*=utf8''memorandum",
+			wantMediaType: "attachment",
+			wantParams:    map[string]string{"filename": "備忘錄.m4a", "title": "memorandum"},
+		},
+		"MultiLineBig5": {
+			arg:           "attachment;\nfilename*0*=big5''%B3%C6a; title*0=utf8''memorandum; filename*2=%BF%FD.m4a; \nfilename*1*=%A7%D1b;",
+			wantMediaType: "attachment",
+			wantParams:    map[string]string{"filename": "備a忘b錄.m4a", "title": "memorandum"},
+		},
+	}
+	for name, testData := range testTable {
+		t.Run(name, testData.run)
+	}
+}
+
+func TestGetEncoding(t *testing.T) {
+	// All MIME charsets with aliases can be found here:
+	// https://www.iana.org/assignments/character-sets/character-sets.xhtml
+	mimesets := map[string][]string{
+		"utf-8": []string{ // MIB 16
+			"utf8",
+			"csutf8",
+			"unicode-1-1-utf-8",
+			"iso-utf-8",
+			"utf8mb4",
+		},
+		"gbk": []string{
+			"gb2312", // MIB 2025
+			//"euc-cn": []string{
+			"euccn",
+			"ibm-euccn",
+		},
+		//"utf7": []string{"utf-7", "unicode-1-1-utf-7"},
+		"iso-8859-2": []string{ // MIB 5
+			"iso-ir-101",
+			"iso_8859-2",
+			"iso8859-2",
+			"latin2",
+			"l2",
+			"csisolatin2",
+			"ibm852",
+			//"FAILEDibm852",
+		},
+		"iso-8859-3": []string{ // MIB 6
+			"iso-ir-109",
+			"iso_8859-3",
+			"latin3",
+			"l3",
+			"csisolatin3",
+		},
+		"iso-8859-4": []string{ // MIB 7
+			"iso-ir-110",
+			"iso_8859-4",
+			"latin4",
+			"l4",
+			"csisolatin4",
+		},
+		"iso-8859-5": []string{ // MIB 8
+			"iso-ir-144",
+			"iso_8859-5",
+			"cyrillic",
+			"csisolatincyrillic",
+		},
+		"iso-8859-6": []string{ // MIB 9
+			"iso-ir-127",
+			"iso_8859-6",
+			"ecma-114",
+			"asmo-708",
+			"arabic",
+			"csisolatinarabic",
+			//"iso-8859-6e": []string{ // MIB 81 just direction
+			"csiso88596e",
+			"iso-8859-6-e",
+			//"iso-8859-6i": []string{ // MIB 82
+			"csiso88596i",
+			"iso-8859-6-i"},
+		"iso-8859-7": []string{ // MIB 10
+			"iso-ir-126",
+			"iso_8859-7",
+			"elot_928",
+			"ecma-118",
+			"greek",
+			"greek8",
+			"csisolatingreek"},
+		"iso-8859-8": []string{ // MIB 11
+			"iso-ir-138",
+			"iso_8859-8",
+			"hebrew",
+			"csisolatinhebrew",
+			//"iso-8859-8e": []string{ // MIB 84 (directionality
+			"csiso88598e",
+			"iso-8859-8-e",
+		},
+		"iso-8859-8-i": []string{ // MIB 85
+			"logical",
+			"csiso88598i",
+			"iso-8859-8-i", // Hebrew, the "i" means right-to-left, probably unnecessary with ISO cleaning above.
+		},
+		"iso-8859-10": []string{ // MIB 13
+			"iso-ir-157",
+			"l6",
+			"iso_8859-10:1992",
+			"csisolatin6",
+			"latin6"},
+		"iso-8859-13": []string{ // MIB 109
+			"csiso885913"},
+		"iso-8859-14": []string{ // MIB 110
+			"iso-ir-199",
+			"iso_8859-14:1998",
+			"iso_8859-14",
+			"latin8",
+			"iso-celtic",
+			"l8",
+			"csiso885914"},
+		"iso-8859-15": []string{ // MIB 111
+			"iso_8859-15",
+			"latin-9",
+			"csiso885915",
+			"ISO8859-15"},
+		"iso-8859-16": []string{ // MIB 112
+			"iso-ir-226",
+			"iso_8859-16:2001",
+			"iso_8859-16",
+			"latin10",
+			"l10",
+			"csiso885916",
+		},
+		"windows-874": []string{ // MIB 2109
+			"cswindows874",
+			"cp874",
+			"iso-8859-11",
+			"tis-620",
+		},
+		"windows-1250": []string{ // MIB 2250
+			"cswindows1250",
+			"cp1250",
+		},
+		"windows-1251": []string{ // MIB 2251
+			"cswindows1251",
+			"cp1251",
+		},
+		"windows-1252": []string{ // MIB 2252
+			"cswindows1252",
+			"cp1252",
+			"3dwindows-1252",
+			"we8mswin1252",
+			"us-ascii",         // MIB 3
+			"ansi_x3.110-1983", // MIB 74 // usascii
+			//"iso-8859-1": []string{ // MIB 4 succeed by win1252
+			"iso8859-1",
+			"iso-ir-100",
+			"iso_8859-1",
+			"latin1",
+			"l1",
+			"ibm819",
+			"cp819",
+			"csisolatin1",
+			"ansi_x3.4-1968",
+			"ansi_x3.4-1986",
+			"cp850",
+			"cp858", // "cp850"  Mostly correct except for the Euro sign.
+			"iso_646.irv:1991",
+			"iso646-us",
+			"us",
+			"ibm367",
+			"cp367",
+			"csascii",
+			"ascii",
+			"iso-ir-6",
+			"we8iso8859p1",
+		},
+		"windows-1253": []string{"cswindows1253", "cp1253"},        // MIB 2253
+		"windows-1254": []string{"cswindows1254", "cp1254"},        // MIB 2254
+		"windows-1255": []string{"cSwindows1255", "cp1255"},        // MIB 2255
+		"windows-1256": []string{"cswIndows1256", "cp1256"},        // MIB 2256
+		"windows-1257": []string{"cswinDows1257", "cp1257"},        // MIB 2257
+		"windows-1258": []string{"cswindoWs1258", "cp1258"},        // MIB 2257
+		"koi8-r":       []string{"cskoi8r", "koi8r"},               // MIB 2084
+		"koi8-u":       []string{"cskoi8u", "koi8u"},               // MIB 2088
+		"macintosh":    []string{"mac", "macroman", "csmacintosh"}, // MIB 2027
+		"big5": []string{
+			"zht16mswin950", // cp950
+			"cp950",
+		},
+		"euc-kr": []string{
+			"euckr", // MIB 38
+			"ibm-euckr",
+			//"uhc": []string{ // Korea
+			"ks_c_5601-1987",
+			"ksc5601",
+			"cp949",
+		},
+		"euc-jp": []string{
+			"eucjp",
+			"ibm-eucjp",
+		},
+		"shift_jis": []string{
+			"CP932",
+			"MS932",
+			"Windows-932",
+			"Windows-31J",
+			"MS_Kanji",
+			"IBM-943",
+			"CP943",
+		},
+		"iso-2022-jp": []string{ // MIB 39
+			"iso2022jp",
+			"csiso2022jp",
+		},
+	}
+
+	for expected, names := range mimesets {
+		expenc, _ := htmlindex.Get(expected)
+		if canonical, err := htmlindex.Name(expenc); canonical != expected || err != nil {
+			t.Fatalf("Error while get canonical name. Expected '%v' but have %v `%#v`: %v", expected, canonical, expenc, err)
+		}
+		for _, name := range names {
+			enc, err := getEncoding(name)
+			if err != nil || enc == nil {
+				t.Errorf("Error while getting encoding for %v returned: '%#v' and error: '%v'", name, enc, err)
+			}
+			if expenc != enc {
+				t.Errorf("For %v expected %v '%v' but have '%v'", name, expected, expenc, enc)
+			}
+		}
+	}
+}
+
+// sample text for UTF8 http://www.columbia.edu/~fdc/utf8/index.html
+func TestEncodeReader(t *testing.T) {
+	// define test data
+	testData := []struct {
+		params   map[string]string
+		original []byte
+		message  string
+	}{
+		// russian
+		{
+			map[string]string{"charset": "koi8-r"},
+			//     а, з, б, у, к, а, а, б, в, г, д, е, ё
+			[]byte{0xC1, 0xDA, 0xC2, 0xD5, 0xCB, 0xC1, 0xC1, 0xC2, 0xD7, 0xC7, 0xC4, 0xC5, 0xA3},
+			"азбукаабвгдеё",
+		},
+		{
+			map[string]string{"charset": "KOI8-R"},
+			[]byte{0xC1, 0xDA, 0xC2, 0xD5, 0xCB, 0xC1, 0xC1, 0xC2, 0xD7, 0xC7, 0xC4, 0xC5, 0xA3},
+			"азбукаабвгдеё",
+		},
+		{
+			map[string]string{"charset": "csKOI8R"},
+			[]byte{0xC1, 0xDA, 0xC2, 0xD5, 0xCB, 0xC1, 0xC1, 0xC2, 0xD7, 0xC7, 0xC4, 0xC5, 0xA3},
+			"азбукаабвгдеё",
+		},
+		{
+			map[string]string{"charset": "koi8-u"},
+			[]byte{0xC1, 0xDA, 0xC2, 0xD5, 0xCB, 0xC1, 0xC1, 0xC2, 0xD7, 0xC7, 0xC4, 0xC5, 0xA3},
+			"азбукаабвгдеё",
+		},
+		{
+			map[string]string{"charset": "iso-8859-5"},
+			//     а    , з    , б    , у    , к    , а    , а    , б    , в    , г    , д    , е    , ё
+			[]byte{0xD0, 0xD7, 0xD1, 0xE3, 0xDA, 0xD0, 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xF1},
+			"азбукаабвгдеё",
+		},
+		{
+			map[string]string{"charset": "csWrong"},
+			[]byte{0xD0, 0xD7, 0xD1, 0xE3, 0xDA, 0xD0, 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6},
+			"",
+		},
+		{
+			map[string]string{"charset": "utf8"},
+			[]byte{0xD0, 0xB0, 0xD0, 0xB7, 0xD0, 0xB1, 0xD1, 0x83, 0xD0, 0xBA, 0xD0, 0xB0, 0xD0, 0xB0, 0xD0, 0xB1, 0xD0, 0xB2, 0xD0, 0xB3, 0xD0, 0xB4, 0xD0, 0xB5, 0xD1, 0x91},
+			"азбукаабвгдеё",
+		},
+		// czechoslovakia
+		{
+			map[string]string{"charset": "windows-1250"},
+			[]byte{225, 228, 232, 233, 236, 244},
+			"áäčéěô",
+		},
+		// umlauts
+		{
+			map[string]string{"charset": "iso-8859-1"},
+			[]byte{196, 203, 214, 220, 228, 235, 246, 252},
+			"ÄËÖÜäëöü",
+		},
+		// latvia
+		{
+			map[string]string{"charset": "iso-8859-4"},
+			[]byte{224, 239, 243, 182, 254},
+			"āīķļū",
+		},
+		{ // encoded by https://www.motobit.com/util/charset-codepage-conversion.asp
+			map[string]string{"charset": "utf7"},
+			[]byte("He wes Leovena+APA-es sone -- li+APA-e him be Drihten.+A6QDtw- +A7MDuwPOA8MDwwOx- +A7wDvwPF- +A60DtAPJA8MDsQO9- +A7UDuwO7A7cDvQO5A7oDrg-. +BCcENQRABD0ENQQ7BDg- +BDgENwQxBEs- +BDcENAQ1BEEETA- +BDg- +BEIEMAQ8-,+BCcENQRABD0ENQQ7BDg- +BDgENwQxBEs- +BDcENAQ1BEEETA- +BDg- +BEIEMAQ8-,+C68LvguuC7ELvwuoC80LpA- +C64Lygu0C78LlQuzC78LsgvH- +C6QLrgu/C7QLzQuuC8oLtAu/- +C6oLywuyC80- +C4cLqQu/C6QLvgu1C6QLwQ- +C44LmQvNC5ULwQuuC80- +C5ULvgujC8sLrgvN-."),
+			"He wes Leovenaðes sone -- liðe him be Drihten.Τη γλώσσα μου έδωσαν ελληνική. Чернели избы здесь и там,Чернели избы здесь и там,யாமறிந்த மொழிகளிலே தமிழ்மொழி போல் இனிதாவது எங்கும் காணோம்.",
+		},
+
+		// iconv -f UTF8 -t GB2312 utf8.txt | hexdump -v -e '"0x" 1/1 "%x, "'
+		{ // encoded by iconv; dump by `cat gb2312.txt | hexdump -v -e '"0x" 1/1 "%x "'` and reformat; text from https://zh.wikipedia.org/wiki/GB_2312
+			map[string]string{"charset": "GB2312"},
+			[]byte{0x47, 0x42, 0x20, 0x32, 0x33, 0x31, 0x32, 0xb5, 0xc4, 0xb3, 0xf6, 0xcf, 0xd6, 0xa3, 0xac, 0xbb, 0xf9, 0xb1, 0xbe, 0xc2, 0xfa, 0xd7, 0xe3, 0xc1, 0xcb, 0xba, 0xba, 0xd7, 0xd6, 0xb5, 0xc4, 0xbc, 0xc6, 0xcb, 0xe3, 0xbb, 0xfa, 0xb4, 0xa6, 0xc0, 0xed, 0xd0, 0xe8, 0xd2, 0xaa, 0xa3, 0xac, 0xcb, 0xfc, 0xcb, 0xf9, 0xca, 0xd5, 0xc2, 0xbc, 0xb5, 0xc4, 0xba, 0xba, 0xd7, 0xd6, 0xd2, 0xd1, 0xbe, 0xad, 0xb8, 0xb2, 0xb8, 0xc7, 0xd6, 0xd0, 0xb9, 0xfa, 0xb4, 0xf3, 0xc2, 0xbd, 0x39, 0x39, 0x2e, 0x37, 0x35, 0x25, 0xb5, 0xc4, 0xca, 0xb9, 0xd3, 0xc3, 0xc6, 0xb5, 0xc2, 0xca, 0xa1, 0xa3, 0xb5, 0xab, 0xb6, 0xd4, 0xd3, 0xda, 0xc8, 0xcb, 0xc3, 0xfb},
+			"GB 2312的出现，基本满足了汉字的计算机处理需要，它所收录的汉字已经覆盖中国大陆99.75%的使用频率。但对于人名",
+		},
+
+		{ // encoded by iconv; text from https://jp.wikipedia.org/wiki/Shift_JIS
+			map[string]string{"charset": "shift-jis"},
+			[]byte{0x95, 0xb6, 0x8e, 0x9a, 0x95, 0x84, 0x8d, 0x86, 0x89, 0xbb, 0x95, 0xfb, 0x8e, 0xae, 0x53, 0x68, 0x69, 0x66, 0x74, 0x5f, 0x4a, 0x49, 0x53, 0x82, 0xcc, 0x90, 0xdd, 0x8c, 0x76, 0x8e, 0xd2, 0x82, 0xe7, 0x82, 0xcd, 0x81, 0x41, 0x90, 0xe6, 0x8d, 0x73, 0x82, 0xb5, 0x82, 0xc4, 0x82, 0xe6, 0x82, 0xad, 0x97, 0x98, 0x97, 0x70, 0x82, 0xb3, 0x82, 0xea, 0x82, 0xc4, 0x82, 0xa2, 0x82, 0xbd, 0x4a, 0x49, 0x53, 0x20, 0x43, 0x20, 0x36, 0x32, 0x32, 0x30, 0x81, 0x69, 0x8c, 0xbb, 0x8d, 0xdd, 0x82, 0xcc, 0x4a, 0x49, 0x53, 0x20, 0x58, 0x20, 0x30, 0x32, 0x30, 0x31, 0x81, 0x6a, 0x82, 0xcc, 0x38, 0x83, 0x72, 0x83, 0x62, 0x83, 0x67, 0x95, 0x84, 0x8d, 0x86, 0x81, 0x69, 0x88, 0xc8, 0x89, 0xba, 0x81, 0x75, 0x89, 0x70, 0x90, 0x94, 0x8e, 0x9a, 0x81, 0x45, 0x94, 0xbc, 0x8a, 0x70, 0x83, 0x4a, 0x83, 0x69, 0x81, 0x76, 0x81, 0x6a, 0x82, 0xc6, 0x81, 0x41, 0x4a, 0x49, 0x53, 0x20, 0x43, 0x20, 0x36, 0x32, 0x32, 0x36, 0x81, 0x69, 0x8c, 0xbb, 0x8d, 0xdd, 0x82, 0xcc, 0x4a, 0x49, 0x53, 0x20, 0x58, 0x20, 0x30, 0x32, 0x30, 0x38, 0x81, 0x41, 0x88, 0xc8, 0x89, 0xba, 0x81, 0x75, 0x8a, 0xbf, 0x8e, 0x9a, 0x81, 0x76, 0x81, 0x6a, 0x82, 0xcc, 0x97, 0xbc, 0x95, 0xb6, 0x8e, 0x9a, 0x8f, 0x57, 0x8d, 0x87, 0x82, 0xf0, 0x95, 0x5c, 0x8c, 0xbb, 0x82, 0xb5, 0x82, 0xe6, 0x82, 0xa4, 0x82, 0xc6, 0x82, 0xb5, 0x82, 0xbd, 0x81, 0x42, 0x82, 0xdc, 0x82, 0xbd, 0x81, 0x41, 0x83, 0x74, 0x83, 0x40, 0x83, 0x43, 0x83, 0x8b, 0x82, 0xcc, 0x91, 0xe5, 0x82, 0xab, 0x82, 0xb3, 0x82, 0xe2, 0x8f, 0x88, 0x97, 0x9d, 0x8e, 0x9e, 0x8a, 0xd4, 0x82, 0xcc, 0x92, 0x5a, 0x8f, 0x6b, 0x82, 0xf0, 0x90, 0x7d, 0x82, 0xe9, 0x82, 0xbd, 0x82, 0xdf, 0x81, 0x41, 0x83, 0x47, 0x83, 0x58, 0x83, 0x50, 0x81, 0x5b, 0x83, 0x76, 0x83, 0x56, 0x81, 0x5b, 0x83, 0x50, 0x83, 0x93, 0x83, 0x58, 0x82, 0xc8, 0x82, 0xb5, 0x82, 0xc5, 0x8d, 0xac, 0x8d, 0xdd, 0x89, 0xc2, 0x94, 0x5c, 0x82, 0xc9, 0x82, 0xb7, 0x82, 0xe9, 0x82, 0xb1, 0x82, 0xc6, 0x82, 0xf0, 0x8a, 0xe9, 0x90, 0x7d, 0x82, 0xb5, 0x82, 0xbd, 0x81, 0x42},
+			"文字符号化方式Shift_JISの設計者らは、先行してよく利用されていたJIS C 6220（現在のJIS X 0201）の8ビット符号（以下「英数字・半角カナ」）と、JIS C 6226（現在のJIS X 0208、以下「漢字」）の両文字集合を表現しようとした。また、ファイルの大きさや処理時間の短縮を図るため、エスケープシーケンスなしで混在可能にすることを企図した。",
+		},
+
+		// add more from mutations of https://en.wikipedia.org/wiki/World_Wide_Web
+
+	}
+
+	// run tests
+	for _, val := range testData {
+		//fmt.Println("Testing ", val)
+		expected := []byte(val.message)
+		decoded, err := DecodeCharset(val.original, val.params)
+		if len(expected) == 0 {
+			if err == nil {
+				t.Error("Expected err but have ", err)
+			} else {
+				//fmt.Println("Expected err: ", err)
+				continue
+			}
+		} else {
+			if err != nil {
+				t.Error("Expected ok but have ", err)
+			}
+		}
+
+		if bytes.Equal(decoded, expected) {
+			// fmt.Println("Succesfull decoding of ", val.params, ":", string(decoded))
+		} else {
+			t.Error("Wrong encoding of ", val.params, ".Expected\n", expected, "\nbut have\n", decoded)
+		}
+		if strings.Compare(val.message, string(decoded)) != 0 {
+			t.Error("Wrong message for ", val.params, ".Expected\n", val.message, "\nbut have\n", string(decoded))
+		}
+	}
+}

pkg/mime/mediaType.go

@@ -0,0 +1,364 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmmime
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"unicode"
+
+	"github.com/sirupsen/logrus"
+)
+
+// changeEncodingAndKeepLastParamDefinition is necessary to modify behaviour
+// provided by the golang standard libraries.
+func changeEncodingAndKeepLastParamDefinition(v string) (out string, err error) {
+	log := logrus.WithField("pkg", "pm-mime")
+
+	out = v // By default don't do anything with that.
+	keepOrig := true
+
+	i := strings.Index(v, ";")
+	if i == -1 {
+		i = len(v)
+	}
+	mediatype := strings.TrimSpace(strings.ToLower(v[0:i]))
+
+	params := map[string]string{}
+	var continuation map[string]map[string]string
+
+	v = v[i:]
+	for len(v) > 0 {
+		v = strings.TrimLeftFunc(v, unicode.IsSpace)
+		if len(v) == 0 {
+			break
+		}
+		key, value, rest := consumeMediaParam(v)
+		if key == "" {
+			break
+		}
+
+		pmap := params
+		if idx := strings.Index(key, "*"); idx != -1 {
+			baseName := key[:idx]
+			if continuation == nil {
+				continuation = make(map[string]map[string]string)
+			}
+			var ok bool
+			if pmap, ok = continuation[baseName]; !ok {
+				continuation[baseName] = make(map[string]string)
+				pmap = continuation[baseName]
+			}
+			if isFirstContinuation(key) {
+				charset, _, err := get2231Charset(value)
+				if err != nil {
+					log.Errorln("Filter params:", err)
+					continue
+				}
+				if charset != "utf-8" && charset != "us-ascii" {
+					keepOrig = false
+				}
+			}
+		}
+		if _, exists := pmap[key]; exists {
+			keepOrig = false
+		}
+		pmap[key] = value
+		v = rest
+	}
+
+	if keepOrig {
+		return
+	}
+
+	if continuation != nil {
+		for paramKey, contMap := range continuation {
+			value, err := mergeContinuations(paramKey, contMap)
+			if err == nil {
+				params[paramKey+"*"] = value
+				continue
+			}
+
+			// Fallback.
+			log.Errorln("Merge param", paramKey, ":", err)
+			for ck, cv := range contMap {
+				params[ck] = cv
+			}
+		}
+	}
+
+	// Merge ;
+	out = mediatype
+	for k, v := range params {
+		out += ";"
+		out += k
+		out += "="
+		out += v
+	}
+
+	return
+}
+
+func isFirstContinuation(key string) bool {
+	if idx := strings.Index(key, "*"); idx != -1 {
+		return key[idx:] == "*" || key[idx:] == "*0*"
+	}
+	return false
+}
+
+// get2231Charset partially from mime/mediatype.go:211 function `decode2231Enc`.
+func get2231Charset(v string) (charset, value string, err error) {
+	sv := strings.SplitN(v, "'", 3)
+	if len(sv) != 3 {
+		err = errors.New("incorrect RFC2231 charset format")
+		return
+	}
+	charset = strings.ToLower(sv[0])
+	value = sv[2]
+	return
+}
+
+func mergeContinuations(paramKey string, contMap map[string]string) (string, error) {
+	var err error
+	var charset, value string
+
+	// Single value.
+	if contValue, ok := contMap[paramKey+"*"]; ok {
+		if charset, value, err = get2231Charset(contValue); err != nil {
+			return "", err
+		}
+	} else {
+		for n := 0; ; n++ {
+			contKey := fmt.Sprintf("%s*%d", paramKey, n)
+			contValue, isLast := contMap[contKey]
+			if !isLast {
+				var ok bool
+				contValue, ok = contMap[contKey+"*"]
+				if !ok {
+					return "", errors.New("not valid RFC2231 continuation")
+				}
+			}
+			if n == 0 {
+				if charset, value, err = get2231Charset(contValue); err != nil || charset == "" {
+					return "", err
+				}
+			} else {
+				value += contValue
+			}
+			if isLast {
+				break
+			}
+		}
+	}
+
+	return convertHexToUTF(charset, value)
+}
+
+// convertHexToUTF converts hex values string with charset to UTF8 in RFC2231 format.
+func convertHexToUTF(charset, value string) (string, error) {
+	raw, err := percentHexUnescape(value)
+	if err != nil {
+		return "", err
+	}
+	utf8, err := DecodeCharset(raw, map[string]string{"charset": charset})
+	return "utf-8''" + percentHexEscape(utf8), err
+}
+
+// consumeMediaParam copy paste mime/mediatype.go:297.
+func consumeMediaParam(v string) (param, value, rest string) {
+	rest = strings.TrimLeftFunc(v, unicode.IsSpace)
+	if !strings.HasPrefix(rest, ";") {
+		return "", "", v
+	}
+
+	rest = rest[1:] // Consume semicolon.
+	rest = strings.TrimLeftFunc(rest, unicode.IsSpace)
+	param, rest = consumeToken(rest)
+	param = strings.ToLower(param)
+	if param == "" {
+		return "", "", v
+	}
+
+	rest = strings.TrimLeftFunc(rest, unicode.IsSpace)
+	if !strings.HasPrefix(rest, "=") {
+		return "", "", v
+	}
+	rest = rest[1:] // Consume equals sign.
+	rest = strings.TrimLeftFunc(rest, unicode.IsSpace)
+	value, rest2 := consumeValue(rest)
+	if value == "" && rest2 == rest {
+		return "", "", v
+	}
+	rest = rest2
+	return param, value, rest
+}
+
+// consumeToken copy paste mime/mediatype.go:238.
+// consumeToken consumes a token from the beginning of the provided string,
+// per RFC 2045 section 5.1 (referenced from 2183), and returns
+// the token consumed and the rest of the string.
+// Returns ("", v) on failure to consume at least one character.
+func consumeToken(v string) (token, rest string) {
+	notPos := strings.IndexFunc(v, isNotTokenChar)
+	if notPos == -1 {
+		return v, ""
+	}
+	if notPos == 0 {
+		return "", v
+	}
+	return v[0:notPos], v[notPos:]
+}
+
+// consumeValue copy paste mime/mediatype.go:253
+// consumeValue consumes a "value" per RFC 2045, where a value is
+// either a 'token' or a 'quoted-string'.  On success, consumeValue
+// returns the value consumed (and de-quoted/escaped, if a
+// quoted-string) and the rest of the string.
+// On failure, returns ("", v).
+func consumeValue(v string) (value, rest string) {
+	if v == "" {
+		return
+	}
+	if v[0] != '"' {
+		return consumeToken(v)
+	}
+
+	// parse a quoted-string
+	buffer := new(strings.Builder)
+	for i := 1; i < len(v); i++ {
+		r := v[i]
+		if r == '"' {
+			return buffer.String(), v[i+1:]
+		}
+		// When MSIE sends a full file path (in "intranet mode"), it does not
+		// escape backslashes: "C:\dev\go\foo.txt", not "C:\\dev\\go\\foo.txt".
+		//
+		// No known MIME generators emit unnecessary backslash escapes
+		// for simple token characters like numbers and letters.
+		//
+		// If we see an unnecessary backslash escape, assume it is from MSIE
+		// and intended as a literal backslash. This makes Go servers deal better
+		// with MSIE without affecting the way they handle conforming MIME
+		// generators.
+		if r == '\\' && i+1 < len(v) && !isTokenChar(rune(v[i+1])) {
+			buffer.WriteByte(v[i+1])
+			i++
+			continue
+		}
+		if r == '\r' || r == '\n' {
+			return "", v
+		}
+		buffer.WriteByte(v[i])
+	}
+	// Did not find end quote.
+	return "", v
+}
+
+// isNotTokenChar copy paste from mime/mediatype.go:234.
+func isNotTokenChar(r rune) bool {
+	return !isTokenChar(r)
+}
+
+// isTokenChar copy paste from mime/grammar.go:19.
+// isTokenChar reports whether rune is in 'token' as defined by RFC 1521 and RFC 2045.
+func isTokenChar(r rune) bool {
+	// token := 1*<any (US-ASCII) CHAR except SPACE, CTLs,
+	//             or tspecials>
+	return r > 0x20 && r < 0x7f && !isTSpecial(r)
+}
+
+// isTSpecial copy paste from mime/grammar.go:13
+// isTSpecial reports whether rune is in 'tspecials' as defined by RFC
+// 1521 and RFC 2045.
+func isTSpecial(r rune) bool {
+	return strings.ContainsRune(`()<>@,;:\"/[]?=`, r)
+}
+
+func percentHexEscape(raw []byte) (out string) {
+	for _, v := range raw {
+		out += fmt.Sprintf("%%%x", v)
+	}
+	return
+}
+
+// percentHexUnescape copy paste from mime/mediatype.go:325.
+func percentHexUnescape(s string) ([]byte, error) {
+	// Count %, check that they're well-formed.
+	percents := 0
+	for i := 0; i < len(s); {
+		if s[i] != '%' {
+			i++
+			continue
+		}
+		percents++
+		if i+2 >= len(s) || !ishex(s[i+1]) || !ishex(s[i+2]) {
+			s = s[i:]
+			if len(s) > 3 {
+				s = s[0:3]
+			}
+			return []byte{}, fmt.Errorf("mime: bogus characters after %%: %q", s)
+		}
+		i += 3
+	}
+	if percents == 0 {
+		return []byte(s), nil
+	}
+
+	t := make([]byte, len(s)-2*percents)
+	j := 0
+	for i := 0; i < len(s); {
+		switch s[i] {
+		case '%':
+			t[j] = unhex(s[i+1])<<4 | unhex(s[i+2])
+			j++
+			i += 3
+		default:
+			t[j] = s[i]
+			j++
+			i++
+		}
+	}
+	return t, nil
+}
+
+// ishex copy paste from mime/mediatype.go:364.
+func ishex(c byte) bool {
+	switch {
+	case '0' <= c && c <= '9':
+		return true
+	case 'a' <= c && c <= 'f':
+		return true
+	case 'A' <= c && c <= 'F':
+		return true
+	}
+	return false
+}
+
+// unhex copy paste from mime/mediatype.go:376.
+func unhex(c byte) byte {
+	switch {
+	case '0' <= c && c <= '9':
+		return c - '0'
+	case 'a' <= c && c <= 'f':
+		return c - 'a' + 10
+	case 'A' <= c && c <= 'F':
+		return c - 'A' + 10
+	}
+	return 0
+}

pkg/mime/parser.go

@@ -0,0 +1,544 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmmime
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"io/ioutil"
+	"mime"
+	"mime/multipart"
+	"net/http"
+	"net/mail"
+	"net/textproto"
+	"regexp"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// VisitAcceptor decides what to do with part which is processed.
+// It is used by MIMEVisitor.
+type VisitAcceptor interface {
+	Accept(partReader io.Reader, header textproto.MIMEHeader, hasPlainSibling bool, isFirst, isLast bool) (err error)
+}
+
+func VisitAll(part io.Reader, h textproto.MIMEHeader, accepter VisitAcceptor) (err error) {
+	mediaType, _, err := getContentType(h)
+	if err != nil {
+		return
+	}
+	return accepter.Accept(part, h, mediaType == "text/plain", true, true)
+}
+
+func IsLeaf(h textproto.MIMEHeader) bool {
+	return !strings.HasPrefix(h.Get("Content-Type"), "multipart/")
+}
+
+// MIMEVisitor is main object to parse (visit) and process (accept) all parts of MIME message.
+type MimeVisitor struct {
+	target VisitAcceptor
+}
+
+// Accept reads part recursively if needed.
+// hasPlainSibling is there when acceptor want to check alternatives.
+func (mv *MimeVisitor) Accept(part io.Reader, h textproto.MIMEHeader, hasPlainSibling bool, isFirst, isLast bool) (err error) {
+	if !isFirst {
+		return
+	}
+
+	parentMediaType, params, err := getContentType(h)
+	if err != nil {
+		return
+	}
+
+	if err = mv.target.Accept(part, h, hasPlainSibling, true, false); err != nil {
+		return
+	}
+
+	if !IsLeaf(h) {
+		var multiparts []io.Reader
+		var multipartHeaders []textproto.MIMEHeader
+		if multiparts, multipartHeaders, err = GetMultipartParts(part, params); err != nil {
+			return
+		}
+		hasPlainChild := false
+		for _, header := range multipartHeaders {
+			mediaType, _, _ := getContentType(header)
+			if mediaType == "text/plain" {
+				hasPlainChild = true
+			}
+		}
+		if hasPlainSibling && parentMediaType == "multipart/related" {
+			hasPlainChild = true
+		}
+
+		for i, p := range multiparts {
+			if err = mv.Accept(p, multipartHeaders[i], hasPlainChild, true, true); err != nil {
+				return
+			}
+			if err = mv.target.Accept(part, h, hasPlainSibling, false, i == (len(multiparts)-1)); err != nil {
+				return
+			}
+		}
+	}
+	return
+}
+
+// NewMIMEVisitor returns a new mime visitor initialised with an acceptor.
+func NewMimeVisitor(targetAccepter VisitAcceptor) *MimeVisitor {
+	return &MimeVisitor{targetAccepter}
+}
+
+func GetRawMimePart(rawdata io.Reader, boundary string) (io.Reader, io.Reader) {
+	b, _ := ioutil.ReadAll(rawdata)
+	tee := bytes.NewReader(b)
+
+	reader := bufio.NewReader(bytes.NewReader(b))
+	byteBoundary := []byte(boundary)
+	bodyBuffer := &bytes.Buffer{}
+	for {
+		line, _, err := reader.ReadLine()
+		if err != nil {
+			return tee, bytes.NewReader(bodyBuffer.Bytes())
+		}
+		if bytes.HasPrefix(line, byteBoundary) {
+			break
+		}
+	}
+	lineEndingLength := 0
+	for {
+		line, isPrefix, err := reader.ReadLine()
+		if err != nil {
+			return tee, bytes.NewReader(bodyBuffer.Bytes())
+		}
+		if bytes.HasPrefix(line, byteBoundary) {
+			break
+		}
+		lineEndingLength = 0
+		bodyBuffer.Write(line)
+		if !isPrefix {
+			reader.UnreadByte()
+			reader.UnreadByte()
+			token, _ := reader.ReadByte()
+			if token == '\r' {
+				lineEndingLength++
+				bodyBuffer.WriteByte(token)
+			}
+			lineEndingLength++
+			bodyBuffer.WriteByte(token)
+		}
+	}
+	ioutil.ReadAll(reader)
+	data := bodyBuffer.Bytes()
+	return tee, bytes.NewReader(data[0 : len(data)-lineEndingLength])
+}
+
+func GetAllChildParts(part io.Reader, h textproto.MIMEHeader) (parts []io.Reader, headers []textproto.MIMEHeader, err error) {
+	mediaType, params, err := getContentType(h)
+	if err != nil {
+		return
+	}
+	if strings.HasPrefix(mediaType, "multipart/") {
+		var multiparts []io.Reader
+		var multipartHeaders []textproto.MIMEHeader
+		if multiparts, multipartHeaders, err = GetMultipartParts(part, params); err != nil {
+			return
+		}
+		if strings.Contains(mediaType, "alternative") {
+			var chosenPart io.Reader
+			var chosenHeader textproto.MIMEHeader
+			if chosenPart, chosenHeader, err = pickAlternativePart(multiparts, multipartHeaders); err != nil {
+				return
+			}
+			var childParts []io.Reader
+			var childHeaders []textproto.MIMEHeader
+			if childParts, childHeaders, err = GetAllChildParts(chosenPart, chosenHeader); err != nil {
+				return
+			}
+			parts = append(parts, childParts...)
+			headers = append(headers, childHeaders...)
+		} else {
+			for i, p := range multiparts {
+				var childParts []io.Reader
+				var childHeaders []textproto.MIMEHeader
+				if childParts, childHeaders, err = GetAllChildParts(p, multipartHeaders[i]); err != nil {
+					return
+				}
+				parts = append(parts, childParts...)
+				headers = append(headers, childHeaders...)
+			}
+		}
+	} else {
+		parts = append(parts, part)
+		headers = append(headers, h)
+	}
+	return
+}
+
+func GetMultipartParts(r io.Reader, params map[string]string) (parts []io.Reader, headers []textproto.MIMEHeader, err error) {
+	mr := multipart.NewReader(r, params["boundary"])
+	parts = []io.Reader{}
+	headers = []textproto.MIMEHeader{}
+	var p *multipart.Part
+	for {
+		p, err = mr.NextPart()
+		if err == io.EOF {
+			err = nil
+			break
+		}
+		if err != nil {
+			return
+		}
+		b, _ := ioutil.ReadAll(p)
+		buffer := bytes.NewBuffer(b)
+
+		parts = append(parts, buffer)
+		headers = append(headers, p.Header)
+	}
+	return
+}
+
+func pickAlternativePart(parts []io.Reader, headers []textproto.MIMEHeader) (part io.Reader, h textproto.MIMEHeader, err error) {
+
+	for i, h := range headers {
+		mediaType, _, err := getContentType(h)
+		if err != nil {
+			continue
+		}
+		if strings.HasPrefix(mediaType, "multipart/") {
+			return parts[i], headers[i], nil
+		}
+	}
+	for i, h := range headers {
+		mediaType, _, err := getContentType(h)
+		if err != nil {
+			continue
+		}
+		if mediaType == "text/html" {
+			return parts[i], headers[i], nil
+		}
+	}
+	for i, h := range headers {
+		mediaType, _, err := getContentType(h)
+		if err != nil {
+			continue
+		}
+		if mediaType == "text/plain" {
+			return parts[i], headers[i], nil
+		}
+	}
+
+	// If we get all the way here, part will be nil.
+	return
+}
+
+// "Parse address comment" as defined in http://tools.wordtothewise.com/rfc/822
+// FIXME: Does not work for address groups.
+// NOTE: This should be removed for go>1.10 (please check).
+func parseAddressComment(raw string) string {
+	parsed := []string{}
+	for _, item := range regexp.MustCompile("[,;]").Split(raw, -1) {
+		re := regexp.MustCompile("[(][^)]*[)]")
+		comments := strings.Join(re.FindAllString(item, -1), " ")
+		comments = strings.Replace(comments, "(", "", -1)
+		comments = strings.Replace(comments, ")", "", -1)
+		withoutComments := re.ReplaceAllString(item, "")
+		addr, err := mail.ParseAddress(withoutComments)
+		if err != nil {
+			continue
+		}
+		if addr.Name == "" {
+			addr.Name = comments
+		}
+		parsed = append(parsed, addr.String())
+	}
+	return strings.Join(parsed, ", ")
+}
+
+func checkHeaders(headers []textproto.MIMEHeader) bool {
+	foundAttachment := false
+
+	for i := 0; i < len(headers); i++ {
+		h := headers[i]
+
+		mediaType, _, _ := getContentType(h)
+
+		if !strings.HasPrefix(mediaType, "text/") {
+			foundAttachment = true
+		} else if foundAttachment {
+			// This means that there is a text part after the first attachment,
+			// so we will have to convert the body from plain->HTML.
+			return true
+		}
+	}
+	return false
+}
+
+func decodePart(partReader io.Reader, header textproto.MIMEHeader) (decodedPart io.Reader) {
+	decodedPart = DecodeContentEncoding(partReader, header.Get("Content-Transfer-Encoding"))
+	if decodedPart == nil {
+		log.Warnf("Unsupported Content-Transfer-Encoding '%v'", header.Get("Content-Transfer-Encoding"))
+		decodedPart = partReader
+	}
+	return
+}
+
+// Assume 'text/plain' if missing.
+func getContentType(header textproto.MIMEHeader) (mediatype string, params map[string]string, err error) {
+	contentType := header.Get("Content-Type")
+	if contentType == "" {
+		contentType = "text/plain"
+	}
+
+	return mime.ParseMediaType(contentType)
+}
+
+// ===================== MIME Printer ===================================
+// Simply print resulting MIME tree into text form.
+// TODO move this to file mime_printer.go.
+
+type stack []string
+
+func (s stack) Push(v string) stack {
+	return append(s, v)
+}
+func (s stack) Pop() (stack, string) {
+	l := len(s)
+	return s[:l-1], s[l-1]
+}
+func (s stack) Peek() string {
+	return s[len(s)-1]
+}
+
+type MIMEPrinter struct {
+	result        *bytes.Buffer
+	boundaryStack stack
+}
+
+func NewMIMEPrinter() (pd *MIMEPrinter) {
+	return &MIMEPrinter{
+		result:        bytes.NewBuffer([]byte("")),
+		boundaryStack: stack{},
+	}
+}
+
+func (pd *MIMEPrinter) Accept(partReader io.Reader, header textproto.MIMEHeader, hasPlainSibling bool, isFirst, isLast bool) (err error) {
+	if isFirst {
+		http.Header(header).Write(pd.result)
+		pd.result.Write([]byte("\n"))
+		if IsLeaf(header) {
+			pd.result.ReadFrom(partReader)
+		} else {
+			_, params, _ := getContentType(header)
+			boundary := params["boundary"]
+			pd.boundaryStack = pd.boundaryStack.Push(boundary)
+			pd.result.Write([]byte("\nThis is a multi-part message in MIME format.\n--" + boundary + "\n"))
+		}
+	} else {
+		if !isLast {
+			pd.result.Write([]byte("\n--" + pd.boundaryStack.Peek() + "\n"))
+		} else {
+			var boundary string
+			pd.boundaryStack, boundary = pd.boundaryStack.Pop()
+			pd.result.Write([]byte("\n--" + boundary + "--\n.\n"))
+		}
+	}
+	return nil
+}
+
+func (pd *MIMEPrinter) String() string {
+	return pd.result.String()
+}
+
+// ======================== PlainText Collector  =========================
+// Collect contents of all non-attachment text/plain parts and return it as a string.
+// TODO move this to file collector_plaintext.go.
+
+type PlainTextCollector struct {
+	target            VisitAcceptor
+	plainTextContents *bytes.Buffer
+}
+
+func NewPlainTextCollector(targetAccepter VisitAcceptor) *PlainTextCollector {
+	return &PlainTextCollector{
+		target:            targetAccepter,
+		plainTextContents: bytes.NewBuffer([]byte("")),
+	}
+}
+
+func (ptc *PlainTextCollector) Accept(partReader io.Reader, header textproto.MIMEHeader, hasPlainSibling bool, isFirst, isLast bool) (err error) {
+	if isFirst {
+		if IsLeaf(header) {
+			mediaType, params, _ := getContentType(header)
+			disp, _, _ := mime.ParseMediaType(header.Get("Content-Disposition"))
+			if mediaType == "text/plain" && disp != "attachment" {
+				partData, _ := ioutil.ReadAll(partReader)
+				decodedPart := decodePart(bytes.NewReader(partData), header)
+
+				if buffer, err := ioutil.ReadAll(decodedPart); err == nil {
+					buffer, err = DecodeCharset(buffer, params)
+					if err != nil {
+						log.Warnln("Decode charset error:", err)
+						return err
+					}
+					ptc.plainTextContents.Write(buffer)
+				}
+
+				err = ptc.target.Accept(bytes.NewReader(partData), header, hasPlainSibling, isFirst, isLast)
+				return
+			}
+		}
+	}
+	err = ptc.target.Accept(partReader, header, hasPlainSibling, isFirst, isLast)
+	return
+}
+
+func (ptc PlainTextCollector) GetPlainText() string {
+	return ptc.plainTextContents.String()
+}
+
+// ======================== Body Collector  ==============
+// Collect contents of all non-attachment parts and return it as a string.
+// TODO move this to file collector_body.go.
+
+type BodyCollector struct {
+	target            VisitAcceptor
+	htmlBodyBuffer    *bytes.Buffer
+	plainBodyBuffer   *bytes.Buffer
+	htmlHeaderBuffer  *bytes.Buffer
+	plainHeaderBuffer *bytes.Buffer
+	hasHtml           bool
+}
+
+func NewBodyCollector(targetAccepter VisitAcceptor) *BodyCollector {
+	return &BodyCollector{
+		target:            targetAccepter,
+		htmlBodyBuffer:    bytes.NewBuffer([]byte("")),
+		plainBodyBuffer:   bytes.NewBuffer([]byte("")),
+		htmlHeaderBuffer:  bytes.NewBuffer([]byte("")),
+		plainHeaderBuffer: bytes.NewBuffer([]byte("")),
+	}
+}
+
+func (bc *BodyCollector) Accept(partReader io.Reader, header textproto.MIMEHeader, hasPlainSibling bool, isFirst, isLast bool) (err error) {
+	// TODO: Collect html and plaintext - if there's html with plain sibling don't include plain/text.
+	if isFirst {
+		if IsLeaf(header) {
+			mediaType, params, _ := getContentType(header)
+			disp, _, _ := mime.ParseMediaType(header.Get("Content-Disposition"))
+			if disp != "attachment" {
+				partData, _ := ioutil.ReadAll(partReader)
+				decodedPart := decodePart(bytes.NewReader(partData), header)
+				if buffer, err := ioutil.ReadAll(decodedPart); err == nil {
+					buffer, err = DecodeCharset(buffer, params)
+					if err != nil {
+						log.Warnln("Decode charset error:", err)
+						return err
+					}
+					if mediaType == "text/html" {
+						bc.hasHtml = true
+						http.Header(header).Write(bc.htmlHeaderBuffer)
+						bc.htmlBodyBuffer.Write(buffer)
+					} else if mediaType == "text/plain" {
+						http.Header(header).Write(bc.plainHeaderBuffer)
+						bc.plainBodyBuffer.Write(buffer)
+					}
+				}
+
+				err = bc.target.Accept(bytes.NewReader(partData), header, hasPlainSibling, isFirst, isLast)
+				return
+			}
+		}
+	}
+	err = bc.target.Accept(partReader, header, hasPlainSibling, isFirst, isLast)
+	return
+}
+
+func (bc *BodyCollector) GetBody() (string, string) {
+	if bc.hasHtml {
+		return bc.htmlBodyBuffer.String(), "text/html"
+	} else {
+		return bc.plainBodyBuffer.String(), "text/plain"
+	}
+}
+
+func (bc *BodyCollector) GetHeaders() string {
+	if bc.hasHtml {
+		return bc.htmlHeaderBuffer.String()
+	} else {
+		return bc.plainHeaderBuffer.String()
+	}
+}
+
+// ======================== Attachments Collector  ==============
+// Collect contents of all attachment parts and return them as a string.
+// TODO move this to file collector_attachment.go.
+
+type AttachmentsCollector struct {
+	target     VisitAcceptor
+	attBuffers []string
+	attHeaders []string
+}
+
+func NewAttachmentsCollector(targetAccepter VisitAcceptor) *AttachmentsCollector {
+	return &AttachmentsCollector{
+		target:     targetAccepter,
+		attBuffers: []string{},
+		attHeaders: []string{},
+	}
+}
+
+func (ac *AttachmentsCollector) Accept(partReader io.Reader, header textproto.MIMEHeader, hasPlainSibling bool, isFirst, isLast bool) (err error) {
+	if isFirst {
+		if IsLeaf(header) {
+			mediaType, params, _ := getContentType(header)
+			disp, _, _ := mime.ParseMediaType(header.Get("Content-Disposition"))
+			if (mediaType != "text/html" && mediaType != "text/plain") || disp == "attachment" {
+				partData, _ := ioutil.ReadAll(partReader)
+				decodedPart := decodePart(bytes.NewReader(partData), header)
+
+				if buffer, err := ioutil.ReadAll(decodedPart); err == nil {
+					buffer, err = DecodeCharset(buffer, params)
+					if err != nil {
+						log.Warnln("Decode charset error:", err)
+						return err
+					}
+					headerBuf := new(bytes.Buffer)
+					http.Header(header).Write(headerBuf)
+					ac.attHeaders = append(ac.attHeaders, headerBuf.String())
+					ac.attBuffers = append(ac.attBuffers, string(buffer))
+				}
+
+				err = ac.target.Accept(bytes.NewReader(partData), header, hasPlainSibling, isFirst, isLast)
+				return
+			}
+		}
+	}
+	err = ac.target.Accept(partReader, header, hasPlainSibling, isFirst, isLast)
+	return
+}
+
+func (ac AttachmentsCollector) GetAttachments() []string {
+	return ac.attBuffers
+}
+
+func (ac AttachmentsCollector) GetAttHeaders() []string {
+	return ac.attHeaders
+}

pkg/mime/parser_test.go

@@ -0,0 +1,228 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmmime
+
+import (
+	"bytes"
+	"fmt"
+
+	"io/ioutil"
+	"net/mail"
+
+	"net/textproto"
+	"strings"
+	"testing"
+)
+
+func minimalParse(mimeBody string) (readBody string, plainContents string, err error) {
+	mm, err := mail.ReadMessage(strings.NewReader(mimeBody))
+	if err != nil {
+		return
+	}
+
+	h := textproto.MIMEHeader(mm.Header)
+	mmBodyData, err := ioutil.ReadAll(mm.Body)
+	if err != nil {
+		return
+	}
+
+	printAccepter := NewMIMEPrinter()
+	plainTextCollector := NewPlainTextCollector(printAccepter)
+	visitor := NewMimeVisitor(plainTextCollector)
+	err = VisitAll(bytes.NewReader(mmBodyData), h, visitor)
+
+	readBody = printAccepter.String()
+	plainContents = plainTextCollector.GetPlainText()
+
+	return readBody, plainContents, err
+}
+
+func androidParse(mimeBody string) (body, headers string, atts, attHeaders []string, err error) {
+	mm, err := mail.ReadMessage(strings.NewReader(mimeBody))
+	if err != nil {
+		return
+	}
+
+	h := textproto.MIMEHeader(mm.Header)
+	mmBodyData, err := ioutil.ReadAll(mm.Body)
+
+	printAccepter := NewMIMEPrinter()
+	bodyCollector := NewBodyCollector(printAccepter)
+	attachmentsCollector := NewAttachmentsCollector(bodyCollector)
+	mimeVisitor := NewMimeVisitor(attachmentsCollector)
+	err = VisitAll(bytes.NewReader(mmBodyData), h, mimeVisitor)
+
+	body, _ = bodyCollector.GetBody()
+	headers = bodyCollector.GetHeaders()
+	atts = attachmentsCollector.GetAttachments()
+	attHeaders = attachmentsCollector.GetAttHeaders()
+
+	return
+}
+
+func TestParseBoundaryIsEmpty(t *testing.T) {
+	testMessage :=
+		`Date: Sun, 10 Mar 2019 11:10:06 -0600
+In-Reply-To: <abcbase64@protonmail.com>
+X-Original-To: enterprise@protonmail.com
+References: <abc64@unicoderns.com> <abc63@protonmail.com> <abc64@protonmail.com> <abc65@mail.gmail.com> <abc66@protonmail.com>
+To: "ProtonMail" <enterprise@protonmail.com>
+X-Pm-Origin: external
+Delivered-To: enterprise@protonmail.com
+Content-Type: multipart/mixed; boundary=ac7e36bd45425e70b4dab2128f34172e4dc3f9ff2eeb47e909267d4252794ec7
+Reply-To: XYZ <xyz@xyz.com>
+Mime-Version: 1.0
+Subject: Encrypted Message
+Return-Path: <xyz@xyz.com>
+From: XYZ <xyz@xyz.com>
+X-Pm-Conversationid-Id: gNX9bDPLmBgFZ-C3Tdlb628cas1Xl0m4dql5nsWzQAEI-WQv0ytfwPR4-PWELEK0_87XuFOgetc239Y0pjPYHQ==
+X-Pm-Date: Sun, 10 Mar 2019 18:10:06 +0100
+Message-Id: <68c11e46-e611-d9e4-edc1-5ec96bac77cc@unicoderns.com>
+X-Pm-Transfer-Encryption: TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)
+X-Pm-External-Id: <68c11e46-e611-d9e4-edc1-5ec96bac77cc@unicoderns.com>
+X-Pm-Internal-Id: _iJ8ETxcqXTSK8IzCn0qFpMUTwvRf-xJUtldRA1f6yHdmXjXzKleG3F_NLjZL3FvIWVHoItTxOuuVXcukwwW3g==
+Openpgp: preference=signencrypt
+User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Thunderbird/60.4.0
+X-Pm-Content-Encryption: end-to-end
+
+--ac7e36bd45425e70b4dab2128f34172e4dc3f9ff2eeb47e909267d4252794ec7
+Content-Disposition: inline
+Content-Transfer-Encoding: quoted-printable
+Content-Type: multipart/mixed; charset=utf-8
+
+Content-Type: multipart/mixed; boundary="xnAIW3Turb9YQZ2rXc2ZGZH45WepHIZyy";
+ protected-headers="v1"
+From: XYZ <xyz@xyz.com>
+To: "ProtonMail" <enterprise@protonmail.com>
+Subject: Encrypted Message
+Message-ID: <68c11e46-e611-d9e4-edc1-5ec96bac77cc@unicoderns.com>
+References: <abc64@unicoderns.com> <abc63@protonmail.com> <abc64@protonmail.com> <abc65@mail.gmail.com> <abc66@protonmail.com>
+In-Reply-To: <abcbase64@protonmail.com>
+
+--xnAIW3Turb9YQZ2rXc2ZGZH45WepHIZyy
+Content-Type: text/rfc822-headers; protected-headers="v1"
+Content-Disposition: inline
+
+From: XYZ <xyz@xyz.com>
+To: ProtonMail <enterprise@protonmail.com>
+Subject: Re: Encrypted Message
+
+--xnAIW3Turb9YQZ2rXc2ZGZH45WepHIZyy
+Content-Type: multipart/alternative;
+ boundary="------------F9E5AA6D49692F51484075E3"
+Content-Language: en-US
+
+This is a multi-part message in MIME format.
+--------------F9E5AA6D49692F51484075E3
+Content-Type: text/plain; charset=utf-8
+Content-Transfer-Encoding: quoted-printable
+
+Hi ...
+
+--------------F9E5AA6D49692F51484075E3
+Content-Type: text/html; charset=utf-8
+Content-Transfer-Encoding: quoted-printable
+
+<html>
+  <head>
+  </head>
+  <body text=3D"#000000" bgcolor=3D"#FFFFFF">
+    <p>Hi ..  </p>
+  </body>
+</html>
+
+--------------F9E5AA6D49692F51484075E3--
+
+--xnAIW3Turb9YQZ2rXc2ZGZH45WepHIZyy--
+
+--ac7e36bd45425e70b4dab2128f34172e4dc3f9ff2eeb47e909267d4252794ec7--
+
+
+`
+
+	body, content, err := minimalParse(testMessage)
+	if err == nil {
+		t.Fatal("should have error but is", err)
+	}
+	t.Log("==BODY==")
+	t.Log(body)
+	t.Log("==CONTENT==")
+	t.Log(content)
+}
+
+func TestParse(t *testing.T) {
+	testMessage :=
+		`From: John Doe <example@example.com>
+MIME-Version: 1.0
+Content-Type: multipart/mixed;
+        boundary="XXXXboundary text"
+
+This is a multipart message in MIME format.
+
+--XXXXboundary text
+Content-Type: text/plain; charset=utf-8
+
+this is the body text
+
+--XXXXboundary text
+Content-Type: text/html; charset=utf-8
+
+<html><body>this is the html body text</body></html>
+
+--XXXXboundary text
+Content-Type: text/plain; charset=utf-8
+Content-Disposition: attachment;
+        filename="test.txt"
+
+this is the attachment text
+
+--XXXXboundary text--
+
+
+`
+	body, heads, att, attHeads, err := androidParse(testMessage)
+	if err != nil {
+		t.Error("parse error", err)
+	}
+
+	fmt.Println("==BODY:")
+	fmt.Println(body)
+	fmt.Println("==BODY HEADERS:")
+	fmt.Println(heads)
+
+	fmt.Println("==ATTACHMENTS:")
+	fmt.Println(att)
+	fmt.Println("==ATTACHMENT HEADERS:")
+	fmt.Println(attHeads)
+}
+
+func TestParseAddressComment(t *testing.T) {
+	parsingExamples := map[string]string{
+		"":                          "",
+		"(Only Comment) here@pm.me": "\"Only Comment\" <here@pm.me>",
+		"Normal Name (With Comment) <here@pm.me>":          "\"Normal Name\" <here@pm.me>",
+		"<Muhammed.(I am the greatest)Ali@(the)Vegas.WBA>": "\"I am the greatest the\" <Muhammed.Ali@Vegas.WBA>",
+	}
+
+	for raw, expected := range parsingExamples {
+		parsed := parseAddressComment(raw)
+		if expected != parsed {
+			t.Errorf("When parsing %q expected %q but have %q", raw, expected, parsed)
+		}
+	}
+}

pkg/mime/utf7Decoder.go

@@ -0,0 +1,188 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmmime
+
+import (
+	"encoding/base64"
+	"errors"
+	"unicode/utf16"
+	"unicode/utf8"
+
+	"golang.org/x/text/encoding"
+	"golang.org/x/text/transform"
+)
+
+// utf7Decoder copied from: https://github.com/cention-sany/utf7/blob/master/utf7.go
+// We need `encoding.Decoder` instead of function `UTF7DecodeBytes`.
+type utf7Decoder struct {
+	transform.NopResetter
+}
+
+// NewUtf7Decoder returns a new decoder for utf7.
+func NewUtf7Decoder() *encoding.Decoder {
+	return &encoding.Decoder{Transformer: utf7Decoder{}}
+}
+
+const (
+	uRepl = '\uFFFD' // Unicode replacement code point
+	u7min = 0x20     // Minimum self-representing UTF-7 value
+	u7max = 0x7E     // Maximum self-representing UTF-7 value
+)
+
+// ErrBadUTF7 is returned to indicate the invalid modified UTF-7 encoding.
+var ErrBadUTF7 = errors.New("utf7: bad utf-7 encoding")
+
+const modifiedbase64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
+
+var u7enc = base64.NewEncoding(modifiedbase64)
+
+func isModifiedBase64(r byte) bool {
+	if r >= 'A' && r <= 'Z' {
+		return true
+	} else if r >= 'a' && r <= 'z' {
+		return true
+	} else if r >= '0' && r <= '9' {
+		return true
+	} else if r == '+' || r == '/' {
+		return true
+	}
+	return false
+}
+
+func (d utf7Decoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	var implicit bool
+	var tmp int
+
+	nd, n := len(dst), len(src)
+	if n == 0 && !atEOF {
+		return 0, 0, transform.ErrShortSrc
+	}
+	for ; nSrc < n; nSrc++ {
+		if nDst >= nd {
+			return nDst, nSrc, transform.ErrShortDst
+		}
+		if c := src[nSrc]; ((c < u7min || c > u7max) &&
+			c != '\t' && c != '\r' && c != '\n') ||
+			c == '~' || c == '\\' {
+			return nDst, nSrc, ErrBadUTF7 // Illegal code point in ASCII mode.
+		} else if c != '+' {
+			dst[nDst] = c // Character is self-representing.
+			nDst++
+			continue
+		}
+		// Found '+'.
+		start := nSrc + 1
+		tmp = nSrc // nSrc still points to '+', tmp points to the end of BASE64.
+
+		// Find the end of the Base64 or "+-" segment.
+		implicit = false
+		for tmp++; tmp < n && src[tmp] != '-'; tmp++ {
+			if !isModifiedBase64(src[tmp]) {
+				if tmp == start {
+					return nDst, tmp, ErrBadUTF7 // '+' next char must modified base64.
+				}
+				// Implicit shift back to ASCII, no need for '-' character.
+				implicit = true
+				break
+			}
+		}
+		if tmp == start {
+			if tmp == n {
+				// Did not find '-' sign and '+' is the last character.
+				// Total nSrc does not include '+'.
+				if atEOF {
+					return nDst, nSrc, ErrBadUTF7 // '+' can not be at the end.
+				}
+				// '+' can not be at the end, the source is too short.
+				return nDst, nSrc, transform.ErrShortSrc
+			}
+			dst[nDst] = '+' // Escape sequence "+-".
+			nDst++
+		} else if tmp == n && !atEOF {
+			// No EOF found, the source is too short.
+			return nDst, nSrc, transform.ErrShortSrc
+		} else if b := utf7dec(src[start:tmp]); len(b) > 0 {
+			if len(b)+nDst > nd {
+				// Need more space in dst for the decoded modified BASE64 unicode.
+				// Total nSrc does not include '+'.
+				return nDst, nSrc, transform.ErrShortDst
+			}
+			copy(dst[nDst:], b) // Control or non-ASCII code points in Base64.
+			nDst += len(b)
+			if implicit {
+				if nDst >= nd {
+					return nDst, tmp, transform.ErrShortDst
+				}
+				dst[nDst] = src[tmp] // Implicit shift.
+				nDst++
+			}
+			if tmp == n {
+				return nDst, tmp, nil
+			}
+		} else {
+			return nDst, nSrc, ErrBadUTF7 // Bad encoding.
+		}
+		nSrc = tmp
+	}
+	return
+}
+
+// utf7dec extracts UTF-16-BE bytes from Base64 data and converts them to UTF-8.
+// A nil slice is returned if the encoding is invalid.
+func utf7dec(b64 []byte) []byte {
+	var b []byte
+
+	// Allocate a single block of memory large enough to store the Base64 data
+	// (if padding is required), UTF-16-BE bytes, and decoded UTF-8 bytes.
+	// Since a 2-byte UTF-16 sequence may expand into a 3-byte UTF-8 sequence,
+	// double the space allocation for UTF-8.
+	if n := len(b64); b64[n-1] == '=' {
+		return nil
+	} else if n&3 == 0 {
+		b = make([]byte, u7enc.DecodedLen(n)*3)
+	} else {
+		n += 4 - n&3
+		b = make([]byte, n+u7enc.DecodedLen(n)*3)
+		copy(b[copy(b, b64):n], []byte("=="))
+		b64, b = b[:n], b[n:]
+	}
+
+	// Decode Base64 into the first 1/3rd of b.
+	n, err := u7enc.Decode(b, b64)
+	if err != nil || n&1 == 1 {
+		return nil
+	}
+
+	// Decode UTF-16-BE into the remaining 2/3rds of b.
+	b, s := b[:n], b[n:]
+	j := 0
+	for i := 0; i < n; i += 2 {
+		r := rune(b[i])<<8 | rune(b[i+1])
+		if utf16.IsSurrogate(r) {
+			if i += 2; i == n {
+				return nil
+			}
+			r2 := rune(b[i])<<8 | rune(b[i+1])
+			if r = utf16.DecodeRune(r, r2); r == uRepl {
+				return nil
+			}
+		}
+		j += utf8.EncodeRune(s[j:], r)
+	}
+	return s[:j]
+}

pkg/parallel/parallel.go

@@ -0,0 +1,136 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package parallel
+
+import (
+	"sync"
+	"time"
+)
+
+// parallelJob is to be used for passing items between input, worker and
+// collector. `idx` is there to know the original order.
+type parallelJob struct {
+	idx   int
+	value interface{}
+}
+
+// RunParallel starts `workers` number of workers and feeds them with `input` data.
+// Each worker calls `process`. Processed data is collected in the same order as
+// the input and is passed in order to the `collect` callback. If an error
+// occurs, the execution is stopped and the error returned.
+// runParallel blocks until everything is done.
+func RunParallel( //nolint[funlen]
+	workers int,
+	input []interface{},
+	process func(interface{}) (interface{}, error),
+	collect func(int, interface{}) error,
+) (resultError error) {
+	wgProcess := &sync.WaitGroup{}
+	wgCollect := &sync.WaitGroup{}
+
+	// Optimise by not executing the code at all if there is no input
+	// or run less workers than requested if there are few inputs.
+	inputLen := len(input)
+	if inputLen == 0 {
+		return nil
+	}
+	if inputLen < workers {
+		workers = inputLen
+	}
+
+	inputChan := make(chan *parallelJob)
+	outputChan := make(chan *parallelJob)
+
+	orderedCollectLock := &sync.Mutex{}
+	orderedCollect := make(map[int]interface{})
+
+	// Feed input channel used by workers with input data with index for ordering.
+	go func() {
+		defer close(inputChan)
+		for idx, item := range input {
+			if resultError != nil {
+				break
+			}
+			inputChan <- &parallelJob{idx, item}
+		}
+	}()
+
+	// Start workers and process all the inputs.
+	wgProcess.Add(workers)
+	for i := 0; i < workers; i++ {
+		go func() {
+			defer wgProcess.Done()
+			for item := range inputChan {
+				if output, err := process(item.value); err != nil {
+					resultError = err
+					break
+				} else {
+					outputChan <- &parallelJob{item.idx, output}
+				}
+			}
+		}()
+	}
+
+	// Collect data into map with the original position in the array.
+	wgCollect.Add(1)
+	go func() {
+		defer wgCollect.Done()
+		for output := range outputChan {
+			orderedCollectLock.Lock()
+			orderedCollect[output.idx] = output.value
+			orderedCollectLock.Unlock()
+		}
+	}()
+
+	// Collect data in the same order as in the input array.
+	wgCollect.Add(1)
+	go func() {
+		defer wgCollect.Done()
+		idx := 0
+		for {
+			if idx >= inputLen || resultError != nil {
+				break
+			}
+			orderedCollectLock.Lock()
+			value, ok := orderedCollect[idx]
+			if ok {
+				if err := collect(idx, value); err != nil {
+					resultError = err
+				}
+				delete(orderedCollect, idx)
+				idx++
+			}
+			orderedCollectLock.Unlock()
+			if !ok {
+				time.Sleep(10 * time.Millisecond)
+			}
+		}
+	}()
+
+	// When input channel is closed, all workers will finish. We need to wait
+	// for all of them and close the output channel only once.
+	wgProcess.Wait()
+	close(outputChan)
+
+	// When workers are done, the last job is to finish collecting data. First
+	// collector is finished when output channel is closed and the second one
+	// when all items are passed to `collect` in the order or after an error.
+	wgCollect.Wait()
+
+	return resultError
+}

pkg/parallel/parallel_test.go

@@ -0,0 +1,131 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package parallel
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"testing"
+	"time"
+
+	r "github.com/stretchr/testify/require"
+)
+
+// nolint[gochecknoglobals]
+var (
+	testInput               = []interface{}{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
+	wantOutput              = []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
+	testProcessSleep        = 100 // ms
+	runParallelTimeOverhead = 100 // ms
+)
+
+func TestParallel(t *testing.T) {
+	workersTests := []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
+
+	for _, workers := range workersTests {
+		workers := workers
+		t.Run(fmt.Sprintf("%d", workers), func(t *testing.T) {
+			collected := make([]int, 0)
+			collect := func(idx int, value interface{}) error {
+				collected = append(collected, value.(int))
+				return nil
+			}
+
+			tstart := time.Now()
+			err := RunParallel(workers, testInput, processSleep, collect)
+			duration := time.Since(tstart)
+
+			r.Nil(t, err)
+			r.Equal(t, wantOutput, collected) // Check the order is always kept.
+
+			wantMinDuration := int(math.Ceil(float64(len(testInput))/float64(workers))) * testProcessSleep
+			wantMaxDuration := wantMinDuration + runParallelTimeOverhead
+			r.True(t, duration.Nanoseconds() > int64(wantMinDuration*1000000), "Duration too short: %v (expected: %v)", duration, wantMinDuration)
+			r.True(t, duration.Nanoseconds() < int64(wantMaxDuration*1000000), "Duration too long: %v (expected: %v)", duration, wantMaxDuration)
+		})
+	}
+}
+
+func TestParallelEmptyInput(t *testing.T) {
+	workersTests := []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
+
+	for _, workers := range workersTests {
+		workers := workers
+		t.Run(fmt.Sprintf("%d", workers), func(t *testing.T) {
+			err := RunParallel(workers, []interface{}{}, processSleep, collectNil)
+			r.Nil(t, err)
+		})
+	}
+}
+
+func TestParallelErrorInProcess(t *testing.T) {
+	workersTests := []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
+
+	for _, workers := range workersTests {
+		workers := workers
+		t.Run(fmt.Sprintf("%d", workers), func(t *testing.T) {
+			var lastCollected int
+			process := func(value interface{}) (interface{}, error) {
+				time.Sleep(10 * time.Millisecond)
+				if value.(int) == 5 {
+					return nil, errors.New("Error")
+				}
+				return value, nil
+			}
+			collect := func(idx int, value interface{}) error {
+				lastCollected = value.(int)
+				return nil
+			}
+
+			err := RunParallel(workers, testInput, process, collect)
+			r.EqualError(t, err, "Error")
+
+			time.Sleep(10 * time.Millisecond)
+			r.True(t, lastCollected < 5, "Last collected cannot be higher that 5, got: %d", lastCollected)
+		})
+	}
+}
+
+func TestParallelErrorInCollect(t *testing.T) {
+	workersTests := []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
+
+	for _, workers := range workersTests {
+		workers := workers
+		t.Run(fmt.Sprintf("%d", workers), func(t *testing.T) {
+			collect := func(idx int, value interface{}) error {
+				if value.(int) == 5 {
+					return errors.New("Error")
+				}
+				return nil
+			}
+
+			err := RunParallel(workers, testInput, processSleep, collect)
+			r.EqualError(t, err, "Error")
+		})
+	}
+}
+
+func processSleep(value interface{}) (interface{}, error) {
+	time.Sleep(time.Duration(testProcessSleep) * time.Millisecond)
+	return value.(int), nil
+}
+
+func collectNil(idx int, value interface{}) error {
+	return nil
+}

pkg/pmapi/Changelog.md

@@ -0,0 +1,309 @@
+# Do not modify this file!
+It is here for historical reasons only. All changes should be documented in the
+Changelog at the root of this repository.
+
+
+# Changelog for API
+> NOTE we are using versioning for go-pmapi in format `major.minor.bugfix`
+> * major stays at version 1 for the forseeable future
+> * minor is increased when a force upgrade happens or in case of major breaking changes
+> * patch is increased when new features are added
+
+## v1.0.16
+
+### Fixed
+* Potential crash when reporting cert pin failure 
+
+## v1.0.15
+
+### Changed
+* Merge only 50 events into one
+* Response header timeout increased from 10s to 30s
+
+### Fixed
+* Make keyring unlocking threadsafe
+
+## v1.0.14
+
+### Added
+* Config for disabling TLS cert fingerprint checking
+
+### Fixed
+* Ensure sensitive stuff is cleared on client logout even if requests fail
+
+## v1.0.13
+
+### Fixed
+* Correctly set Transport in http client
+
+## v1.0.12
+
+### Changed
+* Only `http.RoundTripper` interface is needed instead of full `http.Transport` struct
+
+### Added
+* GODT-61 (and related): Use DoH to find and switch to a proxy server if the API becomes unreachable
+* GODT-67 added random wait to not cause spikes on server after StatusTooManyRequests
+
+### Fixed
+* FirstReadTimeout was wrongly timeout of the whole request including repeating ones, now it's really only timeout for the first read
+
+## v1.0.11
+
+### Added
+* GODT-53 `Message.Type` added with constants `MessageType*`
+
+## v1.0.10
+
+### Added
+* GODT-55 exporting DANGEROUSLYSetUID
+
+### Changed
+* The full communication between clien and API is logged if logrus level is trace
+
+## v1.0.9
+
+### Fixed
+* Use correct address type value (because API starts counting from 1 but we were counting from 0)
+
+## v1.0.8
+
+### Added
+* Introdcution of connection manager
+
+### Fixed
+* Deadlock during the auth-refresh
+* Fixed an issue where some events were being discarded when merging
+
+## v1.0.7
+
+### Changed
+* The given access token is saved during auth refresh if none was available yet
+
+
+## v1.0.6
+
+### Added
+* `ClientConfig.Timeout` to be able to configure the whole timeout of request
+* `ClientConfig.FirstReadTimeout` to be able to configure the timeout of request to the first byte
+* `ClientConfig.MinSpeed` to be able to configure the timeout when the connection is too slow (limitation in minimum bytes per second)
+* Set default timeouts for http.Transport with certificate pinning
+
+### Changed
+* http.Client by default uses ProxyFromEnvironment to support HTTP_PROXY and HTTPS_PROXY environment variables
+
+## v1.0.5
+
+### Added
+* `ContentTypeMultipartEncrypted` MIME content type for encrypted email
+* `MessageCounts` in event struct
+
+## v1.0.4
+
+### Added
+* `PMKeys` for parsing and reading KeyRing
+* `clearableKey` to rewrite memory
+* Proton/backend-communication#25 Unlock with tokens (OneKey2RuleThemAll Phase I)
+
+### Changed
+* Update of gopenpgp: convert JSON to KeyRing in PMAPI
+* `user.KeyRing` -> `user.KeyRing()`
+* typo `client.GetAddresses()`
+
+### Removed
+* `address.KeyRing`
+
+## v1.0.2 v1.0.3
+
+### Changed
+* Fixed capitalisation in a few places
+* Added /metrics API route
+* Changed function names to be compliant with go linter
+* Encrypt with primary key only
+* Fix `client.doBuffered` - closing body before handling unauthorized request
+* go-pm-crypto -> GopenPGP
+* redefine old functions in `keyring.go`
+* `attachment.Decrypt` drops returning signature (does signature check by default)
+* `attachment.Encrypt` is using readers instead of writers
+* `attachment.DetachedSign` drops writer param and returns signature as a reader
+* `message.Decrypt` drops returning signature (does signature check by default)
+* Changed TLS report URL to https://reports.protonmail.ch/reports/tls
+* Moved from current to soon TLS pin
+
+## v1.0.1
+
+### Removed
+* `ClientID` from all auth routes
+* `ErrorDescription` from error
+
+## v1.0.0
+
+### Changed
+* `client.AuthInfo` does return 2FA information only when authenticated, for the first login information available in `Auth.HasTwoFactor`
+* `client.Auth` does not accept 2FA code in favor of `client.Auth2FA`
+* `client.Unlock` supports only new way of unlock with directly available access token
+
+### Added
+* `Res.StatusCode` to pass HTTP status code to responses
+* `Auth.HasTwoFactor` method to determine whether account has enabled 2FA (same as `AuthInfo.HasTwoFactor`)
+* `Auth2FA*` structs for 2FA endpoint
+* `client.Auth2FA` method to fully unlock session with 2FA code
+* `ErrUnauthorized` when request cannot be authorized
+* `ErrBad2FACode` when bad 2FA and user cannot try again
+* `ErrBad2FACodeTryAgain` when bad 2FA but user can try again
+
+## 2019-08-06
+
+### Added
+* Send TLS issue report to API
+* Cert fingerpring with `TLSPinning` struct
+* Check API certificate fingerprint and verify hostname
+
+### Changed
+* Using `AddressID` for `/messge/count` and `/conversations/count`
+* Less of copying of responses from the server in the memory
+
+## 2019-08-01
+* low case for `sirupsen`
+* using go modules
+
+## 2019-07-15
+
+### Changed
+* `client.Auths` field is removed in favor of function `client.SetAuths` which opens possibility to use interface
+
+## 2019-05-18
+
+### Changed
+* proton/backend-communication#11 x-pm-uid sent always for `/auth/refresh`
+* proton/backend-communication#11 UID never changes
+
+## 2019-05-28
+
+### Added
+* New test server patern using callbacks
+* Responses are read from json files
+
+### Changed
+* `auth_tests.go` to new callback server pattern
+* Linter fixes for tests
+
+### Removed
+* `TestClient_Do_expired` due to no effect, use `DoUnauthorized` instead
+
+## 2019-05-24
+* Help functions for test
+* CI with Lint
+
+## 2019-05-23
+* Log userID
+
+## 2019-05-21
+* Fix unlocking user keys
+
+## 2019-04-25
+
+### Changed
+* rename `Uid` -> `UID` proton/backend-communication#11
+
+## 2019-04-09
+
+### Added
+* sending attachments as zip `application/octet-stream`
+* function `ReportReq.AddAttachment()`
+* data memeber `ReportReq.Attachments`
+* general function to report bug `client.Report(req ReportReq)` with object as parameter
+
+### Changed
+* `client.ReportBug` and `client.ReportBugWithClient` functions are obsolete and they uses `client.Report(req ReportReq)`
+* `client.ReportCrash` is obsolete. Use sentry instead
+* `Api`->`API`, `Uid`->`UID`
+
+## 2019-03-13
+* user id in raven
+* add file position of panic sender
+
+## 2019-03-06
+* #30 update `pm-crypto` to store `KeyRing.FirstKeyID`
+* #30 Add key salt to `Auth` object from `GetKeySalts` request
+* #30 Add route `GET /keys/salt`
+* removed unused `PmCrypto`
+
+## 2019-02-20
+* removed unused `decryptAccessToken`
+
+## 2019-01-21
+* #29 Parsing all goroutines from pprof
+* #29 Sentry `Threads` implementation
+* #29 using sentry for crashes
+
+## 2019-01-07
+* refactor `pmapi.DecryptString` -> `pmcrypto.KeyRing.DecryptString`
+* fixed tests
+* `crypto` -> `pmcrypto`
+* refactoring code using repos `go-pm-crypto`, `go-pm-mime` and `go-srp`
+
+
+## 2018-12-10
+* #26 adding `Flags` field to message
+* #26 removing fields deprecated by `Flags`: `IsEncrypted`, `Type`, `IsReplied`, `IsRepliedAll`, `IsForwarded`
+* #26 removing deprecated consts (see #26 for replacement)
+* #26 fixing tests (compiling not working)
+
+## 2018-11-19
+
+### Added
+* Wait and retry from `DoJson` if banned from api
+
+### Changed
+* `ErrNoInternet` -> `ErrAPINotReachable`
+* Adding codes for force upgrade: 5004 and 5005
+* Adding codes for API offline: 7001
+* Adding codes for BansRequests: 85131
+
+## 2018-09-18
+
+### Added
+* `client.decryptAccessToken` if privateKey is received (tested with local api) #23
+
+### Changed
+* added fields to User
+* local config TLS skip verify
+
+## 2018-09-06
+
+### Changed
+* decrypt token only if needed
+
+### Broken
+* Tests are not working
+
+## APIv3 UPDATE (2018-08-01)
+* issue Desktop-Bridge#561
+
+### Added
+* Key flag consts
+* `EventAddress`
+* `MailSettings` object and route call
+* `Client.KeyRingForAddressID`
+* `AuthInfo.HasTwoFactor()`
+* `Auth.HasMailboxPassword()`
+
+### Changed
+* Addresses are part of client
+* Update user updates also addresses
+* `BodyKey` and `AttachmentKey` contains `Key` and `Algorithm`
+* `keyPair` (not use Pubkey) -> `pmKeyObject`
+* lots of indent
+* bugs route
+* two factor (ready to U2F)
+* Reorder some to match order in doc (easier to )
+* omit address Order when empty
+* update user and addresses in `CurrentUser()`
+* `User.Unlock()` -> `Client.UnlockAddresses()`
+* `AuthInfo.Uid`  -> `AuthInfo.Uid()`
+* `User.Addresses`  -> `Client.Addresses()`
+
+### Removed
+* User v3 removed plenty (now in settings)
+* Message v3 removed plenty (Starred is label)

pkg/pmapi/Makefile

@@ -0,0 +1,19 @@
+export GO111MODULE=on
+
+LINTVER="v1.21.0"
+LINTSRC="https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh"
+
+check-has-go:
+	@which go || (echo "Install Go-lang!" && exit 1)
+
+install-dev-dependencies: install-linter
+
+install-linter: check-has-go
+	curl -sfL $(LINTSRC) | sh -s -- -b $(shell go env GOPATH)/bin $(LINTVER)
+
+lint:
+	which golangci-lint || $(MAKE) install-linter
+	golangci-lint run ./...																						\
+
+test:
+	go test -run=${TESTRUN} ./...

pkg/pmapi/addresses.go

@@ -0,0 +1,204 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"fmt"
+	"strings"
+
+	pmcrypto "github.com/ProtonMail/gopenpgp/crypto"
+)
+
+// Address statuses.
+const (
+	DisabledAddress = iota
+	EnabledAddress
+)
+
+// Address receive values.
+const (
+	CannotReceive = iota
+	CanReceive
+)
+
+// Address HasKeys values.
+const (
+	MissingKeys = iota
+	KeysPresent
+)
+
+// Address types.
+const (
+	_ = iota // Skip first.
+	OriginalAddress
+	AliasAddress
+	CustomAddress
+	PremiumAddress
+)
+
+// Address Send values.
+const (
+	NoSendAddress = iota
+	MainSendAddress
+	SecondarySendAddress
+)
+
+// Address represents a user's address.
+type Address struct {
+	ID          string
+	DomainID    string
+	Email       string
+	Send        int
+	Receive     int
+	Status      int
+	Order       int `json:",omitempty"`
+	Type        int
+	DisplayName string
+	Signature   string
+	MemberID    string `json:",omitempty"`
+	MemberName  string `json:",omitempty"`
+
+	HasKeys int
+	Keys    PMKeys
+}
+
+// AddressList is a list of addresses.
+type AddressList []*Address
+
+type AddressesRes struct {
+	Res
+	Addresses AddressList
+}
+
+// KeyRing returns the (possibly unlocked) PMKeys KeyRing.
+func (a *Address) KeyRing() *pmcrypto.KeyRing {
+	return a.Keys.KeyRing
+}
+
+// ByID returns an address by id. Returns nil if no address is found.
+func (l AddressList) ByID(id string) *Address {
+	for _, addr := range l {
+		if addr.ID == id {
+			return addr
+		}
+	}
+	return nil
+}
+
+func (l AddressList) ActiveEmails() (addresses []string) {
+	for _, a := range l {
+		if a.Receive == CanReceive {
+			addresses = append(addresses, a.Email)
+		}
+	}
+	return
+}
+
+// Main gets the main address.
+func (l AddressList) Main() *Address {
+	for _, addr := range l {
+		if addr.Order == 1 {
+			return addr
+		}
+	}
+	return l[0] // Should not happen.
+}
+
+// ByEmail gets an address by email. Returns nil if no address is found.
+func (l AddressList) ByEmail(email string) *Address {
+	email = SanitizeEmail(email)
+	for _, addr := range l {
+		if strings.EqualFold(addr.Email, email) {
+			return addr
+		}
+	}
+	return nil
+}
+
+func SanitizeEmail(email string) string {
+	splitAt := strings.Split(email, "@")
+	if len(splitAt) != 2 {
+		return email
+	}
+	splitPlus := strings.Split(splitAt[0], "+")
+	email = splitPlus[0] + "@" + splitAt[1]
+	return email
+}
+
+func ConstructAddress(headerEmail string, addressEmail string) string {
+	splitAtHeader := strings.Split(headerEmail, "@")
+	if len(splitAtHeader) != 2 {
+		return addressEmail
+	}
+
+	splitPlus := strings.Split(splitAtHeader[0], "+")
+	if len(splitPlus) != 2 {
+		return addressEmail
+	}
+
+	splitAtAddress := strings.Split(addressEmail, "@")
+	if len(splitAtAddress) != 2 {
+		return addressEmail
+	}
+
+	return splitAtAddress[0] + "+" + splitPlus[1] + "@" + splitAtAddress[1]
+}
+
+// GetAddresses requests all of current user addresses (without pagination).
+func (c *Client) GetAddresses() (addresses AddressList, err error) {
+	req, err := NewRequest("GET", "/addresses", nil)
+	if err != nil {
+		return
+	}
+
+	var res AddressesRes
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+
+	return res.Addresses, res.Err()
+}
+
+func (c *Client) Addresses() AddressList {
+	return c.addresses
+}
+
+// UnlockAddresses unlocks all keys for all addresses of current user.
+func (c *Client) UnlockAddresses(passphrase []byte) (err error) {
+	for _, a := range c.addresses {
+		if a.HasKeys == MissingKeys {
+			continue
+		}
+
+		// Unlock the address token using the UserKey, use the unlocked token to unlock the keyring.
+		if err = a.Keys.unlockKeyRing(c.kr, passphrase, c.keyLocker); err != nil {
+			err = fmt.Errorf("pmapi: cannot unlock private key of address %v: %v", a.Email, err)
+			return
+		}
+	}
+
+	return
+}
+
+func (c *Client) KeyRingForAddressID(addrID string) *pmcrypto.KeyRing {
+	addr := c.addresses.ByID(addrID)
+	if addr == nil {
+		addr = c.addresses.Main()
+	}
+	return addr.KeyRing()
+}

pkg/pmapi/addresses_test.go

@@ -0,0 +1,90 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"net/http"
+	"testing"
+)
+
+var testAddressList = AddressList{
+	&Address{
+		ID:     "1",
+		Email:  "root@nsa.gov",
+		Send:   SecondarySendAddress,
+		Status: EnabledAddress,
+		Order:  2,
+	},
+	&Address{
+		ID:     "2",
+		Email:  "root@gchq.gov.uk",
+		Send:   MainSendAddress,
+		Status: EnabledAddress,
+		Order:  1,
+	},
+	&Address{
+		ID:     "3",
+		Email:  "root@protonmail.com",
+		Send:   NoSendAddress,
+		Status: DisabledAddress,
+		Order:  3,
+	},
+}
+
+func routeGetAddresses(tb testing.TB, w http.ResponseWriter, r *http.Request) string {
+	Ok(tb, checkMethodAndPath(r, "GET", "/addresses"))
+	Ok(tb, isAuthReq(r, testUID, testAccessToken))
+	return "addresses/get_response.json"
+}
+
+func routeGetSalts(tb testing.TB, w http.ResponseWriter, r *http.Request) string {
+	Ok(tb, checkMethodAndPath(r, "GET", "/keys/salts"))
+	Ok(tb, isAuthReq(r, testUID, testAccessToken))
+	return "keys/salts/get_response.json"
+}
+
+func TestAddressList(t *testing.T) {
+	input := "1"
+	addr := testAddressList.ByID(input)
+	if addr != testAddressList[0] {
+		t.Errorf("ById(%s) expected:\n%v\n but have:\n%v\n", input, testAddressList[0], addr)
+	}
+
+	input = "42"
+	addr = testAddressList.ByID(input)
+	if addr != nil {
+		t.Errorf("ById expected nil for %s but have : %v\n", input, addr)
+	}
+
+	input = "root@protonmail.com"
+	addr = testAddressList.ByEmail(input)
+	if addr != testAddressList[2] {
+		t.Errorf("ByEmail(%s) expected:\n%v\n but have:\n%v\n", input, testAddressList[2], addr)
+	}
+
+	input = "idontexist@protonmail.com"
+	addr = testAddressList.ByEmail(input)
+	if addr != nil {
+		t.Errorf("ByEmail expected nil for %s but have : %v\n", input, addr)
+	}
+
+	addr = testAddressList.Main()
+	if addr != testAddressList[1] {
+		t.Errorf("Main() expected:\n%v\n but have:\n%v\n", testAddressList[1], addr)
+	}
+}

pkg/pmapi/attachments.go

@@ -0,0 +1,264 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/textproto"
+
+	pmcrypto "github.com/ProtonMail/gopenpgp/crypto"
+)
+
+type header textproto.MIMEHeader
+
+type rawHeader map[string]json.RawMessage
+
+func (h *header) UnmarshalJSON(b []byte) error {
+	if *h == nil {
+		*h = make(header)
+	}
+
+	raw := make(rawHeader)
+	if err := json.Unmarshal(b, &raw); err != nil {
+		return err
+	}
+
+	for k, v := range raw {
+		// Most headers are string because they have only one value.
+		var s string
+		if err := json.Unmarshal(v, &s); err == nil {
+			textproto.MIMEHeader(*h).Set(k, s)
+			continue
+		}
+
+		// If it's not a string, it must be an array of strings.
+		var a []string
+		if err := json.Unmarshal(v, &a); err != nil {
+			return fmt.Errorf("pmapi: attachment header field is neither a string nor an array of strings: %v", err)
+		}
+		for _, vv := range a {
+			textproto.MIMEHeader(*h).Add(k, vv)
+		}
+	}
+
+	return nil
+}
+
+// Attachment represents a message attachment.
+type Attachment struct {
+	ID         string `json:",omitempty"`
+	MessageID  string `json:",omitempty"` // msg v3 ???
+	Name       string `json:",omitempty"`
+	Size       int64  `json:",omitempty"`
+	MIMEType   string `json:",omitempty"`
+	ContentID  string `json:",omitempty"`
+	KeyPackets string `json:",omitempty"`
+	Signature  string `json:",omitempty"`
+
+	Header textproto.MIMEHeader `json:"-"`
+}
+
+// Define a new type to prevent MarshalJSON/UnmarshalJSON infinite loops.
+type attachment Attachment
+
+type rawAttachment struct {
+	attachment
+
+	Header header `json:"Headers,omitempty"`
+}
+
+func (a *Attachment) MarshalJSON() ([]byte, error) {
+	var raw rawAttachment
+	raw.attachment = attachment(*a)
+
+	if a.Header != nil {
+		raw.Header = header(a.Header)
+	}
+
+	return json.Marshal(&raw)
+}
+
+func (a *Attachment) UnmarshalJSON(b []byte) error {
+	var raw rawAttachment
+	if err := json.Unmarshal(b, &raw); err != nil {
+		return err
+	}
+
+	*a = Attachment(raw.attachment)
+
+	if raw.Header != nil {
+		a.Header = textproto.MIMEHeader(raw.Header)
+	}
+
+	return nil
+}
+
+// Decrypt decrypts this attachment's data from r using the keys from kr.
+func (a *Attachment) Decrypt(r io.Reader, kr *pmcrypto.KeyRing) (decrypted io.Reader, err error) {
+	keyPackets, err := base64.StdEncoding.DecodeString(a.KeyPackets)
+	if err != nil {
+		return
+	}
+	return decryptAttachment(kr, keyPackets, r)
+}
+
+// Encrypt encrypts an attachment.
+func (a *Attachment) Encrypt(kr *pmcrypto.KeyRing, att io.Reader) (encrypted io.Reader, err error) {
+	return encryptAttachment(kr, att, a.Name)
+}
+
+func (a *Attachment) DetachedSign(kr *pmcrypto.KeyRing, att io.Reader) (signed io.Reader, err error) {
+	return signAttachment(kr, att)
+}
+
+type CreateAttachmentRes struct {
+	Res
+
+	Attachment *Attachment
+}
+
+func writeAttachment(w *multipart.Writer, att *Attachment, r io.Reader, sig io.Reader) (err error) {
+	// Create metadata fields.
+	if err = w.WriteField("Filename", att.Name); err != nil {
+		return
+	}
+	if err = w.WriteField("MessageID", att.MessageID); err != nil {
+		return
+	}
+	if err = w.WriteField("MIMEType", att.MIMEType); err != nil {
+		return
+	}
+
+	if err = w.WriteField("ContentID", att.ContentID); err != nil {
+		return
+	}
+
+	// And send attachment data.
+	ff, err := w.CreateFormFile("DataPacket", "DataPacket.pgp")
+	if err != nil {
+		return
+	}
+	if _, err = io.Copy(ff, r); err != nil {
+		return
+	}
+
+	// And send attachment data.
+	sigff, err := w.CreateFormFile("Signature", "Signature.pgp")
+	if err != nil {
+		return
+	}
+
+	if _, err = io.Copy(sigff, sig); err != nil {
+		return
+	}
+
+	return err
+}
+
+// CreateAttachment uploads an attachment. It must be already encrypted and contain a MessageID.
+//
+// The returned created attachment contains the new attachment ID and its size.
+func (c *Client) CreateAttachment(att *Attachment, r io.Reader, sig io.Reader) (created *Attachment, err error) {
+	req, w, err := NewMultipartRequest("POST", "/attachments")
+	if err != nil {
+		return
+	}
+
+	// We will write the request as long as it is sent to the API.
+	var res CreateAttachmentRes
+	done := make(chan error, 1)
+	go (func() {
+		done <- c.DoJSON(req, &res)
+	})()
+
+	if err = writeAttachment(w.Writer, att, r, sig); err != nil {
+		return
+	}
+	_ = w.Close()
+
+	if err = <-done; err != nil {
+		return
+	}
+	if err = res.Err(); err != nil {
+		return
+	}
+
+	created = res.Attachment
+	return
+}
+
+type UpdateAttachmentSignatureReq struct {
+	Signature string
+}
+
+func (c *Client) UpdateAttachmentSignature(attachmentID, signature string) (err error) {
+	updateReq := &UpdateAttachmentSignatureReq{signature}
+	req, err := NewJSONRequest("PUT", "/attachments/"+attachmentID+"/signature", updateReq)
+	if err != nil {
+		return
+	}
+
+	var res Res
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+
+	return
+}
+
+// DeleteAttachment removes an attachment. message is the message ID, att is the attachment ID.
+func (c *Client) DeleteAttachment(attID string) (err error) {
+	req, err := NewRequest("DELETE", "/attachments/"+attID, nil)
+	if err != nil {
+		return
+	}
+
+	var res Res
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+
+	err = res.Err()
+	return
+}
+
+// GetAttachment gets an attachment's content. The returned data is encrypted.
+func (c *Client) GetAttachment(id string) (att io.ReadCloser, err error) {
+	if id == "" {
+		err = errors.New("pmapi: cannot get an attachment with an empty id")
+		return
+	}
+
+	req, err := NewRequest("GET", "/attachments/"+id, nil)
+	if err != nil {
+		return
+	}
+
+	res, err := c.Do(req, true)
+	if err != nil {
+		return
+	}
+
+	att = res.Body
+	return
+}

pkg/pmapi/attachments_test.go

@@ -0,0 +1,222 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"mime"
+	"mime/multipart"
+	"net/http"
+	"net/textproto"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var testAttachment = &Attachment{
+	ID:         "y6uKIlc2HdoHPAwPSrvf7dXoZNMYvBgxshYUN67cY5DJjL2O8NYewuvGHcYvCfd8LpEoAI_GdymO0Jr0mHlsEw==",
+	Name:       "croutonmail.txt",
+	Size:       77,
+	MIMEType:   "text/plain",
+	KeyPackets: "wcBMA0fcZ7XLgmf2AQgAiRsOlnm1kSB4/lr7tYe6pBsRGn10GqwUhrwU5PMKOHdCgnO12jO3y3CzP0Yl/jGhAYja9wLDqH8X0sk3tY32u4Sb1Qe5IuzggAiCa4dwOJj5gEFMTHMzjIMPHR7A70XqUxMhmILye8V4KRm/j4c1sxbzA1rM3lYBumQuB5l/ck0Kgt4ZqxHVXHK5Q1l65FHhSXRj8qnunasHa30TYNzP8nmBA8BinnJxpiQ7FGc2umnUhgkFtjm5ixu9vyjr9ukwDTbwAXXfmY+o7tK7kqIXJcmTL6k2UeC6Mz1AagQtRCRtU+bv/3zGojq/trZo9lom3naIeQYa36Ketmcpj2Qwjg==",
+	Header: textproto.MIMEHeader{
+		"Content-Description": {"You'll never believe what's in this text file"},
+		"X-Mailer":            {"Microsoft Outlook 15.0", "Microsoft Live Mail 42.0"},
+	},
+	MessageID: "h3CD-DT7rLoAw1vmpcajvIPAl-wwDfXR2MHtWID3wuQURDBKTiGUAwd6E2WBbS44QQKeXImW-axm6X0hAfcVCA==",
+}
+
+const testAttachmentJSON = `{
+    "ID": "y6uKIlc2HdoHPAwPSrvf7dXoZNMYvBgxshYUN67cY5DJjL2O8NYewuvGHcYvCfd8LpEoAI_GdymO0Jr0mHlsEw==",
+    "Name": "croutonmail.txt",
+    "Size": 77,
+    "MIMEType": "text/plain",
+    "KeyPackets": "wcBMA0fcZ7XLgmf2AQgAiRsOlnm1kSB4/lr7tYe6pBsRGn10GqwUhrwU5PMKOHdCgnO12jO3y3CzP0Yl/jGhAYja9wLDqH8X0sk3tY32u4Sb1Qe5IuzggAiCa4dwOJj5gEFMTHMzjIMPHR7A70XqUxMhmILye8V4KRm/j4c1sxbzA1rM3lYBumQuB5l/ck0Kgt4ZqxHVXHK5Q1l65FHhSXRj8qnunasHa30TYNzP8nmBA8BinnJxpiQ7FGc2umnUhgkFtjm5ixu9vyjr9ukwDTbwAXXfmY+o7tK7kqIXJcmTL6k2UeC6Mz1AagQtRCRtU+bv/3zGojq/trZo9lom3naIeQYa36Ketmcpj2Qwjg==",
+    "Headers": {
+        "content-description": "You'll never believe what's in this text file",
+        "x-mailer": [
+            "Microsoft Outlook 15.0",
+            "Microsoft Live Mail 42.0"
+        ]
+    }
+}
+`
+
+const testAttachmentCleartext = `cc,
+dille.
+`
+
+const testAttachmentEncrypted = `wcBMA0fcZ7XLgmf2AQf/cHhfDRM9zlIuBi+h2W6DKjbbyIHMkgF6ER3JEvn/tSruUH8KTGt0N7Z+a80FFMCuXn1Y1I/nW7MVrNhGuJZAF4OymD8ugvuoAMIQX0eCYEpPXzRIWJBZg82AuowmFMsv8Dgvq4bTZq4cttI3CZcxKUNXuAearmNpmgplUKWj5USmRXK4iGB3VFGjidXkxbElrP4fD5A/rfEZ5aJgCsegqcXxX3MEjWXi9pFzgd/9phOvl1ZFm9U9hNoVAW3QsgmVeihnKaDZUyf2Qsigij21QKAUxw9U3y89eTUIqZAcmIgqeDujA3RWBgJwjtY/lOyhEmkf3AWKzehvf1xtJmCWDtJLAekuH+JfAtTQfMs5nf4zYtMahGbMkwy3Uz/jeEMYdzWY5WvshkbwvaxpqFC+11cqMLBvxik39i1xf+RORZF/91jGMCL9Z9dRMcgB`
+
+const testCreateAttachmentBody = `{
+    "Code": 1000,
+    "Attachment": {"ID": "y6uKIlc2HdoHPAwPSrvf7dXoZNMYvBgxshYUN67cY5DJjL2O8NYewuvGHcYvCfd8LpEoAI_GdymO0Jr0mHlsEw=="}
+}`
+
+const testDeleteAttachmentBody = `{
+    "Code": 1000
+}`
+
+func TestAttachment_UnmarshalJSON(t *testing.T) {
+	att := new(Attachment)
+	if err := json.Unmarshal([]byte(testAttachmentJSON), att); err != nil {
+		t.Fatal("Expected no error while unmarshaling JSON, got:", err)
+	}
+
+	att.MessageID = testAttachment.MessageID // This isn't in the JSON object
+
+	if !reflect.DeepEqual(testAttachment, att) {
+		t.Errorf("Invalid attachment: expected %+v but got %+v", testAttachment, att)
+	}
+}
+
+func TestClient_CreateAttachment(t *testing.T) {
+	s, c := newTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		Ok(t, checkMethodAndPath(r, "POST", "/attachments"))
+
+		contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
+		if err != nil {
+			t.Error("Expected no error while parsing request content type, got:", err)
+		}
+		if contentType != "multipart/form-data" {
+			t.Errorf("Invalid request content type: expected %v but got %v", "multipart/form-data", contentType)
+		}
+
+		mr := multipart.NewReader(r.Body, params["boundary"])
+		form, err := mr.ReadForm(10 * 1024)
+		if err != nil {
+			t.Error("Expected no error while parsing request form, got:", err)
+		}
+		defer Ok(t, form.RemoveAll())
+
+		if form.Value["Filename"][0] != testAttachment.Name {
+			t.Errorf("Invalid attachment filename: expected %v but got %v", testAttachment.Name, form.Value["Filename"][0])
+		}
+		if form.Value["MessageID"][0] != testAttachment.MessageID {
+			t.Errorf("Invalid attachment message id: expected %v but got %v", testAttachment.MessageID, form.Value["MessageID"][0])
+		}
+		if form.Value["MIMEType"][0] != testAttachment.MIMEType {
+			t.Errorf("Invalid attachment message id: expected %v but got %v", testAttachment.MIMEType, form.Value["MIMEType"][0])
+		}
+
+		dataFile, err := form.File["DataPacket"][0].Open()
+		if err != nil {
+			t.Error("Expected no error while opening packets file, got:", err)
+		}
+		defer Ok(t, dataFile.Close())
+
+		b, err := ioutil.ReadAll(dataFile)
+		if err != nil {
+			t.Error("Expected no error while reading packets file, got:", err)
+		}
+		if string(b) != testAttachmentCleartext {
+			t.Errorf("Invalid attachment packets: expected %v but got %v", testAttachment.KeyPackets, string(b))
+		}
+
+		fmt.Fprint(w, testCreateAttachmentBody)
+	}))
+	defer s.Close()
+
+	r := strings.NewReader(testAttachmentCleartext) // In reality, this thing is encrypted
+	created, err := c.CreateAttachment(testAttachment, r, strings.NewReader(""))
+	if err != nil {
+		t.Fatal("Expected no error while creating attachment, got:", err)
+	}
+
+	if created.ID != testAttachment.ID {
+		t.Errorf("Invalid attachment id: expected %v but got %v", testAttachment.ID, created.ID)
+	}
+}
+
+func TestClient_DeleteAttachment(t *testing.T) {
+	s, c := newTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		Ok(t, checkMethodAndPath(r, "DELETE", "/attachments/"+testAttachment.ID))
+
+		b := &bytes.Buffer{}
+		if n, _ := b.ReadFrom(r.Body); n != 0 {
+			t.Fatal("expected no body but have: ", b.String())
+		}
+
+		fmt.Fprint(w, testDeleteAttachmentBody)
+	}))
+	defer s.Close()
+
+	err := c.DeleteAttachment(testAttachment.ID)
+	if err != nil {
+		t.Fatal("Expected no error while deleting attachment, got:", err)
+	}
+}
+
+func TestClient_GetAttachment(t *testing.T) {
+	s, c := newTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		Ok(t, checkMethodAndPath(r, "GET", "/attachments/"+testAttachment.ID))
+
+		fmt.Fprint(w, testAttachmentCleartext)
+	}))
+	defer s.Close()
+
+	r, err := c.GetAttachment(testAttachment.ID)
+	if err != nil {
+		t.Fatal("Expected no error while getting attachment, got:", err)
+	}
+	defer r.Close() //nolint[errcheck]
+
+	// In reality, r contains encrypted data
+	b, err := ioutil.ReadAll(r)
+	if err != nil {
+		t.Fatal("Expected no error while reading attachment, got:", err)
+	}
+
+	if string(b) != testAttachmentCleartext {
+		t.Errorf("Invalid attachment data: expected %q but got %q", testAttachmentCleartext, string(b))
+	}
+}
+
+func TestAttachment_Encrypt(t *testing.T) {
+	data := bytes.NewBufferString(testAttachmentCleartext)
+	r, err := testAttachment.Encrypt(testPublicKeyRing, data)
+	assert.Nil(t, err)
+	b, err := ioutil.ReadAll(r)
+	assert.Nil(t, err)
+
+	// Result is always different, so the best way is to test it by decrypting again.
+	// Another test for decrypting will help us to be sure it's working.
+	dataEnc := bytes.NewBuffer(b)
+	decryptAndCheck(t, dataEnc)
+}
+
+func TestAttachment_Decrypt(t *testing.T) {
+	dataBytes, _ := base64.StdEncoding.DecodeString(testAttachmentEncrypted)
+	dataReader := bytes.NewBuffer(dataBytes)
+	decryptAndCheck(t, dataReader)
+}
+
+func decryptAndCheck(t *testing.T, data io.Reader) {
+	r, err := testAttachment.Decrypt(data, testPrivateKeyRing)
+	assert.Nil(t, err)
+	b, err := ioutil.ReadAll(r)
+	assert.Nil(t, err)
+	assert.Equal(t, testAttachmentCleartext, string(b))
+}

pkg/pmapi/auth.go

@@ -0,0 +1,506 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"crypto/subtle"
+	"encoding/base64"
+	"errors"
+	"net/http"
+	"strings"
+	"time"
+
+	pmcrypto "github.com/ProtonMail/gopenpgp/crypto"
+	"github.com/ProtonMail/proton-bridge/pkg/srp"
+)
+
+var ErrBad2FACode = errors.New("incorrect 2FA code")
+var ErrBad2FACodeTryAgain = errors.New("incorrect 2FA code: please try again")
+
+type AuthInfoReq struct {
+	Username string
+}
+
+type U2FInfo struct {
+	Challenge      string
+	RegisteredKeys []struct {
+		Version   string
+		KeyHandle string
+	}
+}
+
+type TwoFactorInfo struct {
+	Enabled int // 0 for disabled, 1 for OTP, 2 for U2F, 3 for both.
+	TOTP    int
+	U2F     U2FInfo
+}
+
+func (twoFactor *TwoFactorInfo) hasTwoFactor() bool {
+	return twoFactor.Enabled > 0
+}
+
+// AuthInfo contains data used when authenticating a user. It should be
+// provided to Client.Auth(). Each AuthInfo can be used for only one login attempt.
+type AuthInfo struct {
+	TwoFA *TwoFactorInfo `json:"2FA,omitempty"`
+
+	version         int
+	salt            string
+	modulus         string
+	srpSession      string
+	serverEphemeral string
+}
+
+func (a *AuthInfo) HasTwoFactor() bool {
+	if a.TwoFA == nil {
+		return false
+	}
+	return a.TwoFA.hasTwoFactor()
+}
+
+type AuthInfoRes struct {
+	Res
+	AuthInfo
+
+	Modulus         string
+	ServerEphemeral string
+	Version         int
+	Salt            string
+	SRPSession      string
+}
+
+func (res *AuthInfoRes) getAuthInfo() *AuthInfo {
+	info := &res.AuthInfo
+
+	// Some fields in AuthInfo are private, so we need to copy them from AuthRes
+	// (private fields cannot be populated by json).
+	info.version = res.Version
+	info.salt = res.Salt
+	info.modulus = res.Modulus
+	info.srpSession = res.SRPSession
+	info.serverEphemeral = res.ServerEphemeral
+
+	return info
+}
+
+type AuthReq struct {
+	Username        string
+	ClientProof     string
+	ClientEphemeral string
+	SRPSession      string
+}
+
+// Auth contains data after a successful authentication. It should be provided to Client.Unlock().
+type Auth struct {
+	accessToken  string // Read from AuthRes.
+	ExpiresIn    int
+	Scope        string
+	uid          string // Read from AuthRes.
+	RefreshToken string
+	KeySalt      string
+	EventID      string
+	PasswordMode int
+	TwoFA        *TwoFactorInfo `json:"2FA,omitempty"`
+}
+
+func (s *Auth) UID() string {
+	return s.uid
+}
+
+func (s *Auth) HasTwoFactor() bool {
+	if s.TwoFA == nil {
+		return false
+	}
+	return s.TwoFA.hasTwoFactor()
+}
+
+func (s *Auth) HasMailboxPassword() bool {
+	return s.PasswordMode == 2
+}
+
+func (s *Auth) hasFullScope() bool {
+	return strings.Contains(s.Scope, "full")
+}
+
+type AuthRes struct {
+	Res
+	Auth
+
+	AccessToken string
+	TokenType   string
+	UID         string
+
+	ServerProof string
+}
+
+func (res *AuthRes) getAuth() *Auth {
+	auth := &res.Auth
+
+	// Some fields in Auth are private, so we need to copy them from AuthRes
+	// (private fields cannot be populated by json).
+	auth.accessToken = res.AccessToken
+	auth.uid = res.UID
+
+	return auth
+}
+
+type Auth2FAReq struct {
+	TwoFactorCode string
+
+	// Prepared for U2F:
+	// U2F U2FRequest
+}
+
+type Auth2FA struct {
+	Scope string
+}
+
+type Auth2FARes struct {
+	Res
+
+	Scope string
+}
+
+func (res *Auth2FARes) getAuth2FA() *Auth2FA {
+	return &Auth2FA{
+		Scope: res.Scope,
+	}
+}
+
+type AuthRefreshReq struct {
+	ResponseType string
+	GrantType    string
+	RefreshToken string
+	UID          string
+	RedirectURI  string
+	State        string
+}
+
+// SetAuths sets auths channel.
+func (c *Client) SetAuths(auths chan<- *Auth) {
+	c.auths = auths
+}
+
+// AuthInfo gets authentication info for a user.
+func (c *Client) AuthInfo(username string) (info *AuthInfo, err error) {
+	infoReq := &AuthInfoReq{
+		Username: username,
+	}
+
+	req, err := NewJSONRequest("POST", "/auth/info", infoReq)
+	if err != nil {
+		return
+	}
+
+	var infoRes AuthInfoRes
+	if err = c.DoJSON(req, &infoRes); err != nil {
+		return
+	}
+
+	info, err = infoRes.getAuthInfo(), infoRes.Err()
+
+	return
+}
+
+func srpProofsFromInfo(info *AuthInfo, username, password string, fallbackVersion int) (proofs *srp.SrpProofs, err error) {
+	version := info.version
+	if version == 0 {
+		version = fallbackVersion
+	}
+
+	srpAuth, err := srp.NewSrpAuth(version, username, password, info.salt, info.modulus, info.serverEphemeral)
+	if err != nil {
+		return
+	}
+
+	proofs, err = srpAuth.GenerateSrpProofs(2048)
+	return
+}
+
+func (c *Client) tryAuth(username, password string, info *AuthInfo, fallbackVersion int) (res *AuthRes, err error) {
+	proofs, err := srpProofsFromInfo(info, username, password, fallbackVersion)
+	if err != nil {
+		return
+	}
+
+	authReq := &AuthReq{
+		Username:        username,
+		ClientEphemeral: base64.StdEncoding.EncodeToString(proofs.ClientEphemeral),
+		ClientProof:     base64.StdEncoding.EncodeToString(proofs.ClientProof),
+		SRPSession:      info.srpSession,
+	}
+
+	req, err := NewJSONRequest("POST", "/auth", authReq)
+	if err != nil {
+		return
+	}
+
+	var authRes AuthRes
+	if err = c.DoJSON(req, &authRes); err != nil {
+		return
+	}
+
+	if err = authRes.Err(); err != nil {
+		return
+	}
+
+	serverProof, err := base64.StdEncoding.DecodeString(authRes.ServerProof)
+	if err != nil {
+		return
+	}
+
+	if subtle.ConstantTimeCompare(proofs.ExpectedServerProof, serverProof) != 1 {
+		return nil, errors.New("pmapi: bad server proof")
+	}
+
+	res, err = &authRes, authRes.Err()
+	return res, err
+}
+
+func (c *Client) tryFullAuth(username, password string, fallbackVersion int) (info *AuthInfo, authRes *AuthRes, err error) {
+	info, err = c.AuthInfo(username)
+	if err != nil {
+		return
+	}
+	authRes, err = c.tryAuth(username, password, info, fallbackVersion)
+	return
+}
+
+// Auth will authenticate a user.
+func (c *Client) Auth(username, password string, info *AuthInfo) (auth *Auth, err error) {
+	if info == nil {
+		if info, err = c.AuthInfo(username); err != nil {
+			return
+		}
+	}
+
+	authRes, err := c.tryAuth(username, password, info, 2)
+	if err != nil && info.version == 0 && srp.CleanUserName(username) != strings.ToLower(username) {
+		info, authRes, err = c.tryFullAuth(username, password, 1)
+	}
+	if err != nil && info.version == 0 {
+		_, authRes, err = c.tryFullAuth(username, password, 0)
+	}
+	if err != nil {
+		return
+	}
+
+	auth = authRes.getAuth()
+	c.uid = auth.UID()
+	c.accessToken = auth.accessToken
+
+	if c.auths != nil {
+		c.auths <- auth
+	}
+
+	if c.tokenManager != nil {
+		c.tokenManager.SetToken(c.userID, c.uid+":"+auth.RefreshToken)
+		c.log.Info("Set token from auth " + c.uid + ":" + auth.RefreshToken)
+	}
+
+	// Auth has to be fully unlocked to get key salt. During `Auth` it can happen
+	// only to accounts without 2FA. For 2FA accounts, it's done in `Auth2FA`.
+	if auth.hasFullScope() {
+		err = c.setKeySaltToAuth(auth)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	c.expiresAt = time.Now().Add(time.Duration(auth.ExpiresIn) * time.Second)
+	return auth, err
+}
+
+// Auth2FA will authenticate a user into full scope.
+// `Auth` struct contains method `HasTwoFactor` deciding whether this has to be done.
+func (c *Client) Auth2FA(twoFactorCode string, auth *Auth) (*Auth2FA, error) {
+	auth2FAReq := &Auth2FAReq{
+		TwoFactorCode: twoFactorCode,
+	}
+
+	req, err := NewJSONRequest("POST", "/auth/2fa", auth2FAReq)
+	if err != nil {
+		return nil, err
+	}
+
+	var auth2FARes Auth2FARes
+	if err := c.DoJSON(req, &auth2FARes); err != nil {
+		return nil, err
+	}
+
+	if err := auth2FARes.Err(); err != nil {
+		switch auth2FARes.StatusCode {
+		case http.StatusUnauthorized:
+			return nil, ErrBad2FACode
+		case http.StatusUnprocessableEntity:
+			return nil, ErrBad2FACodeTryAgain
+		default:
+			return nil, err
+		}
+	}
+
+	if err := c.setKeySaltToAuth(auth); err != nil {
+		return nil, err
+	}
+
+	return auth2FARes.getAuth2FA(), nil
+}
+
+func (c *Client) setKeySaltToAuth(auth *Auth) error {
+	// KeySalt already set up, no need to do it again.
+	if auth.KeySalt != "" {
+		return nil
+	}
+
+	user, err := c.CurrentUser()
+	if err != nil {
+		return err
+	}
+	salts, err := c.GetKeySalts()
+	if err != nil {
+		return err
+	}
+	for _, s := range salts {
+		if s.ID == user.KeyRing().FirstKeyID {
+			auth.KeySalt = s.KeySalt
+			break
+		}
+	}
+	return nil
+}
+
+// Unlock decrypts the key ring.
+// If the password is invalid, IsUnlockError(err) will return true.
+func (c *Client) Unlock(password string) (kr *pmcrypto.KeyRing, err error) {
+	if _, err = c.CurrentUser(); err != nil {
+		return
+	}
+
+	c.keyLocker.Lock()
+	defer c.keyLocker.Unlock()
+
+	kr = c.user.KeyRing()
+	if err = unlockKeyRingNoErrorWhenAlreadyUnlocked(kr, []byte(password)); err != nil {
+		return
+	}
+
+	c.kr = kr
+	return kr, err
+}
+
+// AuthRefresh will refresh an expired access token.
+func (c *Client) AuthRefresh(uidAndRefreshToken string) (auth *Auth, err error) {
+	// If we don't yet have a saved access token, save this one in case the refresh fails!
+	// That way we can try again later (see handleUnauthorizedStatus).
+	if c.tokenManager != nil {
+		currentAccessToken := c.tokenManager.GetToken(c.userID)
+		if currentAccessToken == "" {
+			c.log.WithField("token", uidAndRefreshToken).
+				Info("Currently have no access token, setting given one")
+			c.tokenManager.SetToken(c.userID, uidAndRefreshToken)
+		}
+	}
+
+	split := strings.Split(uidAndRefreshToken, ":")
+	if len(split) != 2 {
+		err = ErrInvalidToken
+		return
+	}
+
+	refreshReq := &AuthRefreshReq{
+		ResponseType: "token",
+		GrantType:    "refresh_token",
+		RefreshToken: split[1],
+		UID:          split[0],
+		RedirectURI:  "https://protonmail.ch",
+		State:        "random_string",
+	}
+
+	// UID must be set for `x-pm-uid` header field, see backend-communication#11
+	c.uid = split[0]
+
+	req, err := NewJSONRequest("POST", "/auth/refresh", refreshReq)
+	if err != nil {
+		return
+	}
+
+	var res AuthRes
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+	if err = res.Err(); err != nil {
+		return
+	}
+
+	auth = res.getAuth()
+	// UID should never change after auth, see backend-communication#11
+	auth.uid = c.uid
+	if c.auths != nil {
+		c.auths <- auth
+	}
+
+	c.uid = auth.UID()
+	c.accessToken = auth.accessToken
+
+	if c.tokenManager != nil {
+		c.tokenManager.SetToken(c.userID, c.uid+":"+res.RefreshToken)
+		c.log.Info("Set token from auth refresh " + c.uid + ":" + res.RefreshToken)
+	}
+
+	c.expiresAt = time.Now().Add(time.Duration(auth.ExpiresIn) * time.Second)
+	return auth, err
+}
+
+// Logout logs the current user out.
+func (c *Client) Logout() (err error) {
+	req, err := NewRequest("DELETE", "/auth", nil)
+	if err != nil {
+		return
+	}
+
+	var res Res
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+
+	if err = res.Err(); err != nil {
+		return
+	}
+
+	// This can trigger a deadlock! We don't want to do it if the above requests failed (GODT-154).
+	// That's why it's not in the deferred statement above.
+	if c.auths != nil {
+		c.auths <- nil
+	}
+
+	// This should ideally be deferred at the top of this method so that it is executed
+	// regardless of what happens, but we currently don't have a way to prevent ourselves
+	// from using a logged out client. So for now, it's down here, as it was in Charles release.
+	// defer func() {
+	c.uid = ""
+	c.accessToken = ""
+	c.kr = nil
+	// c.addresses = nil
+	c.user = nil
+	if c.tokenManager != nil {
+		c.tokenManager.SetToken(c.userID, "")
+	}
+	// }()
+
+	return err
+}

pkg/pmapi/auth_test.go

@@ -0,0 +1,366 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"encoding/json"
+	"math/rand"
+	"net/http"
+	"testing"
+	"time"
+
+	pmcrypto "github.com/ProtonMail/gopenpgp/crypto"
+	"github.com/ProtonMail/proton-bridge/pkg/srp"
+
+	"github.com/sirupsen/logrus"
+
+	a "github.com/stretchr/testify/assert"
+	r "github.com/stretchr/testify/require"
+)
+
+var aLongTimeAgo = time.Unix(233431200, 0)
+
+var testIdentity = &pmcrypto.Identity{
+	Name:  "UserID",
+	Email: "",
+}
+
+const (
+	testUsername    = "jason"
+	testAPIPassword = "apple"
+
+	testUID             = "729ad6012421d67ad26950dc898bebe3a6e3caa2" //nolint[gosec]
+	testAccessToken     = "de0423049b44243afeec7d9c1d99be7b46da1e8a" //nolint[gosec]
+	testAccessTokenOld  = "feb3159ac63fb05119bcf4480d939278aa746926" //nolint[gosec]
+	testRefreshToken    = "a49b98256745bb497bec20e9b55f5de16f01fb52" //nolint[gosec]
+	testRefreshTokenNew = "b894b4c4f20003f12d486900d8b88c7d68e67235" //nolint[gosec]
+)
+
+var testAuthInfo = &AuthInfo{
+	TwoFA: &TwoFactorInfo{TOTP: 1},
+
+	version:         4,
+	salt:            "yKlc5/CvObfoiw==",
+	modulus:         "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA256\n\nW2z5HBi8RvsfYzZTS7qBaUxxPhsfHJFZpu3Kd6s1JafNrCCH9rfvPLrfuqocxWPgWDH2R8neK7PkNvjxto9TStuY5z7jAzWRvFWN9cQhAKkdWgy0JY6ywVn22+HFpF4cYesHrqFIKUPDMSSIlWjBVmEJZ/MusD44ZT29xcPrOqeZvwtCffKtGAIjLYPZIEbZKnDM1Dm3q2K/xS5h+xdhjnndhsrkwm9U9oyA2wxzSXFL+pdfj2fOdRwuR5nW0J2NFrq3kJjkRmpO/Genq1UW+TEknIWAb6VzJJJA244K/H8cnSx2+nSNZO3bbo6Ys228ruV9A8m6DhxmS+bihN3ttQ==\n-----BEGIN PGP SIGNATURE-----\nVersion: ProtonMail\nComment: https://protonmail.com\n\nwl4EARYIABAFAlwB1j0JEDUFhcTpUY8mAAD8CgEAnsFnF4cF0uSHKkXa1GIa\nGO86yMV4zDZEZcDSJo0fgr8A/AlupGN9EdHlsrZLmTA1vhIx+rOgxdEff28N\nkvNM7qIK\n=q6vu\n-----END PGP SIGNATURE-----\n",
+	srpSession:      "9b2946bbd9055f17c34940abdce0c3d3",
+	serverEphemeral: "5tfigcLKoM0DPWYB+EqYE7QlqsiT63iOVlO5ZX0lTMEILSsrRdVCYrN8L3zkinsAjUZ/cx5wIS7N05k66uZb+ZE3lFOJS2s1BkqLvCrGxYL0e3n5YAnzHYlvCCJKXw/sK57ntfF1OOoblBXX6dw5LjeeDglEep2/DaE0TjD8WUpq4Ls2HlQGn9wrC7dFO2lJXsMhRffxKghiOsdvCLXDmwXginzn/LFezA8KrDsWOBSEGntwpg3s1xFj5h8BqtRHvC0igmoscqgw+3GCMTJ0NZAQ/L+5aJ/0ccL0WBK208ltCNl+/X6Sz0kpyvOP4RqFJhC1auVDJ9AjZQYSYZ1NEQ==",
+}
+
+// testAuth has default values which are adjusted in each test.
+var testAuth = &Auth{
+	EventID:      "NcKPtU5eMNPMrDkIMbEJrgMtC9yQ7Xc5ZBT-tB3UtV1rZ324RWfCIdBI758q0UnsfywS8CkNenIQlWLIX_dUng==",
+	ExpiresIn:    86400,
+	RefreshToken: "feb3159ac63fb05119bcf4480d939278aa746926",
+	Scope:        "full mail payments reset keys",
+
+	accessToken: testAccessToken,
+	uid:         testUID,
+}
+
+var testAuth2FA = &Auth2FA{
+	Scope: "full mail payments reset keys",
+}
+
+var testAuthRefreshReq = AuthRefreshReq{
+	ResponseType: "token",
+	GrantType:    "refresh_token",
+	RefreshToken: testRefreshToken,
+	UID:          testUID,
+	RedirectURI:  "https://protonmail.ch",
+	State:        "random_string",
+}
+
+var testAuthReq = AuthReq{
+	Username:        testUsername,
+	ClientProof:     "axfvYdl9iXZjY6zQ+hBYmY7X3TDc/9JtSvrmyZXhDxjxkXB3Hro27t1KItmFIJloItY5sLZDs0eEEZJI34oFZD4ViSG0kfB7ZXcCZ9Jse+U5OFu4vdnPTGolnSofRMEs1NR6ePXzH7mQ10qoq43ity3ve2vmhQNuJNlHAPynKf2WqKOgxq7mmkBzEpXES4mIhwwgVbOygKcUSvguz5E5g13ATF0ZX2d9SJWAbZ262Tks+h99Cdk/dOfgLQhr0nO/r0cpwP84W2RWU2Q34LNkKuuQHkjmxelgBleGq54tCbhoCAYPP6vapgrQjNoVAC/dkjIIAoNL9bJSIynFM5znAA==",
+	ClientEphemeral: "mK+eSMosfZO/Cs5s+vcbjpsN7F8UAObwlKKnCy/z9FpoMRM2PfTe5ywLBgffmLYaapPq7XOxaqaj08kcZLHcM1fIA2JQZZTKPnESN1qAQztJ3/YHMI0op6yBgzx9803OjIznjCD2B3XBSMOHIG4oG0UwocsIX32hiMnYlMMkt8NGrityPlnmEbxpRna3fu9LEZ+v0uo6PjKCrO7+9E3uaMi64HadXBfyx2raBFFwA+yh7FvE7U+hl3AJclEre4d8pmfhMdxXze1soJI8fMuqaa07rY0r0rF5mLLTuqTIGRFkU1qG9loq9+IMsSwgkt1P3ghW63JK7Y6LWdDy0d6cAg==",
+	SRPSession:      "9b2946bbd9055f17c34940abdce0c3d3",
+}
+
+var testAuth2FAReq = Auth2FAReq{
+	TwoFactorCode: "424242",
+}
+
+func init() {
+	logrus.SetLevel(logrus.DebugLevel)
+	srp.RandReader = rand.New(rand.NewSource(42))
+}
+
+func TestClient_AuthInfo(t *testing.T) {
+	finish, c := newTestServerCallbacks(t,
+		func(tb testing.TB, w http.ResponseWriter, r *http.Request) string {
+			Ok(t, checkMethodAndPath(r, "POST", "/auth/info"))
+
+			var infoReq AuthInfoReq
+			Ok(t, json.NewDecoder(r.Body).Decode(&infoReq))
+			Equals(t, infoReq.Username, testUsername)
+
+			return "/auth/info/post_response.json"
+		},
+	)
+	defer finish()
+
+	info, err := c.AuthInfo(testCurrentUser.Name)
+	Ok(t, err)
+	Equals(t, testAuthInfo, info)
+}
+
+// TestClient_Auth reflects changes from proton/backend-communcation#3.
+func TestClient_Auth(t *testing.T) {
+	srp.RandReader = rand.New(rand.NewSource(42))
+	finish, c := newTestServerCallbacks(t,
+		func(tb testing.TB, w http.ResponseWriter, req *http.Request) string {
+			a.Nil(t, checkMethodAndPath(req, "POST", "/auth"))
+
+			var authReq AuthReq
+			r.Nil(t, json.NewDecoder(req.Body).Decode(&authReq))
+			r.Equal(t, testAuthReq, authReq)
+
+			return "/auth/post_response.json"
+		},
+		routeGetUsers,
+		routeGetAddresses,
+		routeGetSalts,
+	)
+	defer finish()
+
+	auth, err := c.Auth(testUsername, testAPIPassword, testAuthInfo)
+	r.Nil(t, err)
+
+	r.True(t, c.user.KeyRing().FirstKeyID != "", "Parsing First key ID issue")
+
+	exp := &Auth{}
+	*exp = *testAuth
+	exp.accessToken = testAccessToken
+	exp.RefreshToken = testRefreshToken
+	exp.KeySalt = "abc"
+	a.Equal(t, exp, auth)
+}
+
+func TestClient_Auth2FA(t *testing.T) {
+	finish, c := newTestServerCallbacks(t,
+		func(tb testing.TB, w http.ResponseWriter, r *http.Request) string {
+			Ok(t, checkMethodAndPath(r, "POST", "/auth/2fa"))
+
+			var info2FAReq Auth2FAReq
+			Ok(t, json.NewDecoder(r.Body).Decode(&info2FAReq))
+			Equals(t, info2FAReq.TwoFactorCode, testAuth2FAReq.TwoFactorCode)
+
+			return "/auth/2fa/post_response.json"
+		},
+		routeGetUsers,
+		routeGetAddresses,
+		routeGetSalts,
+	)
+	defer finish()
+
+	c.uid = testUID
+	c.accessToken = testAccessToken
+	auth2FA, err := c.Auth2FA(testAuth2FAReq.TwoFactorCode, testAuth)
+	Ok(t, err)
+
+	Equals(t, testAuth2FA, auth2FA)
+}
+
+func TestClient_Auth2FA_Fail(t *testing.T) {
+	finish, c := newTestServerCallbacks(t,
+		func(tb testing.TB, w http.ResponseWriter, r *http.Request) string {
+			Ok(t, checkMethodAndPath(r, "POST", "/auth/2fa"))
+
+			var info2FAReq Auth2FAReq
+			Ok(t, json.NewDecoder(r.Body).Decode(&info2FAReq))
+			Equals(t, info2FAReq.TwoFactorCode, testAuth2FAReq.TwoFactorCode)
+
+			return "/auth/2fa/post_401_bad_password.json"
+		},
+	)
+	defer finish()
+
+	c.uid = testUID
+	c.accessToken = testAccessToken
+	_, err := c.Auth2FA(testAuth2FAReq.TwoFactorCode, testAuth)
+	Equals(t, ErrBad2FACode, err)
+}
+
+func TestClient_Auth2FA_Retry(t *testing.T) {
+	finish, c := newTestServerCallbacks(t,
+		func(tb testing.TB, w http.ResponseWriter, r *http.Request) string {
+			Ok(t, checkMethodAndPath(r, "POST", "/auth/2fa"))
+
+			var info2FAReq Auth2FAReq
+			Ok(t, json.NewDecoder(r.Body).Decode(&info2FAReq))
+			Equals(t, info2FAReq.TwoFactorCode, testAuth2FAReq.TwoFactorCode)
+
+			return "/auth/2fa/post_422_bad_password.json"
+		},
+	)
+	defer finish()
+
+	c.uid = testUID
+	c.accessToken = testAccessToken
+	_, err := c.Auth2FA(testAuth2FAReq.TwoFactorCode, testAuth)
+	Equals(t, ErrBad2FACodeTryAgain, err)
+}
+
+func TestClient_Unlock(t *testing.T) {
+	finish, c := newTestServerCallbacks(t,
+		routeGetUsers,
+		routeGetAddresses,
+	)
+	defer finish()
+	c.uid = testUID
+	c.accessToken = testAccessToken
+
+	_, err := c.Unlock("wrong")
+	a.True(t, IsUnlockError(err), "expected error, pasword is wrong")
+
+	_, err = c.Unlock(testMailboxPassword)
+	a.Nil(t, err)
+	a.Equal(t, testUID, c.uid)
+	a.Equal(t, testAccessToken, c.accessToken)
+
+	// second try should not fail because there is an unlocked key already
+	_, err = c.Unlock("wrong")
+	a.Nil(t, err)
+}
+
+func TestClient_Unlock_EncPrivKey(t *testing.T) {
+	finish, c := newTestServerCallbacks(t,
+		routeGetUsers,
+		routeGetAddresses,
+	)
+	defer finish()
+	c.uid = testUID
+	c.accessToken = testAccessToken
+
+	_, err := c.Unlock(testMailboxPassword)
+	Ok(t, err)
+	Equals(t, testUID, c.uid)
+	Equals(t, testAccessToken, c.accessToken)
+}
+
+func routeAuthRefresh(tb testing.TB, w http.ResponseWriter, r *http.Request) string {
+	Ok(tb, checkMethodAndPath(r, "POST", "/auth/refresh"))
+	Ok(tb, checkHeader(r.Header, "x-pm-uid", testUID))
+
+	var refreshReq AuthRefreshReq
+	Ok(tb, json.NewDecoder(r.Body).Decode(&refreshReq))
+	Equals(tb, testAuthRefreshReq, refreshReq)
+
+	return "/auth/refresh/post_response.json"
+}
+
+// TestClient_AuthRefresh reflects changes from proton/backend-communcation#11.
+func TestClient_AuthRefresh(t *testing.T) {
+	finish, c := newTestServerCallbacks(t,
+		routeAuthRefresh,
+	)
+	defer finish()
+	c.uid = "" // Testing that we always send correct `x-pm-uid`.
+	c.accessToken = "oldToken"
+
+	auth, err := c.AuthRefresh(testUID + ":" + testRefreshToken)
+	Ok(t, err)
+
+	exp := &Auth{}
+	*exp = *testAuth
+	exp.accessToken = testAccessToken
+	exp.KeySalt = ""
+	exp.EventID = ""
+	exp.ExpiresIn = 360000
+	exp.RefreshToken = testRefreshTokenNew
+	Equals(t, exp, auth)
+}
+
+func routeAuthRefreshHasUID(tb testing.TB, w http.ResponseWriter, r *http.Request) string {
+	Ok(tb, checkMethodAndPath(r, "POST", "/auth/refresh"))
+	Ok(tb, checkHeader(r.Header, "x-pm-uid", testUID))
+
+	var refreshReq AuthRefreshReq
+	Ok(tb, json.NewDecoder(r.Body).Decode(&refreshReq))
+	Equals(tb, testAuthRefreshReq, refreshReq)
+
+	return "/auth/refresh/post_resp_has_uid.json"
+}
+
+// TestClient_AuthRefresh reflects changes from proton/backend-communcation#3.
+func TestClient_AuthRefresh_HasUID(t *testing.T) {
+	finish, c := newTestServerCallbacks(t,
+		routeAuthRefreshHasUID,
+	)
+	defer finish()
+	c.uid = testUID
+	c.accessToken = "oldToken"
+
+	auth, err := c.AuthRefresh(testUID + ":" + testRefreshToken)
+	Ok(t, err)
+
+	exp := &Auth{}
+	*exp = *testAuth
+	exp.accessToken = testAccessToken
+	exp.KeySalt = ""
+	exp.EventID = ""
+	exp.ExpiresIn = 360000
+	exp.RefreshToken = testRefreshTokenNew
+	Equals(t, exp, auth)
+}
+
+func TestClient_Logout(t *testing.T) {
+	finish, c := newTestServerCallbacks(t,
+		func(tb testing.TB, w http.ResponseWriter, r *http.Request) string {
+			Ok(t, checkMethodAndPath(r, "DELETE", "/auth"))
+			Ok(t, isAuthReq(r, testUID, testAccessToken))
+			return "auth/delete_response.json"
+		},
+	)
+	defer finish()
+	c.uid = testUID
+	c.accessToken = testAccessToken
+
+	Ok(t, c.Logout())
+}
+
+func TestClient_DoUnauthorized(t *testing.T) {
+	finish, c := newTestServerCallbacks(t,
+		func(tb testing.TB, w http.ResponseWriter, r *http.Request) string {
+			Ok(t, checkMethodAndPath(r, "GET", "/"))
+			return httpResponse(http.StatusUnauthorized)
+		},
+		routeAuthRefresh,
+		func(tb testing.TB, w http.ResponseWriter, r *http.Request) string {
+			Ok(t, checkMethodAndPath(r, "GET", "/"))
+			Ok(t, isAuthReq(r, testUID, testAccessToken))
+			return httpResponse(http.StatusOK)
+		},
+	)
+	defer finish()
+
+	c.uid = testUID
+	c.accessToken = testAccessTokenOld
+	c.expiresAt = aLongTimeAgo
+	c.tokenManager = NewTokenManager()
+	c.tokenManager.tokenMap[c.userID] = testUID + ":" + testRefreshToken
+
+	req, err := NewRequest("GET", "/", nil)
+	Ok(t, err)
+
+	res, err := c.Do(req, true)
+	Ok(t, err)
+
+	defer Ok(t, res.Body.Close())
+}

pkg/pmapi/auth_test_export.go

@@ -0,0 +1,23 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+// DANGEROUSLYSetUID SHOULD NOT be used!!! This is only for testing purposes.
+func (s *Auth) DANGEROUSLYSetUID(uid string) {
+	s.uid = uid
+}

pkg/pmapi/bugs.go

@@ -0,0 +1,217 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"archive/zip"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"net/textproto"
+	"runtime"
+	"strings"
+)
+
+// ClientType is required by API.
+const (
+	EmailClientType = iota + 1
+	VPNClientType
+)
+
+type reportAtt struct {
+	name, filename string
+	body           io.Reader
+}
+
+// ReportReq stores data for report.
+type ReportReq struct {
+	OS                string      `json:",omitempty"`
+	OSVersion         string      `json:",omitempty"`
+	Browser           string      `json:",omitempty"`
+	BrowserVersion    string      `json:",omitempty"`
+	BrowserExtensions string      `json:",omitempty"`
+	Resolution        string      `json:",omitempty"`
+	DisplayMode       string      `json:",omitempty"`
+	Client            string      `json:",omitempty"`
+	ClientVersion     string      `json:",omitempty"`
+	ClientType        int         `json:",omitempty"`
+	Title             string      `json:",omitempty"`
+	Description       string      `json:",omitempty"`
+	Username          string      `json:",omitempty"`
+	Email             string      `json:",omitempty"`
+	Country           string      `json:",omitempty"`
+	ISP               string      `json:",omitempty"`
+	Debug             string      `json:",omitempty"`
+	Attachments       []reportAtt `json:",omitempty"`
+}
+
+// AddAttachment to report.
+func (rep *ReportReq) AddAttachment(name, filename string, r io.Reader) {
+	rep.Attachments = append(rep.Attachments, reportAtt{name: name, filename: filename, body: r})
+}
+
+func writeMultipartReport(w *multipart.Writer, rep *ReportReq) error { // nolint[funlen]
+	fieldData := map[string]string{
+		"OS":                rep.OS,
+		"OSVersion":         rep.OSVersion,
+		"Browser":           rep.Browser,
+		"BrowserVersion":    rep.BrowserVersion,
+		"BrowserExtensions": rep.BrowserExtensions,
+		"Resolution":        rep.Resolution,
+		"DisplayMode":       rep.DisplayMode,
+		"Client":            rep.Client,
+		"ClientVersion":     rep.ClientVersion,
+		"ClientType":        "1",
+		"Title":             rep.Title,
+		"Description":       rep.Description,
+		"Username":          rep.Username,
+		"Email":             rep.Email,
+		"Country":           rep.Country,
+		"ISP":               rep.ISP,
+		"Debug":             rep.Debug,
+	}
+
+	for field, data := range fieldData {
+		if data == "" {
+			continue
+		}
+		if err := w.WriteField(field, data); err != nil {
+			return err
+		}
+	}
+
+	quoteEscaper := strings.NewReplacer("\\", "\\\\", `"`, "\\\"")
+
+	for _, att := range rep.Attachments {
+		h := make(textproto.MIMEHeader)
+		h.Set("Content-Disposition",
+			fmt.Sprintf(`form-data; name="%s"; filename="%s"`,
+				quoteEscaper.Replace(att.name), quoteEscaper.Replace(att.filename+".zip")))
+		h.Set("Content-Type", "application/octet-stream")
+		//h.Set("Content-Transfere-Encoding", "base64")
+		attWr, err := w.CreatePart(h)
+		if err != nil {
+			return err
+		}
+
+		zipArch := zip.NewWriter(attWr)
+		zipWr, err := zipArch.Create(att.filename)
+		//b64 := base64.NewEncoder(base64.StdEncoding, zipWr)
+		if err != nil {
+			return err
+		}
+		_, err = io.Copy(zipWr, att.body)
+		if err != nil {
+			return err
+		}
+		err = zipArch.Close()
+		//err = b64.Close()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Report sends request as json or multipart (if has attachment).
+func (c *Client) Report(rep ReportReq) (err error) {
+	rep.Client = c.config.ClientID
+	rep.ClientVersion = c.config.AppVersion
+	rep.ClientType = EmailClientType
+
+	var req *http.Request
+	var w *MultipartWriter
+	if len(rep.Attachments) > 0 {
+		req, w, err = NewMultipartRequest("POST", "/reports/bug")
+	} else {
+		req, err = NewJSONRequest("POST", "/reports/bug", rep)
+	}
+	if err != nil {
+		return
+	}
+
+	var res Res
+	done := make(chan error, 1)
+	go func() {
+		done <- c.DoJSON(req, &res)
+	}()
+
+	if w != nil {
+		err = writeMultipartReport(w.Writer, &rep)
+		if err != nil {
+			c.log.Errorln("report write: ", err)
+			return
+		}
+		err = w.Close()
+		if err != nil {
+			c.log.Errorln("report close: ", err)
+			return
+		}
+	}
+
+	if err = <-done; err != nil {
+		return
+	}
+
+	return res.Err()
+}
+
+// ReportBug is old. Use Report instead.
+func (c *Client) ReportBug(os, osVersion, title, description, username, email string) (err error) {
+	return c.ReportBugWithEmailClient(os, osVersion, title, description, username, email, "")
+}
+
+// ReportBugWithEmailClient is old. Use Report instead.
+func (c *Client) ReportBugWithEmailClient(os, osVersion, title, description, username, email, emailClient string) (err error) {
+	bugReq := ReportReq{
+		OS:          os,
+		OSVersion:   osVersion,
+		Browser:     emailClient,
+		Title:       title,
+		Description: description,
+		Username:    username,
+		Email:       email,
+	}
+
+	return c.Report(bugReq)
+}
+
+// ReportCrash is old. Use sentry instead.
+func (c *Client) ReportCrash(stacktrace string) (err error) {
+	crashReq := ReportReq{
+		Client:        c.config.ClientID,
+		ClientVersion: c.config.AppVersion,
+		ClientType:    EmailClientType,
+		OS:            runtime.GOOS,
+		Debug:         stacktrace,
+	}
+	req, err := NewJSONRequest("POST", "/reports/crash", crashReq)
+	if err != nil {
+		return
+	}
+
+	var res Res
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+
+	err = res.Err()
+	return
+}

pkg/pmapi/bugs_test.go

@@ -0,0 +1,180 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"runtime"
+	"strings"
+	"testing"
+)
+
+var testBugsReportReq = ReportReq{
+	OS:            "Mac OSX",
+	OSVersion:     "10.11.6",
+	Client:        "demoapp",
+	ClientVersion: "GoPMAPI_1.0.14",
+	ClientType:    1,
+	Title:         "Big Bug",
+	Description:   "Cannot fetch new messages",
+	Username:      "apple",
+	Email:         "apple@gmail.com",
+}
+
+var testBugsReportReqWithEmailClient = ReportReq{
+	OS:            "Mac OSX",
+	OSVersion:     "10.11.6",
+	Browser:       "AppleMail",
+	Client:        "demoapp",
+	ClientVersion: "GoPMAPI_1.0.14",
+	ClientType:    1,
+	Title:         "Big Bug",
+	Description:   "Cannot fetch new messages",
+	Username:      "Apple",
+	Email:         "apple@gmail.com",
+}
+
+var testBugsCrashReq = ReportReq{
+	OS:            runtime.GOOS,
+	Client:        "demoapp",
+	ClientVersion: "GoPMAPI_1.0.14",
+	ClientType:    1,
+	Debug:         "main.func·001()\n/Users/sunny/Code/Go/src/scratch/stack.go:21 +0xabruntime.panic(0x80b80, 0x2101fb150)\n/usr/local/Cellar/go/1.2/libexec/src/pkg/runtime/panic.c:248 +0x106\nmain.inner()/Users/sunny/Code/Go/src/scratch/stack.go:27 +0x68\nmain.outer()\n/Users/sunny/Code/Go/src/scratch/stack.go:13 +0x1a\nmain.main()\n/Users/sunny/Code/Go/src/scratch/stack.go:9 +0x1a",
+}
+
+const testBugsBody = `{
+    "Code": 1000
+}
+`
+
+const testAttachmentJSONZipped = "PK\x03\x04\x14\x00\b\x00\b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\b\x00\x00\x00last.log\\Rَ\xaaH\x00}ﯨ\xf8r\x1f\xeeܖED;\xe9\ap\x03\x11\x11\x97\x0e8\x99L\xb0(\xa1\xa0\x16\x85b\x91I\xff\xfbD{\x99\xc9}\xab:K\x9d\xa4\xce\xf9\xe7\t\x00\x00z\xf6\xb4\xf7\x02z\xb7a\xe5\xd8\x04*V̭\x8d\xd1lvE}\xd6\xe3\x80\x1f\xd7nX\x9bI[\xa6\xe1a=\xd4a\xa8M\x97\xd9J\xf1F\xeb\x105U\xbd\xb0`XO\xce\xf1hu\x99q\xc3\xfe{\x11ߨ'-\v\x89Z\xa4\x9c5\xaf\xaf\xbd?>R\xd6\x11E\xf7\x1cX\xf0JpF#L\x9eE+\xbe\xe8\x1d\xee\ued2e\u007f\xde]\u06dd\xedo\x97\x87E\xa0V\xf4/$\xc2\xecK\xed\xa0\xdb&\x829\x12\xe5\x9do\xa0\xe9\x1a\xd2\x19\x1e\xf5`\x95гb\xf8\x89\x81\xb7\xa5G\x18\x95\xf3\x9d9\xe8\x93B\x17!\x1a^\xccr\xbb`\xb2\xb4\xb86\x87\xb4h\x0e\xda\xc6u<+\x9e$̓\x95\xccSo\xea\xa4\xdbH!\xe9g\x8b\xd4\b\xb3hܬ\xa6Wk\x14He\xae\x8aPU\xaa\xc1\xee$\xfbH\xb3\xab.I\f<\x89\x06q\xe3-3-\x99\xcdݽ\xe5v\x99\xedn\xac\xadn\xe8Rp=\xb4nJ\xed\xd5\r\x8d\xde\x06Ζ\xf6\xb3\x01\x94\xcb\xf6\xd4\x19r\xe1\xaa$4+\xeaW\xa6F\xfa0\x97\x9cD\f\x8e\xd7\xd6z\v,G\xf3e2\xd4\xe6V\xba\v\xb6\xd9\xe8\xca*\x16\x95V\xa4J\xfbp\xddmF\x8c\x9a\xc6\xc8Č-\xdb\v\xf6\xf5\xf9\x02*\x15e\x874\xc9\xe7\"\xa3\x1an\xabq}ˊq\x957\xd3\xfd\xa91\x82\xe0Lß\\\x17\x8e\x9e_\xed`\t\xe9~5̕\x03\x9a\f\xddN6\xa2\xc4\x17\xdb\xc9V\x1c~\x9e\xea\xbe\xda-xv\xed\x8b\xe2\xc8DŽS\x95E6\xf2\xc3H\x1d:HPx\xc9\x14\xbfɒ\xff\xea\xb4P\x14\xa3\xe2\xfe\xfd\x1f+z\x80\x903\x81\x98\xf8\x15\xa3\x12\x16\xf8\"0g\xf7~B^\xfd \x040T\xa3\x02\x9c\x10\xc1\xa8F\xa0I#\xf1\xa3\x04\x98\x01\x91\xe2\x12\xdc;\x06gL\xd0g\xc0\xe3\xbd\xf6\xd7}&\xa8轀?\xbfяy`X\xf0\x92\x9f\x05\xf0*A8ρ\xac=K\xff\xf3\xfe\xa6Z\xe1\x1a\x017\xc2\x04\f\x94g\xa9\xf7-\xfb\xebqz\u007fz\u007f\xfa7\x00\x00\xff\xffPK\a\b\xf5\\\v\xe5I\x02\x00\x00\r\x03\x00\x00PK\x01\x02\x14\x00\x14\x00\b\x00\b\x00\x00\x00\x00\x00\xf5\\\v\xe5I\x02\x00\x00\r\x03\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00last.logPK\x05\x06\x00\x00\x00\x00\x01\x00\x01\x006\x00\x00\x00\u007f\x02\x00\x00\x00\x00" //nolint[misspell]
+
+func TestClient_BugReport(t *testing.T) {
+	s, c := newTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		Ok(t, checkMethodAndPath(r, "POST", "/reports/bug"))
+		Ok(t, isAuthReq(r, testUID, testAccessToken))
+
+		var bugsReportReq ReportReq
+		Ok(t, json.NewDecoder(r.Body).Decode(&bugsReportReq))
+		Equals(t, testBugsReportReq, bugsReportReq)
+
+		fmt.Fprint(w, testBugsBody)
+	}))
+	defer s.Close()
+	c.uid = testUID
+	c.accessToken = testAccessToken
+
+	Ok(t, c.ReportBug(
+		testBugsReportReq.OS,
+		testBugsReportReq.OSVersion,
+		testBugsReportReq.Title,
+		testBugsReportReq.Description,
+		testBugsReportReq.Username,
+		testBugsReportReq.Email,
+	))
+}
+
+func TestClient_BugReportWithAttachment(t *testing.T) {
+	s, c := newTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		Ok(t, checkMethodAndPath(r, "POST", "/reports/bug"))
+		Ok(t, isAuthReq(r, testUID, testAccessToken))
+
+		Ok(t, r.ParseMultipartForm(10*1024))
+
+		for field, expected := range map[string]string{
+			"OS":            testBugsReportReq.OS,
+			"OSVersion":     testBugsReportReq.OSVersion,
+			"Client":        testBugsReportReq.Client,
+			"ClientVersion": testBugsReportReq.ClientVersion,
+			"ClientType":    fmt.Sprintf("%d", testBugsReportReq.ClientType),
+			"Title":         testBugsReportReq.Title,
+			"Description":   testBugsReportReq.Description,
+			"Username":      testBugsReportReq.Username,
+			"Email":         testBugsReportReq.Email,
+		} {
+			if r.PostFormValue(field) != expected {
+				t.Errorf("Field %q has %q but expected %q", field, r.PostFormValue(field), expected)
+			}
+		}
+
+		attReader, err := r.MultipartForm.File["log"][0].Open()
+		Ok(t, err)
+
+		log, err := ioutil.ReadAll(attReader)
+		Ok(t, err)
+
+		Equals(t, []byte(testAttachmentJSONZipped), log)
+
+		fmt.Fprint(w, testBugsBody)
+	}))
+	defer s.Close()
+	c.uid = testUID
+	c.accessToken = testAccessToken
+
+	rep := testBugsReportReq
+	rep.AddAttachment("log", "last.log", strings.NewReader(testAttachmentJSON))
+
+	Ok(t, c.Report(rep))
+}
+
+func TestClient_BugReportWithEmailClient(t *testing.T) {
+	s, c := newTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		Ok(t, checkMethodAndPath(r, "POST", "/reports/bug"))
+		Ok(t, isAuthReq(r, testUID, testAccessToken))
+
+		var bugsReportReq ReportReq
+		Ok(t, json.NewDecoder(r.Body).Decode(&bugsReportReq))
+		Equals(t, testBugsReportReqWithEmailClient, bugsReportReq)
+
+		fmt.Fprint(w, testBugsBody)
+	}))
+	defer s.Close()
+	c.uid = testUID
+	c.accessToken = testAccessToken
+
+	Ok(t, c.ReportBugWithEmailClient(
+		testBugsReportReqWithEmailClient.OS,
+		testBugsReportReqWithEmailClient.OSVersion,
+		testBugsReportReqWithEmailClient.Title,
+		testBugsReportReqWithEmailClient.Description,
+		testBugsReportReqWithEmailClient.Username,
+		testBugsReportReqWithEmailClient.Email,
+		testBugsReportReqWithEmailClient.Browser,
+	))
+}
+
+func TestClient_BugsCrash(t *testing.T) {
+	s, c := newTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		Ok(t, checkMethodAndPath(r, "POST", "/reports/crash"))
+		Ok(t, isAuthReq(r, testUID, testAccessToken))
+
+		var bugsCrashReq ReportReq
+		Ok(t, json.NewDecoder(r.Body).Decode(&bugsCrashReq))
+		Equals(t, testBugsCrashReq, bugsCrashReq)
+
+		fmt.Fprint(w, testBugsBody)
+	}))
+	defer s.Close()
+	c.uid = testUID
+	c.accessToken = testAccessToken
+
+	Ok(t, c.ReportCrash(testBugsCrashReq.Debug))
+}

pkg/pmapi/client.go

@@ -0,0 +1,503 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"net/http"
+	"reflect"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	pmcrypto "github.com/ProtonMail/gopenpgp/crypto"
+	"github.com/jaytaylor/html2text"
+	"github.com/sirupsen/logrus"
+)
+
+// Version of the API.
+const Version = 3
+
+// API return codes.
+const (
+	ForceUpgradeBadAPIVersion = 5003
+	ForceUpgradeInvalidAPI    = 5004
+	ForceUpgradeBadAppVersion = 5005
+	APIOffline                = 7001
+	ImportMessageTooLong      = 36022
+	BansRequests              = 85131
+)
+
+// The output errors.
+var (
+	ErrInvalidToken       = errors.New("refresh token invalid")
+	ErrAPINotReachable    = errors.New("cannot reach the server")
+	ErrUpgradeApplication = errors.New("application upgrade required")
+)
+
+type ErrUnauthorized struct {
+	error
+}
+
+func (err *ErrUnauthorized) Error() string {
+	return fmt.Sprintf("unauthorized access: %+v", err.error.Error())
+}
+
+type TokenManager struct {
+	tokensLocker sync.Locker
+	tokenMap     map[string]string
+}
+
+func NewTokenManager() *TokenManager {
+	tm := &TokenManager{
+		tokensLocker: &sync.Mutex{},
+		tokenMap:     map[string]string{},
+	}
+	return tm
+}
+
+func (tm *TokenManager) GetToken(userID string) string {
+	tm.tokensLocker.Lock()
+	defer tm.tokensLocker.Unlock()
+
+	return tm.tokenMap[userID]
+}
+
+func (tm *TokenManager) SetToken(userID, token string) {
+	tm.tokensLocker.Lock()
+	defer tm.tokensLocker.Unlock()
+
+	tm.tokenMap[userID] = token
+}
+
+// ClientConfig contains Client configuration.
+type ClientConfig struct {
+	// The client application name and version.
+	AppVersion string
+
+	// The client ID.
+	ClientID string
+
+	TokenManager *TokenManager
+
+	// Transport specifies the mechanism by which individual HTTP requests are made.
+	// If nil, http.DefaultTransport is used.
+	Transport http.RoundTripper
+
+	// Timeout specifies the timeout from request to getting response headers to our API.
+	// Passed to http.Client, empty means no timeout.
+	Timeout time.Duration
+
+	// FirstReadTimeout specifies the timeout from getting response to the first read of body response.
+	// This timeout is applied only when MinSpeed is used.
+	// Default is 5 minutes.
+	FirstReadTimeout time.Duration
+
+	// MinSpeed specifies minimum Bytes per second or the request will be canceled.
+	// Zero means no limitation.
+	MinSpeed int64
+}
+
+// Client to communicate with API.
+type Client struct {
+	auths chan<- *Auth // Channel that sends Auth responses back to the bridge.
+
+	log    *logrus.Entry
+	config *ClientConfig
+	client *http.Client
+	conrep ConnectionReporter
+
+	uid           string
+	accessToken   string
+	userID        string // Twice here because Username is not unique.
+	requestLocker sync.Locker
+	keyLocker     sync.Locker
+
+	tokenManager *TokenManager
+	expiresAt    time.Time
+	user         *User
+	addresses    AddressList
+	kr           *pmcrypto.KeyRing
+}
+
+// NewClient creates a new API client.
+func NewClient(cfg *ClientConfig, userID string) *Client {
+	hc := &http.Client{
+		Timeout: cfg.Timeout,
+	}
+	if cfg.Transport != nil {
+		cfgTransport, ok := cfg.Transport.(*http.Transport)
+		if ok {
+			// In future use Clone here.
+			// https://go-review.googlesource.com/c/go/+/174597/
+			transport := &http.Transport{}
+			*transport = *cfgTransport //nolint
+			if transport.Proxy == nil {
+				transport.Proxy = http.ProxyFromEnvironment
+			}
+			hc.Transport = transport
+		} else {
+			hc.Transport = cfg.Transport
+		}
+	} else if defaultTransport != nil {
+		hc.Transport = defaultTransport
+	}
+
+	log := logrus.WithFields(logrus.Fields{
+		"pkg":    "pmapi",
+		"userID": userID,
+	})
+
+	return &Client{
+		log:           log,
+		config:        cfg,
+		client:        hc,
+		tokenManager:  cfg.TokenManager,
+		userID:        userID,
+		requestLocker: &sync.Mutex{},
+		keyLocker:     &sync.Mutex{},
+	}
+}
+
+// SetConnectionReporter sets the connection reporter used by the client to report when
+// internet connection is lost.
+func (c *Client) SetConnectionReporter(conrep ConnectionReporter) {
+	c.conrep = conrep
+}
+
+// reportLostConnection reports that the internet connection has been lost using the connection reporter.
+// If the connection reporter has not been set, this does nothing.
+func (c *Client) reportLostConnection() {
+	if c.conrep != nil {
+		err := c.conrep.NotifyConnectionLost()
+		if err != nil {
+			logrus.WithError(err).Error("Failed to notify of lost connection")
+		}
+	}
+}
+
+// Do makes an API request. It does not check for HTTP status code errors.
+func (c *Client) Do(req *http.Request, retryUnauthorized bool) (res *http.Response, err error) {
+	// Copy the request body in case we need to retry it.
+	var bodyBuffer []byte
+	if req.Body != nil {
+		defer req.Body.Close() //nolint[errcheck]
+		bodyBuffer, err = ioutil.ReadAll(req.Body)
+
+		if err != nil {
+			return nil, err
+		}
+
+		r := bytes.NewReader(bodyBuffer)
+		req.Body = ioutil.NopCloser(r)
+	}
+
+	return c.doBuffered(req, bodyBuffer, retryUnauthorized)
+}
+
+// If needed it retries using req and buffered body.
+func (c *Client) doBuffered(req *http.Request, bodyBuffer []byte, retryUnauthorized bool) (res *http.Response, err error) { // nolint[funlen]
+	isAuthReq := strings.Contains(req.URL.Path, "/auth")
+
+	req.Header.Set("x-pm-appversion", c.config.AppVersion)
+	req.Header.Set("x-pm-apiversion", strconv.Itoa(Version))
+
+	if c.uid != "" {
+		req.Header.Set("x-pm-uid", c.uid)
+	}
+
+	if c.accessToken != "" {
+		req.Header.Set("Authorization", "Bearer "+c.accessToken)
+	}
+
+	c.log.Debugln("Requesting ", req.Method, req.URL.RequestURI())
+	if logrus.GetLevel() == logrus.TraceLevel {
+		head := ""
+		for i, v := range req.Header {
+			head += i + ": "
+			head += strings.Join(v, "")
+			head += "\n"
+		}
+		c.log.Tracef("REQHEAD \n%s", head)
+		c.log.Tracef("REQBODY '%s'", string(bodyBuffer))
+	}
+
+	hasBody := len(bodyBuffer) > 0
+	if res, err = c.client.Do(req); err != nil {
+		if res == nil {
+			c.log.WithError(err).Error("Cannot get response")
+			err = ErrAPINotReachable
+			c.reportLostConnection()
+		}
+		return
+	}
+
+	resDate := res.Header.Get("Date")
+	if resDate != "" {
+		if serverTime, err := http.ParseTime(resDate); err == nil {
+			pmcrypto.GetGopenPGP().UpdateTime(serverTime.Unix())
+		}
+	}
+
+	if res.StatusCode == http.StatusUnauthorized {
+		if hasBody {
+			r := bytes.NewReader(bodyBuffer)
+			req.Body = ioutil.NopCloser(r)
+		}
+
+		if !isAuthReq {
+			_, _ = io.Copy(ioutil.Discard, res.Body)
+			_ = res.Body.Close()
+			return c.handleStatusUnauthorized(req, bodyBuffer, res, retryUnauthorized)
+		}
+	}
+
+	// Retry induced by HTTP status code>
+	retryAfter := 10
+	doRetry := res.StatusCode == http.StatusTooManyRequests
+	if doRetry {
+		if headerAfter, err := strconv.Atoi(res.Header.Get("Retry-After")); err == nil && headerAfter > 0 {
+			retryAfter = headerAfter
+		}
+		// To avoid spikes when all clients retry at the same time, we add some random wait.
+		retryAfter += rand.Intn(10)
+
+		if hasBody {
+			r := bytes.NewReader(bodyBuffer)
+			req.Body = ioutil.NopCloser(r)
+		}
+
+		c.log.Warningf("Retrying %s after %ds induced by http code %d", req.URL.Path, retryAfter, res.StatusCode)
+		time.Sleep(time.Duration(retryAfter) * time.Second)
+		_, _ = io.Copy(ioutil.Discard, res.Body)
+		_ = res.Body.Close()
+		return c.doBuffered(req, bodyBuffer, false)
+	}
+
+	return res, err
+}
+
+// DoJSON performs the request and unmarshals the response as JSON into data.
+// If the API returns a non-2xx HTTP status code, the error returned will contain status
+// and response as plaintext. API errors must be checked by the caller.
+// It is performed buffered, in case we need to retry.
+func (c *Client) DoJSON(req *http.Request, data interface{}) error {
+	// Copy the request body in case we need to retry it
+	var reqBodyBuffer []byte
+
+	if req.Body != nil {
+		defer req.Body.Close() //nolint[errcheck]
+		var err error
+		if reqBodyBuffer, err = ioutil.ReadAll(req.Body); err != nil {
+			return err
+		}
+
+		req.Body = ioutil.NopCloser(bytes.NewReader(reqBodyBuffer))
+	}
+
+	return c.doJSONBuffered(req, reqBodyBuffer, data)
+}
+
+// doJSONBuffered performs a buffered json request (see DoJSON for more information).
+func (c *Client) doJSONBuffered(req *http.Request, reqBodyBuffer []byte, data interface{}) error { // nolint[funlen]
+	req.Header.Set("Accept", "application/vnd.protonmail.v1+json")
+
+	var cancelRequest context.CancelFunc
+	if c.config.MinSpeed > 0 {
+		var ctx context.Context
+		ctx, cancelRequest = context.WithCancel(req.Context())
+		defer func() {
+			cancelRequest()
+		}()
+		req = req.WithContext(ctx)
+	}
+
+	res, err := c.doBuffered(req, reqBodyBuffer, false)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close() //nolint[errcheck]
+
+	var resBody []byte
+	if c.config.MinSpeed == 0 {
+		resBody, err = ioutil.ReadAll(res.Body)
+	} else {
+		resBody, err = c.readAllMinSpeed(res.Body, cancelRequest)
+	}
+
+	// The server response may contain data which we want to have in memory
+	// for as little time as possible (such as keys). Go is garbage collected,
+	// so we are not in charge of when the memory will actually be cleared.
+	// We can at least try to rewrite the original data to mitigate this problem.
+	defer func() {
+		for i := 0; i < len(resBody); i++ {
+			resBody[i] = byte(65)
+		}
+	}()
+
+	if logrus.GetLevel() == logrus.TraceLevel {
+		head := ""
+		for i, v := range res.Header {
+			head += i + ": "
+			head += strings.Join(v, "")
+			head += "\n"
+		}
+		c.log.Tracef("RESHEAD \n%s", head)
+		c.log.Tracef("RESBODY '%s'", resBody)
+	}
+
+	if err != nil {
+		return err
+	}
+
+	// Retry induced by API code.
+	errCode := &Res{}
+	if err := json.Unmarshal(resBody, errCode); err == nil {
+		if errCode.Code == BansRequests {
+			retryAfter := 3
+			c.log.Warningf("Retrying %s after %ds induced by API code %d", req.URL.Path, retryAfter, errCode.Code)
+			time.Sleep(time.Duration(retryAfter) * time.Second)
+			if len(reqBodyBuffer) > 0 {
+				req.Body = ioutil.NopCloser(bytes.NewReader(reqBodyBuffer))
+			}
+			return c.doJSONBuffered(req, reqBodyBuffer, data)
+		}
+	}
+
+	if err := json.Unmarshal(resBody, data); err != nil {
+		// Check to see if this is due to a non 2xx HTTP status code.
+		if res.StatusCode != http.StatusOK {
+			r := bytes.NewReader(bytes.ReplaceAll(resBody, []byte("\n"), []byte("\\n")))
+			plaintext, err := html2text.FromReader(r)
+			if err == nil {
+				return fmt.Errorf("Error: \n\n" + res.Status + "\n\n" + plaintext)
+			}
+		}
+
+		if errJS, ok := err.(*json.SyntaxError); ok {
+			return fmt.Errorf("invalid json %v (offset:%d) ", errJS.Error(), errJS.Offset)
+		}
+
+		return fmt.Errorf("unmarshal fail: %v ", err)
+	}
+
+	// Set StatusCode in case data struct supports that field.
+	// It's safe to set StatusCode, server returns Code. StatusCode should be preferred over Code.
+	dataValue := reflect.ValueOf(data).Elem()
+	statusCodeField := dataValue.FieldByName("StatusCode")
+	if statusCodeField.IsValid() && statusCodeField.CanSet() && statusCodeField.Kind() == reflect.Int {
+		statusCodeField.SetInt(int64(res.StatusCode))
+	}
+
+	if res.StatusCode != http.StatusOK {
+		c.log.Warnf("request %s %s NOT OK: %s", req.Method, req.URL.Path, res.Status)
+	}
+
+	return nil
+}
+
+func (c *Client) readAllMinSpeed(data io.Reader, cancelRequest context.CancelFunc) ([]byte, error) {
+	firstReadTimeout := c.config.FirstReadTimeout
+	if firstReadTimeout == 0 {
+		firstReadTimeout = 5 * time.Minute
+	}
+	timer := time.AfterFunc(firstReadTimeout, func() {
+		cancelRequest()
+	})
+	var buffer bytes.Buffer
+	for {
+		_, err := io.CopyN(&buffer, data, c.config.MinSpeed)
+		timer.Stop()
+		timer.Reset(1 * time.Second)
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return nil, err
+		}
+	}
+	return ioutil.ReadAll(&buffer)
+}
+
+func (c *Client) refreshAccessToken() (err error) {
+	c.log.Debug("Refreshing token")
+	refreshToken := c.tokenManager.GetToken(c.userID)
+	c.log.WithField("token", refreshToken).Info("Current refresh token")
+	if refreshToken == "" {
+		if c.auths != nil {
+			c.auths <- nil
+		}
+		if c.tokenManager != nil {
+			c.tokenManager.SetToken(c.userID, "")
+		}
+		return ErrInvalidToken
+	}
+
+	auth, err := c.AuthRefresh(refreshToken)
+	if err != nil {
+		c.log.WithError(err).WithField("auths", c.auths).Debug("Token refreshing failed")
+		// The refresh failed, so we should log the user out.
+		// A nil value in the Auths channel will trigger this.
+		if c.auths != nil {
+			c.auths <- nil
+		}
+		if c.tokenManager != nil {
+			c.tokenManager.SetToken(c.userID, "")
+		}
+		return
+	}
+	c.uid = auth.UID()
+	c.accessToken = auth.accessToken
+	return err
+}
+
+func (c *Client) handleStatusUnauthorized(req *http.Request, reqBodyBuffer []byte, res *http.Response, retry bool) (retryRes *http.Response, err error) {
+	c.log.Info("Handling unauthorized status")
+
+	// If this is not a retry, then it is the first time handling status unauthorized,
+	// so try again without refreshing the access token.
+	if !retry {
+		c.log.Debug("Handling unauthorized status by retrying")
+		c.requestLocker.Lock()
+		defer c.requestLocker.Unlock()
+
+		_, _ = io.Copy(ioutil.Discard, res.Body)
+		_ = res.Body.Close()
+		return c.doBuffered(req, reqBodyBuffer, true)
+	}
+
+	// This is already a retry, so we will try to refresh the access token before trying again.
+	if err = c.refreshAccessToken(); err != nil {
+		c.log.WithError(err).Warn("Cannot refresh token")
+		err = &ErrUnauthorized{err}
+		return
+	}
+	_, err = io.Copy(ioutil.Discard, res.Body)
+	if err != nil {
+		c.log.WithError(err).Warn("Failed to read out response body")
+	}
+	_ = res.Body.Close()
+	return c.doBuffered(req, reqBodyBuffer, true)
+}

pkg/pmapi/client_test.go

@@ -0,0 +1,215 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+var testClientConfig = &ClientConfig{
+	AppVersion:       "GoPMAPI_1.0.14",
+	ClientID:         "demoapp",
+	FirstReadTimeout: 500 * time.Millisecond,
+	MinSpeed:         256,
+}
+
+func newTestClient() *Client {
+	c := NewClient(testClientConfig, "tester")
+	c.tokenManager = NewTokenManager()
+	return c
+}
+
+func TestClient_Do(t *testing.T) {
+	const testResBody = "Hello World!"
+
+	var receivedReq *http.Request
+	s, c := newTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		receivedReq = r
+		fmt.Fprint(w, testResBody)
+	}))
+	defer s.Close()
+
+	req, err := NewRequest("GET", "/", nil)
+	if err != nil {
+		t.Fatal("Expected no error while creating request, got:", err)
+	}
+
+	res, err := c.Do(req, true)
+	if err != nil {
+		t.Fatal("Expected no error while executing request, got:", err)
+	}
+
+	b, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		t.Fatal("Expected no error while reading response, got:", err)
+	}
+	require.Nil(t, res.Body.Close())
+
+	if string(b) != testResBody {
+		t.Fatalf("Invalid response body: expected %v, got %v", testResBody, string(b))
+	}
+
+	h := receivedReq.Header
+	if h.Get("x-pm-appversion") != testClientConfig.AppVersion {
+		t.Fatalf("Invalid app version header: expected %v, got %v", testClientConfig.AppVersion, h.Get("x-pm-appversion"))
+	}
+	if h.Get("x-pm-apiversion") != fmt.Sprintf("%v", Version) {
+		t.Fatalf("Invalid api version header: expected %v, got %v", Version, h.Get("x-pm-apiversion"))
+	}
+	if h.Get("x-pm-uid") != "" {
+		t.Fatalf("Expected no uid header when not authenticated, got %v", h.Get("x-pm-uid"))
+	}
+	if h.Get("Authorization") != "" {
+		t.Fatalf("Expected no authentication header when not authenticated, got %v", h.Get("Authorization"))
+	}
+}
+
+func TestClient_DoRetryAfter(t *testing.T) {
+	testStart := time.Now()
+	secondAttemptTime := time.Now()
+
+	finish, c := newTestServerCallbacks(t,
+		func(tb testing.TB, w http.ResponseWriter, req *http.Request) string {
+			w.Header().Set("content-type", "application/json;charset=utf-8")
+			w.Header().Set("Retry-After", "1")
+			w.WriteHeader(http.StatusTooManyRequests)
+			return ""
+		},
+		func(tb testing.TB, w http.ResponseWriter, req *http.Request) string {
+			w.Header().Set("content-type", "application/json;charset=utf-8")
+			w.WriteHeader(http.StatusOK)
+			secondAttemptTime = time.Now()
+			return "/HTTP_200.json"
+		},
+	)
+	defer finish()
+
+	require.Nil(t, c.SendSimpleMetric("some_category", "some_action", "some_label"))
+	waitedTime := secondAttemptTime.Sub(testStart)
+	isInRange := 1*time.Second < waitedTime && waitedTime <= 11*time.Second
+	require.True(t, isInRange, "Waited time: %v", waitedTime)
+}
+
+type slowTransport struct {
+	transport      http.RoundTripper
+	firstBodySleep time.Duration
+}
+
+func (t *slowTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	resp, err := t.transport.RoundTrip(req)
+	if err == nil {
+		resp.Body = &slowReadCloser{
+			req:            req,
+			readCloser:     resp.Body,
+			firstBodySleep: t.firstBodySleep,
+		}
+	}
+	return resp, err
+}
+
+type slowReadCloser struct {
+	req            *http.Request
+	readCloser     io.ReadCloser
+	firstBodySleep time.Duration
+}
+
+func (r *slowReadCloser) Read(p []byte) (n int, err error) {
+	// Normally timeout is processed by Read function.
+	// It's hard to test slow connection; we need to manually
+	// check when context is Done, because otherwise timeout
+	// happens only during failed Read which will not happen
+	// in this artificial environment.
+	select {
+	case <-r.req.Context().Done():
+		return 0, context.Canceled
+	case <-time.After(r.firstBodySleep):
+	}
+	return r.readCloser.Read(p)
+}
+
+func (r *slowReadCloser) Close() error {
+	return r.readCloser.Close()
+}
+
+func TestClient_FirstReadTimeout(t *testing.T) {
+	requestTimeout := testClientConfig.FirstReadTimeout + 1*time.Second
+
+	finish, c := newTestServerCallbacks(t,
+		func(tb testing.TB, w http.ResponseWriter, req *http.Request) string {
+			return "/HTTP_200.json"
+		},
+	)
+	defer finish()
+
+	c.client.Transport = &slowTransport{
+		transport:      c.client.Transport,
+		firstBodySleep: requestTimeout,
+	}
+
+	started := time.Now()
+	err := c.SendSimpleMetric("some_category", "some_action", "some_label")
+	require.Error(t, err, "cannot reach the server")
+	require.True(t, time.Since(started) < requestTimeout, "Actual waited time: %v", time.Since(started))
+}
+
+func TestClient_MinSpeedTimeout(t *testing.T) {
+	finish, c := newTestServerCallbacks(t,
+		routeSlow(2*time.Second),
+	)
+	defer finish()
+
+	err := c.SendSimpleMetric("some_category", "some_action", "some_label")
+	require.Error(t, err, "cannot reach the server")
+}
+
+func TestClient_MinSpeedNoTimeout(t *testing.T) {
+	finish, c := newTestServerCallbacks(t,
+		routeSlow(500*time.Millisecond),
+	)
+	defer finish()
+
+	err := c.SendSimpleMetric("some_category", "some_action", "some_label")
+	require.Nil(t, err)
+}
+
+func routeSlow(delay time.Duration) func(tb testing.TB, w http.ResponseWriter, req *http.Request) string {
+	return func(tb testing.TB, w http.ResponseWriter, req *http.Request) string {
+		w.Header().Set("content-type", "application/json;charset=utf-8")
+		w.WriteHeader(http.StatusOK)
+
+		_, _ = w.Write([]byte("{\"code\":1000,\"key\":\""))
+		for chunk := 1; chunk <= 10; chunk++ {
+			// We need to write enough bytes which enforce flushing data
+			// because writer used by httptest does not implement Flusher.
+			for i := 1; i <= 10000; i++ {
+				_, _ = w.Write([]byte("a"))
+			}
+			time.Sleep(delay)
+		}
+		_, _ = w.Write([]byte("\"}"))
+		return ""
+	}
+}

pkg/pmapi/config.go

@@ -0,0 +1,43 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"net/http"
+	"runtime"
+)
+
+// RootURL is the API root URL.
+//
+// This can be changed using build flags: pmapi_local for "http://localhost/api",
+// pmapi_dev or pmapi_prod. Default is pmapi_prod.
+var RootURL = "https://api.protonmail.ch" //nolint[gochecknoglobals]
+
+// CurrentUserAgent is the default User-Agent for go-pmapi lib. This can be changed to program
+// version and email client.
+// e.g. Bridge/1.0.4 (Windows) MicrosoftOutlook/16.0.9330.2087
+var CurrentUserAgent = "GoPMAPI/1.0.14 (" + runtime.GOOS + "; no client)" //nolint[gochecknoglobals]
+
+// The HTTP transport to use by default.
+var defaultTransport = &http.Transport{ //nolint[gochecknoglobals]
+	Proxy: http.ProxyFromEnvironment,
+}
+
+// checkTLSCerts controls whether TLS certs are checked against known fingerprints.
+// The default is for this to always be done.
+var checkTLSCerts = true //nolint[gochecknoglobals]

pkg/pmapi/config_dev.go

@@ -0,0 +1,24 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+// +build pmapi_dev
+
+package pmapi
+
+func init() {
+	RootURL = "https://dev.protonmail.com/api"
+}

pkg/pmapi/config_local.go

@@ -0,0 +1,37 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+// +build pmapi_local
+
+package pmapi
+
+import (
+	"crypto/tls"
+	"net/http"
+)
+
+func init() {
+	// Use port above 1000 which doesn't need root access to start anything on it.
+	// Now the port is rounded pi. :-)
+	RootURL = "http://127.0.0.1:3142/api"
+
+	// TLS certificate is self-signed
+	defaultTransport = &http.Transport{
+		Proxy:           http.ProxyFromEnvironment,
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+}

pkg/pmapi/config_nopin.go

@@ -0,0 +1,25 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+// +build pmapi_nopin
+
+package pmapi
+
+func init() {
+	// This config disables TLS cert checking.
+	checkTLSCerts = false
+}

pkg/pmapi/conrep.go

@@ -0,0 +1,23 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+// ConnectionReporter provides a way to report when internet connection is lost.
+type ConnectionReporter interface {
+	NotifyConnectionLost() error
+}

pkg/pmapi/contacts.go

@@ -0,0 +1,430 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"errors"
+	"net/url"
+	"strconv"
+)
+
+type Card struct {
+	Type      int
+	Data      string
+	Signature string
+}
+
+const (
+	CardEncrypted = 1
+	CardSigned    = 2
+)
+
+type Contact struct {
+	ID         string
+	Name       string
+	UID        string
+	Size       int64
+	CreateTime int64
+	ModifyTime int64
+	LabelIDs   []string
+
+	ContactEmails []ContactEmail
+	Cards         []Card
+}
+
+type ContactEmail struct {
+	ID        string
+	Name      string
+	Email     string
+	Type      []string
+	Defaults  int
+	Order     int
+	ContactID string
+	LabelIDs  []string
+}
+
+var errVerificationFailed = errors.New("signature verification failed")
+
+//================= Public utility functions ======================
+
+func (c *Client) EncryptAndSignCards(cards []Card) ([]Card, error) {
+	var err error
+	for i := range cards {
+		card := &cards[i]
+		if isEncryptedCardType(card.Type) {
+			if isSignedCardType(card.Type) {
+				if card.Signature, err = c.sign(card.Data); err != nil {
+					return nil, err
+				}
+			}
+
+			if card.Data, err = c.encrypt(card.Data, nil); err != nil {
+				return nil, err
+			}
+		} else if isSignedCardType(card.Type) {
+			if card.Signature, err = c.sign(card.Data); err != nil {
+				return nil, err
+			}
+		}
+	}
+	return cards, nil
+}
+
+func (c *Client) DecryptAndVerifyCards(cards []Card) ([]Card, error) {
+	for i := range cards {
+		card := &cards[i]
+		if isEncryptedCardType(card.Type) {
+			signedCard, err := c.decrypt(card.Data)
+			if err != nil {
+				return nil, err
+			}
+			card.Data = signedCard
+		}
+		if isSignedCardType(card.Type) {
+			err := c.verify(card.Data, card.Signature)
+			if err != nil {
+				return cards, errVerificationFailed
+			}
+		}
+	}
+	return cards, nil
+}
+
+//====================== READ ===========================
+
+type ContactsListRes struct {
+	Res
+	Contacts []*Contact
+}
+
+// GetContacts gets all contacts.
+func (c *Client) GetContacts(page int, pageSize int) (contacts []*Contact, err error) {
+	v := url.Values{}
+	v.Set("Page", strconv.Itoa(page))
+	if pageSize > 0 {
+		v.Set("PageSize", strconv.Itoa(pageSize))
+	}
+	req, err := NewRequest("GET", "/contacts?"+v.Encode(), nil)
+
+	if err != nil {
+		return
+	}
+
+	var res ContactsListRes
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+
+	contacts, err = res.Contacts, res.Err()
+	return
+}
+
+// GetContactByID gets contact details specified by contact ID.
+func (c *Client) GetContactByID(id string) (contactDetail Contact, err error) {
+	req, err := NewRequest("GET", "/contacts/"+id, nil)
+
+	if err != nil {
+		return
+	}
+
+	type ContactRes struct {
+		Res
+		Contact Contact
+	}
+	var res ContactRes
+
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+
+	contactDetail, err = res.Contact, res.Err()
+	return
+}
+
+// GetContactsForExport gets contacts in vCard format, signed and encrypted.
+func (c *Client) GetContactsForExport(page int, pageSize int) (contacts []Contact, err error) {
+	v := url.Values{}
+	v.Set("Page", strconv.Itoa(page))
+	if pageSize > 0 {
+		v.Set("PageSize", strconv.Itoa(pageSize))
+	}
+
+	req, err := NewRequest("GET", "/contacts/export?"+v.Encode(), nil)
+
+	if err != nil {
+		return
+	}
+
+	type ContactsDetailsRes struct {
+		Res
+		Contacts []Contact
+	}
+	var res ContactsDetailsRes
+
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+
+	contacts, err = res.Contacts, res.Err()
+	return
+}
+
+type ContactsEmailsRes struct {
+	Res
+	ContactEmails []ContactEmail
+	Total         int
+}
+
+// GetAllContactsEmails gets all emails from all contacts.
+func (c *Client) GetAllContactsEmails(page int, pageSize int) (contactsEmails []ContactEmail, err error) {
+	v := url.Values{}
+	v.Set("Page", strconv.Itoa(page))
+	if pageSize > 0 {
+		v.Set("PageSize", strconv.Itoa(pageSize))
+	}
+
+	req, err := NewRequest("GET", "/contacts/emails?"+v.Encode(), nil)
+	if err != nil {
+		return
+	}
+
+	var res ContactsEmailsRes
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+
+	contactsEmails, err = res.ContactEmails, res.Err()
+	return
+}
+
+// GetContactEmailByEmail gets all emails from all contacts matching a specified email string.
+func (c *Client) GetContactEmailByEmail(email string, page int, pageSize int) (contactEmails []ContactEmail, err error) {
+	v := url.Values{}
+	v.Set("Page", strconv.Itoa(page))
+	if pageSize > 0 {
+		v.Set("PageSize", strconv.Itoa(pageSize))
+	}
+	v.Set("Email", email)
+
+	req, err := NewRequest("GET", "/contacts/emails?"+v.Encode(), nil)
+	if err != nil {
+		return
+	}
+
+	var res ContactsEmailsRes
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+
+	contactEmails, err = res.ContactEmails, res.Err()
+	return
+}
+
+//============================ CREATE ====================================
+
+type CardsList struct {
+	Cards []Card
+}
+
+type ContactsCards struct {
+	Contacts []CardsList
+}
+
+type SingleContactResponse struct {
+	Res
+	Contact Contact
+}
+
+type IndexedContactResponse struct {
+	Index    int
+	Response SingleContactResponse
+}
+
+type AddContactsResponse struct {
+	Res
+	Responses []IndexedContactResponse
+}
+
+type AddContactsReq struct {
+	ContactsCards
+	Overwrite int
+	Groups    int
+	Labels    int
+}
+
+// AddContacts adds contacts specified by cards. Performs signing and encrypting based on card type.
+func (c *Client) AddContacts(cards ContactsCards, overwrite int, groups int, labels int) (res *AddContactsResponse, err error) {
+	reqBody := AddContactsReq{
+		ContactsCards: cards,
+		Overwrite:     overwrite,
+		Groups:        groups,
+		Labels:        labels,
+	}
+
+	req, err := NewJSONRequest("POST", "/contacts", reqBody)
+	if err != nil {
+		return
+	}
+
+	var addContactsRes AddContactsResponse
+	if err = c.DoJSON(req, &addContactsRes); err != nil {
+		return
+	}
+
+	res, err = &addContactsRes, addContactsRes.Err()
+	return
+}
+
+// ================================= UPDATE =======================================
+
+type UpdateContactResponse struct {
+	Res
+	Contact Contact
+}
+
+type UpdateContactReq struct {
+	Cards []Card
+}
+
+// UpdateContact updates contact identified by contact ID. Modified contact is specified by cards.
+func (c *Client) UpdateContact(id string, cards []Card) (res *UpdateContactResponse, err error) {
+	reqBody := UpdateContactReq{
+		Cards: cards,
+	}
+	req, err := NewJSONRequest("PUT", "/contacts/"+id, reqBody)
+	if err != nil {
+		return
+	}
+	var updateContactRes UpdateContactResponse
+	if err = c.DoJSON(req, &updateContactRes); err != nil {
+		return
+	}
+
+	res, err = &updateContactRes, updateContactRes.Err()
+	return
+}
+
+type SingleIDResponse struct {
+	Res
+	ID string
+}
+
+type UpdateContactGroupsResponse struct {
+	Res
+	Response SingleIDResponse
+}
+
+func (c *Client) AddContactGroups(groupID string, contactEmailIDs []string) (res *UpdateContactGroupsResponse, err error) {
+	return c.modifyContactGroups(groupID, addContactGroupsAction, contactEmailIDs)
+}
+
+func (c *Client) RemoveContactGroups(groupID string, contactEmailIDs []string) (res *UpdateContactGroupsResponse, err error) {
+	return c.modifyContactGroups(groupID, removeContactGroupsAction, contactEmailIDs)
+}
+
+const (
+	removeContactGroupsAction = 0
+	addContactGroupsAction    = 1
+)
+
+type ModifyContactGroupsReq struct {
+	LabelID         string
+	Action          int
+	ContactEmailIDs []string
+}
+
+func (c *Client) modifyContactGroups(groupID string, modifyContactGroupsAction int, contactEmailIDs []string) (res *UpdateContactGroupsResponse, err error) {
+	reqBody := ModifyContactGroupsReq{
+		LabelID:         groupID,
+		Action:          modifyContactGroupsAction,
+		ContactEmailIDs: contactEmailIDs,
+	}
+	req, err := NewJSONRequest("PUT", "/contacts/group", reqBody)
+	if err != nil {
+		return
+	}
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+	err = res.Err()
+	return
+}
+
+// ================================= DELETE =======================================
+
+type DeleteReq struct {
+	IDs []string
+}
+
+// DeleteContacts deletes contacts specified by an array of contact IDs.
+func (c *Client) DeleteContacts(ids []string) (err error) {
+	deleteReq := DeleteReq{
+		IDs: ids,
+	}
+
+	req, err := NewJSONRequest("PUT", "/contacts/delete", deleteReq)
+	if err != nil {
+		return
+	}
+
+	type DeleteContactsRes struct {
+		Res
+		Responses []struct {
+			ID       string
+			Response Res
+		}
+	}
+	var res DeleteContactsRes
+
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+	if err = res.Err(); err != nil {
+		return
+	}
+	return
+}
+
+// DeleteAllContacts deletes all contacts.
+func (c *Client) DeleteAllContacts() (err error) {
+	req, err := NewRequest("DELETE", "/contacts", nil)
+	if err != nil {
+		return
+	}
+
+	var res Res
+
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+	if err = res.Err(); err != nil {
+		return
+	}
+
+	return
+}
+
+//===================== Private utility methods =======================
+
+func isSignedCardType(cardType int) bool {
+	return (cardType & CardSigned) == CardSigned
+}
+
+func isEncryptedCardType(cardType int) bool {
+	return (cardType & CardEncrypted) == CardEncrypted
+}

pkg/pmapi/contacts_test.go

@@ -0,0 +1,677 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var (
+	CleartextCard       = 0
+	EncryptedCard       = 1
+	SignedCard          = 2
+	EncryptedSignedCard = 3
+)
+
+var testAddContactsReq = AddContactsReq{
+	ContactsCards: ContactsCards{
+		Contacts: []CardsList{
+			{
+				Cards: []Card{
+					{
+						Type: 2,
+						Data: `BEGIN:VCARD
+VERSION:4.0
+FN;TYPE=fn:Bob
+item1.EMAIL:bob.tester@protonmail.com
+UID:proton-web-cd974706-5cde-0e53-e131-c49c88a92ece
+END:VCARD
+`,
+						Signature: ``,
+					},
+				},
+			},
+		},
+	},
+	Overwrite: 0,
+	Groups:    0,
+	Labels:    0,
+}
+
+var testAddContactsResponseBody = `{
+    "Code": 1001,
+    "Responses": [
+        {
+            "Index": 0,
+            "Response": {
+                "Code": 1000,
+                "Contact": {
+                    "ID": "EU7qYvPAdgJ-zl53hw_btO1WG8TN2FYh2cTIFq1_T6KqulwgxF8CzPjVk_RBUdEejtLvfynlelVNoZwMK_9X2g==",
+                    "Name": "Bob",
+                    "UID": "proton-web-cd974706-5cde-0e53-e131-c49c88a92ece",
+                    "Size": 139,
+                    "CreateTime": 1517319495,
+                    "ModifyTime": 1517319495,
+                    "ContactEmails": [
+                        {
+                            "ID": "VT4NoPeQPk48_vg0CVmk63n5mB6CZn9q-P_DYODhOUemhuzUkgBFGF1MktVArjX5zsVdfVlEBFObvt0_K5NwPg==",
+                            "Name": "Bob",
+                            "Email": "bob.tester@protonmail.com",
+                            "Type": [],
+                            "Defaults": 1,
+                            "Order": 1,
+                            "ContactID": "EU7qYvPAdgJ-zl53hw_btO1WG8TN2FYh2cTIFq1_T6KqulwgxF8CzPjVk_RBUdEejtLvfynlelVNoZwMK_9X2g==",
+                            "LabelIDs": []
+                        }
+                    ],
+                    "LabelIDs": []
+                }
+            }
+        }
+    ]
+}`
+
+var testContactCreated = &AddContactsResponse{
+	Res: Res{
+		Code:       1001,
+		StatusCode: 200,
+	},
+	Responses: []IndexedContactResponse{
+		{
+			Index: 0,
+			Response: SingleContactResponse{
+				Res: Res{
+					Code: 1000,
+				},
+				Contact: Contact{
+					ID:         "EU7qYvPAdgJ-zl53hw_btO1WG8TN2FYh2cTIFq1_T6KqulwgxF8CzPjVk_RBUdEejtLvfynlelVNoZwMK_9X2g==",
+					Name:       "Bob",
+					UID:        "proton-web-cd974706-5cde-0e53-e131-c49c88a92ece",
+					Size:       139,
+					CreateTime: 1517319495,
+					ModifyTime: 1517319495,
+					ContactEmails: []ContactEmail{
+						{
+							ID:        "VT4NoPeQPk48_vg0CVmk63n5mB6CZn9q-P_DYODhOUemhuzUkgBFGF1MktVArjX5zsVdfVlEBFObvt0_K5NwPg==",
+							Name:      "Bob",
+							Email:     "bob.tester@protonmail.com",
+							Type:      []string{},
+							Defaults:  1,
+							Order:     1,
+							ContactID: "EU7qYvPAdgJ-zl53hw_btO1WG8TN2FYh2cTIFq1_T6KqulwgxF8CzPjVk_RBUdEejtLvfynlelVNoZwMK_9X2g==",
+							LabelIDs:  []string{},
+						},
+					},
+					LabelIDs: []string{},
+				},
+			},
+		},
+	},
+}
+
+var testContactUpdated = &UpdateContactResponse{
+	Res: Res{
+		Code:       1000,
+		StatusCode: 200,
+	},
+	Contact: Contact{
+		ID:         "l4PrVkmDsIIDba9aln829uwPK0nnyWZHnFtrsyb7CJsYgrD6JTVTuuoaVmaANfO2jIVxzZ2vtbt74rznGjjwFQ==",
+		Name:       "Bob",
+		UID:        "proton-web-cd974706-5cde-0e53-e131-c49c88a92ece",
+		Size:       303,
+		CreateTime: 1517416603,
+		ModifyTime: 1517416656,
+		ContactEmails: []ContactEmail{
+			{
+				ID:        "14n6vuf1zbeo3zsYzgV471S6xJ9gzl7-VZ8tcOTQq6ifBlNEre0SUdUM7sXh6e2Q_4NhJZaU9c7jLdB1HCV6dA==",
+				Name:      "Bob",
+				Email:     "bob.changed.tester@protonmail.com",
+				Type:      []string{},
+				Defaults:  1,
+				Order:     1,
+				ContactID: "l4PrVkmDsIIDba9aln829uwPK0nnyWZHnFtrsyb7CJsYgrD6JTVTuuoaVmaANfO2jIVxzZ2vtbt74rznGjjwFQ==",
+				LabelIDs:  []string{},
+			},
+		},
+		LabelIDs: []string{},
+	},
+}
+
+func TestContact_AddContact(t *testing.T) {
+	s, c := newTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		Ok(t, checkMethodAndPath(r, "POST", "/contacts"))
+
+		var addContactsReq AddContactsReq
+		if err := json.NewDecoder(r.Body).Decode(&addContactsReq); err != nil {
+			t.Error("Expecting no error while reading request body, got:", err)
+		}
+		if !reflect.DeepEqual(testAddContactsReq.ContactsCards, addContactsReq.ContactsCards) {
+			t.Errorf("Invalid contacts request: expected %+v but got %+v", testAddContactsReq.ContactsCards, addContactsReq.ContactsCards)
+		}
+
+		fmt.Fprint(w, testAddContactsResponseBody)
+	}))
+	defer s.Close()
+
+	created, err := c.AddContacts(testAddContactsReq.ContactsCards, 0, 0, 0)
+	if err != nil {
+		t.Fatal("Expected no error while adding contact, got:", err)
+	}
+
+	if !reflect.DeepEqual(created, testContactCreated) {
+		t.Fatalf("Invalid created contact: expected %+v, got %+v", testContactCreated, created)
+	}
+}
+
+var testGetContactsResponseBody = `{
+    "Code": 1000,
+    "Contacts": [
+        {
+            "ID": "s_SN9y1q0jczjYCH4zhvfOdHv1QNovKhnJ9bpDcTE0u7WCr2Z-NV9uubHXvOuRozW-HRVam6bQupVYRMC3BCqg==",
+            "Name": "Alice",
+            "UID": "proton-web-98c8de5e-4536-140b-9ab0-bd8ab6a2050b",
+            "Size": 243,
+            "CreateTime": 1517395498,
+            "ModifyTime": 1517395498,
+            "LabelIDs": []
+        },
+        {
+            "ID": "c6CWuyEE6mMRApAxvvCO9MQKydTU8Do1iikL__M5MoWWjDEebzChAUx-73qa1jTV54RzFO5p9pLBPsIIgCwpww==",
+            "Name": "Bob",
+            "UID": "proton-web-cd974706-5cde-0e53-e131-c49c88a92ece",
+            "Size": 303,
+            "CreateTime": 1517394677,
+            "ModifyTime": 1517394678,
+            "LabelIDs": []
+        }
+    ],
+    "Total": 2
+}`
+
+var testGetContacts = []*Contact{
+
+	{
+		ID:         "s_SN9y1q0jczjYCH4zhvfOdHv1QNovKhnJ9bpDcTE0u7WCr2Z-NV9uubHXvOuRozW-HRVam6bQupVYRMC3BCqg==",
+		Name:       "Alice",
+		UID:        "proton-web-98c8de5e-4536-140b-9ab0-bd8ab6a2050b",
+		Size:       243,
+		CreateTime: 1517395498,
+		ModifyTime: 1517395498,
+		LabelIDs:   []string{},
+	},
+	{
+		ID:         "c6CWuyEE6mMRApAxvvCO9MQKydTU8Do1iikL__M5MoWWjDEebzChAUx-73qa1jTV54RzFO5p9pLBPsIIgCwpww==",
+		Name:       "Bob",
+		UID:        "proton-web-cd974706-5cde-0e53-e131-c49c88a92ece",
+		Size:       303,
+		CreateTime: 1517394677,
+		ModifyTime: 1517394678,
+		LabelIDs:   []string{},
+	},
+}
+
+func TestContact_GetContacts(t *testing.T) {
+	s, c := newTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		Ok(t, checkMethodAndPath(r, "GET", "/contacts?Page=0&PageSize=1000"))
+
+		fmt.Fprint(w, testGetContactsResponseBody)
+	}))
+	defer s.Close()
+
+	contacts, err := c.GetContacts(0, 1000)
+	if err != nil {
+		t.Fatal("Expected no error while getting contacts, got:", err)
+	}
+
+	if !reflect.DeepEqual(contacts, testGetContacts) {
+		t.Fatalf("Invalid created contact: expected %+v, got %+v", testGetContacts, contacts)
+	}
+}
+
+var testGetContactByIDResponseBody = `{
+    "Code": 1000,
+    "Contact": {
+        "ID": "s_SN9y1q0jczjYCH4zhvfOdHv1QNovKhnJ9bpDcTE0u7WCr2Z-NV9uubHXvOuRozW-HRVam6bQupVYRMC3BCqg==",
+        "Name": "Alice",
+        "UID": "proton-web-98c8de5e-4536-140b-9ab0-bd8ab6a2050b",
+        "Size": 243,
+        "CreateTime": 1517395498,
+        "ModifyTime": 1517395498,
+        "Cards": [
+            {
+                "Type": 3,
+                "Data": "-----BEGIN PGP MESSAGE-----\nVersion: ProtonMail\nComment: https://protonmail.com\n\nwcBMA1vYAFKnBP8gAQf/RnOQRpo8DVJHQSJRgckEaUQvdMcADiM4L23diyiS\nQfclby/Ve2WInmvZc2RJ3rWENfeqyDZE6krQT642pKiW09GOIyVIjl+hje9y\nE4HBX0AIAWv7QhhKX6UZcM5dYSFbV3j3QxQB8A4Thng2G6ltotMTlbtcHbhu\n96Lt6ngA1tngXLSF5seyflnoiSQ5gLi2qVzrd95dIP6D4Ottcp929/4hDGmq\nPyxw9dColx6gVd1bmIDSI6ewkET4Grmo6QYqjSvjqLOf0PqHKzqypSFLkI5l\nmmnWKYTQCgl9wX+hq6Qz5E+m/BtbkdeX0YxYUss2e+oSAzJmnfdETErG9U5z\n3NJqAc3sgdwDzfWHBzogAxAbDHiqrF6zMlR5SFvZ6nRU7M2DTOE5dJhf+zOp\n1WSKn5LR46LGyt0m5wJPDjaGyQdPffAO4EULvwhGENe10UxRjY1qcUmjYOtS\nunl/vh3afI9PC1jj+HHJD2VgCA==\n=UpcY\n-----END PGP MESSAGE-----\n",
+                "Signature": "-----BEGIN PGP SIGNATURE-----\nVersion: ProtonMail\nComment: https://protonmail.com\n\nwsBcBAEBCAAQBQJacZ4pCRDMO9BwcW4mpAAA6h0H/2+97koXzly5pu9hpbaW\n75d1Q976RjMr5DjAx6tKFtSzznel8YfWgvA6OQmMGdPY8ae7/+3mwCJZYWy/\nXVvUfCSflmYpSIKGfP+Vm1XezWY1W84DGhiFj5n8sdaWisv3bpFwFf1YR3Ae\noBoZ4ufNzaQALRqGPMgXETtXZCtzuL/+0vGSKj5SLECiRcSE4jCPEVRy2bcl\nWJyB9r4VmcjF042OMHxphXoYmTEWvgigyaQFHNORu5cK9EHfHpCG6IcjGbdx\n+9Px5YnDY1ix+YpBKePGSTlLE0u6ow0VTUrdvNjl7IUBaRcfJcIIdgCBOTMw\n1uQ/yeyP46V5AFXFnIKeZeQ=\n=FlOf\n-----END PGP SIGNATURE-----\n"
+            },
+            {
+                "Type": 2,
+                "Data": "BEGIN:VCARD\nVERSION:4.0\nFN;TYPE=fn:Alice\nitem1.EMAIL:alice@protonmail.com\nUID:proton-web-98c8de5e-4536-140b-9ab0-bd8ab6a2050b\nEND:VCARD",
+                "Signature": "-----BEGIN PGP SIGNATURE-----\nVersion: ProtonMail\nComment: https://protonmail.com\n\nwsBcBAEBCAAQBQJacZ4qCRDMO9BwcW4mpAAA3jUIAJ88mIyO8Yj0+evSFXnK\nNxNdjNgn7t1leY0BWlh1nkK76XrZEPipdw2QU8cOcZzn1Wby2SGfZVkwoPc4\nzAhPT4WKbkFVqXhDry5399kLwGYJCxdEcw/oPyYj+YgpQKMxhTrQq21tbEwr\n7JDRBXgi3Cckh/XsteFHOIiAVnM7BV6zFudipnYxa4uNF0Bf4VbUZx1Mm0Wb\nMJaGsO5reqQUQzDPO5TdSAZ8qGSdjVv7RESgUu5DckcDSsnB987Zbh9uFc22\nfPYmb6zA0cEZh3dAjpDPT7cg8hlvfYBb+kP3sLFyLiIkdEG8Pcagjf0k+l76\nr1IsPlYBx2LJmsJf+WDNlj8=\n=Xn+3\n-----END PGP SIGNATURE-----\n"
+            }
+        ],
+        "ContactEmails": [
+            {
+                "ID": "4m2sBxLq4McqD0D330Kuy5xG-yyDNXyLEjG5_RYcjy9X-3qHGNP07DNOWLY40TYtUAQr4fAVp8zOcZ_z2o6H-A==",
+                "Name": "Alice",
+                "Email": "alice@protonmail.com",
+                "Type": [],
+                "Defaults": 1,
+                "Order": 1,
+                "ContactID": "s_SN9y1q0jczjYCH4zhvfOdHv1QNovKhnJ9bpDcTE0u7WCr2Z-NV9uubHXvOuRozW-HRVam6bQupVYRMC3BCqg==",
+                "LabelIDs": []
+            }
+        ],
+        "LabelIDs": []
+    }
+}`
+
+var testGetContactByID = Contact{
+	ID:         "s_SN9y1q0jczjYCH4zhvfOdHv1QNovKhnJ9bpDcTE0u7WCr2Z-NV9uubHXvOuRozW-HRVam6bQupVYRMC3BCqg==",
+	Name:       "Alice",
+	UID:        "proton-web-98c8de5e-4536-140b-9ab0-bd8ab6a2050b",
+	Size:       243,
+	CreateTime: 1517395498,
+	ModifyTime: 1517395498,
+	Cards: []Card{
+		{
+			Type:      3,
+			Data:      "-----BEGIN PGP MESSAGE-----\nVersion: ProtonMail\nComment: https://protonmail.com\n\nwcBMA1vYAFKnBP8gAQf/RnOQRpo8DVJHQSJRgckEaUQvdMcADiM4L23diyiS\nQfclby/Ve2WInmvZc2RJ3rWENfeqyDZE6krQT642pKiW09GOIyVIjl+hje9y\nE4HBX0AIAWv7QhhKX6UZcM5dYSFbV3j3QxQB8A4Thng2G6ltotMTlbtcHbhu\n96Lt6ngA1tngXLSF5seyflnoiSQ5gLi2qVzrd95dIP6D4Ottcp929/4hDGmq\nPyxw9dColx6gVd1bmIDSI6ewkET4Grmo6QYqjSvjqLOf0PqHKzqypSFLkI5l\nmmnWKYTQCgl9wX+hq6Qz5E+m/BtbkdeX0YxYUss2e+oSAzJmnfdETErG9U5z\n3NJqAc3sgdwDzfWHBzogAxAbDHiqrF6zMlR5SFvZ6nRU7M2DTOE5dJhf+zOp\n1WSKn5LR46LGyt0m5wJPDjaGyQdPffAO4EULvwhGENe10UxRjY1qcUmjYOtS\nunl/vh3afI9PC1jj+HHJD2VgCA==\n=UpcY\n-----END PGP MESSAGE-----\n",
+			Signature: "-----BEGIN PGP SIGNATURE-----\nVersion: ProtonMail\nComment: https://protonmail.com\n\nwsBcBAEBCAAQBQJacZ4pCRDMO9BwcW4mpAAA6h0H/2+97koXzly5pu9hpbaW\n75d1Q976RjMr5DjAx6tKFtSzznel8YfWgvA6OQmMGdPY8ae7/+3mwCJZYWy/\nXVvUfCSflmYpSIKGfP+Vm1XezWY1W84DGhiFj5n8sdaWisv3bpFwFf1YR3Ae\noBoZ4ufNzaQALRqGPMgXETtXZCtzuL/+0vGSKj5SLECiRcSE4jCPEVRy2bcl\nWJyB9r4VmcjF042OMHxphXoYmTEWvgigyaQFHNORu5cK9EHfHpCG6IcjGbdx\n+9Px5YnDY1ix+YpBKePGSTlLE0u6ow0VTUrdvNjl7IUBaRcfJcIIdgCBOTMw\n1uQ/yeyP46V5AFXFnIKeZeQ=\n=FlOf\n-----END PGP SIGNATURE-----\n",
+		},
+		{
+			Type:      2,
+			Data:      "BEGIN:VCARD\nVERSION:4.0\nFN;TYPE=fn:Alice\nitem1.EMAIL:alice@protonmail.com\nUID:proton-web-98c8de5e-4536-140b-9ab0-bd8ab6a2050b\nEND:VCARD",
+			Signature: "-----BEGIN PGP SIGNATURE-----\nVersion: ProtonMail\nComment: https://protonmail.com\n\nwsBcBAEBCAAQBQJacZ4qCRDMO9BwcW4mpAAA3jUIAJ88mIyO8Yj0+evSFXnK\nNxNdjNgn7t1leY0BWlh1nkK76XrZEPipdw2QU8cOcZzn1Wby2SGfZVkwoPc4\nzAhPT4WKbkFVqXhDry5399kLwGYJCxdEcw/oPyYj+YgpQKMxhTrQq21tbEwr\n7JDRBXgi3Cckh/XsteFHOIiAVnM7BV6zFudipnYxa4uNF0Bf4VbUZx1Mm0Wb\nMJaGsO5reqQUQzDPO5TdSAZ8qGSdjVv7RESgUu5DckcDSsnB987Zbh9uFc22\nfPYmb6zA0cEZh3dAjpDPT7cg8hlvfYBb+kP3sLFyLiIkdEG8Pcagjf0k+l76\nr1IsPlYBx2LJmsJf+WDNlj8=\n=Xn+3\n-----END PGP SIGNATURE-----\n",
+		},
+	},
+	ContactEmails: []ContactEmail{
+		{
+			ID:        "4m2sBxLq4McqD0D330Kuy5xG-yyDNXyLEjG5_RYcjy9X-3qHGNP07DNOWLY40TYtUAQr4fAVp8zOcZ_z2o6H-A==",
+			Name:      "Alice",
+			Email:     "alice@protonmail.com",
+			Type:      []string{},
+			Defaults:  1,
+			Order:     1,
+			ContactID: "s_SN9y1q0jczjYCH4zhvfOdHv1QNovKhnJ9bpDcTE0u7WCr2Z-NV9uubHXvOuRozW-HRVam6bQupVYRMC3BCqg==",
+			LabelIDs:  []string{},
+		},
+	},
+	LabelIDs: []string{},
+}
+
+func TestContact_GetContactById(t *testing.T) {
+	s, c := newTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		Ok(t, checkMethodAndPath(r, "GET", "/contacts/s_SN9y1q0jczjYCH4zhvfOdHv1QNovKhnJ9bpDcTE0u7WCr2Z-NV9uubHXvOuRozW-HRVam6bQupVYRMC3BCqg=="))
+
+		fmt.Fprint(w, testGetContactByIDResponseBody)
+	}))
+	defer s.Close()
+
+	contact, err := c.GetContactByID("s_SN9y1q0jczjYCH4zhvfOdHv1QNovKhnJ9bpDcTE0u7WCr2Z-NV9uubHXvOuRozW-HRVam6bQupVYRMC3BCqg==")
+	if err != nil {
+		t.Fatal("Expected no error while getting contacts, got:", err)
+	}
+
+	if !reflect.DeepEqual(contact, testGetContactByID) {
+		t.Fatalf("Invalid got contact: expected %+v, got %+v", testGetContactByID, contact)
+	}
+}
+
+var testGetContactsForExportResponseBody = `{
+    "Code": 1000,
+    "Contacts": [
+        {
+            "ID": "c6CWuyEE6mMRApAxvvCO9MQKydTU8Do1iikL__M5MoWWjDEebzChAUx-73qa1jTV54RzFO5p9pLBPsIIgCwpww==",
+            "Cards": [
+                {
+                    "Type": 2,
+                    "Data": "BEGIN:VCARD\nVERSION:4.0\nFN;TYPE=fn:Bob\nitem1.EMAIL:bob.changed.tester@protonmail.com\nUID:proton-web-cd974706-5cde-0e53-e131-c49c88a92ece\nEND:VCARD\n",
+                    "Signature": "-----BEGIN PGP SIGNATURE-----\nVersion: ProtonMail\nComment: https://protonmail.com\n\nwsBcBAEBCAAQBQJacZr2CRDMO9BwcW4mpAAAtxwIAFGgPO+xH4PHppffQC1R\nxCp/Bjzaq5rDUE3ZMKVJ1sFqGVlq2bP5CIN4w2XCe/MuZ+z2o87fSEtt2n7i\n0/8Ah35u4czn7t8FZoW8u9WwHPURa8gUbP3fYpVASBY1Bt2fUxJrSUYn5KQp\njJM/DgF99bhIjOTuhx9IN7DFKG647Arq+GJh9M6RJNxkb3CBfcCVUXoIwMB7\nnM/fA1r+mcl8dQam0WKVJgy9aO2XUUR62w1SpqJlXY3z8hKvXjjskzU3DQk5\net07RLVQvhy2nCZePsM+TJzL8OBbTa1aF/p1xPe+HND7t3ZCm9tQOY+UhK5H\nbhPbQY48KGdci1dTcm2HbsQ=\n=iOnV\n-----END PGP SIGNATURE-----\n"
+                },
+                {
+                    "Type": 3,
+                    "Data": "-----BEGIN PGP MESSAGE-----\nVersion: ProtonMail\nComment: https://protonmail.com\n\nwcBMA1vYAFKnBP8gAQgABsQWnqZadqrHDN43McGhEYfJjOB66R5HhkQAUavP\nHaAHpJciGxfz6tbztQu4C6kdMA80ElbD8c+bJqalw6ZbT4seoP4TTQLykD1n\n0LuNBlaW4x8kfd8rZzFdckk/dY2PruX6byAjSZslnZlZSwp99AJJbvJtfXRR\nzunKMbDieRkaApGZYT25wT5mz1embpXFesvO4nDkOEQCa0uyti3mNSLhYlf/\ntbaOS3WM9VYM9eB9YRZGzJNxMtTxOsd45tBlGCHnCzWEUnJdqZuYzH2QOky7\nMckXhk6YwyemYi/q7OOgSYEg/0lCs2EK3b//14yPDx8Bj5G7rZrnDgsP+BHj\nu9KaAZb2pSBPQoJ2DY3Y4A2Sg8GjaX5CMO9D6GKJkZSYkXddQgcmw7sVPUS+\n+5JaPXlfxoJOOn9kj9A6LDC6eMhYaLujG1BKcZ16DB0jqfwMnPLJ+bYEdatr\nKMvd9rbdsDwQ/tfk11VvHpiEBCNZjxM2+bdBLl9q2EXaLXi+dz/rJg5C0A9u\nNS2CzCUvg6+jNUzHo/RBfRXvlNV8tw==\n=mE2b\n-----END PGP MESSAGE-----\n",
+                    "Signature": "-----BEGIN PGP SIGNATURE-----\nVersion: ProtonMail\nComment: https://protonmail.com\n\nwsBcBAEBCAAQBQJacZr2CRDMO9BwcW4mpAAApucIAD/uwWuV6DOg127XIPG6\n/jluL8jmwyCJX9noL6S8ZVMOymziKSh4/P1QyMPC5SL4lMPEiuaEdyetfBkU\n+5hW3tcZ+ptxmDi59SVYqmXTVewPgeB7t8c5nbzCuVuzA7ZAo8HAXHzFVQDS\nj9fKVGjZzQkmlwdcfnkXHAF0Ejilv9wxOOYgqVDuzm7JXVF3Um7nAgGKTJE5\n5CNnrEjmJGapj96mQFwXzET/kAhNIBw9tL5FAkDlKImdw8C0w9sXdvDu3yVM\ntvUZ5o2rR6ft0SC1byFso49vgJ/syeK6P2pPzltZJbsp4MvmlPUB0/G1XRU+\nI7q4IOWCvs8RD88ADmOty2o=\n=hyZE\n-----END PGP SIGNATURE-----\n"
+                }
+            ]
+        },
+        {
+            "ID": "s_SN9y1q0jczjYCH4zhvfOdHv1QNovKhnJ9bpDcTE0u7WCr2Z-NV9uubHXvOuRozW-HRVam6bQupVYRMC3BCqg==",
+            "Cards": [
+                {
+                    "Type": 3,
+                    "Data": "-----BEGIN PGP MESSAGE-----\nVersion: ProtonMail\nComment: https://protonmail.com\n\nwcBMA1vYAFKnBP8gAQf/RnOQRpo8DVJHQSJRgckEaUQvdMcADiM4L23diyiS\nQfclby/Ve2WInmvZc2RJ3rWENfeqyDZE6krQT642pKiW09GOIyVIjl+hje9y\nE4HBX0AIAWv7QhhKX6UZcM5dYSFbV3j3QxQB8A4Thng2G6ltotMTlbtcHbhu\n96Lt6ngA1tngXLSF5seyflnoiSQ5gLi2qVzrd95dIP6D4Ottcp929/4hDGmq\nPyxw9dColx6gVd1bmIDSI6ewkET4Grmo6QYqjSvjqLOf0PqHKzqypSFLkI5l\nmmnWKYTQCgl9wX+hq6Qz5E+m/BtbkdeX0YxYUss2e+oSAzJmnfdETErG9U5z\n3NJqAc3sgdwDzfWHBzogAxAbDHiqrF6zMlR5SFvZ6nRU7M2DTOE5dJhf+zOp\n1WSKn5LR46LGyt0m5wJPDjaGyQdPffAO4EULvwhGENe10UxRjY1qcUmjYOtS\nunl/vh3afI9PC1jj+HHJD2VgCA==\n=UpcY\n-----END PGP MESSAGE-----\n",
+                    "Signature": "-----BEGIN PGP SIGNATURE-----\nVersion: ProtonMail\nComment: https://protonmail.com\n\nwsBcBAEBCAAQBQJacZ4pCRDMO9BwcW4mpAAA6h0H/2+97koXzly5pu9hpbaW\n75d1Q976RjMr5DjAx6tKFtSzznel8YfWgvA6OQmMGdPY8ae7/+3mwCJZYWy/\nXVvUfCSflmYpSIKGfP+Vm1XezWY1W84DGhiFj5n8sdaWisv3bpFwFf1YR3Ae\noBoZ4ufNzaQALRqGPMgXETtXZCtzuL/+0vGSKj5SLECiRcSE4jCPEVRy2bcl\nWJyB9r4VmcjF042OMHxphXoYmTEWvgigyaQFHNORu5cK9EHfHpCG6IcjGbdx\n+9Px5YnDY1ix+YpBKePGSTlLE0u6ow0VTUrdvNjl7IUBaRcfJcIIdgCBOTMw\n1uQ/yeyP46V5AFXFnIKeZeQ=\n=FlOf\n-----END PGP SIGNATURE-----\n"
+                },
+                {
+                    "Type": 2,
+                    "Data": "BEGIN:VCARD\nVERSION:4.0\nFN;TYPE=fn:Alice\nitem1.EMAIL:alice@protonmail.com\nUID:proton-web-98c8de5e-4536-140b-9ab0-bd8ab6a2050b\nEND:VCARD",
+                    "Signature": "-----BEGIN PGP SIGNATURE-----\nVersion: ProtonMail\nComment: https://protonmail.com\n\nwsBcBAEBCAAQBQJacZ4qCRDMO9BwcW4mpAAA3jUIAJ88mIyO8Yj0+evSFXnK\nNxNdjNgn7t1leY0BWlh1nkK76XrZEPipdw2QU8cOcZzn1Wby2SGfZVkwoPc4\nzAhPT4WKbkFVqXhDry5399kLwGYJCxdEcw/oPyYj+YgpQKMxhTrQq21tbEwr\n7JDRBXgi3Cckh/XsteFHOIiAVnM7BV6zFudipnYxa4uNF0Bf4VbUZx1Mm0Wb\nMJaGsO5reqQUQzDPO5TdSAZ8qGSdjVv7RESgUu5DckcDSsnB987Zbh9uFc22\nfPYmb6zA0cEZh3dAjpDPT7cg8hlvfYBb+kP3sLFyLiIkdEG8Pcagjf0k+l76\nr1IsPlYBx2LJmsJf+WDNlj8=\n=Xn+3\n-----END PGP SIGNATURE-----\n"
+                }
+            ]
+        }
+    ],
+    "Total": 2
+}`
+
+var testGetContactsForExport = []Contact{
+	{
+		ID: "c6CWuyEE6mMRApAxvvCO9MQKydTU8Do1iikL__M5MoWWjDEebzChAUx-73qa1jTV54RzFO5p9pLBPsIIgCwpww==",
+		Cards: []Card{
+			{
+				Type:      2,
+				Data:      "BEGIN:VCARD\nVERSION:4.0\nFN;TYPE=fn:Bob\nitem1.EMAIL:bob.changed.tester@protonmail.com\nUID:proton-web-cd974706-5cde-0e53-e131-c49c88a92ece\nEND:VCARD\n",
+				Signature: "-----BEGIN PGP SIGNATURE-----\nVersion: ProtonMail\nComment: https://protonmail.com\n\nwsBcBAEBCAAQBQJacZr2CRDMO9BwcW4mpAAAtxwIAFGgPO+xH4PHppffQC1R\nxCp/Bjzaq5rDUE3ZMKVJ1sFqGVlq2bP5CIN4w2XCe/MuZ+z2o87fSEtt2n7i\n0/8Ah35u4czn7t8FZoW8u9WwHPURa8gUbP3fYpVASBY1Bt2fUxJrSUYn5KQp\njJM/DgF99bhIjOTuhx9IN7DFKG647Arq+GJh9M6RJNxkb3CBfcCVUXoIwMB7\nnM/fA1r+mcl8dQam0WKVJgy9aO2XUUR62w1SpqJlXY3z8hKvXjjskzU3DQk5\net07RLVQvhy2nCZePsM+TJzL8OBbTa1aF/p1xPe+HND7t3ZCm9tQOY+UhK5H\nbhPbQY48KGdci1dTcm2HbsQ=\n=iOnV\n-----END PGP SIGNATURE-----\n",
+			},
+			{
+				Type:      3,
+				Data:      "-----BEGIN PGP MESSAGE-----\nVersion: ProtonMail\nComment: https://protonmail.com\n\nwcBMA1vYAFKnBP8gAQgABsQWnqZadqrHDN43McGhEYfJjOB66R5HhkQAUavP\nHaAHpJciGxfz6tbztQu4C6kdMA80ElbD8c+bJqalw6ZbT4seoP4TTQLykD1n\n0LuNBlaW4x8kfd8rZzFdckk/dY2PruX6byAjSZslnZlZSwp99AJJbvJtfXRR\nzunKMbDieRkaApGZYT25wT5mz1embpXFesvO4nDkOEQCa0uyti3mNSLhYlf/\ntbaOS3WM9VYM9eB9YRZGzJNxMtTxOsd45tBlGCHnCzWEUnJdqZuYzH2QOky7\nMckXhk6YwyemYi/q7OOgSYEg/0lCs2EK3b//14yPDx8Bj5G7rZrnDgsP+BHj\nu9KaAZb2pSBPQoJ2DY3Y4A2Sg8GjaX5CMO9D6GKJkZSYkXddQgcmw7sVPUS+\n+5JaPXlfxoJOOn9kj9A6LDC6eMhYaLujG1BKcZ16DB0jqfwMnPLJ+bYEdatr\nKMvd9rbdsDwQ/tfk11VvHpiEBCNZjxM2+bdBLl9q2EXaLXi+dz/rJg5C0A9u\nNS2CzCUvg6+jNUzHo/RBfRXvlNV8tw==\n=mE2b\n-----END PGP MESSAGE-----\n",
+				Signature: "-----BEGIN PGP SIGNATURE-----\nVersion: ProtonMail\nComment: https://protonmail.com\n\nwsBcBAEBCAAQBQJacZr2CRDMO9BwcW4mpAAApucIAD/uwWuV6DOg127XIPG6\n/jluL8jmwyCJX9noL6S8ZVMOymziKSh4/P1QyMPC5SL4lMPEiuaEdyetfBkU\n+5hW3tcZ+ptxmDi59SVYqmXTVewPgeB7t8c5nbzCuVuzA7ZAo8HAXHzFVQDS\nj9fKVGjZzQkmlwdcfnkXHAF0Ejilv9wxOOYgqVDuzm7JXVF3Um7nAgGKTJE5\n5CNnrEjmJGapj96mQFwXzET/kAhNIBw9tL5FAkDlKImdw8C0w9sXdvDu3yVM\ntvUZ5o2rR6ft0SC1byFso49vgJ/syeK6P2pPzltZJbsp4MvmlPUB0/G1XRU+\nI7q4IOWCvs8RD88ADmOty2o=\n=hyZE\n-----END PGP SIGNATURE-----\n",
+			},
+		},
+	},
+	{
+		ID: "s_SN9y1q0jczjYCH4zhvfOdHv1QNovKhnJ9bpDcTE0u7WCr2Z-NV9uubHXvOuRozW-HRVam6bQupVYRMC3BCqg==",
+		Cards: []Card{
+			{
+				Type:      3,
+				Data:      "-----BEGIN PGP MESSAGE-----\nVersion: ProtonMail\nComment: https://protonmail.com\n\nwcBMA1vYAFKnBP8gAQf/RnOQRpo8DVJHQSJRgckEaUQvdMcADiM4L23diyiS\nQfclby/Ve2WInmvZc2RJ3rWENfeqyDZE6krQT642pKiW09GOIyVIjl+hje9y\nE4HBX0AIAWv7QhhKX6UZcM5dYSFbV3j3QxQB8A4Thng2G6ltotMTlbtcHbhu\n96Lt6ngA1tngXLSF5seyflnoiSQ5gLi2qVzrd95dIP6D4Ottcp929/4hDGmq\nPyxw9dColx6gVd1bmIDSI6ewkET4Grmo6QYqjSvjqLOf0PqHKzqypSFLkI5l\nmmnWKYTQCgl9wX+hq6Qz5E+m/BtbkdeX0YxYUss2e+oSAzJmnfdETErG9U5z\n3NJqAc3sgdwDzfWHBzogAxAbDHiqrF6zMlR5SFvZ6nRU7M2DTOE5dJhf+zOp\n1WSKn5LR46LGyt0m5wJPDjaGyQdPffAO4EULvwhGENe10UxRjY1qcUmjYOtS\nunl/vh3afI9PC1jj+HHJD2VgCA==\n=UpcY\n-----END PGP MESSAGE-----\n",
+				Signature: "-----BEGIN PGP SIGNATURE-----\nVersion: ProtonMail\nComment: https://protonmail.com\n\nwsBcBAEBCAAQBQJacZ4pCRDMO9BwcW4mpAAA6h0H/2+97koXzly5pu9hpbaW\n75d1Q976RjMr5DjAx6tKFtSzznel8YfWgvA6OQmMGdPY8ae7/+3mwCJZYWy/\nXVvUfCSflmYpSIKGfP+Vm1XezWY1W84DGhiFj5n8sdaWisv3bpFwFf1YR3Ae\noBoZ4ufNzaQALRqGPMgXETtXZCtzuL/+0vGSKj5SLECiRcSE4jCPEVRy2bcl\nWJyB9r4VmcjF042OMHxphXoYmTEWvgigyaQFHNORu5cK9EHfHpCG6IcjGbdx\n+9Px5YnDY1ix+YpBKePGSTlLE0u6ow0VTUrdvNjl7IUBaRcfJcIIdgCBOTMw\n1uQ/yeyP46V5AFXFnIKeZeQ=\n=FlOf\n-----END PGP SIGNATURE-----\n",
+			},
+			{
+				Type:      2,
+				Data:      "BEGIN:VCARD\nVERSION:4.0\nFN;TYPE=fn:Alice\nitem1.EMAIL:alice@protonmail.com\nUID:proton-web-98c8de5e-4536-140b-9ab0-bd8ab6a2050b\nEND:VCARD",
+				Signature: "-----BEGIN PGP SIGNATURE-----\nVersion: ProtonMail\nComment: https://protonmail.com\n\nwsBcBAEBCAAQBQJacZ4qCRDMO9BwcW4mpAAA3jUIAJ88mIyO8Yj0+evSFXnK\nNxNdjNgn7t1leY0BWlh1nkK76XrZEPipdw2QU8cOcZzn1Wby2SGfZVkwoPc4\nzAhPT4WKbkFVqXhDry5399kLwGYJCxdEcw/oPyYj+YgpQKMxhTrQq21tbEwr\n7JDRBXgi3Cckh/XsteFHOIiAVnM7BV6zFudipnYxa4uNF0Bf4VbUZx1Mm0Wb\nMJaGsO5reqQUQzDPO5TdSAZ8qGSdjVv7RESgUu5DckcDSsnB987Zbh9uFc22\nfPYmb6zA0cEZh3dAjpDPT7cg8hlvfYBb+kP3sLFyLiIkdEG8Pcagjf0k+l76\nr1IsPlYBx2LJmsJf+WDNlj8=\n=Xn+3\n-----END PGP SIGNATURE-----\n",
+			},
+		},
+	},
+}
+
+func TestContact_GetContactsForExport(t *testing.T) {
+	s, c := newTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		Ok(t, checkMethodAndPath(r, "GET", "/contacts/export?Page=0&PageSize=1000"))
+
+		fmt.Fprint(w, testGetContactsForExportResponseBody)
+	}))
+	defer s.Close()
+
+	contacts, err := c.GetContactsForExport(0, 1000)
+	if err != nil {
+		t.Fatal("Expected no error while getting contacts for export, got:", err)
+	}
+
+	if !reflect.DeepEqual(contacts, testGetContactsForExport) {
+		t.Fatalf("Invalid contact for export: expected %+v, got %+v", testGetContactsForExport, contacts)
+	}
+}
+
+var testGetContactsEmailsResponseBody = `{
+    "Code": 1000,
+    "ContactEmails": [
+        {
+            "ID": "Hgyz1tG0OiC2v_hMIVOa6juMOAp_recWNzWII7a79Tfwdx08Jy3FJY0_Y_UtFYwbi6mN-Xx1sOI9_GmUGAcwWg==",
+            "Name": "Bob",
+            "Email": "bob.changed.tester@protonmail.com",
+            "Type": [],
+            "Defaults": 1,
+            "Order": 1,
+            "ContactID": "c6CWuyEE6mMRApAxvvCO9MQKydTU8Do1iikL__M5MoWWjDEebzChAUx-73qa1jTV54RzFO5p9pLBPsIIgCwpww==",
+            "LabelIDs": []
+        },
+        {
+            "ID": "4m2sBxLq4McqD0D330Kuy5xG-yyDNXyLEjG5_RYcjy9X-3qHGNP07DNOWLY40TYtUAQr4fAVp8zOcZ_z2o6H-A==",
+            "Name": "Alice",
+            "Email": "alice@protonmail.com",
+            "Type": [],
+            "Defaults": 1,
+            "Order": 1,
+            "ContactID": "s_SN9y1q0jczjYCH4zhvfOdHv1QNovKhnJ9bpDcTE0u7WCr2Z-NV9uubHXvOuRozW-HRVam6bQupVYRMC3BCqg==",
+            "LabelIDs": []
+        }
+    ],
+    "Total": 2
+}`
+
+var testGetContactsEmails = []ContactEmail{
+	{
+		ID:        "Hgyz1tG0OiC2v_hMIVOa6juMOAp_recWNzWII7a79Tfwdx08Jy3FJY0_Y_UtFYwbi6mN-Xx1sOI9_GmUGAcwWg==",
+		Name:      "Bob",
+		Email:     "bob.changed.tester@protonmail.com",
+		Type:      []string{},
+		Defaults:  1,
+		Order:     1,
+		ContactID: "c6CWuyEE6mMRApAxvvCO9MQKydTU8Do1iikL__M5MoWWjDEebzChAUx-73qa1jTV54RzFO5p9pLBPsIIgCwpww==",
+		LabelIDs:  []string{},
+	},
+	{
+		ID:        "4m2sBxLq4McqD0D330Kuy5xG-yyDNXyLEjG5_RYcjy9X-3qHGNP07DNOWLY40TYtUAQr4fAVp8zOcZ_z2o6H-A==",
+		Name:      "Alice",
+		Email:     "alice@protonmail.com",
+		Type:      []string{},
+		Defaults:  1,
+		Order:     1,
+		ContactID: "s_SN9y1q0jczjYCH4zhvfOdHv1QNovKhnJ9bpDcTE0u7WCr2Z-NV9uubHXvOuRozW-HRVam6bQupVYRMC3BCqg==",
+		LabelIDs:  []string{},
+	},
+}
+
+func TestContact_GetAllContactsEmails(t *testing.T) {
+	s, c := newTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		Ok(t, checkMethodAndPath(r, "GET", "/contacts/emails?Page=0&PageSize=1000"))
+
+		fmt.Fprint(w, testGetContactsEmailsResponseBody)
+	}))
+	defer s.Close()
+
+	contactsEmails, err := c.GetAllContactsEmails(0, 1000)
+	if err != nil {
+		t.Fatal("Expected no error while getting contacts for export, got:", err)
+	}
+
+	if !reflect.DeepEqual(contactsEmails, testGetContactsEmails) {
+		t.Fatalf("Invalid contact for export: expected %+v, got %+v", testGetContactsEmails, contactsEmails)
+	}
+}
+
+var testUpdateContactReq = UpdateContactReq{
+	Cards: []Card{
+		{
+			Type: 2,
+			Data: `BEGIN:VCARD
+VERSION:4.0
+FN;TYPE=fn:Bob
+item1.EMAIL:bob.changed.tester@protonmail.com
+UID:proton-web-cd974706-5cde-0e53-e131-c49c88a92ece
+END:VCARD
+`,
+			Signature: ``,
+		},
+	},
+}
+
+var testUpdateContactResponseBody = `{
+    "Code": 1000,
+    "Contact": {
+        "ID": "l4PrVkmDsIIDba9aln829uwPK0nnyWZHnFtrsyb7CJsYgrD6JTVTuuoaVmaANfO2jIVxzZ2vtbt74rznGjjwFQ==",
+        "Name": "Bob",
+        "UID": "proton-web-cd974706-5cde-0e53-e131-c49c88a92ece",
+        "Size": 303,
+        "CreateTime": 1517416603,
+        "ModifyTime": 1517416656,
+        "ContactEmails": [
+            {
+                "ID": "14n6vuf1zbeo3zsYzgV471S6xJ9gzl7-VZ8tcOTQq6ifBlNEre0SUdUM7sXh6e2Q_4NhJZaU9c7jLdB1HCV6dA==",
+                "Name": "Bob",
+                "Email": "bob.changed.tester@protonmail.com",
+                "Type": [],
+                "Defaults": 1,
+                "Order": 1,
+                "ContactID": "l4PrVkmDsIIDba9aln829uwPK0nnyWZHnFtrsyb7CJsYgrD6JTVTuuoaVmaANfO2jIVxzZ2vtbt74rznGjjwFQ==",
+                "LabelIDs": []
+            }
+        ],
+        "LabelIDs": []
+    }
+}`
+
+func TestContact_UpdateContact(t *testing.T) {
+	s, c := newTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		Ok(t, checkMethodAndPath(r, "PUT", "/contacts/l4PrVkmDsIIDba9aln829uwPK0nnyWZHnFtrsyb7CJsYgrD6JTVTuuoaVmaANfO2jIVxzZ2vtbt74rznGjjwFQ=="))
+
+		var updateContactReq UpdateContactReq
+		if err := json.NewDecoder(r.Body).Decode(&updateContactReq); err != nil {
+			t.Error("Expecting no error while reading request body, got:", err)
+		}
+		if !reflect.DeepEqual(testUpdateContactReq.Cards, updateContactReq.Cards) {
+			t.Errorf("Invalid contacts request: expected %+v but got %+v", testUpdateContactReq.Cards, updateContactReq.Cards)
+		}
+
+		fmt.Fprint(w, testUpdateContactResponseBody)
+	}))
+	defer s.Close()
+
+	created, err := c.UpdateContact("l4PrVkmDsIIDba9aln829uwPK0nnyWZHnFtrsyb7CJsYgrD6JTVTuuoaVmaANfO2jIVxzZ2vtbt74rznGjjwFQ==", testUpdateContactReq.Cards)
+	if err != nil {
+		t.Fatal("Expected no error while updating contact, got:", err)
+	}
+
+	if !reflect.DeepEqual(created, testContactUpdated) {
+		t.Fatalf("Invalid updated contact: expected\n%+v\ngot\n%+v\n", testContactUpdated, created)
+	}
+}
+
+var testDeleteContactsReq = DeleteReq{
+	IDs: []string{
+		"s_SN9y1q0jczjYCH4zhvfOdHv1QNovKhnJ9bpDcTE0u7WCr2Z-NV9uubHXvOuRozW-HRVam6bQupVYRMC3BCqg==",
+	},
+}
+
+var testDeleteContactsResponseBody = `{
+    "Code": 1001,
+    "Responses": [
+        {
+            "ID": "s_SN9y1q0jczjYCH4zhvfOdHv1QNovKhnJ9bpDcTE0u7WCr2Z-NV9uubHXvOuRozW-HRVam6bQupVYRMC3BCqg==",
+            "Response": {
+                "Code": 1000
+            }
+        }
+    ]
+}`
+
+func TestContact_DeleteContacts(t *testing.T) {
+	s, c := newTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		Ok(t, checkMethodAndPath(r, "PUT", "/contacts/delete"))
+
+		var deleteContactsReq DeleteReq
+		if err := json.NewDecoder(r.Body).Decode(&deleteContactsReq); err != nil {
+			t.Error("Expecting no error while reading request body, got:", err)
+		}
+		if !reflect.DeepEqual(testDeleteContactsReq.IDs, deleteContactsReq.IDs) {
+			t.Errorf("Invalid delete contacts request: expected %+v but got %+v", deleteContactsReq.IDs, testDeleteContactsReq.IDs)
+		}
+
+		fmt.Fprint(w, testDeleteContactsResponseBody)
+	}))
+	defer s.Close()
+
+	err := c.DeleteContacts([]string{"s_SN9y1q0jczjYCH4zhvfOdHv1QNovKhnJ9bpDcTE0u7WCr2Z-NV9uubHXvOuRozW-HRVam6bQupVYRMC3BCqg=="})
+	if err != nil {
+		t.Fatal("Expected no error while getting contacts for export, got:", err)
+	}
+}
+
+var testDeleteAllResponseBody = `{
+    "Code": 1000
+}`
+
+func TestContact_DeleteAllContacts(t *testing.T) {
+	s, c := newTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		Ok(t, checkMethodAndPath(r, "DELETE", "/contacts"))
+
+		fmt.Fprint(w, testDeleteAllResponseBody)
+	}))
+	defer s.Close()
+
+	err := c.DeleteAllContacts()
+	if err != nil {
+		t.Fatal("Expected no error while getting contacts for export, got:", err)
+	}
+}
+
+func TestContact_isSignedCardType(t *testing.T) {
+	if !isSignedCardType(SignedCard) || !isSignedCardType(EncryptedSignedCard) {
+		t.Fatal("isSignedCardType shouldn't return false for signed card types")
+	}
+	if isSignedCardType(CleartextCard) || isSignedCardType(EncryptedCard) {
+		t.Fatal("isSignedCardType shouldn't return true for non-signed card types")
+	}
+}
+
+func TestContact_isEncryptedCardType(t *testing.T) {
+	if !isEncryptedCardType(EncryptedCard) || !isEncryptedCardType(EncryptedSignedCard) {
+		t.Fatal("isEncryptedCardType shouldn't return false for encrypted card types")
+	}
+	if isEncryptedCardType(CleartextCard) || isEncryptedCardType(SignedCard) {
+		t.Fatal("isEncryptedCardType shouldn't return true for non-encrypted card types")
+	}
+}
+
+var testCardsEncrypted = []Card{
+	{
+		Type:      EncryptedSignedCard,
+		Data:      "-----BEGIN PGP MESSAGE-----\nVersion: GopenPGP 0.0.1 (ddacebe0)\nComment: https://gopenpgp.org\n\nwcBMA0fcZ7XLgmf2AQf/fLKA6ZCkDxumpDoUoFQfO86B9LFuqGEJq+voP12C6UXo\nfB2nTy/K4+VosLKYOkU9sW1PZOCL+i00z+zkqUZ6jchbZBpzwy/UCTmpPRw5zrmr\nW6bZCwwgqJSGVWrvcrDA3bW9cn/HHqQqU6jNeXIF+IuhTscRAJVGehJZYWjr1lgB\nToJhg4+//Bgp/Fxzz8Fej/fsokgOlRJ8xcZKYx0rKL/+Il0u2jnd08kJTegpaY+6\nBlsYBzfYq25WkS02iy02wHbt6XD7AxFDi4WDjsM8bryLSm/KNWrejqfDYb/tMAKa\nKNJqK39/EUewzp1gHEXiGmdDEIFTKCHTDTPV84mwf9I1Ae4yoLs+ilYE6sSk7DCh\nPSWjDC8lpKzmw93slsejTG93HJKQPcZ0rLBpv6qPZX6widNYjDE=\n=QFxr\n-----END PGP MESSAGE-----",
+		Signature: "-----BEGIN PGP SIGNATURE-----\nVersion: GopenPGP 0.0.1 (ddacebe0)\nComment: https://gopenpgp.org\n\nwsBcBAABCgAQBQJdZQ1kCRA+tiWe3yHfJAAA9nMH/0X7pS8TGt6Ox0BewRh0vjfQ\n9LPLwbOiHdj97LNqutZcLlDTfm9SPH82221ZpVILWhB0u2kFeNUGihVbjAqJGYJn\nEk2TELLwn8csYRy9r5JkyUirqrvh7jgl4vs1yt8O/3Yb4ARudOoZr8Yrb4+NVNe0\nCcwQJnH/fJPtF1hbarKwtKtCo3IFwTis4pc8qWJRpBH61z1mO0Yr/LIh85QndhnF\nnZ/3MkWOY0kp2gl4ptqtNUw7z+JJ4LLVdT3ycdVK7GVTZmIG90y5KKxwJvrwbS7/\n8rmPGPQ5diLEMrzuKC2plXT6Pdy0ShtZxie2C3JY86e7ol7xvl0pNqxzOrj424w=\n=AOTG\n-----END PGP SIGNATURE-----",
+	},
+}
+
+var testCardsCleartext = []Card{
+	{
+		Type:      EncryptedSignedCard,
+		Data:      "data",
+		Signature: "-----BEGIN PGP SIGNATURE-----\nVersion: GopenPGP 0.0.1 (ddacebe0)\nComment: https://gopenpgp.org\n\nwsBcBAABCgAQBQJdZQ1kCRA+tiWe3yHfJAAA9nMH/0X7pS8TGt6Ox0BewRh0vjfQ\n9LPLwbOiHdj97LNqutZcLlDTfm9SPH82221ZpVILWhB0u2kFeNUGihVbjAqJGYJn\nEk2TELLwn8csYRy9r5JkyUirqrvh7jgl4vs1yt8O/3Yb4ARudOoZr8Yrb4+NVNe0\nCcwQJnH/fJPtF1hbarKwtKtCo3IFwTis4pc8qWJRpBH61z1mO0Yr/LIh85QndhnF\nnZ/3MkWOY0kp2gl4ptqtNUw7z+JJ4LLVdT3ycdVK7GVTZmIG90y5KKxwJvrwbS7/\n8rmPGPQ5diLEMrzuKC2plXT6Pdy0ShtZxie2C3JY86e7ol7xvl0pNqxzOrj424w=\n=AOTG\n-----END PGP SIGNATURE-----",
+	},
+}
+
+func TestClient_Encrypt(t *testing.T) {
+	c := newTestClient()
+	c.kr = testPrivateKeyRing
+
+	cardEncrypted, err := c.EncryptAndSignCards(testCardsCleartext)
+	assert.Nil(t, err)
+
+	// Result is always different, so the best way is to test it by decrypting again.
+	// Another test for decrypting will help us to be sure it's working.
+	cardCleartext, err := c.DecryptAndVerifyCards(cardEncrypted)
+	assert.Nil(t, err)
+	assert.Equal(t, testCardsCleartext[0].Data, cardCleartext[0].Data)
+}
+
+func TestClient_Decrypt(t *testing.T) {
+	c := newTestClient()
+	c.kr = testPrivateKeyRing
+
+	cardCleartext, err := c.DecryptAndVerifyCards(testCardsEncrypted)
+	assert.Nil(t, err)
+	assert.Equal(t, testCardsCleartext[0].Data, cardCleartext[0].Data)
+}

pkg/pmapi/conversations.go

@@ -0,0 +1,51 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+// ConversationsCount have same structure as MessagesCount.
+type ConversationsCount MessagesCount
+
+// ConversationsCountsRes holds response from server.
+type ConversationsCountsRes struct {
+	Res
+
+	Counts []*ConversationsCount
+}
+
+// Conversation contains one body and multiple metadata.
+type Conversation struct{}
+
+// CountConversations counts conversations by label.
+func (c *Client) CountConversations(addressID string) (counts []*ConversationsCount, err error) {
+	reqURL := "/conversations/count"
+	if addressID != "" {
+		reqURL += ("?AddressID=" + addressID)
+	}
+	req, err := NewRequest("GET", reqURL, nil)
+	if err != nil {
+		return
+	}
+
+	var res ConversationsCountsRes
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+
+	counts, err = res.Counts, res.Err()
+	return
+}

pkg/pmapi/dialer_with_proxy.go

@@ -0,0 +1,373 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"encoding/pem"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+// TLSReport is inspired by https://tools.ietf.org/html/rfc7469#section-3.
+type TLSReport struct {
+	//  DateTime of observed pin validation in time.RFC3339 format.
+	DateTime string `json:"date-time"`
+
+	// Hostname to which the UA made original request that failed pin validation.
+	Hostname string `json:"hostname"`
+
+	// Port to which the UA made original request that failed pin validation.
+	Port int `json:"port"`
+
+	// EffectiveExpirationDate for noted pins in time.RFC3339 format.
+	EffectiveExpirationDate string `json:"effective-expiration-date"`
+
+	// IncludeSubdomains indicates whether or not the UA has noted the
+	// includeSubDomains directive for the Known Pinned Host.
+	IncludeSubdomains bool `json:"include-subdomains"`
+
+	// NotedHostname indicates the hostname that the UA noted when it noted
+	// the Known Pinned Host. This field allows operators to understand why
+	// Pin Validation was performed for, e.g., foo.example.com when the
+	// noted Known Pinned Host was example.com with includeSubDomains set.
+	NotedHostname string `json:"noted-hostname"`
+
+	// ServedCertificateChain is the certificate chain, as served by
+	// the Known Pinned Host during TLS session setup.  It is provided as an
+	// array of strings; each string pem1, ... pemN is the Privacy-Enhanced
+	// Mail (PEM) representation of each X.509 certificate as described in
+	// [RFC7468].
+	ServedCertificateChain []string `json:"served-certificate-chain"`
+
+	// ValidatedCertificateChain is the certificate chain, as
+	// constructed by the UA during certificate chain verification.  (This
+	// may differ from the served-certificate-chain.)  It is provided as an
+	// array of strings; each string pem1, ... pemN is the PEM
+	// representation of each X.509 certificate as described in [RFC7468].
+	// UAs that build certificate chains in more than one way during the
+	// validation process SHOULD send the last chain built.  In this way,
+	// they can avoid keeping too much state during the validation process.
+	ValidatedCertificateChain []string `json:"validated-certificate-chain"`
+
+	// The known-pins are the Pins that the UA has noted for the Known
+	// Pinned Host.  They are provided as an array of strings with the
+	// syntax: known-pin = token "=" quoted-string
+	// e.g.:
+	// ```
+	// "known-pins": [
+	//   'pin-sha256="d6qzRu9zOECb90Uez27xWltNsj0e1Md7GkYYkVoZWmM="',
+	//   "pin-sha256=\"E9CZ9INDbd+2eRQozYqqbQ2yXLVKB9+xcprMF+44U1g=\""
+	// ]
+	// ```
+	KnownPins []string `json:"known-pins"`
+
+	// AppVersion is used to set `x-pm-appversion` json format from datatheorem/TrustKit.
+	AppVersion string `json:"app-version"`
+}
+
+// ErrTLSMatch indicates that no TLS fingerprint match could be found.
+var ErrTLSMatch = fmt.Errorf("TLS fingerprint match not found")
+
+// DialerWithPinning will provide dial function which checks the fingerprints of public cert
+// received from contacted server. If no match found among know pinse it will report using
+// ReportCertIssueLocal.
+type DialerWithPinning struct {
+	// isReported will stop reporting if true.
+	isReported bool
+
+	// report stores known pins.
+	report TLSReport
+
+	// When reportURI is not empty the tls issue report will be send to this URI.
+	reportURI string
+
+	// ReportCertIssueLocal is used send signal to application about certificate issue.
+	// It is used only if set.
+	ReportCertIssueLocal func()
+
+	// proxyManager manages API proxies.
+	proxyManager *proxyManager
+
+	// A logger for logging messages.
+	log logrus.FieldLogger
+}
+
+func NewDialerWithPinning(reportURI string, report TLSReport) *DialerWithPinning {
+	log := logrus.WithField("pkg", "pmapi/tls-pinning")
+
+	proxyManager := newProxyManager(dohProviders, proxyQuery)
+
+	return &DialerWithPinning{
+		isReported:   false,
+		reportURI:    reportURI,
+		report:       report,
+		proxyManager: proxyManager,
+		log:          log,
+	}
+}
+
+func NewPMAPIPinning(appVersion string) *DialerWithPinning {
+	return NewDialerWithPinning(
+		"https://reports.protonmail.ch/reports/tls",
+		TLSReport{
+			EffectiveExpirationDate:   time.Now().Add(365 * 24 * 60 * 60 * time.Second).Format(time.RFC3339),
+			IncludeSubdomains:         false,
+			ValidatedCertificateChain: []string{},
+			ServedCertificateChain:    []string{},
+			AppVersion:                appVersion,
+
+			// NOTE: the proxy pins are the same for all proxy servers, guaranteed by infra team ;)
+			KnownPins: []string{
+				`pin-sha256="drtmcR2kFkM8qJClsuWgUzxgBkePfRCkRpqUesyDmeE="`, // current
+				`pin-sha256="YRGlaY0jyJ4Jw2/4M8FIftwbDIQfh8Sdro96CeEel54="`, // hot
+				`pin-sha256="AfMENBVvOS8MnISprtvyPsjKlPooqh8nMB/pvCrpJpw="`, // cold
+				`pin-sha256="EU6TS9MO0L/GsDHvVc9D5fChYLNy5JdGYpJw0ccgetM="`, // proxy main
+				`pin-sha256="iKPIHPnDNqdkvOnTClQ8zQAIKG0XavaPkcEo0LBAABA="`, // proxy backup 1
+				`pin-sha256="MSlVrBCdL0hKyczvgYVSRNm88RicyY04Q2y5qrBt0xA="`, // proxy backup 2
+				`pin-sha256="C2UxW0T1Ckl9s+8cXfjXxlEqwAfPM4HiW2y3UdtBeCw="`, // proxy backup 3
+			},
+		},
+	)
+}
+
+func (p *DialerWithPinning) reportCertIssue(connState tls.ConnectionState) {
+	p.isReported = true
+
+	if p.ReportCertIssueLocal != nil {
+		go p.ReportCertIssueLocal()
+	}
+
+	if p.reportURI != "" {
+		p.report.NotedHostname = connState.ServerName
+		p.report.ServedCertificateChain = marshalCert7468(connState.PeerCertificates)
+
+		if len(connState.VerifiedChains) > 0 {
+			p.report.ServedCertificateChain = marshalCert7468(
+				connState.VerifiedChains[len(connState.VerifiedChains)-1],
+			)
+		}
+
+		go p.reportCertIssueRemote()
+	}
+}
+
+func (p *DialerWithPinning) reportCertIssueRemote() {
+	b, err := json.Marshal(p.report)
+	if err != nil {
+		p.log.Errorf("marshal request: %v", err)
+		return
+	}
+
+	req, err := http.NewRequest("POST", p.reportURI, bytes.NewReader(b))
+	if err != nil {
+		p.log.Errorf("create request: %v", err)
+	}
+
+	req.Header.Add("Content-Type", "application/json")
+	req.Header.Set("User-Agent", CurrentUserAgent)
+	req.Header.Set("x-pm-apiversion", strconv.Itoa(Version))
+	req.Header.Set("x-pm-appversion", p.report.AppVersion)
+
+	p.log.Debugf("report req: %+v\n", req)
+
+	c := &http.Client{}
+	res, err := c.Do(req)
+	p.log.Debugf("res: %+v\nerr: %v", res, err)
+	if err != nil {
+		return
+	}
+	_, _ = ioutil.ReadAll(res.Body)
+	if res.StatusCode != http.StatusOK {
+		p.log.Errorf("response status: %v", res.Status)
+	}
+	_ = res.Body.Close()
+}
+
+func certFingerprint(cert *x509.Certificate) string {
+	hash := sha256.Sum256(cert.RawSubjectPublicKeyInfo)
+	return fmt.Sprintf(`pin-sha256=%q`, base64.StdEncoding.EncodeToString(hash[:]))
+}
+
+func marshalCert7468(certs []*x509.Certificate) (pemCerts []string) {
+	var buffer bytes.Buffer
+	for _, cert := range certs {
+		if err := pem.Encode(&buffer, &pem.Block{
+			Type:  "CERTIFICATE",
+			Bytes: cert.Raw,
+		}); err != nil {
+			logrus.WithField("pkg", "pmapi/tls-pinning").Errorf("encoding TLS cert: %v", err)
+		}
+		pemCerts = append(pemCerts, buffer.String())
+		buffer.Reset()
+	}
+
+	return pemCerts
+}
+
+func (p *DialerWithPinning) TransportWithPinning() *http.Transport {
+	return &http.Transport{
+		Proxy:                 http.ProxyFromEnvironment,
+		DialTLS:               p.dialAndCheckFingerprints,
+		MaxIdleConns:          100,
+		IdleConnTimeout:       5 * time.Minute,
+		ExpectContinueTimeout: 500 * time.Millisecond,
+
+		// GODT-126: this was initially 10s but logs from users showed a significant number
+		// were hitting this timeout, possibly due to flaky wifi taking >10s to reconnect.
+		// Bumping to 30s for now to avoid this problem.
+		ResponseHeaderTimeout: 30 * time.Second,
+
+		// If we allow up to 30 seconds for response headers, it is reasonable to allow up
+		// to 30 seconds for the TLS handshake to take place.
+		TLSHandshakeTimeout: 30 * time.Second,
+	}
+}
+
+// dialAndCheckFingerprint to set as http.Transport.DialTLS.
+//
+// * note that when DialTLS is not nil the Transport.TLSClientConfig and Transport.TLSHandshakeTimeout are ignored.
+// * dialAndCheckFingerprints fails if certificate is not valid (not signed by authority or not matching hostname).
+// * dialAndCheckFingerprints will pass if certificate pin does not have a match, but will send notification using
+//   p.ReportCertIssueLocal() and p.reportCertIssueRemote() if they are not nil.
+func (p *DialerWithPinning) dialAndCheckFingerprints(network, address string) (conn net.Conn, err error) {
+	// If DoH is enabled, we hardfail on fingerprint mismatches.
+	if globalIsDoHAllowed() && p.isReported {
+		return nil, ErrTLSMatch
+	}
+
+	// Try to dial the given address but use a proxy if necessary.
+	if conn, err = p.dialWithProxyFallback(network, address); err != nil {
+		return
+	}
+
+	// If cert issue was already reported, we don't want to check fingerprints anymore.
+	if p.isReported {
+		return nil, ErrTLSMatch
+	}
+
+	// Check the cert fingerprint to ensure it is known.
+	if err = p.checkFingerprints(conn); err != nil {
+		p.log.WithError(err).Error("Error checking cert fingerprints")
+		return
+	}
+
+	return
+}
+
+// dialWithProxyFallback tries to dial the given address but falls back to alternative proxies if need be.
+func (p *DialerWithPinning) dialWithProxyFallback(network, address string) (conn net.Conn, err error) {
+	var host, port string
+	if host, port, err = net.SplitHostPort(address); err != nil {
+		return
+	}
+
+	// Try to dial, and if it succeeds, then just return.
+	if conn, err = p.dial(network, address); err == nil {
+		return
+	}
+
+	// If DoH is not allowed, give up. Or, if we are dialing something other than the API
+	// (e.g. we dial protonmail.com/... to check for updates), there's also no point in
+	// continuing since a proxy won't help us reach that.
+	if !globalIsDoHAllowed() || host != stripProtocol(GlobalGetRootURL()) {
+		return
+	}
+
+	// Find a new proxy.
+	var proxy string
+	if proxy, err = p.proxyManager.findProxy(); err != nil {
+		return
+	}
+
+	// Switch to the proxy.
+	p.log.WithField("proxy", proxy).Debug("Switching to proxy")
+	p.proxyManager.useProxy(proxy)
+
+	// Retry dial with proxy.
+	return p.dial(network, net.JoinHostPort(proxy, port))
+}
+
+// dial returns a connection to the given address using the given network.
+func (p *DialerWithPinning) dial(network, address string) (conn net.Conn, err error) {
+	var port string
+	if p.report.Hostname, port, err = net.SplitHostPort(address); err != nil {
+		return
+	}
+	if p.report.Port, err = strconv.Atoi(port); err != nil {
+		return
+	}
+	p.report.DateTime = time.Now().Format(time.RFC3339)
+
+	dialer := &net.Dialer{Timeout: 10 * time.Second}
+
+	// If we are not dialing the standard API then we should skip cert verification checks.
+	var tlsConfig *tls.Config = nil
+	if address != stripProtocol(globalOriginalURL) {
+		tlsConfig = &tls.Config{InsecureSkipVerify: true} // nolint[gosec]
+	}
+
+	return tls.DialWithDialer(dialer, network, address, tlsConfig)
+}
+
+func (p *DialerWithPinning) checkFingerprints(conn net.Conn) (err error) {
+	if !checkTLSCerts {
+		return
+	}
+
+	connState := conn.(*tls.Conn).ConnectionState()
+
+	hasFingerprintMatch := false
+	for _, peerCert := range connState.PeerCertificates {
+		fingerprint := certFingerprint(peerCert)
+
+		for i, pin := range p.report.KnownPins {
+			if pin == fingerprint {
+				hasFingerprintMatch = true
+
+				if i != 0 {
+					p.log.Warnf("Matched fingerprint (%q) was not primary pinned key (was key #%d)", fingerprint, i)
+				}
+
+				break
+			}
+		}
+
+		if hasFingerprintMatch {
+			break
+		}
+	}
+
+	if !hasFingerprintMatch {
+		p.reportCertIssue(connState)
+		return ErrTLSMatch
+	}
+
+	return err
+}

pkg/pmapi/dialer_with_proxy_test.go

@@ -0,0 +1,126 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+const liveAPI = "https://api.protonmail.ch"
+
+var testLiveConfig = &ClientConfig{
+	AppVersion: "Bridge_1.2.4-test",
+	ClientID:   "Bridge",
+}
+
+func newTestDialerWithPinning() (*int, *DialerWithPinning) {
+	called := 0
+	p := NewPMAPIPinning(testLiveConfig.AppVersion)
+	p.ReportCertIssueLocal = func() { called++ }
+	testLiveConfig.Transport = p.TransportWithPinning()
+	return &called, p
+}
+
+func TestTLSPinValid(t *testing.T) {
+	called, _ := newTestDialerWithPinning()
+
+	RootURL = liveAPI
+	client := NewClient(testLiveConfig, "pmapi"+t.Name())
+
+	_, err := client.AuthInfo("this.address.is.disabled")
+	Ok(t, err)
+
+	Equals(t, 0, *called)
+}
+
+func TestTLSPinBackup(t *testing.T) {
+	called, p := newTestDialerWithPinning()
+	p.report.KnownPins[1] = p.report.KnownPins[0]
+	p.report.KnownPins[0] = ""
+
+	RootURL = liveAPI
+	client := NewClient(testLiveConfig, "pmapi"+t.Name())
+
+	_, err := client.AuthInfo("this.address.is.disabled")
+	Ok(t, err)
+
+	Equals(t, 0, *called)
+}
+
+func _TestTLSPinNoMatch(t *testing.T) { // nolint[unused]
+	called, p := newTestDialerWithPinning()
+	for i := 0; i < len(p.report.KnownPins); i++ {
+		p.report.KnownPins[i] = "testing"
+	}
+
+	RootURL = liveAPI
+	client := NewClient(testLiveConfig, "pmapi"+t.Name())
+
+	_, err := client.AuthInfo("this.address.is.disabled")
+	Ok(t, err)
+
+	// check that it will be called only once per session
+	client = NewClient(testLiveConfig, "pmapi"+t.Name())
+	_, err = client.AuthInfo("this.address.is.disabled")
+	Ok(t, err)
+
+	Equals(t, 1, *called)
+}
+
+func _TestTLSPinInvalid(t *testing.T) { // nolint[unused]
+	ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		writeJSONResponsefromFile(t, w, "/auth/info/post_response.json", 0)
+	}))
+	defer ts.Close()
+
+	called, _ := newTestDialerWithPinning()
+
+	client := NewClient(testLiveConfig, "pmapi"+t.Name())
+
+	RootURL = liveAPI
+	_, err := client.AuthInfo("this.address.is.disabled")
+	Ok(t, err)
+
+	RootURL = ts.URL
+	_, err = client.AuthInfo("this.address.is.disabled")
+	Assert(t, err != nil, "error is expected but have %v", err)
+
+	Equals(t, 1, *called)
+}
+
+func _TestTLSSignedCertWrongPublicKey(t *testing.T) { // nolint[unused]
+	_, dialer := newTestDialerWithPinning()
+	_, err := dialer.dialAndCheckFingerprints("tcp", "rsa4096.badssl.com:443")
+	Assert(t, err != nil, "expected dial to fail because of wrong public key: ", err.Error())
+}
+
+func _TestTLSSignedCertTrustedPublicKey(t *testing.T) { // nolint[unused]
+	_, dialer := newTestDialerWithPinning()
+	dialer.report.KnownPins = append(dialer.report.KnownPins, `pin-sha256="W8/42Z0ffufwnHIOSndT+eVzBJSC0E8uTIC8O6mEliQ="`)
+	_, err := dialer.dialAndCheckFingerprints("tcp", "rsa4096.badssl.com:443")
+	Assert(t, err == nil, "expected dial to succeed because public key is known and cert is signed by CA: ", err.Error())
+}
+
+func _TestTLSSelfSignedCertTrustedPublicKey(t *testing.T) { // nolint[unused]
+	_, dialer := newTestDialerWithPinning()
+	dialer.report.KnownPins = append(dialer.report.KnownPins, `pin-sha256="9SLklscvzMYj8f+52lp5ze/hY0CFHyLSPQzSpYYIBm8="`)
+	_, err := dialer.dialAndCheckFingerprints("tcp", "self-signed.badssl.com:443")
+	Assert(t, err == nil, "expected dial to succeed because public key is known despite cert being self-signed: ", err.Error())
+}

pkg/pmapi/events.go

@@ -0,0 +1,237 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/mail"
+)
+
+// Event represents changes since the last check.
+type Event struct {
+	// The current event ID.
+	EventID string
+	// If set to one, all cached data must be fetched again.
+	Refresh int
+	// If set to one, fetch more events.
+	More int
+	// Changes applied to messages.
+	Messages []*EventMessage
+	// Counts of messages per labels.
+	MessageCounts []*MessagesCount
+	// Changes applied to labels.
+	Labels []*EventLabel
+	// Current user status.
+	User User
+	// Changes to addresses.
+	Addresses []*EventAddress
+	// Messages to show to the user.
+	Notices []string
+}
+
+// EventAction is the action that created a change.
+type EventAction int
+
+const (
+	EventDelete      EventAction = iota // Item has been deleted.
+	EventCreate                         // Item has been created.
+	EventUpdate                         // Item has been updated.
+	EventUpdateFlags                    // For messages: flags have been updated.
+)
+
+// Flags for event refresh.
+const (
+	EventRefreshMail    = 1
+	EventRefreshContact = 2
+	EventRefreshAll     = 255
+)
+
+// maxNumberOfMergedEvents limits how many events are merged into one. It means
+// when GetEvent is called and event returns there is more events, it will
+// automatically fetch next one and merge it up to this number of events.
+const maxNumberOfMergedEvents = 50
+
+// EventItem is an item that has changed.
+type EventItem struct {
+	ID     string
+	Action EventAction
+}
+
+// EventMessage is a message that has changed.
+type EventMessage struct {
+	EventItem
+
+	// If the message has been created, the new message.
+	Created *Message `json:"-"`
+	// If the message has been updated, the updated fields.
+	Updated *EventMessageUpdated `json:"-"`
+}
+
+// eventMessage defines a new type to prevent MarshalJSON/UnmarshalJSON infinite loops.
+type eventMessage EventMessage
+
+type rawEventMessage struct {
+	eventMessage
+
+	// This will be parsed depending on the action.
+	Message json.RawMessage `json:",omitempty"`
+}
+
+func (em *EventMessage) UnmarshalJSON(b []byte) (err error) {
+	var raw rawEventMessage
+	if err := json.Unmarshal(b, &raw); err != nil {
+		return err
+	}
+	*em = EventMessage(raw.eventMessage)
+
+	switch em.Action {
+	case EventCreate:
+		em.Created = &Message{ID: raw.ID}
+		return json.Unmarshal(raw.Message, em.Created)
+	case EventUpdate, EventUpdateFlags:
+		em.Updated = &EventMessageUpdated{ID: raw.ID}
+		return json.Unmarshal(raw.Message, em.Updated)
+	}
+	return nil
+}
+
+func (em *EventMessage) MarshalJSON() ([]byte, error) {
+	var raw rawEventMessage
+	raw.eventMessage = eventMessage(*em)
+
+	var err error
+	switch em.Action {
+	case EventCreate:
+		raw.Message, err = json.Marshal(em.Created)
+	case EventUpdate, EventUpdateFlags:
+		raw.Message, err = json.Marshal(em.Updated)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return json.Marshal(raw)
+}
+
+// EventMessageUpdated contains changed fields for an updated message.
+type EventMessageUpdated struct {
+	ID string
+
+	Subject *string
+	Unread  *int
+	Flags   *int64
+	Sender  *mail.Address
+	ToList  *[]*mail.Address
+	CCList  *[]*mail.Address
+	BCCList *[]*mail.Address
+	Time    int64
+
+	// Fields only present for EventUpdateFlags.
+	LabelIDs        []string
+	LabelIDsAdded   []string
+	LabelIDsRemoved []string
+}
+
+// EventLabel is a label that has changed.
+type EventLabel struct {
+	EventItem
+	Label *Label
+}
+
+// EventAddress is an address that has changed.
+type EventAddress struct {
+	EventItem
+	Address *Address
+}
+
+type EventRes struct {
+	Res
+	*Event
+}
+
+type LatestEventRes struct {
+	Res
+	*Event
+}
+
+// GetEvent returns a summary of events that occurred since last. To get the latest event,
+// provide an empty last value. The latest event is always empty.
+func (c *Client) GetEvent(last string) (event *Event, err error) {
+	return c.getEvent(last, 1)
+}
+
+func (c *Client) getEvent(last string, numberOfMergedEvents int) (event *Event, err error) {
+	var req *http.Request
+	if last == "" {
+		req, err = NewRequest("GET", "/events/latest", nil)
+		if err != nil {
+			return
+		}
+
+		var res LatestEventRes
+		if err = c.DoJSON(req, &res); err != nil {
+			return
+		}
+
+		event, err = res.Event, res.Err()
+	} else {
+		req, err = NewRequest("GET", "/events/"+last, nil)
+		if err != nil {
+			return
+		}
+
+		var res EventRes
+		if err = c.DoJSON(req, &res); err != nil {
+			return
+		}
+
+		event, err = res.Event, res.Err()
+		if err != nil {
+			return
+		}
+
+		if event.More == 1 && numberOfMergedEvents < maxNumberOfMergedEvents {
+			var moreEvents *Event
+			if moreEvents, err = c.getEvent(event.EventID, numberOfMergedEvents+1); err != nil {
+				return
+			}
+			event = mergeEvents(event, moreEvents)
+		}
+	}
+
+	return event, err
+}
+
+// mergeEvents combines an old events and a new events object.
+// This is not as simple as just blindly joining the two because some things should only be taken from the new events.
+func mergeEvents(eventsOld *Event, eventsNew *Event) (mergedEvents *Event) {
+	mergedEvents = &Event{
+		EventID:       eventsNew.EventID,
+		Refresh:       eventsOld.Refresh | eventsNew.Refresh,
+		More:          eventsNew.More,
+		Messages:      append(eventsOld.Messages, eventsNew.Messages...),
+		MessageCounts: append(eventsOld.MessageCounts, eventsNew.MessageCounts...),
+		Labels:        append(eventsOld.Labels, eventsNew.Labels...),
+		User:          eventsNew.User,
+		Addresses:     append(eventsOld.Addresses, eventsNew.Addresses...),
+		Notices:       append(eventsOld.Notices, eventsNew.Notices...),
+	}
+
+	return
+}

pkg/pmapi/events_test.go

@@ -0,0 +1,524 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"fmt"
+	"net/http"
+	"regexp"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestClient_GetEvent(t *testing.T) {
+	s, c := newTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.NoError(t, checkMethodAndPath(r, "GET", "/events/latest"))
+		fmt.Fprint(w, testEventBody)
+	}))
+	defer s.Close()
+
+	event, err := c.GetEvent("")
+	require.NoError(t, err)
+	require.Equal(t, testEvent, event)
+}
+
+func TestClient_GetEvent_withID(t *testing.T) {
+	s, c := newTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.NoError(t, checkMethodAndPath(r, "GET", "/events/"+testEvent.EventID))
+		fmt.Fprint(w, testEventBody)
+	}))
+	defer s.Close()
+
+	event, err := c.GetEvent(testEvent.EventID)
+	require.NoError(t, err)
+	require.Equal(t, testEvent, event)
+}
+
+// We first call GetEvent with id of eventID1, which returns More=1 so we fetch with id eventID2.
+func TestClient_GetEvent_mergeEvents(t *testing.T) {
+	s, c := newTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.RequestURI() {
+		case "/events/eventID1":
+			assert.NoError(t, checkMethodAndPath(r, "GET", "/events/eventID1"))
+			fmt.Fprint(w, testEventBodyMore1)
+		case "/events/eventID2":
+			assert.NoError(t, checkMethodAndPath(r, "GET", "/events/eventID2"))
+			fmt.Fprint(w, testEventBodyMore2)
+		default:
+			t.Fail()
+		}
+	}))
+	defer s.Close()
+
+	event, err := c.GetEvent("eventID1")
+	require.NoError(t, err)
+	require.Equal(t, testEventMerged, event)
+}
+
+func TestClient_GetEvent_mergeMaxNumberOfEvents(t *testing.T) {
+	numberOfCalls := 0
+
+	s, c := newTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		numberOfCalls++
+
+		re := regexp.MustCompile(`/eventID([0-9]+)`)
+		eventIDString := re.FindStringSubmatch(r.URL.RequestURI())[1]
+		eventID, err := strconv.Atoi(eventIDString)
+		require.NoError(t, err)
+
+		if numberOfCalls > maxNumberOfMergedEvents*2 {
+			require.Fail(t, "Too many calls!")
+		}
+
+		fmt.Println("")
+
+		body := strings.ReplaceAll(testEventBodyMore1, "eventID2", "eventID"+strconv.Itoa(eventID+1))
+		fmt.Fprint(w, body)
+	}))
+	defer s.Close()
+
+	event, err := c.GetEvent("eventID1")
+	require.NoError(t, err)
+	require.Equal(t, maxNumberOfMergedEvents, numberOfCalls)
+	require.Equal(t, 1, event.More)
+}
+
+var (
+	testEventMessageUpdateUnread = 0
+
+	testEvent = &Event{
+		EventID: "eventID1",
+		Refresh: 0,
+		Messages: []*EventMessage{
+			{
+				EventItem: EventItem{ID: "hdI7aIgUO1hFplCIcJHB0jShRVsAzS0AB75wGCaiNVeIHXLmaUnt4eJ8l7c7L6uk4g0ZdXhGWG5gfh6HHgAZnw==", Action: EventCreate},
+				Created: &Message{
+					ID:      "hdI7aIgUO1hFplCIcJHB0jShRVsAzS0AB75wGCaiNVeIHXLmaUnt4eJ8l7c7L6uk4g0ZdXhGWG5gfh6HHgAZnw==",
+					Subject: "Hey there",
+				},
+			},
+			{
+				EventItem: EventItem{ID: "bSFLAimPSfGz2Kj0aV3l3AyXsof_Vf7sfrrMJ8ifgGJe-f2NG2eLaEGXLytjMhq9wnLMtkoZpO2uBXM4nOVa5g==", Action: EventUpdateFlags},
+				Updated: &EventMessageUpdated{
+					ID:              "bSFLAimPSfGz2Kj0aV3l3AyXsof_Vf7sfrrMJ8ifgGJe-f2NG2eLaEGXLytjMhq9wnLMtkoZpO2uBXM4nOVa5g==",
+					Unread:          &testEventMessageUpdateUnread,
+					Time:            1472391377,
+					LabelIDsAdded:   []string{ArchiveLabel},
+					LabelIDsRemoved: []string{InboxLabel},
+				},
+			},
+			{
+				EventItem: EventItem{ID: "XRBMBYnSkaEJWtqFACp2kjlNc-7GjzX3SnPcOtWK4PyLG11Nhsg0uxPYjTXoClQfB-EHVDl9gE3w2PVuj93jBg==", Action: EventDelete},
+			},
+		},
+		MessageCounts: []*MessagesCount{
+			{
+				LabelID: "0",
+				Total:   19,
+				Unread:  2,
+			},
+			{
+				LabelID: "6",
+				Total:   1,
+				Unread:  0,
+			},
+		},
+		Notices: []string{"Server will be down in 2min because of a NSA attack"},
+	}
+
+	testEventMerged = &Event{
+		EventID: "eventID3",
+		Refresh: 1,
+		Messages: []*EventMessage{
+			{
+				EventItem: EventItem{ID: "msgID1", Action: EventCreate},
+				Created: &Message{
+					ID:      "id",
+					Subject: "Hey there",
+				},
+			},
+			{
+				EventItem: EventItem{ID: "msgID2", Action: EventCreate},
+				Created: &Message{
+					ID:      "id",
+					Subject: "Hey there again",
+				},
+			},
+		},
+		MessageCounts: []*MessagesCount{
+			{
+				LabelID: "label1",
+				Total:   19,
+				Unread:  2,
+			},
+			{
+				LabelID: "label2",
+				Total:   1,
+				Unread:  0,
+			},
+			{
+				LabelID: "label2",
+				Total:   2,
+				Unread:  1,
+			},
+			{
+				LabelID: "label3",
+				Total:   1,
+				Unread:  0,
+			},
+		},
+		Notices: []string{"Server will be down in 2min because of a NSA attack", "Just kidding lol"},
+		Labels: []*EventLabel{
+			{
+				EventItem: EventItem{
+					ID:     "labelID1",
+					Action: 1,
+				},
+				Label: &Label{
+					ID:   "id",
+					Name: "Event Label 1",
+				},
+			},
+			{
+				EventItem: EventItem{
+					ID:     "labelID2",
+					Action: 1,
+				},
+				Label: &Label{
+					ID:   "id",
+					Name: "Event Label 2",
+				},
+			},
+		},
+		User: User{
+			ID:        "userID1",
+			Name:      "user",
+			UsedSpace: 23456,
+		},
+		Addresses: []*EventAddress{
+			{
+				EventItem: EventItem{
+					ID:     "addressID1",
+					Action: 2,
+				},
+				Address: &Address{
+					ID:          "id",
+					DisplayName: "address 1",
+				},
+			},
+			{
+				EventItem: EventItem{
+					ID:     "addressID2",
+					Action: 2,
+				},
+				Address: &Address{
+					ID:          "id",
+					DisplayName: "address 2",
+				},
+			},
+		},
+	}
+)
+
+const (
+	testEventBody = `{
+    "EventID": "eventID1",
+    "Refresh": 0,
+    "Messages": [
+        {
+            "ID": "hdI7aIgUO1hFplCIcJHB0jShRVsAzS0AB75wGCaiNVeIHXLmaUnt4eJ8l7c7L6uk4g0ZdXhGWG5gfh6HHgAZnw==",
+            "Action": 1,
+            "Message": {
+                "ID": "hdI7aIgUO1hFplCIcJHB0jShRVsAzS0AB75wGCaiNVeIHXLmaUnt4eJ8l7c7L6uk4g0ZdXhGWG5gfh6HHgAZnw==",
+                "Subject": "Hey there"
+            }
+        },
+        {
+            "ID": "bSFLAimPSfGz2Kj0aV3l3AyXsof_Vf7sfrrMJ8ifgGJe-f2NG2eLaEGXLytjMhq9wnLMtkoZpO2uBXM4nOVa5g==",
+            "Action": 3,
+            "Message": {
+                "ConversationID": "2oX3EILYRuZ0IRBVlzMg1oV5eazQL67sFIHlcR8bjickPn7K4id4sJZuAB6n0pdtI3hRIVsjCpgWfRm8c_x3IQ==",
+                "Unread": 0,
+                "Time": 1472391377,
+                "Location": 6,
+                "LabelIDsAdded": [
+                    "6"
+                ],
+                "LabelIDsRemoved": [
+                    "0"
+                ]
+            }
+        },
+        {
+            "ID": "XRBMBYnSkaEJWtqFACp2kjlNc-7GjzX3SnPcOtWK4PyLG11Nhsg0uxPYjTXoClQfB-EHVDl9gE3w2PVuj93jBg==",
+            "Action": 0
+        }
+    ],
+    "Conversations": [
+        {
+            "ID": "2oX3EILYRuZ0IRBVlzMg1oV5eazQL67sFIHlcR8bjickPn7K4id4sJZuAB6n0pdtI3hRIVsjCpgWfRm8c_x3IQ==",
+            "Action": 1,
+            "Conversation": {
+                "ID": "2oX3EILYRuZ0IRBVlzMg1oV5eazQL67sFIHlcR8bjickPn7K4id4sJZuAB6n0pdtI3hRIVsjCpgWfRm8c_x3IQ==",
+                "Order": 1616,
+                "Subject": "Hey there",
+                "Senders": [
+                    {
+                        "Address": "apple@protonmail.com",
+                        "Name": "apple@protonmail.com"
+                    }
+                ],
+                "Recipients": [
+                    {
+                        "Address": "apple@protonmail.com",
+                        "Name": "apple@protonmail.com"
+                    }
+                ],
+                "NumMessages": 1,
+                "NumUnread": 1,
+                "NumAttachments": 0,
+                "ExpirationTime": 0,
+                "TotalSize": 636,
+                "AddressID": "QMJs2dzTx7uqpH5PNgIzjULywU4gO9uMBhEMVFOAVJOoUml54gC0CCHtW9qYwzH-zYbZwMv3MFYncPjW1Usq7Q==",
+                "LabelIDs": [
+                    "0"
+                ],
+                "Labels": [
+                    {
+                        "Count": 1,
+                        "NumMessages": 1,
+                        "NumUnread": 1,
+                        "ID": "0"
+                    }
+                ]
+            }
+        }
+    ],
+    "Total": {
+        "Locations": [
+            {
+                "Location": 0,
+                "Count": 19
+            },
+            {
+                "Location": 1,
+                "Count": 16
+            },
+            {
+                "Location": 2,
+                "Count": 16
+            },
+            {
+                "Location": 3,
+                "Count": 17
+            },
+            {
+                "Location": 6,
+                "Count": 1
+            }
+        ],
+        "Labels": [
+            {
+                "LabelID": "LLz8ysmVxwr4dF6mWpClePT0SpSWOEvzTdq17RydSl4ndMckvY1K63HeXDzn03BJQwKYvgf-eWT8Qfd9WVuIEQ==",
+                "Count": 2
+            },
+            {
+                "LabelID": "BvbqbySUPo9uWW_eR8tLA13NUsQMz3P4Zhw4UnpvrKqURnrHlE6L2Au0nplHfHlVXFgGz4L4hJ9-BYllOL-L5g==",
+                "Count": 2
+            }
+        ],
+        "Starred": 3
+    },
+    "Unread": {
+        "Locations": [
+            {
+                "Location": 0,
+                "Count": 2
+            },
+            {
+                "Location": 1,
+                "Count": 0
+            },
+            {
+                "Location": 2,
+                "Count": 0
+            },
+            {
+                "Location": 3,
+                "Count": 0
+            },
+            {
+                "Location": 6,
+                "Count": 0
+            }
+        ],
+        "Labels": [
+            {
+                "LabelID": "LLz8ysmVxwr4dF6mWpClePT0SpSWOEvzTdq17RydSl4ndMckvY1K63HeXDzn03BJQwKYvgf-eWT8Qfd9WVuIEQ==",
+                "Count": 0
+            },
+            {
+                "LabelID": "BvbqbySUPo9uWW_eR8tLA13NUsQMz3P4Zhw4UnpvrKqURnrHlE6L2Au0nplHfHlVXFgGz4L4hJ9-BYllOL-L5g==",
+                "Count": 0
+            }
+        ],
+        "Starred": 0
+    },
+    "MessageCounts": [
+        {
+            "LabelID": "0",
+            "Total": 19,
+            "Unread": 2
+        },
+        {
+            "LabelID": "6",
+            "Total": 1,
+            "Unread": 0
+        }
+    ],
+    "ConversationCounts": [
+        {
+            "LabelID": "0",
+            "Total": 19,
+            "Unread": 2
+        },
+        {
+            "LabelID": "6",
+            "Total": 1,
+            "Unread": 0
+        }
+    ],
+    "UsedSpace": 7552905,
+    "Notices": ["Server will be down in 2min because of a NSA attack"],
+    "Code": 1000
+}
+`
+
+	testEventBodyMore1 = `{
+    "EventID": "eventID2",
+		"More": 1,
+    "Refresh": 1,
+    "Messages": [
+        {
+            "ID": "msgID1",
+            "Action": 1,
+            "Message": {
+                "ID": "id",
+                "Subject": "Hey there"
+            }
+        }
+    ],
+    "MessageCounts": [
+        {
+            "LabelID": "label1",
+            "Total": 19,
+            "Unread": 2
+        },
+        {
+            "LabelID": "label2",
+            "Total": 1,
+            "Unread": 0
+        }
+    ],
+		"Labels": [
+        {
+            "ID":"labelID1",
+            "Action":1,
+            "Label":{
+                "ID":"id",
+                "Name":"Event Label 1"
+            }
+        }
+    ],
+		"User": {
+        "ID": "userID1",
+        "Name": "user",
+        "UsedSpace": 12345
+    },
+		"Addresses": [
+        {
+            "ID": "addressID1",
+            "Action": 2,
+            "Address": {
+                "ID": "id",
+                "DisplayName": "address 1"
+            }
+        }
+    ],
+    "Notices": ["Server will be down in 2min because of a NSA attack"]
+}
+`
+
+	testEventBodyMore2 = `{
+    "EventID": "eventID3",
+    "Refresh": 0,
+    "Messages": [
+        {
+            "ID": "msgID2",
+            "Action": 1,
+            "Message": {
+                "ID": "id",
+                "Subject": "Hey there again"
+            }
+        }
+    ],
+    "MessageCounts": [
+        {
+            "LabelID": "label2",
+            "Total": 2,
+            "Unread": 1
+        },
+        {
+            "LabelID": "label3",
+            "Total": 1,
+            "Unread": 0
+        }
+    ],
+		"Labels": [
+        {
+            "ID":"labelID2",
+            "Action":1,
+            "Label":{
+                "ID":"id",
+                "Name":"Event Label 2"
+            }
+        }
+    ],
+		"User": {
+        "ID": "userID1",
+        "Name": "user",
+        "UsedSpace": 23456
+    },
+		"Addresses": [
+        {
+            "ID": "addressID2",
+            "Action": 2,
+            "Address": {
+                "ID": "id",
+                "DisplayName": "address 2"
+            }
+        }
+    ],
+    "Notices": ["Just kidding lol"]
+}
+`
+)

pkg/pmapi/import.go

@@ -0,0 +1,157 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"encoding/json"
+	"io"
+	"mime/multipart"
+	"strconv"
+)
+
+// Import errors.
+const (
+	ImportMessageTooLarge = 36022
+)
+
+// ImportReq is an import request.
+type ImportReq struct {
+	// A list of messages that will be imported.
+	Messages []*ImportMsgReq
+}
+
+// WriteTo writes the import request to a multipart writer.
+func (req *ImportReq) WriteTo(w *multipart.Writer) (err error) {
+	// Create Metadata field.
+	mw, err := w.CreateFormField("Metadata")
+	if err != nil {
+		return
+	}
+
+	// Build metadata.
+	metadata := map[string]*ImportMsgReq{}
+	for i, msg := range req.Messages {
+		name := strconv.Itoa(i)
+		metadata[name] = msg
+	}
+
+	// Write metadata.
+	if err = json.NewEncoder(mw).Encode(metadata); err != nil {
+		return
+	}
+
+	// Write messages.
+	for i, msg := range req.Messages {
+		name := strconv.Itoa(i)
+
+		var fw io.Writer
+		if fw, err = w.CreateFormFile(name, name+".eml"); err != nil {
+			return err
+		}
+
+		if _, err = fw.Write(msg.Body); err != nil {
+			return
+		}
+	}
+
+	return err
+}
+
+// ImportMsgReq is a request to import a message. All fields are optional except AddressID and Body.
+type ImportMsgReq struct {
+	// The address where the message will be imported.
+	AddressID string
+	// The full MIME message.
+	Body []byte `json:"-"`
+
+	// 0: read, 1: unread.
+	Unread int
+	// 1 if the message has been replied.
+	IsReplied int
+	// 1 if the message has been replied to all.
+	IsRepliedAll int
+	// 1 if the message has been forwarded.
+	IsForwarded int
+	// The time when the message was received as a Unix time.
+	Time int64
+	// The type of the imported message.
+	Flags int64
+	// The labels to apply to the imported message. Must contain at least one system label.
+	LabelIDs []string
+}
+
+// ImportRes is a response to an import request.
+type ImportRes struct {
+	Res
+
+	Responses []struct {
+		Name     string
+		Response struct {
+			Res
+			MessageID string
+		}
+	}
+}
+
+// ImportMsgRes is a response to a single message import request.
+type ImportMsgRes struct {
+	// The error encountered while importing the message, if any.
+	Error error
+	// The newly created message ID.
+	MessageID string
+}
+
+// Import imports messages to the user's account.
+func (c *Client) Import(reqs []*ImportMsgReq) (resps []*ImportMsgRes, err error) {
+	importReq := &ImportReq{Messages: reqs}
+
+	req, w, err := NewMultipartRequest("POST", "/import")
+	if err != nil {
+		return
+	}
+
+	// We will write the request as long as it is sent to the API.
+	var importRes ImportRes
+	done := make(chan error, 1)
+	go (func() {
+		done <- c.DoJSON(req, &importRes)
+	})()
+
+	// Write the request.
+	if err = importReq.WriteTo(w.Writer); err != nil {
+		return
+	}
+	_ = w.Close()
+
+	if err = <-done; err != nil {
+		return
+	}
+	if err = importRes.Err(); err != nil {
+		return
+	}
+
+	resps = make([]*ImportMsgRes, len(importRes.Responses))
+	for i, r := range importRes.Responses {
+		resps[i] = &ImportMsgRes{
+			Error:     r.Response.Err(),
+			MessageID: r.Response.MessageID,
+		}
+	}
+
+	return resps, err
+}

pkg/pmapi/import_test.go

@@ -0,0 +1,155 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"mime"
+	"mime/multipart"
+	"net/http"
+	"reflect"
+	"testing"
+)
+
+var testImportReqs = []*ImportMsgReq{
+	{
+		AddressID: "QMJs2dzTx7uqpH5PNgIzjULywU4gO9uMBhEMVFOAVJOoUml54gC0CCHtW9qYwzH-zYbZwMv3MFYncPjW1Usq7Q==",
+		Body:      []byte("Hello World!"),
+		Unread:    0,
+		Flags:     FlagReceived | FlagImported,
+		LabelIDs:  []string{ArchiveLabel},
+	},
+}
+
+const testImportBody = `{
+    "Code": 1001,
+    "Responses": [{
+        "Name": "0",
+        "Response": {"Code": 1000, "MessageID": "UKjSNz95KubYjrYmfbv1mbIfGxzY6D64mmHmVpWhkeEau-u0PIS4ru5IFMHgX6WjKpWYKCht3oiOtL5-wZChNg=="}
+    }]
+}`
+
+var testImportRes = &ImportMsgRes{
+	Error:     nil,
+	MessageID: "UKjSNz95KubYjrYmfbv1mbIfGxzY6D64mmHmVpWhkeEau-u0PIS4ru5IFMHgX6WjKpWYKCht3oiOtL5-wZChNg==",
+}
+
+func TestClient_Import(t *testing.T) { // nolint[funlen]
+	s, c := newTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		Ok(t, checkMethodAndPath(r, "POST", "/import"))
+
+		contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
+		if err != nil {
+			t.Error("Expected no error while parsing request content type, got:", err)
+		}
+		if contentType != "multipart/form-data" {
+			t.Errorf("Invalid request content type: expected %v but got %v", "multipart/form-data", contentType)
+		}
+
+		mr := multipart.NewReader(r.Body, params["boundary"])
+
+		// First part is metadata.
+		p, err := mr.NextPart()
+		if err != nil {
+			t.Error("Expected no error while reading first part of request body, got:", err)
+		}
+
+		contentDisp, params, err := mime.ParseMediaType(p.Header.Get("Content-Disposition"))
+		if err != nil {
+			t.Error("Expected no error while parsing part content disposition, got:", err)
+		}
+		if contentDisp != "form-data" {
+			t.Errorf("Invalid part content disposition: expected %v but got %v", "form-data", contentType)
+		}
+		if params["name"] != "Metadata" {
+			t.Errorf("Invalid part name: expected %v but got %v", "Metadata", params["name"])
+		}
+
+		metadata := map[string]*ImportMsgReq{}
+		if err := json.NewDecoder(p).Decode(&metadata); err != nil {
+			t.Error("Expected no error while parsing metadata json, got:", err)
+		}
+
+		if len(metadata) != 1 {
+			t.Errorf("Expected metadata to contain exactly one item, got %v", metadata)
+		}
+
+		req := metadata["0"]
+		if metadata["0"] == nil {
+			t.Errorf("Expected metadata to contain one item indexed by 0, got %v", metadata)
+		}
+
+		// No Body in metadata.
+		expected := *testImportReqs[0]
+		expected.Body = nil
+		if !reflect.DeepEqual(&expected, req) {
+			t.Errorf("Invalid message metadata: expected %v, got %v", &expected, req)
+		}
+
+		// Second part is message body.
+		p, err = mr.NextPart()
+		if err != nil {
+			t.Error("Expected no error while reading second part of request body, got:", err)
+		}
+
+		contentDisp, params, err = mime.ParseMediaType(p.Header.Get("Content-Disposition"))
+		if err != nil {
+			t.Error("Expected no error while parsing part content disposition, got:", err)
+		}
+		if contentDisp != "form-data" {
+			t.Errorf("Invalid part content disposition: expected %v but got %v", "form-data", contentType)
+		}
+		if params["name"] != "0" {
+			t.Errorf("Invalid part name: expected %v but got %v", "0", params["name"])
+		}
+
+		b, err := ioutil.ReadAll(p)
+		if err != nil {
+			t.Error("Expected no error while reading second part body, got:", err)
+		}
+
+		if string(b) != string(testImportReqs[0].Body) {
+			t.Errorf("Invalid message body: expected %v but got %v", string(testImportReqs[0].Body), string(b))
+		}
+
+		// No more parts.
+		_, err = mr.NextPart()
+		if err != io.EOF {
+			t.Error("Expected no more parts but error was not EOF, got:", err)
+		}
+
+		fmt.Fprint(w, testImportBody)
+	}))
+	defer s.Close()
+
+	imported, err := c.Import(testImportReqs)
+	if err != nil {
+		t.Fatal("Expected no error while importing, got:", err)
+	}
+
+	if len(imported) != 1 {
+		t.Fatalf("Expected exactly one imported message, got %v", len(imported))
+	}
+
+	if !reflect.DeepEqual(testImportRes, imported[0]) {
+		t.Errorf("Invalid response for imported message: expected %+v but got %+v", testImportRes, imported[0])
+	}
+}

pkg/pmapi/key.go

@@ -0,0 +1,138 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+
+	pmcrypto "github.com/ProtonMail/gopenpgp/crypto"
+)
+
+// Flags
+const (
+	UseToVerifyFlag = 1 << iota
+	UseToEncryptFlag
+)
+
+type PublicKeyRes struct {
+	Res
+
+	RecipientType int
+	MIMEType      string
+	Keys          []PublicKey
+}
+
+type PublicKey struct {
+	Flags     int
+	PublicKey string
+}
+
+// PublicKeys returns the public keys of the given email addresses.
+func (c *Client) PublicKeys(emails []string) (keys map[string]*pmcrypto.KeyRing, err error) {
+	if len(emails) == 0 {
+		err = fmt.Errorf("pmapi: cannot get public keys: no email address provided")
+		return
+	}
+	keys = map[string]*pmcrypto.KeyRing{}
+
+	for _, email := range emails {
+		email = url.QueryEscape(email)
+
+		var req *http.Request
+		if req, err = NewRequest("GET", "/keys?Email="+email, nil); err != nil {
+			return
+		}
+
+		var res PublicKeyRes
+		if err = c.DoJSON(req, &res); err != nil {
+			return
+		}
+
+		for _, key := range res.Keys {
+			if key.Flags&UseToEncryptFlag == UseToEncryptFlag {
+				var kr *pmcrypto.KeyRing
+				if kr, err = pmcrypto.ReadArmoredKeyRing(strings.NewReader(key.PublicKey)); err != nil {
+					return
+				}
+				keys[email] = kr
+			}
+		}
+	}
+
+	return keys, err
+}
+
+const (
+	RecipientInternal = 1
+	RecipientExternal = 2
+)
+
+// GetPublicKeysForEmail returns all sending public keys for the given email address.
+func (c *Client) GetPublicKeysForEmail(email string) (keys []PublicKey, internal bool, err error) {
+	email = url.QueryEscape(email)
+
+	var req *http.Request
+	if req, err = NewRequest("GET", "/keys?Email="+email, nil); err != nil {
+		return
+	}
+
+	var res PublicKeyRes
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+
+	internal = res.RecipientType == RecipientInternal
+
+	for _, key := range res.Keys {
+		if key.Flags&UseToEncryptFlag == UseToEncryptFlag {
+			keys = append(keys, key)
+		}
+	}
+	return
+}
+
+// KeySalt contains id and salt for key.
+type KeySalt struct {
+	ID, KeySalt string
+}
+
+// KeySaltRes is used to unmarshal API response.
+type KeySaltRes struct {
+	Res
+	KeySalts []KeySalt
+}
+
+// GetKeySalts sends request to get list of key salts (n.b. locked route).
+func (c *Client) GetKeySalts() (keySalts []KeySalt, err error) {
+	var req *http.Request
+	if req, err = NewRequest("GET", "/keys/salts", nil); err != nil {
+		return
+	}
+
+	var res KeySaltRes
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+
+	keySalts = res.KeySalts
+
+	return
+}

pkg/pmapi/keyring.go

@@ -0,0 +1,295 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"sync"
+
+	pmcrypto "github.com/ProtonMail/gopenpgp/crypto"
+)
+
+// clearableKey is a region of memory intended to hold a private key and which can be securely
+// cleared by calling clear().
+type clearableKey []byte
+
+// UnmarshalJSON Removes quotation and unescapes CR, LF.
+func (pk *clearableKey) UnmarshalJSON(b []byte) (err error) {
+	b = bytes.Trim(b, "\"")
+	b = bytes.ReplaceAll(b, []byte("\\n"), []byte("\n"))
+	b = bytes.ReplaceAll(b, []byte("\\r"), []byte("\r"))
+	*pk = b
+	return
+}
+
+// clear irreversibly destroys the full range of `clearableKey` by filling it with zeros to ensure
+// nobody can see what was in there (e.g. while waiting for the garbage collector to clean it up).
+func (pk *clearableKey) clear() {
+	for b := range *pk {
+		(*pk)[b] = 0
+	}
+}
+
+type PMKey struct {
+	ID          string
+	Version     int
+	Flags       int
+	Fingerprint string
+	Primary     int
+	Token       *string `json:",omitempty"`
+	Signature   *string `json:",omitempty"`
+}
+
+type PMKeys struct {
+	Keys    []PMKey
+	KeyRing *pmcrypto.KeyRing
+}
+
+func (k *PMKeys) UnmarshalJSON(b []byte) (err error) {
+	var rawKeys []struct {
+		PMKey
+		PrivateKey clearableKey
+	}
+	if err = json.Unmarshal(b, &rawKeys); err != nil {
+		return
+	}
+
+	k.KeyRing = &pmcrypto.KeyRing{}
+	for _, rawKey := range rawKeys {
+		err = k.KeyRing.ReadFrom(bytes.NewReader(rawKey.PrivateKey), true)
+		rawKey.PrivateKey.clear()
+		if err != nil {
+			return
+		}
+		k.Keys = append(k.Keys, rawKey.PMKey)
+	}
+	if len(k.Keys) > 0 {
+		k.KeyRing.FirstKeyID = k.Keys[0].ID
+	}
+	return
+}
+
+// unlockKeyRing tries to unlock them with the provided keyRing using the token
+// and if the token is not available it will use passphrase. It will not fail
+// if keyring contains at least one unlocked private key.
+func (k *PMKeys) unlockKeyRing(userKeyring *pmcrypto.KeyRing, passphrase []byte, locker sync.Locker) (err error) {
+	locker.Lock()
+	defer locker.Unlock()
+
+	if k == nil {
+		err = errors.New("keys is a nil object")
+		return
+	}
+
+	for _, key := range k.Keys {
+		if key.Token == nil || key.Signature == nil {
+			if err = unlockKeyRingNoErrorWhenAlreadyUnlocked(k.KeyRing, passphrase); err != nil {
+				return
+			}
+			continue
+		}
+
+		message, err := pmcrypto.NewPGPMessageFromArmored(*key.Token)
+		if err != nil {
+			return err
+		}
+
+		signature, err := pmcrypto.NewPGPSignatureFromArmored(*key.Signature)
+		if err != nil {
+			return err
+		}
+
+		if userKeyring == nil {
+			return errors.New("userkey required to decrypt tokens but wasn't provided")
+		}
+		token, err := userKeyring.Decrypt(message, nil, 0)
+		if err != nil {
+			return err
+		}
+
+		err = userKeyring.VerifyDetached(token, signature, 0)
+		if err != nil {
+			return err
+		}
+
+		err = unlockKeyRingNoErrorWhenAlreadyUnlocked(k.KeyRing, token.GetBinary())
+		if err != nil {
+			return fmt.Errorf("wrong token: %v", err)
+		}
+	}
+
+	return nil
+}
+
+type unlockError struct {
+	error
+}
+
+func (err *unlockError) Error() string {
+	return "Invalid mailbox password (" + err.error.Error() + ")"
+}
+
+// IsUnlockError checks whether the error is due to failure to unlock (which is represented by an unexported type).
+func IsUnlockError(err error) bool {
+	_, ok := err.(*unlockError)
+	return ok
+}
+
+func unlockKeyRingNoErrorWhenAlreadyUnlocked(kr *pmcrypto.KeyRing, passphrase []byte) (err error) {
+	if err = kr.Unlock(passphrase); err != nil {
+		// Do not fail if it has already unlocked keys.
+		hasUnlockedKey := false
+		for _, e := range kr.GetEntities() {
+			if e.PrivateKey != nil && !e.PrivateKey.Encrypted {
+				hasUnlockedKey = true
+				break
+			}
+			for _, se := range e.Subkeys {
+				if se.PrivateKey != nil && (!se.Sig.FlagsValid || se.Sig.FlagEncryptStorage || se.Sig.FlagEncryptCommunications) && !e.PrivateKey.Encrypted {
+					hasUnlockedKey = true
+					break
+				}
+			}
+			if hasUnlockedKey {
+				break
+			}
+		}
+		if !hasUnlockedKey {
+			err = &unlockError{err}
+			return
+		}
+		err = nil
+	}
+	return
+}
+
+// ErrNoKeyringAvailable represents an error caused by a keyring being nil or having no entities.
+var ErrNoKeyringAvailable = errors.New("no keyring available")
+
+func (c *Client) encrypt(plain string, signer *pmcrypto.KeyRing) (armored string, err error) {
+	return encrypt(c.kr, plain, signer)
+}
+
+func encrypt(encrypter *pmcrypto.KeyRing, plain string, signer *pmcrypto.KeyRing) (armored string, err error) {
+	if encrypter == nil || encrypter.FirstKey() == nil {
+		return "", ErrNoKeyringAvailable
+	}
+	plainMessage := pmcrypto.NewPlainMessageFromString(plain)
+	// We use only primary key to encrypt the message. Our keyring contains all keys (primary, old and deacivated ones).
+	pgpMessage, err := encrypter.FirstKey().Encrypt(plainMessage, signer)
+	if err != nil {
+		return
+	}
+	return pgpMessage.GetArmored()
+}
+
+func (c *Client) decrypt(armored string) (plain string, err error) {
+	return decrypt(c.kr, armored)
+}
+
+func decrypt(decrypter *pmcrypto.KeyRing, armored string) (plainBody string, err error) {
+	if decrypter == nil {
+		return "", ErrNoKeyringAvailable
+	}
+	pgpMessage, err := pmcrypto.NewPGPMessageFromArmored(armored)
+	if err != nil {
+		return
+	}
+	plainMessage, err := decrypter.Decrypt(pgpMessage, nil, 0)
+	if err != nil {
+		return
+	}
+	return plainMessage.GetString(), nil
+}
+
+func (c *Client) sign(plain string) (armoredSignature string, err error) {
+	if c.kr == nil {
+		return "", ErrNoKeyringAvailable
+	}
+	plainMessage := pmcrypto.NewPlainMessageFromString(plain)
+	pgpSignature, err := c.kr.SignDetached(plainMessage)
+	if err != nil {
+		return
+	}
+	return pgpSignature.GetArmored()
+}
+
+func (c *Client) verify(plain, amroredSignature string) (err error) {
+	plainMessage := pmcrypto.NewPlainMessageFromString(plain)
+	pgpSignature, err := pmcrypto.NewPGPSignatureFromArmored(amroredSignature)
+	if err != nil {
+		return
+	}
+	verifyTime := int64(0) // By default it will use current timestamp.
+	return c.kr.VerifyDetached(plainMessage, pgpSignature, verifyTime)
+}
+
+func encryptAttachment(kr *pmcrypto.KeyRing, data io.Reader, filename string) (encrypted io.Reader, err error) {
+	if kr == nil || kr.FirstKey() == nil {
+		return nil, ErrNoKeyringAvailable
+	}
+	dataBytes, err := ioutil.ReadAll(data)
+	if err != nil {
+		return
+	}
+	plainMessage := pmcrypto.NewPlainMessage(dataBytes)
+	// We use only primary key to encrypt the message. Our keyring contains all keys (primary, old and deacivated ones).
+	pgpSplitMessage, err := kr.FirstKey().EncryptAttachment(plainMessage, filename)
+	if err != nil {
+		return
+	}
+	packets := append(pgpSplitMessage.KeyPacket, pgpSplitMessage.DataPacket...)
+	return bytes.NewReader(packets), nil
+}
+
+func decryptAttachment(kr *pmcrypto.KeyRing, keyPackets []byte, data io.Reader) (decrypted io.Reader, err error) {
+	if kr == nil {
+		return nil, ErrNoKeyringAvailable
+	}
+	dataBytes, err := ioutil.ReadAll(data)
+	if err != nil {
+		return
+	}
+	pgpSplitMessage := pmcrypto.NewPGPSplitMessage(keyPackets, dataBytes)
+	plainMessage, err := kr.DecryptAttachment(pgpSplitMessage)
+	if err != nil {
+		return
+	}
+	return plainMessage.NewReader(), nil
+}
+
+func signAttachment(encrypter *pmcrypto.KeyRing, data io.Reader) (signature io.Reader, err error) {
+	if encrypter == nil {
+		return nil, ErrNoKeyringAvailable
+	}
+	dataBytes, err := ioutil.ReadAll(data)
+	if err != nil {
+		return
+	}
+	plainMessage := pmcrypto.NewPlainMessage(dataBytes)
+	sig, err := encrypter.SignDetached(plainMessage)
+	if err != nil {
+		return
+	}
+	return bytes.NewReader(sig.GetBinary()), nil
+}

pkg/pmapi/keyring_test.go

@@ -0,0 +1,94 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"encoding/json"
+	"strings"
+	"sync"
+	"testing"
+
+	pmcrypto "github.com/ProtonMail/gopenpgp/crypto"
+	"github.com/stretchr/testify/assert"
+)
+
+func loadPMKeys(jsonKeys string) (keys *PMKeys) {
+	_ = json.Unmarshal([]byte(jsonKeys), &keys)
+	return
+}
+
+func TestPMKeys_GetKeyRingAndUnlock(t *testing.T) {
+	addrKeysWithTokens := loadPMKeys(readTestFile("keyring_addressKeysWithTokens_JSON", false))
+	addrKeysWithoutTokens := loadPMKeys(readTestFile("keyring_addressKeysWithoutTokens_JSON", false))
+	addrKeysPrimaryHasToken := loadPMKeys(readTestFile("keyring_addressKeysPrimaryHasToken_JSON", false))
+	addrKeysSecondaryHasToken := loadPMKeys(readTestFile("keyring_addressKeysSecondaryHasToken_JSON", false))
+
+	userKey, err := pmcrypto.ReadArmoredKeyRing(strings.NewReader(readTestFile("keyring_userKey", false)))
+	assert.NoError(t, err, "Expected not to receive an error unlocking user key")
+
+	type args struct {
+		userKeyring *pmcrypto.KeyRing
+		passphrase  []byte
+	}
+	tests := []struct {
+		name string
+		keys *PMKeys
+		args args
+	}{
+		{
+			name: "AddressKeys locked with tokens",
+			keys: addrKeysWithTokens,
+			args: args{userKeyring: userKey, passphrase: []byte("testpassphrase")},
+		},
+		{
+			name: "AddressKeys locked with passphrase, not tokens",
+			keys: addrKeysWithoutTokens,
+			args: args{userKeyring: userKey, passphrase: []byte("testpassphrase")},
+		},
+		{
+			name: "AddressKeys, primary locked with token, secondary with passphrase",
+			keys: addrKeysPrimaryHasToken,
+			args: args{userKeyring: userKey, passphrase: []byte("testpassphrase")},
+		},
+		{
+			name: "AddressKeys, primary locked with passphrase, secondary with token",
+			keys: addrKeysSecondaryHasToken,
+			args: args{userKeyring: userKey, passphrase: []byte("testpassphrase")},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tempLocker := &sync.Mutex{}
+
+			err := tt.keys.unlockKeyRing(tt.args.userKeyring, tt.args.passphrase, tempLocker) // nolint[scopelint]
+			if !assert.NoError(t, err) {
+				return
+			}
+
+			// assert at least one key has been decrypted
+			atLeastOneDecrypted := false
+			for _, e := range tt.keys.KeyRing.GetEntities() { // nolint[scopelint]
+				if !e.PrivateKey.Encrypted {
+					atLeastOneDecrypted = true
+					break
+				}
+			}
+			assert.True(t, atLeastOneDecrypted)
+		})
+	}
+}

pkg/pmapi/labels.go

@@ -0,0 +1,177 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import "fmt"
+
+// System labels
+const (
+	InboxLabel     = "0"
+	AllDraftsLabel = "1"
+	AllSentLabel   = "2"
+	TrashLabel     = "3"
+	SpamLabel      = "4"
+	AllMailLabel   = "5"
+	ArchiveLabel   = "6"
+	SentLabel      = "7"
+	DraftLabel     = "8"
+	StarredLabel   = "10"
+
+	LabelTypeMailbox      = 1
+	LabelTypeContactGroup = 2
+)
+
+// IsSystemLabel checks if a label is a pre-defined system label.
+func IsSystemLabel(label string) bool {
+	switch label {
+	case InboxLabel, DraftLabel, SentLabel, TrashLabel, SpamLabel, ArchiveLabel, StarredLabel, AllMailLabel, AllSentLabel, AllDraftsLabel:
+		return true
+	}
+	return false
+}
+
+// LabelColors provides the RGB values of the available label colors.
+var LabelColors = []string{ //nolint[gochecknoglobals]
+	"#7272a7",
+	"#cf5858",
+	"#c26cc7",
+	"#7569d1",
+	"#69a9d1",
+	"#5ec7b7",
+	"#72bb75",
+	"#c3d261",
+	"#e6c04c",
+	"#e6984c",
+	"#8989ac",
+	"#cf7e7e",
+	"#c793ca",
+	"#9b94d1",
+	"#a8c4d5",
+	"#97c9c1",
+	"#9db99f",
+	"#c6cd97",
+	"#e7d292",
+	"#dfb286",
+}
+
+type LabelAction int
+
+const (
+	RemoveLabel LabelAction = iota
+	AddLabel
+)
+
+// Label for message.
+type Label struct {
+	ID        string
+	Name      string
+	Color     string
+	Order     int `json:",omitempty"`
+	Display   int // Not used for now, leave it empty.
+	Exclusive int
+	Type      int
+	Notify    int
+}
+
+type LabelListRes struct {
+	Res
+	Labels []*Label
+}
+
+func (c *Client) ListLabels() (labels []*Label, err error) {
+	return c.ListLabelType(LabelTypeMailbox)
+}
+
+func (c *Client) ListContactGroups() (labels []*Label, err error) {
+	return c.ListLabelType(LabelTypeContactGroup)
+}
+
+// ListLabelType lists all labels created by the user.
+func (c *Client) ListLabelType(labelType int) (labels []*Label, err error) {
+	req, err := NewRequest("GET", fmt.Sprintf("/labels?%d", labelType), nil)
+	if err != nil {
+		return
+	}
+
+	var res LabelListRes
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+
+	labels, err = res.Labels, res.Err()
+	return
+}
+
+type LabelReq struct {
+	*Label
+}
+
+type LabelRes struct {
+	Res
+	Label *Label
+}
+
+// CreateLabel creates a new label.
+func (c *Client) CreateLabel(label *Label) (created *Label, err error) {
+	labelReq := &LabelReq{label}
+	req, err := NewJSONRequest("POST", "/labels", labelReq)
+	if err != nil {
+		return
+	}
+
+	var res LabelRes
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+
+	created, err = res.Label, res.Err()
+	return
+}
+
+// UpdateLabel updates a label.
+func (c *Client) UpdateLabel(label *Label) (updated *Label, err error) {
+	labelReq := &LabelReq{label}
+	req, err := NewJSONRequest("PUT", "/labels/"+label.ID, labelReq)
+	if err != nil {
+		return
+	}
+
+	var res LabelRes
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+
+	updated, err = res.Label, res.Err()
+	return
+}
+
+// DeleteLabel deletes a label.
+func (c *Client) DeleteLabel(id string) (err error) {
+	req, err := NewRequest("DELETE", "/labels/"+id, nil)
+	if err != nil {
+		return
+	}
+
+	var res Res
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+
+	err = res.Err()
+	return
+}

pkg/pmapi/labels_test.go

@@ -0,0 +1,186 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"reflect"
+	"testing"
+)
+
+const testLabelsBody = `{
+    "Labels": [
+        {
+            "ID": "LLz8ysmVxwr4dF6mWpClePT0SpSWOEvzTdq17RydSl4ndMckvY1K63HeXDzn03BJQwKYvgf-eWT8Qfd9WVuIEQ==",
+            "Name": "CroutonMail is awesome :)",
+            "Color": "#7272a7",
+            "Display": 0,
+            "Order": 1,
+            "Type": 1
+        },
+        {
+            "ID": "BvbqbySUPo9uWW_eR8tLA13NUsQMz3P4Zhw4UnpvrKqURnrHlE6L2Au0nplHfHlVXFgGz4L4hJ9-BYllOL-L5g==",
+            "Name": "Royal sausage",
+            "Color": "#cf5858",
+            "Display": 1,
+            "Order": 2,
+            "Type": 1
+        }
+    ],
+    "Code": 1000
+}
+`
+
+var testLabels = []*Label{
+	{ID: "LLz8ysmVxwr4dF6mWpClePT0SpSWOEvzTdq17RydSl4ndMckvY1K63HeXDzn03BJQwKYvgf-eWT8Qfd9WVuIEQ==", Name: "CroutonMail is awesome :)", Color: "#7272a7", Order: 1, Display: 0, Type: LabelTypeMailbox},
+	{ID: "BvbqbySUPo9uWW_eR8tLA13NUsQMz3P4Zhw4UnpvrKqURnrHlE6L2Au0nplHfHlVXFgGz4L4hJ9-BYllOL-L5g==", Name: "Royal sausage", Color: "#cf5858", Order: 2, Display: 1, Type: LabelTypeMailbox},
+}
+
+var testLabelReq = LabelReq{&Label{
+	Name:    "sava",
+	Color:   "#c26cc7",
+	Display: 1,
+}}
+
+const testCreateLabelBody = `{
+    "Label": {
+        "ID": "otkpEZzG--8dMXvwyLXLQWB72hhBhNGzINjH14rUDfywvOyeN01cDxDrS3Koifxf6asA7Xcwtldm0r_MCmWiAQ==",
+        "Name": "sava",
+        "Color": "#c26cc7",
+        "Display": 1,
+        "Order": 3,
+        "Type": 1
+    },
+    "Code": 1000
+}
+`
+
+var testLabelCreated = &Label{
+	ID:      "otkpEZzG--8dMXvwyLXLQWB72hhBhNGzINjH14rUDfywvOyeN01cDxDrS3Koifxf6asA7Xcwtldm0r_MCmWiAQ==",
+	Name:    "sava",
+	Color:   "#c26cc7",
+	Order:   3,
+	Display: 1,
+	Type:    LabelTypeMailbox,
+}
+
+const testDeleteLabelBody = `{
+    "Code": 1000
+}
+`
+
+func TestClient_ListLabels(t *testing.T) {
+	s, c := newTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		Ok(t, checkMethodAndPath(r, "GET", "/labels?1"))
+
+		fmt.Fprint(w, testLabelsBody)
+	}))
+	defer s.Close()
+
+	labels, err := c.ListLabels()
+	if err != nil {
+		t.Fatal("Expected no error while listing labels, got:", err)
+	}
+
+	if !reflect.DeepEqual(labels, testLabels) {
+		for i, l := range testLabels {
+			t.Errorf("expected %d: %#v\n", i, l)
+		}
+		for i, l := range labels {
+			t.Errorf("got %d: %#v\n", i, l)
+		}
+		t.Fatalf("Not same")
+	}
+}
+
+func TestClient_CreateLabel(t *testing.T) {
+	s, c := newTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		Ok(t, checkMethodAndPath(r, "POST", "/labels"))
+
+		body := &bytes.Buffer{}
+		_, err := body.ReadFrom(r.Body)
+		Ok(t, err)
+
+		if bytes.Contains(body.Bytes(), []byte("Order")) {
+			t.Fatal("Body contains `Order`: ", body.String())
+		}
+
+		var labelReq LabelReq
+		if err := json.NewDecoder(body).Decode(&labelReq); err != nil {
+			t.Error("Expecting no error while reading request body, got:", err)
+		}
+		if !reflect.DeepEqual(testLabelReq.Label, labelReq.Label) {
+			t.Errorf("Invalid label request: expected %+v but got %+v", testLabelReq.Label, labelReq.Label)
+		}
+
+		fmt.Fprint(w, testCreateLabelBody)
+	}))
+	defer s.Close()
+
+	created, err := c.CreateLabel(testLabelReq.Label)
+	if err != nil {
+		t.Fatal("Expected no error while creating label, got:", err)
+	}
+
+	if !reflect.DeepEqual(created, testLabelCreated) {
+		t.Fatalf("Invalid created label: expected %+v, got %+v", testLabelCreated, created)
+	}
+}
+
+func TestClient_UpdateLabel(t *testing.T) {
+	s, c := newTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		Ok(t, checkMethodAndPath(r, "PUT", "/labels/"+testLabelCreated.ID))
+
+		var labelReq LabelReq
+		if err := json.NewDecoder(r.Body).Decode(&labelReq); err != nil {
+			t.Error("Expecting no error while reading request body, got:", err)
+		}
+		if !reflect.DeepEqual(testLabelCreated, labelReq.Label) {
+			t.Errorf("Invalid label request: expected %+v but got %+v", testLabelCreated, labelReq.Label)
+		}
+
+		fmt.Fprint(w, testCreateLabelBody)
+	}))
+	defer s.Close()
+
+	updated, err := c.UpdateLabel(testLabelCreated)
+	if err != nil {
+		t.Fatal("Expected no error while updating label, got:", err)
+	}
+
+	if !reflect.DeepEqual(updated, testLabelCreated) {
+		t.Fatalf("Invalid updated label: expected %+v, got %+v", testLabelCreated, updated)
+	}
+}
+
+func TestClient_DeleteLabel(t *testing.T) {
+	s, c := newTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		Ok(t, checkMethodAndPath(r, "DELETE", "/labels/"+testLabelCreated.ID))
+
+		fmt.Fprint(w, testDeleteLabelBody)
+	}))
+	defer s.Close()
+
+	err := c.DeleteLabel(testLabelCreated.ID)
+	if err != nil {
+		t.Fatal("Expected no error while deleting label, got:", err)
+	}
+}

pkg/pmapi/messages.go

@@ -0,0 +1,810 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"bytes"
+	"crypto/aes"
+	"crypto/cipher"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"io"
+	"net/http"
+	"net/mail"
+	"net/url"
+	"strconv"
+	"strings"
+
+	pmcrypto "github.com/ProtonMail/gopenpgp/crypto"
+	"golang.org/x/crypto/openpgp/packet"
+)
+
+// Header types.
+const (
+	MessageHeader       = "-----BEGIN PGP MESSAGE-----"
+	MessageTail         = "-----END PGP MESSAGE-----"
+	MessageHeaderLegacy = "---BEGIN ENCRYPTED MESSAGE---"
+	MessageTailLegacy   = "---END ENCRYPTED MESSAGE---"
+	RandomKeyHeader     = "---BEGIN ENCRYPTED RANDOM KEY---"
+	RandomKeyTail       = "---END ENCRYPTED RANDOM KEY---"
+)
+
+// Sort types.
+const (
+	SortByTo      = "To"
+	SortByFrom    = "From"
+	SortBySubject = "Subject"
+	SortBySize    = "Size"
+	SortByTime    = "Time"
+	SortByID      = "ID"
+	SortDesc      = true
+	SortAsc       = false
+)
+
+// Message actions.
+const (
+	ActionReply    = 0
+	ActionReplyAll = 1
+	ActionForward  = 2
+)
+
+// Message flag definitions.
+const (
+	FlagReceived   = 1
+	FlagSent       = 2
+	FlagInternal   = 4
+	FlagE2E        = 8
+	FlagAuto       = 16
+	FlagReplied    = 32
+	FlagRepliedAll = 64
+	FlagForwarded  = 128
+
+	FlagAutoreplied = 256
+	FlagImported    = 512
+	FlagOpened      = 1024
+	FlagReceiptSent = 2048
+)
+
+// Draft flags.
+const (
+	FlagReceiptRequest = 1 << 16
+	FlagPublicKey      = 1 << 17
+	FlagSign           = 1 << 18
+)
+
+// Spam flags.
+const (
+	FlagSpfFail        = 1 << 24
+	FlagDkimFail       = 1 << 25
+	FlagDmarcFail      = 1 << 26
+	FlagHamManual      = 1 << 27
+	FlagSpamAuto       = 1 << 28
+	FlagSpamManual     = 1 << 29
+	FlagPhishingAuto   = 1 << 30
+	FlagPhishingManual = 1 << 31
+)
+
+// Message flag masks.
+const (
+	FlagMaskGeneral = 4095
+	FlagMaskDraft   = FlagReceiptRequest * 7
+	FlagMaskSpam    = FlagSpfFail * 255
+	FlagMask        = FlagMaskGeneral | FlagMaskDraft | FlagMaskSpam
+)
+
+// INTERNAL, AUTO are immutable. E2E is immutable except for drafts on send.
+const (
+	FlagMaskAdd = 4067 + (16777216 * 168)
+)
+
+// Content types.
+const (
+	ContentTypeMultipartMixed     = "multipart/mixed"
+	ContentTypeMultipartEncrypted = "multipart/encrypted"
+	ContentTypePlainText          = "text/plain"
+	ContentTypeHTML               = "text/html"
+)
+
+// LabelsOperation is the operation to apply to labels.
+type LabelsOperation int
+
+const (
+	KeepLabels    LabelsOperation = iota // Do nothing.
+	ReplaceLabels                        // Replace current labels with new ones.
+	AddLabels                            // Add new labels to current ones.
+	RemoveLabels                         // Remove specified labels from current ones.
+)
+
+const (
+	MessageTypeInbox int = iota
+	MessageTypeDraft
+	MessageTypeSent
+	MessageTypeInboxAndSent
+)
+
+// Due to API limitations, we shouldn't make requests with more than 100 message IDs at a time.
+const messageIDPageSize = 100
+
+// Message structure.
+type Message struct {
+	ID             string `json:",omitempty"`
+	Order          int64  `json:",omitempty"`
+	ConversationID string `json:",omitempty"` // only filter
+	Subject        string
+	Unread         int
+	Type           int
+	Flags          int64
+	Sender         *mail.Address
+	ReplyTo        *mail.Address   `json:",omitempty"`
+	ReplyTos       []*mail.Address `json:",omitempty"`
+	ToList         []*mail.Address
+	CCList         []*mail.Address
+	BCCList        []*mail.Address
+	Time           int64 // Unix time
+	Size           int64
+	NumAttachments int
+	ExpirationTime int64 // Unix time
+	SpamScore      int
+	AddressID      string
+	Body           string `json:",omitempty"`
+	Attachments    []*Attachment
+	LabelIDs       []string
+	ExternalID     string
+	Header         mail.Header
+	MIMEType       string
+}
+
+// NewMessage initializes a new message.
+func NewMessage() *Message {
+	return &Message{
+		ToList:      []*mail.Address{},
+		CCList:      []*mail.Address{},
+		BCCList:     []*mail.Address{},
+		Attachments: []*Attachment{},
+		LabelIDs:    []string{},
+	}
+}
+
+// Define a new type to prevent MarshalJSON/UnmarshalJSON infinite loops.
+type message Message
+
+type rawMessage struct {
+	message
+
+	Header string `json:",omitempty"`
+}
+
+func (m *Message) MarshalJSON() ([]byte, error) {
+	var raw rawMessage
+	raw.message = message(*m)
+
+	b := &bytes.Buffer{}
+	_ = http.Header(m.Header).Write(b)
+	raw.Header = b.String()
+
+	return json.Marshal(&raw)
+}
+
+func (m *Message) UnmarshalJSON(b []byte) error {
+	var raw rawMessage
+	if err := json.Unmarshal(b, &raw); err != nil {
+		return err
+	}
+
+	*m = Message(raw.message)
+
+	if raw.Header != "" && raw.Header != "(No Header)" {
+		msg, err := mail.ReadMessage(strings.NewReader(raw.Header + "\r\n\r\n"))
+		if err == nil {
+			m.Header = msg.Header
+		}
+	}
+
+	return nil
+}
+
+func (m *Message) IsBodyEncrypted() bool {
+	trimmedBody := strings.TrimSpace(m.Body)
+	return strings.HasPrefix(trimmedBody, MessageHeader) &&
+		strings.HasSuffix(trimmedBody, MessageTail)
+}
+
+func (m *Message) IsLegacyMessage() bool {
+	return strings.Contains(m.Body, RandomKeyHeader) &&
+		strings.Contains(m.Body, RandomKeyTail) &&
+		strings.Contains(m.Body, MessageHeaderLegacy) &&
+		strings.Contains(m.Body, MessageTailLegacy) &&
+		strings.Contains(m.Body, MessageHeader) &&
+		strings.Contains(m.Body, MessageTail)
+}
+
+func (m *Message) Decrypt(kr *pmcrypto.KeyRing) (err error) {
+	if m.IsLegacyMessage() {
+		return m.DecryptLegacy(kr)
+	}
+
+	if !m.IsBodyEncrypted() {
+		return
+	}
+
+	armored := strings.TrimSpace(m.Body)
+	body, err := decrypt(kr, armored)
+	if err != nil {
+		return
+	}
+
+	m.Body = body
+	return
+}
+
+func (m *Message) DecryptLegacy(kr *pmcrypto.KeyRing) (err error) {
+	randomKeyStart := strings.Index(m.Body, RandomKeyHeader) + len(RandomKeyHeader)
+	randomKeyEnd := strings.Index(m.Body, RandomKeyTail)
+	randomKey := m.Body[randomKeyStart:randomKeyEnd]
+
+	signedKey, err := decrypt(kr, strings.TrimSpace(randomKey))
+	if err != nil {
+		return
+	}
+	bytesKey, err := decodeBase64UTF8(signedKey)
+	if err != nil {
+		return
+	}
+
+	messageStart := strings.Index(m.Body, MessageHeaderLegacy) + len(MessageHeaderLegacy)
+	messageEnd := strings.Index(m.Body, MessageTailLegacy)
+	message := m.Body[messageStart:messageEnd]
+	bytesMessage, err := decodeBase64UTF8(message)
+	if err != nil {
+		return
+	}
+
+	block, err := aes.NewCipher(bytesKey)
+	if err != nil {
+		return
+	}
+
+	prefix := make([]byte, block.BlockSize()+2)
+	bytesMessageReader := bytes.NewReader(bytesMessage)
+
+	_, err = io.ReadFull(bytesMessageReader, prefix)
+	if err != nil {
+		return
+	}
+	s := packet.NewOCFBDecrypter(block, prefix, packet.OCFBResync)
+	if s == nil {
+		err = errors.New("pmapi: incorrect key for legacy decryption")
+		return
+	}
+
+	reader := cipher.StreamReader{S: s, R: bytesMessageReader}
+	buf := new(bytes.Buffer)
+	_, _ = buf.ReadFrom(reader)
+	plaintextBytes := buf.Bytes()
+
+	plaintext := ""
+	for i := 0; i < len(plaintextBytes); i++ {
+		plaintext += string(plaintextBytes[i])
+	}
+	bytesPlaintext, err := decodeBase64UTF8(plaintext)
+	if err != nil {
+		return
+	}
+
+	m.Body = string(bytesPlaintext)
+	return err
+}
+
+func decodeBase64UTF8(input string) (output []byte, err error) {
+	input = strings.TrimSpace(input)
+	decodedMessage, err := base64.StdEncoding.DecodeString(input)
+	if err != nil {
+		return
+	}
+	utf8DecodedMessage := []rune(string(decodedMessage))
+	output = make([]byte, len(utf8DecodedMessage))
+	for i := 0; i < len(utf8DecodedMessage); i++ {
+		output[i] = byte(int(utf8DecodedMessage[i]))
+	}
+	return
+}
+
+func (m *Message) Encrypt(encrypter, signer *pmcrypto.KeyRing) (err error) {
+	if m.IsBodyEncrypted() {
+		err = errors.New("pmapi: trying to encrypt an already encrypted message")
+		return
+	}
+
+	m.Body, err = encrypt(encrypter, m.Body, signer)
+	return
+}
+
+func (m *Message) Has(flag int64) bool {
+	return (m.Flags & flag) == flag
+}
+
+// MessagesCount contains message counts for one label.
+type MessagesCount struct {
+	LabelID string
+	Total   int
+	Unread  int
+}
+
+// MessagesFilter contains fields to filter messages.
+type MessagesFilter struct {
+	Page           int
+	PageSize       int
+	Limit          int
+	LabelID        string
+	Sort           string // Time by default (Time, To, From, Subject, Size).
+	Desc           *bool
+	Begin          int64 // Unix time.
+	End            int64 // Unix time.
+	BeginID        string
+	EndID          string
+	Keyword        string
+	To             string
+	From           string
+	Subject        string
+	ConversationID string
+	AddressID      string
+	ID             []string
+	Attachments    *bool
+	Unread         *bool
+	ExternalID     string // MIME Message-Id (only valid for messages).
+	AutoWildcard   *bool
+}
+
+func (filter *MessagesFilter) urlValues() url.Values { // nolint[funlen]
+	v := url.Values{}
+
+	if filter.Page != 0 {
+		v.Set("Page", strconv.Itoa(filter.Page))
+	}
+	if filter.PageSize != 0 {
+		v.Set("PageSize", strconv.Itoa(filter.PageSize))
+	}
+	if filter.Limit != 0 {
+		v.Set("Limit", strconv.Itoa(filter.Limit))
+	}
+	if filter.LabelID != "" {
+		v.Set("LabelID", filter.LabelID)
+	}
+	if filter.Sort != "" {
+		v.Set("Sort", filter.Sort)
+	}
+	if filter.Desc != nil {
+		if *filter.Desc {
+			v.Set("Desc", "1")
+		} else {
+			v.Set("Desc", "0")
+		}
+	}
+	if filter.Begin != 0 {
+		v.Set("Begin", strconv.Itoa(int(filter.Begin)))
+	}
+	if filter.End != 0 {
+		v.Set("End", strconv.Itoa(int(filter.End)))
+	}
+	if filter.BeginID != "" {
+		v.Set("BeginID", filter.BeginID)
+	}
+	if filter.EndID != "" {
+		v.Set("EndID", filter.EndID)
+	}
+	if filter.Keyword != "" {
+		v.Set("Keyword", filter.Keyword)
+	}
+	if filter.To != "" {
+		v.Set("To", filter.To)
+	}
+	if filter.From != "" {
+		v.Set("From", filter.From)
+	}
+	if filter.Subject != "" {
+		v.Set("Subject", filter.Subject)
+	}
+	if filter.ConversationID != "" {
+		v.Set("ConversationID", filter.ConversationID)
+	}
+	if filter.AddressID != "" {
+		v.Set("AddressID", filter.AddressID)
+	}
+	if len(filter.ID) > 0 {
+		for _, id := range filter.ID {
+			v.Add("ID[]", id)
+		}
+	}
+	if filter.Attachments != nil {
+		if *filter.Attachments {
+			v.Set("Attachments", "1")
+		} else {
+			v.Set("Attachments", "0")
+		}
+	}
+	if filter.Unread != nil {
+		if *filter.Unread {
+			v.Set("Unread", "1")
+		} else {
+			v.Set("Unread", "0")
+		}
+	}
+	if filter.ExternalID != "" {
+		v.Set("ExternalID", filter.ExternalID)
+	}
+	if filter.AutoWildcard != nil {
+		if *filter.AutoWildcard {
+			v.Set("AutoWildcard", "1")
+		} else {
+			v.Set("AutoWildcard", "0")
+		}
+	}
+
+	return v
+}
+
+type MessagesListRes struct {
+	Res
+
+	Total    int
+	Messages []*Message
+}
+
+// ListMessages gets message metadata.
+func (c *Client) ListMessages(filter *MessagesFilter) (msgs []*Message, total int, err error) {
+	req, err := NewRequest("GET", "/messages", nil)
+	if err != nil {
+		return
+	}
+
+	req.URL.RawQuery = filter.urlValues().Encode()
+	var res MessagesListRes
+	if err = c.DoJSON(req, &res); err != nil {
+		// If the URI was too long and we searched with IDs, we will try again without the API IDs.
+		if strings.Contains(err.Error(), "api returned: 414") && len(filter.ID) > 0 {
+			filter.ID = []string{}
+			return c.ListMessages(filter)
+		}
+		return
+	}
+
+	msgs, total, err = res.Messages, res.Total, res.Err()
+	return
+}
+
+type MessagesCountsRes struct {
+	Res
+
+	Counts []*MessagesCount
+}
+
+// CountMessages counts messages by label.
+func (c *Client) CountMessages(addressID string) (counts []*MessagesCount, err error) {
+	reqURL := "/messages/count"
+	if addressID != "" {
+		reqURL += ("?AddressID=" + addressID)
+	}
+	req, err := NewRequest("GET", reqURL, nil)
+	if err != nil {
+		return
+	}
+
+	var res MessagesCountsRes
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+
+	counts, err = res.Counts, res.Err()
+	return
+}
+
+type MessageRes struct {
+	Res
+
+	Message *Message
+}
+
+// GetMessage retrieves a message.
+func (c *Client) GetMessage(id string) (msg *Message, err error) {
+	req, err := NewRequest("GET", "/messages/"+id, nil)
+	if err != nil {
+		return
+	}
+
+	var res MessageRes
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+
+	msg, err = res.Message, res.Err()
+	return
+}
+
+type SendMessageReq struct {
+	ExpirationTime int64 `json:",omitempty"`
+	// AutoSaveContacts int `json:",omitempty"`
+
+	// Data for encrypted recipients.
+	Packages []*MessagePackage
+}
+
+// Message package types.
+const (
+	InternalPackage         = 1
+	EncryptedOutsidePackage = 2
+	ClearPackage            = 4
+	PGPInlinePackage        = 8
+	PGPMIMEPackage          = 16
+	ClearMIMEPackage        = 32
+)
+
+// Signature types.
+const (
+	NoSignature  = 0
+	YesSignature = 1
+)
+
+type MessagePackage struct {
+	Addresses      map[string]*MessageAddress
+	Type           int
+	MIMEType       string
+	Body           string             // base64-encoded encrypted data packet.
+	BodyKey        AlgoKey            // base64-encoded session key (only if cleartext recipients).
+	AttachmentKeys map[string]AlgoKey // Only include if cleartext & attachments.
+}
+
+type MessageAddress struct {
+	Type                 int
+	BodyKeyPacket        string // base64-encoded key packet.
+	Signature            int    // 0 = None, 1 = Detached, 2 = Attached/Armored
+	AttachmentKeyPackets map[string]string
+}
+
+type AlgoKey struct {
+	Key       string
+	Algorithm string
+}
+
+type SendMessageRes struct {
+	Res
+
+	Sent *Message
+
+	// Parent is only present if the sent message has a parent (reply/reply all/forward).
+	Parent *Message
+}
+
+func (c *Client) SendMessage(id string, sendReq *SendMessageReq) (sent, parent *Message, err error) {
+	if id == "" {
+		err = errors.New("pmapi: cannot send message with an empty id")
+		return
+	}
+
+	if sendReq.Packages == nil {
+		sendReq.Packages = []*MessagePackage{}
+	}
+
+	req, err := NewJSONRequest("POST", "/messages/"+id, sendReq)
+	if err != nil {
+		return
+	}
+
+	var res SendMessageRes
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+
+	sent, parent, err = res.Sent, res.Parent, res.Err()
+	return
+}
+
+const (
+	DraftActionReply    = 0
+	DraftActionReplyAll = 1
+	DraftActionForward  = 2
+)
+
+type DraftReq struct {
+	Message              *Message
+	ParentID             string `json:",omitempty"`
+	Action               int
+	AttachmentKeyPackets []string
+}
+
+func (c *Client) CreateDraft(m *Message, parent string, action int) (created *Message, err error) {
+	createReq := &DraftReq{Message: m, ParentID: parent, Action: action, AttachmentKeyPackets: []string{}}
+
+	req, err := NewJSONRequest("POST", "/messages", createReq)
+	if err != nil {
+		return
+	}
+
+	var res MessageRes
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+
+	created, err = res.Message, res.Err()
+	return
+}
+
+type MessagesActionReq struct {
+	IDs []string
+}
+
+type MessagesActionRes struct {
+	Res
+
+	Responses []struct {
+		ID       string
+		Response Res
+	}
+}
+
+func (res MessagesActionRes) Err() error {
+	if err := res.Res.Err(); err != nil {
+		return err
+	}
+
+	for _, msgRes := range res.Responses {
+		if err := msgRes.Response.Err(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// doMessagesAction performs paged requests to doMessagesActionInner.
+// This can eventually be done in parallel though.
+func (c *Client) doMessagesAction(action string, ids []string) (err error) {
+	for len(ids) > messageIDPageSize {
+		var requestIDs []string
+		requestIDs, ids = ids[:messageIDPageSize], ids[messageIDPageSize:]
+		if err = c.doMessagesActionInner(action, requestIDs); err != nil {
+			return
+		}
+	}
+
+	return c.doMessagesActionInner(action, ids)
+}
+
+// doMessagesActionInner is the non-paged inner method of doMessagesAction.
+// You should not call this directly unless you know what you are doing (it can overload the server).
+func (c *Client) doMessagesActionInner(action string, ids []string) (err error) {
+	actionReq := &MessagesActionReq{IDs: ids}
+	req, err := NewJSONRequest("PUT", "/messages/"+action, actionReq)
+	if err != nil {
+		return
+	}
+
+	var res MessagesActionRes
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+
+	err = res.Err()
+
+	return
+}
+
+func (c *Client) MarkMessagesRead(ids []string) error {
+	return c.doMessagesAction("read", ids)
+}
+
+func (c *Client) MarkMessagesUnread(ids []string) error {
+	return c.doMessagesAction("unread", ids)
+}
+
+func (c *Client) DeleteMessages(ids []string) error {
+	return c.doMessagesAction("delete", ids)
+}
+
+func (c *Client) UndeleteMessages(ids []string) error {
+	return c.doMessagesAction("undelete", ids)
+}
+
+type LabelMessagesReq struct {
+	LabelID string
+	IDs     []string
+}
+
+// LabelMessages labels the given message IDs with the given label.
+// The requests are performed paged; this can eventually be done in parallel.
+func (c *Client) LabelMessages(ids []string, label string) (err error) {
+	for len(ids) > messageIDPageSize {
+		var requestIDs []string
+		requestIDs, ids = ids[:messageIDPageSize], ids[messageIDPageSize:]
+		if err = c.labelMessages(requestIDs, label); err != nil {
+			return
+		}
+	}
+
+	return c.labelMessages(ids, label)
+}
+
+func (c *Client) labelMessages(ids []string, label string) (err error) {
+	labelReq := &LabelMessagesReq{LabelID: label, IDs: ids}
+	req, err := NewJSONRequest("PUT", "/messages/label", labelReq)
+	if err != nil {
+		return
+	}
+
+	var res MessagesActionRes
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+
+	err = res.Err()
+	return
+}
+
+// UnlabelMessages removes the given label from the given message IDs.
+// The requests are performed paged; this can eventually be done in parallel.
+func (c *Client) UnlabelMessages(ids []string, label string) (err error) {
+	for len(ids) > messageIDPageSize {
+		var requestIDs []string
+		requestIDs, ids = ids[:messageIDPageSize], ids[messageIDPageSize:]
+		if err = c.unlabelMessages(requestIDs, label); err != nil {
+			return
+		}
+	}
+
+	return c.unlabelMessages(ids, label)
+}
+
+func (c *Client) unlabelMessages(ids []string, label string) (err error) {
+	labelReq := &LabelMessagesReq{LabelID: label, IDs: ids}
+	req, err := NewJSONRequest("PUT", "/messages/unlabel", labelReq)
+	if err != nil {
+		return
+	}
+
+	var res MessagesActionRes
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+
+	err = res.Err()
+	return
+}
+
+func (c *Client) EmptyFolder(labelID, addressID string) (err error) {
+	if labelID == "" {
+		return errors.New("pmapi: labelID parameter is empty string")
+	}
+	reqURL := "/messages/empty?LabelID=" + labelID
+	if addressID != "" {
+		reqURL += ("&AddressID=" + addressID)
+	}
+
+	req, err := NewRequest("DELETE", reqURL, nil)
+
+	if err != nil {
+		return
+	}
+
+	var res Res
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+
+	err = res.Err()
+	return
+}

pkg/pmapi/messages_test.go

@@ -0,0 +1,223 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+	"testing"
+
+	pmcrypto "github.com/ProtonMail/gopenpgp/crypto"
+	"github.com/stretchr/testify/assert"
+)
+
+const testMessageCleartext = `<div>jeej saas<br></div><div><br></div><div class="protonmail_signature_block"><div>Sent from <a href="https://protonmail.ch">ProtonMail</a>, encrypted email based in Switzerland.<br></div><div><br></div></div>`
+const testMessageCleartextLegacy = `<div>flkasjfkjasdklfjasd<br></div><div>fasd<br></div><div>jfasjdfjasd<br></div><div>fj<br></div><div>asdfj<br></div><div>sadjf<br></div><div>sadjf<br></div><div>asjdf<br></div><div>jasd<br></div><div>fj<br></div><div>asdjf<br></div><div>asdjfsad<br></div><div>fasdlkfjasdjfkljsadfljsdfjsdljflkdsjfkljsdlkfjsdlk<br></div><div>jasfd<br></div><div>jsd<br></div><div>jf<br></div><div>sdjfjsdf<br></div><div><br></div><div>djfskjsladf<br></div><div>asd<br></div><div>fja<br></div><div>sdjfajsf<br></div><div>jas<br></div><div>fas<br></div><div>fj<br></div><div>afj<br></div><div>ajf<br></div><div>af<br></div><div>asdfasdfasd<br></div><div>Sent from <a href="https://protonmail.ch">ProtonMail</a>, encrypted email based in Switzerland.<br></div><div>dshfljsadfasdf<br></div><div>as<br></div><div>df<br></div><div>asd<br></div><div>fasd<br></div><div>f<br></div><div>asd<br></div><div>fasdflasdklfjsadlkjf</div><div>asd<br></div><div>fasdlkfjasdlkfjklasdjflkasjdflaslkfasdfjlasjflkasflksdjflkjasdf<br></div><div>asdflkasdjflajsfljaslkflasf<br></div><div>asdfkas<br></div><div>dfjas<br></div><div>djf<br></div><div>asjf<br></div><div>asj<br></div><div>faj<br></div><div>f<br></div><div>afj<br></div><div>sdjaf<br></div><div>jas<br></div><div>sdfj<br></div><div>ajf<br></div><div>aj<br></div><div>ajsdafafdaaf<br></div><div>a<br></div><div>f<br></div><div>lasl;ga<br></div><div>sags<br></div><div>ad<br></div><div>gags<br></div><div>g<br></div><div>ga<br></div><div>a<br></div><div>gg<br></div><div>a<br></div><div>ag<br></div><div>ag<br></div><div>agga.g.ga,ag.ag./ga<br></div><div><br></div><div>dsga<br></div><div>sg<br></div><div><br></div><div>gasga\g\g\g\g\g\n\y\t\r\\r\r\\n\n\n\<br></div><div><br></div><div><br></div><div>sd<br></div><div>asdf<br></div><div>asdf<br></div><div>dsa<br></div><div>fasd<br></div><div>f</div>`
+
+const testMessageEncrypted = `-----BEGIN PGP MESSAGE-----
+Version: OpenPGP.js v1.2.0
+Comment: http://openpgpjs.org
+
+wcBMA0fcZ7XLgmf2AQf+JPulpEOWwmY/Sfze8rBpYvrO2cebSSkjCgapFfXG
+CI4PA+rb+WGkn9uBJf3FgEEg76c2ZqGh9zXTyrdHyFLm8ekarvxzgLpvcei/
+p18IzcxsWnaM+1uknL4bKUtK3298gIl6xrfc4eVEA8tqUPUkSLSGk7uggjhj
+zEYR4zIgMa0c6sMVcZ1Idvy9gGsTIvvcZJ4h1lKVUl8gba+qr1D76RaAf5xS
+SBT74q9HhgfEMZwk6hXAp4MYY5h+lIsuhFu5kQ9fhZKU0PWS7ljddv854ZxS
+9gHKPBerv4NBjkkCLp9xa2QNjDnu1fNlzlJpfCavp6wDdC83GiT61VRHPE4s
+J9LASwFwgOrPmB8Mi867AQM0dddbj4Qe5ghlUcF1XnybkwfHqvQA1QT50d5n
+ddFyxwIjvI/Nsn8MTCSnmrWCrjQ7v8JC73NyGxO5k6ZlUnc6BQVie78QJo5a
+ftzl5b6nwlCYuXI8R6N/t5MXzrC5GwR8nvjH6kgbUVTLL1hO2Sbgyq5bBKLW
+jjylTsZDHUGi4OX7q7eet5/RhKusWdvR0cHEaZAVD6BhTNN0mFBJ5bM1SINI
+9gxJVqKJe7j4nJP4PGZBJrokZihhiBS/WEbJdvS54frYajGKjMavB3VhFP6k
+qi5aiqGJKOJOV/G8yIwtdtxac3UL34eWo69U39Zx2mNfSXCzSjuafCr1nmAS
+4g==
+=Uw3B
+-----END PGP MESSAGE-----
+`
+
+const testMessageEncryptedLegacy = `---BEGIN ENCRYPTED MESSAGE---esK5w7TCgVnDj8KQHBvDvhJObcOvw6/Cv2/CjMOpw5UES8KQwq/CiMOpI3MrexLDimzDmsKqVmwQw7vDkcKlRgXCosOpwoJgV8KEBCslSGbDtsOlw5gow7NxG8OSw6JNPlYuwrHCg8K5w6vDi8Kww5V5wo/Dl8KgwpnCi8Kww7nChMKdw5FHwoxmCGbCm8O6wpDDmRVEWsO7wqnCtVnDlMKORDbDnjbCqcOnNMKEwoPClFlaw6k1w5TDpcOGJsOUw5Unw5fCrcK3XnLCoRBBwo/DpsKAJiTDrUHDuGEQXz/DjMOhTCN7esO5ZjVIQSoFZMOyF8Kgw6nChcKmw6fCtcOBcW7Ck8KJwpTDnCzCnz3DjFY7wp5jUsOhw7XDosKQNsOUBmLDksKzPcO4fE/Dmw1GecKew4/CmcOJTFXDsB5uMcOFd1vDmX9ow4bDpCPDoU3Drw8oScKOXznDisKfYF3DvMKoEy0DDmzDhlHDjwIyC8OzRS/CnEZ4woM9w5cnw51fw6MZMAzDk8O3CDXDoyHDvzlFwqDCg8KsTnAiaMOsIyfCmUEaw6nChMK5TMOxG8KEHUNIwo1seMOXw5HDhyVawrzCr8KmFWHDpMO3asKpwrQbbMOlwoMew4t1Jz51wp9Jw6kGWcOzc8KgwpLCpsOHOMOgYB3DiMOxLcOQB8K7AcOyWF3CmnwfK8Kxw6XDm2TCiT/CnVTCg8Omw7Ngwp3CuUAHw6/CjRLDgcKsU8O/w6gXJ0cIw6pZMcOxEWETwpd4w58Mwr5SBMKORQjCi3FYcULDgx09w5M7SH7DrMKrw4gnXMKjwqUrBMOLwqQyF0nDhcKuwqTDqsO2w7LCnGjCvkbDgDgcw54xAkEiQMKUFlzDkMOew73CmkU4wrnCjw3DvsKaW8K0InA+w4sPSXfDuhbClMKgUcKeCMORw5ZYJcKnNEzDoMOhw7MYCX4DwqIQwoHCvsOaB1UAI8KVw6LCvcOTw53CuSgow4kZdHw5aRkYw7ZyV8OsP0LCh8KnwpIuw4p1NisoEcKcwrjDhcOtMzdvw5rDmsK3IAdAw7M4J8K+w6zCmR3CuMKUw4lqw6osPMObw53Dg8K3wqLCrsKZwr8mPcK4w4QWw5LCnwZeH1bDgwwiXcKbUhHDk1DDk0MLwoDDqMKXw5skNsKAAcOFw77Di8KNGCBzP8OcwrI5wodQQwQyw5V0wrInwrPDt8O+T8KbNsKVw7Mzw7HCsMOjwpcewoPCuMOUEsOow6QZVDjDpgbDlMOBGDXCtMOmw6jDuMKfw4nDlWTDq8Kqd0TDvwPCpSzDlA4JO3EHwrlBWcK5w7DCscOwCMK2wpsvwrYNIcOgBBXChMK0w6nCosKWEVd+w7cEal5hIcO4SWrCu0TDrW5Yw4XCmBgCwpc7YVwIwqPCi8OlGDzDmyJ/woHCscOtw4zDuC7CpUXCrDAJwp7Cj8KxPX3CrhDCvVB2w7PCosKbw7F+V11hY8Omwq1eQcO8w4wcRMKBJ2LDgW/DomXDhwkgAlxmQcKew6HDq8Ouw6ASeG/DlcKgUcKmLMOowpQWNcKJJcKDa3XDksK/woHCo3d6wrHDpMOqwqs/UUXCjUpnwrHCmsOyJx4bwoHChAnDi0TCpjLDrBvCvEghw5VtfhPCk8K5KsKIw75FCsOyDsKtV17CicOjwqAnF8OHHC0qMsOEwrgEwr13c8KZw4fDn8KXw73CksKAw4QTGRgIG8KMMXwpwrRBT2DDq8K3AsOQXl/DqMKYMivClsKiXcOhGkvDmsK9w77Cmmpvwrhsd8Kaw7bDgQ/DuCU2CyTDtjnCgn/DiMOtSyPDnsOfVTstccO6EVXDrj03MUHDvDDCgsO7BFQFEX3DszIyw7Rsw7pNwpjCs8OCLR9UbsOlw5USw73DiWJqVXTCl2tFw7FaAcKaw7l5a3Mvw5TCpMKCwpbDi3fCi8KHwrfDugUZwo5hw7fChsKDw5ZhPjA7w7HDjcO9wrrCjUbDoy4JXA1JICRDw49UNsOYOsK9FGE5wqhAw67DumnDqW0cwqbCu8OedEbDqcOfw50MVH8twpVLH8O3LsKvacKJw75xTMKkOcOJw4/DvsOYwqRwZcOnwqfCm2XCnRJFwqEgX8KLPsKfwpQWw6nChm82w6hME10KTRhGw5LCj1stPiXClsO8w7rCocOLw6lFw7tAZ8K0O3wswpZ4wqvCmMOFwpzDhMKVRRQjw53CikECPMOKZcOOwoAKcMK7WMO3K8Okw4bCjgrCisKLRsKewqzDvmtnw584wrtiw6RFVsKPecOpIhx7TsKzw4TCisKyw6nCqcK+w6fChsKxw5kWSsOgfD7CkRfCncKGKMOubsKoBA9Fe2YHwrx4aQNSG8Kpw5zDrMO1FMOPZcKSIVnDrHxOBsKyBcKmYwQMOl7CiRvCnDNVw7NaesOoPR3CrnQEwr9Xw600BSFYECnDgi1OFS7DoFYJw4M6wrzCog09WFPCmiHDogjDpQFjdsKKIsOWFsKXd0TDjXU3CsONRX3DssOrw4HDmX0Mw7rDiENvwpPCghsXacK2w6XCkMOICcKVw4nCkMO8RcOUw4zCn1VJw752RAUawqhdw5dEwqbDh0wAMH/DlTrChC/DosOoGsOPw5nClTcyw5XDlsKhNsKAcBINwpxUAi8Rw5Jvwpckwq4uBy0nw51dP2UGbidATX1FLMKFw5zDsQxewp3DlMKwwo3CrhBPJGR7cVHCnTUnwrDDksO0AcO5T3jCm245OnUVUT8WD1HDhTnCqnbCt8OjMDvCsAzCjsKSwoDDlDhtw7cFwpsDaS7CvVLDu0zDnlvDlMOEwrnCgVzCgcOZN8Oxwp0LSMKswq/DrMK9fcKTL1zDgcOvwofCtWAoL0IKR8OWwqpPw6QfVsKcwqxTXGEPKCFydX4Mw5jDmcOEWlPCgMKDPcOJw7HDgcOMahzCjMO7HyPDo8K3Y8OswqPDgSQ+w6wfw67Cr8O/w61oMsO+woTDrnECI2TDuMK5wrzDusOHw5/CosKFwrciQF3Csj5aw7DDpMKwZMK3Z8KlRBIcLcKvM2/CtBk8JMKWwqVyw6RNwoUhwoDCsXbCrD04wpQ4F8KOcMKIw7PDtMKqZRTCjsKSOMOKCMKYQ8OhwqZ1dGrChcKXLSnDiT7CrEjCihckNcOXw63CkUYpT8KTwq7CgMKiw7PCqmBzwq/Crz50XcKEGlLCrUBjw6ASVsObD8K9wpZ6eBHCi2FTMVcDSzvDgwtxw5ZJHlF5woDDtsKTwovChMOyYMKOSCt7w7hGDDsFaMOewrrCjRbDrGPDg2rCpsO3wo8IEMO9wqjCrG0mRXHDocKJwqQYdsKOw7UUwqIUwq/CqUlKW8ObwpcZGizCpgd4dAZBXMOYw5s5w6HDvkEgw6sbRxAwwoBSOyXCjDPDpsKlwrPCrl/DqsOswoJJDWzDp8Ocw5nDrE5FWm3DncKVwpnCqMKiwoDDmMONQcOEwpwRwonCsh0Tw7FCw6Nfw7U7wp7DnMKnfMOHCMOnw4TClcOVwrzCiiddUj3CmsOgwqvDhxfDjsOMWcKDZnvDocObw77Do1rDgMKHVsKCLcOXRMOHD0RNwpEdwozCrBnDqBYWwojCiVzCjTTCqcO5wqgAwqhhw7tnw5ZuOcOYNGTDiR1GAEzDuE0PeErDnlQlfsOjw6UGWUUNw6TCmgx8NMKzDMKgL8O3esKDwprDoTl8wrbDvVDCvU4Iw5sAwr/DugcoR8KMw4hNeMKSw7Jmw4rDjG8NbcO8w7jCs8OvfFXCoBBNfcOqNsK0EQLCncKPw53DrsOiwolvwqjCr8OZDsORw47DiyA+VcOMSg5wworDgGx0w7sgKMOyDMOyZRkgw43CqUHDicKfwpDCo8OII8KvKsOxDcKoFsOaw7HCgXTDssK7B8KIwoNcw4zCu8KBw4vCvFjDkWLDl8OyB8O/w4oYw5DCslzDk2kDw7jDgcOJw4jComXDkwdfw61xw53Cv8KPf11iwq0kKsKDw7nCmiVNF0NqLMKvwqvDjhQ3ZXbDomvDs8OKQQ7CocOnwr1Fw7xZRMK6w41cw5DDgzzCthIoAMOBQcOPbcOPVx/Cm8OYw7pHwo/CvCxhCcKVw7vChShnw6rClUQ7w6dbZMOrw4hpw7lZXMOxw5pnUXHDiMOLDxrDiA/DtMKqw6zDjXRJwp07BsKEwoTClBHCritDYXgzT3RWDcOlw4lfw4Vbw7fCj8K0w4AnwqjCrxPDpCVXF8KbY8OMPwQvwqdaw6E8w4AHPcKbNGl8wpQMX2PDp0pJfcOyGsOUXkNww5jCg8Obwo7DryjCisKeYiQ/XUzDvRvDncOtCMKJwqxHw6LDh8KwwrV7LGPCkcKOIXbCv8KHwpnDi1keQkLDssOSw7XCk8K+w7YdSMKAQmbDo8KPw7xywpnCsgANNTJYScKkNAvDo8KZw6Ayw6tmC8KaTsKEbcOZTx3DilrDtUjDi8OWV8K/wrocwpNKLlYbbcOmPcKPwrvCsTpLey5Xw58XJBPCo8KEPWJrwqZJX1fCncKDw4AZw4hWw5pTw7pidlzDtMO6w7t9DcK+R8KefMOfETvCskgjOgHCqcK7UgHCgsOfwrt8bcKQw5FeZcOiw4Faw7hRTjDDocOuEMOoEm04NQTCrCjDvMOaNDV6V8OHc8OTdMOndCh7HMOqw7HDnlzCl3MqwpjDiiDDtcKmCknCuBcQwobDvcOUN2LDmsOeHMOmPMKeH0nCt0nDgsO8w73CkRDDmMOuacO9w5J1KsKswqY7UMKyHHzDjMOjw5QOSWUhw4jCpMKJw4DCtcKNdcKPLcOFJsOqQ14=---END ENCRYPTED MESSAGE---||---BEGIN ENCRYPTED RANDOM KEY--------BEGIN PGP MESSAGE-----
+Version: OpenPGP.js v0.9.0
+Comment: http://openpgpjs.org
+
+wcBMA2tjJVxNCRhtAQf/YzkQoUqaqa3NJ/c1apIF/dsl7yJ4GdVrC3/w7lxE
+2CO5ioQD4s6QMWP2Y9dOdVl2INwz8eXOds9NS+1nMs4SoMbrpJnAjx8Cthti
+1Z/8eWMU023LYahds8BYM0T435K/2tTB5GTA4uTl2y8Xzz2PbptQ4PrUDaII
++egeQQyPA0yuoRDwpaeTiaBYOSa06YYuK5Agr0buQAxRIMCxI2o+fucjoabv
+FsQHKGu20U5GlJroSIyIVVkaH3evhNti/AnYX1HuokcGEQNsF5vo4SjWcH23
+2P86EIV+w5lUWC1FN9vZCyvbvyuqLHQMtqKVn4GBOkIc3bYQ0jru3a0FG4Cx
+bNJ0ASps2+p3Vxe0d+so2iFV92ByQ+0skyCUwCNUlwOV5V5f2fy1ImXk4mXI
+cO/bcbqRxx3pG9gkPIh43FoQktTT+tsJ5vS53qfaLGdhCYfkrWjsKu+2P9Xg
++Cr8clh6NTblhfkoAS1gzjA3XgsgEFrtP+OGqwg=
+=c5WU
+-----END PGP MESSAGE-----
+---END ENCRYPTED RANDOM KEY---
+`
+
+const testMessageSigned = `-----BEGIN PGP MESSAGE-----
+Version: OpenPGP.js v4.5.3
+Comment: https://openpgpjs.org
+
+wcBMA0fcZ7XLgmf2AQgAgnHOlcAwVu2AnVfi2fIQHSkTQ0OFnZMJMRR3MJ1q
+HtUW8jkSLcurL0Sn/tBFLIIR4YT2tQMzV7cvZzZyBEuZM4OYnDp8xSmoszPh
+Gc/nvYG0A0pmKAQkL27v05Dul8oUWA0APT51urghH2Pzm7NdOMtTKIE4LQjS
+mBfQ6Cf14uKV0xGS9v2dSFjFxxXEEpMQ+k60NCKRYClN2LVVxf3OKXbuugds
+m2GUGn3CuFsiabosIUv4EcdE3aD9HbNo+PIWLJWRJIYJSc5+FWcbwXuIIFgC
+XX1s7OV53ceZJnhjCmDE0N2ZOLLAYWED2zRvUa+CAqG+hZgc/3Ia+UmJUVuZ
+BNLAugFuRsOVgh3olUIz0vazHhyGG0XIsNqmRm0U9SIfhWkPPHBmU6Xht6Qw
+EvLbBfKTYHxX01yQUNgIv4S/TULeQuUjZQfsNYNXXGepS+jiCoIdEgUwpvre
+OMFGsypwQXVCFYO/GQdYanMQRTckEexyBY4hGYVrevDM1yG/zGJIdbfI2L+1
+1cz76jI8PtzL+S0zcVkevLcjjsHm2Je959uSida9jara7Bymr0y56UdoXoWX
+4vZ0kQNo58eEEV0zg7dit4lDvwcuSZMW6K//xNtRQ4QX7/EDtlcYqBJXPwJY
+eQSBVeYbeUbZ+PHJdu5gbI85BJNE2dKcS1bdOhEU2lPLYpvmMpPdot9TwnJb
+dN3l8yDyhScGvTIZqlxhU7HCM9VHAS0bDqCUoO8EruztUSgjMI+gKC9+xdVU
+yrkF7K23UNLWflROMv4cp0LDRB57619Y2w5lY/MG5bS0jSfMWBwnJG2AF28c
+2tYKnHw6rpZXvXnlDmEDT8suTzuTGA==
+=Sir8
+-----END PGP MESSAGE-----
+`
+
+const testMessageSigner = `-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: OpenPGP.js v0.7.1
+Comment: http://openpgpjs.org
+
+xsBNBFSI0BMBB/9td6B5RDzVSFTlFzYOS4JxIb5agtNW1rbA4FeLoC47bGLR
+8E42IA6aKcO4H0vOZ1lFms0URiKk1DjCMXn3AUErbxqiV5IATRZLwliH6vwy
+PI6j5rtGF8dyxYfwmLtoUNkDcPdcFEb4NCdowsN7e8tKU0bcpouZcQhAqawC
+9nEdaG/gS5w+2k4hZX2lOKS1EF5SvP48UadlspEK2PLAIp5wB9XsFS9ey2wu
+elzkSfDh7KUAlteqFGSMqIgYH62/gaKm+TcckfZeyiMHWFw6sfrcFQ3QOZPq
+ahWt0Rn9XM5xBAxx5vW0oceuQ1vpvdfFlM5ix4gn/9w6MhmStaCee8/fABEB
+AAHNBlVzZXJJRMLAcgQQAQgAJgUCVIjQHQYLCQgHAwIJEASDR1Fk7GNTBBUI
+AgoDFgIBAhsDAh4BAADmhAf/Yt0mCfWqQ25NNGUN14pKKgnPm68zwj1SmMGa
+pU7+7ItRpoFNaDwV5QYiQSLC1SvSb1ZeKoY928GPKfqYyJlBpTPL9zC1OHQj
+9+2yYauHjYW9JWQM7hst2S2LBcdiQPOs3ybWPaO9yaccV4thxKOCPvyClaS5
+b9T4Iv9GEVZQIUvArkwI8hyzIi6skRgxflGheq1O+S1W4Gzt2VtYvo8g8r6W
+GzAGMw2nrs2h0+vUr+dLDgIbFCTc5QU99d5jE/e5Hw8iqBxv9tqB1hVATf8T
+wC8aU5MTtxtabOiBgG0PsBs6oIwjFqEjpOIza2/AflPZfo7stp6IiwbwvTHo
+1NlHoM7ATQRUiNAdAQf/eOLJYxX4lUQUzrNQgASDNE8gJPj7ywcGzySyqr0Y
+5rbG57EjtKMIgZrpzJRpSCuRbBjfsltqJ5Q9TBAbPO+oR3rue0LqPKMnmr/q
+KsHswBJRfsb/dbktUNmv/f7R9IVyOuvyP6RgdGeloxdGNeWiZSA6AZYI+WGc
+xaOvVDPz8thtnML4G4MUhXxxNZ7JzQ0Lfz6mN8CCkblIP5xpcJsyRU7lUsGD
+EJGZX0JH/I8bRVN1Xu08uFinIkZyiXRJ5ZGgF3Dns6VbIWmbttY54tBELtk+
+5g9pNSl9qiYwiCdwuZrA//NmD3xlZIN8sG4eM7ZUibZ23vEq+bUt1++6Mpba
+GQARAQABwsBfBBgBCAATBQJUiNAfCRAEg0dRZOxjUwIbDAAAlpMH/085qZdO
+mGRAlbvViUNhF2rtHvCletC48WHGO1ueSh9VTxalkP21YAYLJ4JgJzArJ7tH
+lEeiKiHm8YU9KhLe11Yv/o3AiKIAQjJiQluvk+mWdMcddB4fBjL6ttMTRAXe
+gHnjtMoamHbSZdeUTUadv05Fl6ivWtpXlODG4V02YvDiGBUbDosdGXEqDtpT
+g6MYlj3QMvUiUNQvt7YGMJS8A9iQ9qBNzErgRW8L6CON2RmpQ/wgwP5nwUHz
+JjY51d82Vj8bZeI8LdsX41SPoUhyC7kmNYpw9ZRy7NlrCt8dBIOB4/BKEJ2G
+ClW54lp9eeOfYTsdTSbn9VaSO0E6m2/Q4Tk=
+=WFtr
+-----END PGP PUBLIC KEY BLOCK-----`
+
+func TestMessage_IsBodyEncrypted(t *testing.T) {
+	msg := &Message{Body: testMessageEncrypted}
+	Assert(t, msg.IsBodyEncrypted(), "the body should be encrypted")
+
+	msg.Body = testMessageCleartext
+	Assert(t, !msg.IsBodyEncrypted(), "the body should not be encrypted")
+}
+
+func TestMessage_Decrypt(t *testing.T) {
+	msg := &Message{Body: testMessageEncrypted}
+	err := msg.Decrypt(testPrivateKeyRing)
+	Ok(t, err)
+	Equals(t, testMessageCleartext, msg.Body)
+}
+
+func TestMessage_Decrypt_Legacy(t *testing.T) {
+	testPrivateKeyLegacy := readTestFile("testPrivateKeyLegacy", false)
+	testPrivateKeyRingLegacy, err := pmcrypto.ReadArmoredKeyRing(strings.NewReader(testPrivateKeyLegacy))
+	Ok(t, err)
+
+	Ok(t, testPrivateKeyRingLegacy.Unlock([]byte(testMailboxPasswordLegacy)))
+
+	msg := &Message{Body: testMessageEncryptedLegacy}
+
+	err = msg.Decrypt(testPrivateKeyRingLegacy)
+	Ok(t, err)
+
+	Equals(t, testMessageCleartextLegacy, msg.Body)
+}
+
+func TestMessage_Decrypt_signed(t *testing.T) {
+	msg := &Message{Body: testMessageSigned}
+	err := msg.Decrypt(testPrivateKeyRing)
+	Ok(t, err)
+	Equals(t, testMessageCleartext, msg.Body)
+}
+
+func TestMessage_Encrypt(t *testing.T) {
+	signer, err := pmcrypto.ReadArmoredKeyRing(strings.NewReader(testMessageSigner))
+	Ok(t, err)
+
+	msg := &Message{Body: testMessageCleartext}
+	Ok(t, msg.Encrypt(testPrivateKeyRing, testPrivateKeyRing))
+
+	err = msg.Decrypt(testPrivateKeyRing)
+	Ok(t, err)
+
+	Equals(t, testMessageCleartext, msg.Body)
+	Equals(t, testIdentity, signer.Identities()[0])
+}
+
+func routeLabelMessages(tb testing.TB, w http.ResponseWriter, r *http.Request) string {
+	Ok(tb, checkMethodAndPath(r, "PUT", "/messages/label"))
+
+	return "messages/label/put_response.json"
+}
+
+func TestMessage_LabelMessages_NoPaging(t *testing.T) {
+	// This should be only enough IDs to produce one page.
+	testIDs := []string{}
+	for i := 0; i < messageIDPageSize-1; i++ {
+		testIDs = append(testIDs, fmt.Sprintf("%v", i))
+	}
+
+	// There should be enough IDs to produce just one page so the endpoint should be called once.
+	finish, c := newTestServerCallbacks(t,
+		routeLabelMessages,
+	)
+	defer finish()
+
+	c.uid = testUID
+	c.accessToken = testAccessToken
+
+	assert.NoError(t, c.LabelMessages(testIDs, "mylabel"))
+}
+
+func TestMessage_LabelMessages_Paging(t *testing.T) {
+	// This should be enough IDs to produce three pages.
+	testIDs := []string{}
+	for i := 0; i < 3*messageIDPageSize; i++ {
+		testIDs = append(testIDs, fmt.Sprintf("%v", i))
+	}
+
+	// There should be enough IDs to produce three pages so the endpoint should be called three times.
+	finish, c := newTestServerCallbacks(t,
+		routeLabelMessages,
+		routeLabelMessages,
+		routeLabelMessages,
+	)
+	defer finish()
+
+	c.uid = testUID
+	c.accessToken = testAccessToken
+
+	assert.NoError(t, c.LabelMessages(testIDs, "mylabel"))
+}

pkg/pmapi/metrics.go

@@ -0,0 +1,43 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"net/url"
+)
+
+// SendSimpleMetric makes a simple GET request to send a simple metrics report.
+func (c *Client) SendSimpleMetric(category, action, label string) (err error) {
+	v := url.Values{}
+	v.Set("Category", category)
+	v.Set("Action", action)
+	v.Set("Label", label)
+
+	req, err := NewRequest("GET", "/metrics?"+v.Encode(), nil)
+	if err != nil {
+		return
+	}
+
+	var res Res
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+
+	err = res.Err()
+	return
+}

pkg/pmapi/metrics_test.go

@@ -0,0 +1,43 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"fmt"
+	"net/http"
+	"testing"
+)
+
+const testSendSimpleMetricsBody = `{
+    "Code": 1000
+}
+`
+
+func TestClient_SendSimpleMetric(t *testing.T) {
+	s, c := newTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		Ok(t, checkMethodAndPath(r, "GET", "/metrics?Action=some_action&Category=some_category&Label=some_label"))
+
+		fmt.Fprint(w, testSendSimpleMetricsBody)
+	}))
+	defer s.Close()
+
+	err := c.SendSimpleMetric("some_category", "some_action", "some_label")
+	if err != nil {
+		t.Fatal("Expected no error while sending simple metric, got:", err)
+	}
+}

pkg/pmapi/passwords.go

@@ -0,0 +1,47 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"encoding/base64"
+	"errors"
+
+	"github.com/jameskeane/bcrypt"
+)
+
+func HashMailboxPassword(password, keySalt string) (hashedPassword string, err error) {
+	if keySalt == "" {
+		hashedPassword = password
+		return
+	}
+	decodedSalt, err := base64.StdEncoding.DecodeString(keySalt)
+	if err != nil {
+		return
+	}
+	encodedSalt := base64.NewEncoding("./ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789").WithPadding(base64.NoPadding).EncodeToString(decodedSalt)
+	hashResult, err := bcrypt.Hash(password, "$2y$10$"+encodedSalt)
+	if err != nil {
+		return
+	}
+	if len(hashResult) != 60 {
+		err = errors.New("pmapi: invalid mailbox password hash")
+		return
+	}
+	hashedPassword = hashResult[len(hashResult)-31:]
+	return
+}

pkg/pmapi/pmapi_test.go

@@ -0,0 +1,62 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"io/ioutil"
+	"strings"
+
+	pmcrypto "github.com/ProtonMail/gopenpgp/crypto"
+)
+
+const testMailboxPassword = "apple"
+const testMailboxPasswordLegacy = "123"
+
+var (
+	testPrivateKeyRing *pmcrypto.KeyRing
+	testPublicKeyRing  *pmcrypto.KeyRing
+)
+
+func init() {
+	testPrivateKey := readTestFile("testPrivateKey", false)
+	testPublicKey := readTestFile("testPublicKey", false)
+
+	var err error
+	if testPrivateKeyRing, err = pmcrypto.ReadArmoredKeyRing(strings.NewReader(testPrivateKey)); err != nil {
+		panic(err)
+	}
+
+	if testPublicKeyRing, err = pmcrypto.ReadArmoredKeyRing(strings.NewReader(testPublicKey)); err != nil {
+		panic(err)
+	}
+
+	if err := testPrivateKeyRing.Unlock([]byte(testMailboxPassword)); err != nil {
+		panic(err)
+	}
+}
+
+func readTestFile(name string, trimNewlines bool) string { // nolint[unparam]
+	data, err := ioutil.ReadFile("testdata/" + name)
+	if err != nil {
+		panic(err)
+	}
+	if trimNewlines {
+		return strings.TrimRight(string(data), "\n")
+	}
+	return string(data)
+}

pkg/pmapi/proxy.go

@@ -0,0 +1,304 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"crypto/tls"
+	"encoding/base64"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/go-resty/resty/v2"
+	"github.com/miekg/dns"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	proxyRevertTime    = 24 * time.Hour
+	proxySearchTimeout = 30 * time.Second
+	proxyQueryTimeout  = 10 * time.Second
+	proxyLookupWait    = 5 * time.Second
+	proxyQuery         = "dMFYGSLTQOJXXI33ONVQWS3BOMNUA.protonpro.xyz"
+)
+
+var dohProviders = []string{ //nolint[gochecknoglobals]
+	"https://dns11.quad9.net/dns-query",
+	"https://dns.google/dns-query",
+}
+
+// globalAllowDoH controls whether or not to enable use of DoH/Proxy in pmapi.
+var globalAllowDoH = false // nolint[golint]
+
+// globalProxyMutex allows threadsafe modification of proxy state.
+var globalProxyMutex = sync.RWMutex{} // nolint[golint]
+
+// globalOriginalURL backs up the original API url so it can be restored later.
+var globalOriginalURL = RootURL // nolint[golint]
+
+// globalIsDoHAllowed returns whether or not to use DoH.
+func globalIsDoHAllowed() bool { // nolint[golint]
+	globalProxyMutex.RLock()
+	defer globalProxyMutex.RUnlock()
+
+	return globalAllowDoH
+}
+
+// GlobalAllowDoH enables DoH.
+func GlobalAllowDoH() { // nolint[golint]
+	globalProxyMutex.Lock()
+	defer globalProxyMutex.Unlock()
+
+	globalAllowDoH = true
+}
+
+// GlobalDisallowDoH disables DoH and sets the RootURL back to what it was.
+func GlobalDisallowDoH() { // nolint[golint]
+	globalProxyMutex.Lock()
+	defer globalProxyMutex.Unlock()
+
+	globalAllowDoH = false
+	RootURL = globalOriginalURL
+}
+
+// globalSetRootURL sets the global RootURL.
+func globalSetRootURL(url string) { // nolint[golint]
+	globalProxyMutex.Lock()
+	defer globalProxyMutex.Unlock()
+
+	RootURL = url
+}
+
+// GlobalGetRootURL returns the global RootURL.
+func GlobalGetRootURL() (url string) { // nolint[golint]
+	globalProxyMutex.RLock()
+	defer globalProxyMutex.RUnlock()
+
+	return RootURL
+}
+
+// isProxyEnabled returns whether or not we are currently using a proxy.
+func isProxyEnabled() bool { // nolint[golint]
+	return globalOriginalURL != GlobalGetRootURL()
+}
+
+// proxyManager manages known proxies.
+type proxyManager struct {
+	// dohLookup is used to look up the given query at the given DoH provider, returning the TXT records>
+	dohLookup func(query, provider string) (urls []string, err error)
+
+	providers  []string // List of known doh providers.
+	query      string   // The query string used to find proxies.
+	proxyCache []string // All known proxies, cached in case DoH providers are unreachable.
+
+	useDuration                time.Duration // How much time to use the proxy before returning to the original API.
+	findTimeout, lookupTimeout time.Duration // Timeouts for DNS query and proxy search.
+
+	lastLookup time.Time // The time at which we last attempted to find a proxy.
+}
+
+// newProxyManager creates a new proxyManager that queries the given DoH providers
+// to retrieve DNS records for the given query string.
+func newProxyManager(providers []string, query string) (p *proxyManager) { // nolint[unparam]
+	p = &proxyManager{
+		providers:     providers,
+		query:         query,
+		useDuration:   proxyRevertTime,
+		findTimeout:   proxySearchTimeout,
+		lookupTimeout: proxyQueryTimeout,
+	}
+
+	// Use the default DNS lookup method; this can be overridden if necessary.
+	p.dohLookup = p.defaultDoHLookup
+
+	return
+}
+
+// findProxy returns a new proxy domain which is not equal to the current RootURL.
+// It returns an error if the process takes longer than ProxySearchTime.
+func (p *proxyManager) findProxy() (proxy string, err error) {
+	if time.Now().Before(p.lastLookup.Add(proxyLookupWait)) {
+		return "", errors.New("not looking for a proxy, too soon")
+	}
+
+	p.lastLookup = time.Now()
+
+	proxyResult := make(chan string)
+	errResult := make(chan error)
+	go func() {
+		if err = p.refreshProxyCache(); err != nil {
+			logrus.WithError(err).Warn("Failed to refresh proxy cache, cache may be out of date")
+		}
+
+		for _, proxy := range p.proxyCache {
+			if proxy != stripProtocol(GlobalGetRootURL()) && p.canReach(proxy) {
+				proxyResult <- proxy
+				return
+			}
+		}
+
+		errResult <- errors.New("no proxy available")
+	}()
+
+	select {
+	case <-time.After(p.findTimeout):
+		logrus.Error("Timed out finding a proxy server")
+		return "", errors.New("timed out finding a proxy")
+
+	case proxy = <-proxyResult:
+		logrus.WithField("proxy", proxy).Info("Found proxy server")
+		return
+
+	case err = <-errResult:
+		logrus.WithError(err).Error("Failed to find available proxy server")
+		return
+	}
+}
+
+// useProxy sets the proxy server to use. It returns to the original RootURL after 24 hours.
+func (p *proxyManager) useProxy(proxy string) {
+	if !isProxyEnabled() {
+		p.disableProxyAfter(p.useDuration)
+	}
+
+	globalSetRootURL(https(proxy))
+}
+
+// disableProxyAfter disables the proxy after the given amount of time.
+func (p *proxyManager) disableProxyAfter(d time.Duration) {
+	go func() {
+		<-time.After(d)
+		globalSetRootURL(globalOriginalURL)
+	}()
+}
+
+// refreshProxyCache loads the latest proxies from the known providers.
+func (p *proxyManager) refreshProxyCache() error {
+	logrus.Info("Refreshing proxy cache")
+
+	for _, provider := range p.providers {
+		if proxies, err := p.dohLookup(p.query, provider); err == nil {
+			p.proxyCache = proxies
+
+			// We also want to allow bridge to switch back to the standard API at any time.
+			p.proxyCache = append(p.proxyCache, globalOriginalURL)
+
+			logrus.WithField("proxies", proxies).Info("Available proxies")
+
+			return nil
+		}
+	}
+
+	return errors.New("lookup failed with all DoH providers")
+}
+
+// canReach returns whether we can reach the given url.
+// NOTE: we skip cert verification to stop it complaining that cert name doesn't match hostname.
+func (p *proxyManager) canReach(url string) bool {
+	pinger := resty.New().
+		SetHostURL(https(url)).
+		SetTimeout(p.lookupTimeout).
+		SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}) // nolint[gosec]
+
+	if _, err := pinger.R().Get("/tests/ping"); err != nil {
+		return false
+	}
+
+	return true
+}
+
+// defaultDoHLookup is the default implementation of the proxy manager's DoH lookup.
+// It looks up DNS TXT records for the given query URL using the given DoH provider.
+// It returns a list of all found TXT records.
+// If the whole process takes more than ProxyQueryTime then an error is returned.
+func (p *proxyManager) defaultDoHLookup(query, dohProvider string) (data []string, err error) {
+	dataResult := make(chan []string)
+	errResult := make(chan error)
+	go func() {
+		// Build new DNS request in RFC1035 format.
+		dnsRequest := new(dns.Msg).SetQuestion(dns.Fqdn(query), dns.TypeTXT)
+
+		// Pack the DNS request message into wire format.
+		rawRequest, err := dnsRequest.Pack()
+		if err != nil {
+			errResult <- errors.Wrap(err, "failed to pack DNS request")
+			return
+		}
+
+		// Encode wire-format DNS request message as base64url (RFC4648) without padding chars.
+		encodedRequest := base64.RawURLEncoding.EncodeToString(rawRequest)
+
+		// Make DoH request to the given DoH provider.
+		rawResponse, err := resty.New().R().SetQueryParam("dns", encodedRequest).Get(dohProvider)
+		if err != nil {
+			errResult <- errors.Wrap(err, "failed to make DoH request")
+			return
+		}
+
+		// Unpack the DNS response.
+		dnsResponse := new(dns.Msg)
+		if err = dnsResponse.Unpack(rawResponse.Body()); err != nil {
+			errResult <- errors.Wrap(err, "failed to unpack DNS response")
+			return
+		}
+
+		// Pick out the TXT answers.
+		for _, answer := range dnsResponse.Answer {
+			if t, ok := answer.(*dns.TXT); ok {
+				data = append(data, t.Txt...)
+			}
+		}
+
+		dataResult <- data
+	}()
+
+	select {
+	case <-time.After(p.lookupTimeout):
+		logrus.WithField("provider", dohProvider).Error("Timed out querying DNS records")
+		return []string{}, errors.New("timed out querying DNS records")
+
+	case data = <-dataResult:
+		logrus.WithField("data", data).Info("Received TXT records")
+		return
+
+	case err = <-errResult:
+		logrus.WithField("provider", dohProvider).WithError(err).Error("Failed to query DNS records")
+		return
+	}
+}
+
+func stripProtocol(url string) string {
+	if strings.HasPrefix(url, "https://") {
+		return strings.TrimPrefix(url, "https://")
+	}
+
+	if strings.HasPrefix(url, "http://") {
+		return strings.TrimPrefix(url, "http://")
+	}
+
+	return url
+}
+
+func https(url string) string {
+	if !strings.HasPrefix(url, "https://") && !strings.HasPrefix(url, "http://") {
+		url = "https://" + url
+	}
+
+	return url
+}

pkg/pmapi/proxy_test.go

@@ -0,0 +1,304 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	TestDoHQuery       = "dMFYGSLTQOJXXI33ONVQWS3BOMNUA.protonpro.xyz"
+	TestQuad9Provider  = "https://dns11.quad9.net/dns-query"
+	TestGoogleProvider = "https://dns.google/dns-query"
+)
+
+func TestProxyManager_FindProxy(t *testing.T) {
+	blockAPI()
+	defer unblockAPI()
+
+	proxy := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+	defer proxy.Close()
+
+	p := newProxyManager([]string{"not used"}, "not used")
+	p.dohLookup = func(q, p string) ([]string, error) { return []string{proxy.URL}, nil }
+
+	url, err := p.findProxy()
+	require.NoError(t, err)
+	require.Equal(t, proxy.URL, url)
+}
+
+func TestProxyManager_FindProxy_ChooseReachableProxy(t *testing.T) {
+	blockAPI()
+	defer unblockAPI()
+
+	badProxy := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+	goodProxy := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+
+	// Close the bad proxy first so it isn't reachable; we should then choose the good proxy.
+	badProxy.Close()
+	defer goodProxy.Close()
+
+	p := newProxyManager([]string{"not used"}, "not used")
+	p.dohLookup = func(q, p string) ([]string, error) { return []string{badProxy.URL, goodProxy.URL}, nil }
+
+	url, err := p.findProxy()
+	require.NoError(t, err)
+	require.Equal(t, goodProxy.URL, url)
+}
+
+func TestProxyManager_FindProxy_FailIfNoneReachable(t *testing.T) {
+	blockAPI()
+	defer unblockAPI()
+
+	badProxy := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+	anotherBadProxy := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+
+	// Close the proxies to simulate them not being reachable.
+	badProxy.Close()
+	anotherBadProxy.Close()
+
+	p := newProxyManager([]string{"not used"}, "not used")
+	p.dohLookup = func(q, p string) ([]string, error) { return []string{badProxy.URL, anotherBadProxy.URL}, nil }
+
+	_, err := p.findProxy()
+	require.Error(t, err)
+}
+
+func TestProxyManager_FindProxy_LookupTimeout(t *testing.T) {
+	blockAPI()
+	defer unblockAPI()
+
+	proxy := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+	defer proxy.Close()
+
+	p := newProxyManager([]string{"not used"}, "not used")
+	p.lookupTimeout = time.Second
+	p.dohLookup = func(q, p string) ([]string, error) { time.Sleep(2 * time.Second); return nil, nil }
+
+	// The findProxy should fail because lookup takes 2 seconds but we only allow 1 second.
+	_, err := p.findProxy()
+	require.Error(t, err)
+}
+
+func TestProxyManager_FindProxy_FindTimeout(t *testing.T) {
+	blockAPI()
+	defer unblockAPI()
+
+	slowProxy := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		time.Sleep(2 * time.Second)
+	}))
+	defer slowProxy.Close()
+
+	p := newProxyManager([]string{"not used"}, "not used")
+	p.findTimeout = time.Second
+	p.dohLookup = func(q, p string) ([]string, error) { return []string{slowProxy.URL}, nil }
+
+	// The findProxy should fail because lookup takes 2 seconds but we only allow 1 second.
+	_, err := p.findProxy()
+	require.Error(t, err)
+}
+
+func TestProxyManager_UseProxy(t *testing.T) {
+	blockAPI()
+	defer unblockAPI()
+
+	proxy := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+	defer proxy.Close()
+
+	p := newProxyManager([]string{"not used"}, "not used")
+	p.dohLookup = func(q, p string) ([]string, error) { return []string{proxy.URL}, nil }
+
+	url, err := p.findProxy()
+	require.NoError(t, err)
+
+	p.useProxy(url)
+	require.Equal(t, proxy.URL, GlobalGetRootURL())
+}
+
+func TestProxyManager_UseProxy_MultipleTimes(t *testing.T) {
+	blockAPI()
+	defer unblockAPI()
+
+	proxy1 := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+	defer proxy1.Close()
+	proxy2 := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+	defer proxy2.Close()
+	proxy3 := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+	defer proxy3.Close()
+
+	p := newProxyManager([]string{"not used"}, "not used")
+
+	p.dohLookup = func(q, p string) ([]string, error) { return []string{proxy1.URL}, nil }
+	url, err := p.findProxy()
+	require.NoError(t, err)
+	p.useProxy(url)
+	require.Equal(t, proxy1.URL, GlobalGetRootURL())
+
+	// Have to wait so as to not get rejected.
+	time.Sleep(proxyLookupWait)
+
+	p.dohLookup = func(q, p string) ([]string, error) { return []string{proxy2.URL}, nil }
+	url, err = p.findProxy()
+	require.NoError(t, err)
+	p.useProxy(url)
+	require.Equal(t, proxy2.URL, GlobalGetRootURL())
+
+	// Have to wait so as to not get rejected.
+	time.Sleep(proxyLookupWait)
+
+	p.dohLookup = func(q, p string) ([]string, error) { return []string{proxy3.URL}, nil }
+	url, err = p.findProxy()
+	require.NoError(t, err)
+	p.useProxy(url)
+	require.Equal(t, proxy3.URL, GlobalGetRootURL())
+}
+
+func TestProxyManager_UseProxy_RevertAfterTime(t *testing.T) {
+	blockAPI()
+	defer unblockAPI()
+
+	proxy := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+	defer proxy.Close()
+
+	p := newProxyManager([]string{"not used"}, "not used")
+	p.useDuration = time.Second
+	p.dohLookup = func(q, p string) ([]string, error) { return []string{proxy.URL}, nil }
+
+	url, err := p.findProxy()
+	require.NoError(t, err)
+	require.Equal(t, proxy.URL, url)
+
+	p.useProxy(url)
+	require.Equal(t, proxy.URL, GlobalGetRootURL())
+
+	time.Sleep(2 * time.Second)
+	require.Equal(t, globalOriginalURL, GlobalGetRootURL())
+}
+
+func TestProxyManager_UseProxy_RevertIfProxyStopsWorkingAndOriginalAPIIsReachable(t *testing.T) {
+	// Don't block the API here because we want it to be working so the test can find it.
+	defer unblockAPI()
+
+	proxy := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+	defer proxy.Close()
+
+	p := newProxyManager([]string{"not used"}, "not used")
+	p.dohLookup = func(q, p string) ([]string, error) { return []string{proxy.URL}, nil }
+
+	url, err := p.findProxy()
+	require.NoError(t, err)
+	require.Equal(t, proxy.URL, url)
+
+	p.useProxy(url)
+	require.Equal(t, proxy.URL, GlobalGetRootURL())
+
+	// Simulate that the proxy stops working.
+	proxy.Close()
+	time.Sleep(proxyLookupWait)
+
+	// We should now find the original API URL if it is working again.
+	url, err = p.findProxy()
+	require.NoError(t, err)
+	require.Equal(t, globalOriginalURL, url)
+
+	p.useProxy(url)
+	require.Equal(t, globalOriginalURL, GlobalGetRootURL())
+}
+
+func TestProxyManager_UseProxy_FindSecondAlternativeIfFirstFailsAndAPIIsStillBlocked(t *testing.T) {
+	blockAPI()
+	defer unblockAPI()
+
+	proxy1 := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+	defer proxy1.Close()
+	proxy2 := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+	defer proxy2.Close()
+
+	p := newProxyManager([]string{"not used"}, "not used")
+	p.dohLookup = func(q, p string) ([]string, error) { return []string{proxy1.URL, proxy2.URL}, nil }
+
+	// Find a proxy.
+	url, err := p.findProxy()
+	require.NoError(t, err)
+	p.useProxy(url)
+	require.Equal(t, proxy1.URL, GlobalGetRootURL())
+
+	// Have to wait so as to not get rejected.
+	time.Sleep(proxyLookupWait)
+
+	// The proxy stops working and the protonmail API is still blocked.
+	proxy1.Close()
+
+	// Should switch to the second proxy because both the first proxy and the protonmail API are blocked.
+	url, err = p.findProxy()
+	require.NoError(t, err)
+	p.useProxy(url)
+	require.Equal(t, proxy2.URL, GlobalGetRootURL())
+}
+
+func TestProxyManager_DoHLookup_Quad9(t *testing.T) {
+	p := newProxyManager([]string{TestQuad9Provider, TestGoogleProvider}, TestDoHQuery)
+
+	records, err := p.dohLookup(TestDoHQuery, TestQuad9Provider)
+	require.NoError(t, err)
+	require.NotEmpty(t, records)
+}
+
+func TestProxyManager_DoHLookup_Google(t *testing.T) {
+	p := newProxyManager([]string{TestQuad9Provider, TestGoogleProvider}, TestDoHQuery)
+
+	records, err := p.dohLookup(TestDoHQuery, TestGoogleProvider)
+	require.NoError(t, err)
+	require.NotEmpty(t, records)
+}
+
+func TestProxyManager_DoHLookup_FindProxy(t *testing.T) {
+	p := newProxyManager([]string{TestQuad9Provider, TestGoogleProvider}, TestDoHQuery)
+
+	url, err := p.findProxy()
+	require.NoError(t, err)
+	require.NotEmpty(t, url)
+}
+
+func TestProxyManager_DoHLookup_FindProxyFirstProviderUnreachable(t *testing.T) {
+	p := newProxyManager([]string{"https://unreachable", TestGoogleProvider}, TestDoHQuery)
+
+	url, err := p.findProxy()
+	require.NoError(t, err)
+	require.NotEmpty(t, url)
+}
+
+// testAPIURLBackup is used to hold the globalOriginalURL because we clear it for test purposes and need to restore it.
+var testAPIURLBackup = globalOriginalURL
+
+// blockAPI prevents tests from reaching the standard API, forcing them to find a proxy.
+func blockAPI() {
+	globalSetRootURL("")
+	globalOriginalURL = ""
+}
+
+// unblockAPI allow tests to reach the standard API again.
+func unblockAPI() {
+	globalOriginalURL = testAPIURLBackup
+	globalSetRootURL(globalOriginalURL)
+}

pkg/pmapi/req.go

@@ -0,0 +1,90 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"mime/multipart"
+	"net/http"
+)
+
+// NewRequest creates a new request.
+func NewRequest(method, path string, body io.Reader) (req *http.Request, err error) {
+	req, err = http.NewRequest(method, GlobalGetRootURL()+path, body)
+	if req != nil {
+		req.Header.Set("User-Agent", CurrentUserAgent)
+	}
+	return
+}
+
+// NewJSONRequest create a new JSON request.
+func NewJSONRequest(method, path string, body interface{}) (*http.Request, error) {
+	b, err := json.Marshal(body)
+	if err != nil {
+		panic(err)
+	}
+
+	req, err := NewRequest(method, path, bytes.NewReader(b))
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Add("Content-Type", "application/json")
+
+	return req, nil
+}
+
+type MultipartWriter struct {
+	*multipart.Writer
+
+	c io.Closer
+}
+
+func (w *MultipartWriter) Close() error {
+	if err := w.Writer.Close(); err != nil {
+		return err
+	}
+	return w.c.Close()
+}
+
+// NewMultipartRequest creates a new multipart request.
+//
+// The multipart request is written as long as it is sent to the API. That means
+// that writing the request and sending it MUST be done in parallel. If the
+// request fails, subsequent writes to the multipart writer will fail with an
+// io.ErrClosedPipe error.
+func NewMultipartRequest(method, path string) (req *http.Request, w *MultipartWriter, err error) {
+	// The pipe will connect the multipart writer and the HTTP request body.
+	pr, pw := io.Pipe()
+
+	// pw needs to be closed once the multipart writer is closed.
+	w = &MultipartWriter{
+		multipart.NewWriter(pw),
+		pw,
+	}
+
+	req, err = NewRequest(method, path, pr)
+	if err != nil {
+		return
+	}
+
+	req.Header.Add("Content-Type", w.FormDataContentType())
+	return
+}

pkg/pmapi/res.go

@@ -0,0 +1,72 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+// Common response codes.
+const (
+	CodeOk = 1000
+)
+
+// Res is an API response.
+type Res struct {
+	// The response code is the code from the body JSON. It's still used,
+	// but preference is to use HTTP status code instead for new changes.
+	Code       int
+	StatusCode int
+
+	// The error, if there is any.
+	*ResError
+}
+
+// Err returns error if the response is an error. Otherwise, returns nil.
+func (res Res) Err() error {
+	if res.ResError == nil {
+		return nil
+	}
+
+	if res.Code == ForceUpgradeBadAPIVersion ||
+		res.Code == ForceUpgradeInvalidAPI ||
+		res.Code == ForceUpgradeBadAppVersion {
+		return ErrUpgradeApplication
+	}
+
+	if res.Code == APIOffline {
+		return ErrAPINotReachable
+	}
+
+	return &Error{
+		Code:         res.Code,
+		ErrorMessage: res.ResError.Error,
+	}
+}
+
+type ResError struct {
+	Error string
+}
+
+// Error is an API error.
+type Error struct {
+	// The error code.
+	Code int
+	// The error message.
+	ErrorMessage string `json:"Error"`
+}
+
+func (err Error) Error() string {
+	return err.ErrorMessage
+}

pkg/pmapi/sentry.go

@@ -0,0 +1,173 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"fmt"
+	"regexp"
+	"runtime"
+	"runtime/pprof"
+	"strconv"
+	"strings"
+
+	"github.com/getsentry/raven-go"
+)
+
+const fileParseError = "[file parse error]"
+
+var isGoroutine = regexp.MustCompile("^goroutine [[:digit:]]+.*") //nolint[gochecknoglobals]
+
+// SentryThreads implements standard sentry thread report.
+type SentryThreads struct {
+	Values []Thread `json:"values"`
+}
+
+// Class specifier.
+func (s *SentryThreads) Class() string { return "threads" }
+
+// Thread wraps a single stacktrace.
+type Thread struct {
+	ID         int               `json:"id"`
+	Name       string            `json:"name"`
+	Crashed    bool              `json:"crashed"`
+	Stacktrace *raven.Stacktrace `json:"stacktrace"`
+}
+
+// TraceAllRoutines traces all goroutines and saves them to the current object.
+func (s *SentryThreads) TraceAllRoutines() {
+	s.Values = []Thread{}
+	goroutines := &strings.Builder{}
+	_ = pprof.Lookup("goroutine").WriteTo(goroutines, 2)
+
+	thread := Thread{ID: -1}
+	var frame *raven.StacktraceFrame
+	for _, v := range strings.Split(goroutines.String(), "\n") {
+		// Ignore empty lines.
+		if v == "" {
+			continue
+		}
+
+		// New routine.
+		if isGoroutine.MatchString(v) {
+			if thread.ID >= 0 {
+				s.Values = append(s.Values, thread)
+			}
+			thread = Thread{ID: thread.ID + 1, Name: v, Crashed: thread.ID == -1, Stacktrace: &raven.Stacktrace{Frames: []*raven.StacktraceFrame{}}}
+			continue
+		}
+
+		// New function.
+		if frame == nil {
+			frame = &raven.StacktraceFrame{Function: v}
+			continue
+		}
+
+		// Set filename and add frame.
+		if frame.Filename == "" {
+			fld := strings.Fields(v)
+			if len(fld) != 2 {
+				frame.Filename = fileParseError
+				frame.AbsolutePath = v
+			} else {
+				frame.Filename = fld[0]
+				sp := strings.Split(fld[0], ":")
+				if len(sp) > 1 {
+					i, err := strconv.Atoi(sp[len(sp)-1])
+					if err == nil {
+						frame.Filename = strings.Join(sp[:len(sp)-1], ":")
+						frame.Lineno = i
+					}
+				}
+			}
+			if frame.AbsolutePath == "" && frame.Filename != fileParseError {
+				frame.AbsolutePath = frame.Filename
+				if sp := strings.Split(frame.Filename, "/"); len(sp) > 1 {
+					frame.Filename = sp[len(sp)-1]
+				}
+			}
+			thread.Stacktrace.Frames = append([]*raven.StacktraceFrame{frame}, thread.Stacktrace.Frames...)
+			frame = nil
+			continue
+		}
+	}
+	// Add last thread.
+	s.Values = append(s.Values, thread)
+}
+
+func findPanicSender(s *SentryThreads, err error) string {
+	out := "error nil"
+	if err != nil {
+		out = err.Error()
+	}
+	for _, thread := range s.Values {
+		if !thread.Crashed {
+			continue
+		}
+		for i, fr := range thread.Stacktrace.Frames {
+			if strings.HasSuffix(fr.Filename, "panic.go") && strings.HasPrefix(fr.Function, "panic") {
+				// Next frame if any.
+				j := 0
+				if i > j {
+					j = i - 1
+				}
+
+				// Directory and filename.
+				fname := thread.Stacktrace.Frames[j].AbsolutePath
+				if sp := strings.Split(fname, "/"); len(sp) > 2 {
+					fname = strings.Join(sp[len(sp)-2:], "/")
+				}
+
+				// Line number.
+				if ln := thread.Stacktrace.Frames[j].Lineno; ln > 0 {
+					fname = fmt.Sprintf("%s:%d", fname, ln)
+				}
+
+				out = fmt.Sprintf("%s: %s", fname, out)
+				break // Just first panic.
+			}
+		}
+	}
+	return out
+}
+
+// ReportSentryCrash reports a sentry crash with stacktrace from all goroutines.
+func (c *Client) ReportSentryCrash(reportErr error) (err error) {
+	if reportErr == nil {
+		return
+	}
+	tags := map[string]string{
+		"OS":        runtime.GOOS,
+		"Client":    c.config.ClientID,
+		"Version":   c.config.AppVersion,
+		"UserAgent": CurrentUserAgent,
+		"UserID":    c.userID,
+	}
+
+	threads := &SentryThreads{}
+	threads.TraceAllRoutines()
+	errorWithFile := findPanicSender(threads, reportErr)
+	packet := raven.NewPacket(errorWithFile, threads)
+
+	eventID, ch := raven.Capture(packet, tags)
+	if err = <-ch; err == nil {
+		c.log.Warn("Reported error with id: ", eventID)
+	} else {
+		c.log.Errorf("Can not report `%s` due to `%s`", reportErr.Error(), err.Error())
+	}
+	return err
+}

pkg/pmapi/sentry_test.go

@@ -0,0 +1,65 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/getsentry/raven-go"
+)
+
+func TestSentryCrashReport(t *testing.T) {
+	c := NewClient(testClientConfig, "bridgetest")
+	if err := c.ReportSentryCrash(errors.New("Testing crash report - api proxy; goroutines with threads, find origin")); err != nil {
+		t.Fatal("Expected no error while report, but have", err)
+	}
+}
+
+func (s *SentryThreads) TraceAllRoutinesTest() {
+	s.Values = []Thread{
+		{
+			ID:      0,
+			Name:    "goroutine 20 [running]",
+			Crashed: true,
+			Stacktrace: &raven.Stacktrace{
+				Frames: []*raven.StacktraceFrame{
+					{
+						Filename: "/home/dev/build/go-1.10.2/go/src/runtime/pprof/pprof.go",
+						Function: "runtime/pprof.writeGoroutineStacks(0x9b7de0, 0xc4203e2900, 0xd0, 0xd0)",
+						Lineno:   650,
+					},
+				},
+			},
+		},
+		{
+			ID:      1,
+			Name:    "goroutine 20 [chan receive]",
+			Crashed: false,
+			Stacktrace: &raven.Stacktrace{
+				Frames: []*raven.StacktraceFrame{
+					{
+						Filename: "/home/dev/build/go-1.10.2/go/src/testing/testing.go",
+						Function: "testing.(*T).Run(0xc4203e42d0, 0x90f445, 0x15, 0x97d358, 0x47a501)",
+						Lineno:   825,
+					},
+				},
+			},
+		},
+	}
+}

pkg/pmapi/server_test.go

@@ -0,0 +1,166 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"reflect"
+	"regexp"
+	"runtime"
+	"strconv"
+	"testing"
+
+	"github.com/hashicorp/go-multierror"
+)
+
+var (
+	colRed     = "\033[1;31m"
+	colNon     = "\033[0;39m"
+	reHTTPCode = regexp.MustCompile(`(HTTP|get|post|put|delete)_(\d{3}).*.json`)
+)
+
+// Assert fails the test if the condition is false.
+func Assert(tb testing.TB, condition bool, msg string, v ...interface{}) {
+	if !condition {
+		_, file, line, _ := runtime.Caller(1)
+		vv := []interface{}{filepath.Base(file), line, colRed}
+		vv = append(vv, v...)
+		vv = append(vv, colNon)
+		fmt.Printf("%s:%d: %s"+msg+"%s\n\n", vv...)
+		tb.FailNow()
+	}
+}
+
+// Ok fails the test if an err is not nil.
+func Ok(tb testing.TB, err error) {
+	if err != nil {
+		_, file, line, _ := runtime.Caller(1)
+		fmt.Printf("%s:%d: %sunexpected error: %s%s\n\n", filepath.Base(file), line, colRed, err.Error(), colNon)
+		tb.FailNow()
+	}
+}
+
+// Equals fails the test if exp is not equal to act.
+func Equals(tb testing.TB, exp, act interface{}) {
+	if !reflect.DeepEqual(exp, act) {
+		_, file, line, _ := runtime.Caller(1)
+		fmt.Printf("%s:%d:\n\n%s\texp: %#v\n\n\tgot: %#v%s\n\n", filepath.Base(file), line, colRed, exp, act, colNon)
+		tb.FailNow()
+	}
+}
+
+// newTestServer is old function and should be replaced everywhere by newTestServerCallbacks.
+func newTestServer(h http.Handler) (*httptest.Server, *Client) {
+	s := httptest.NewServer(h)
+	RootURL = s.URL
+
+	return s, newTestClient()
+}
+
+func newTestServerCallbacks(tb testing.TB, callbacks ...func(testing.TB, http.ResponseWriter, *http.Request) string) (func(), *Client) {
+	reqNum := 0
+	_, file, line, _ := runtime.Caller(1)
+	file = filepath.Base(file)
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		reqNum++
+		if reqNum > len(callbacks) {
+			fmt.Printf(
+				"%s:%d: %sServer was requeted %d times which is more requests than expected %d%s\n\n",
+				file, line, colRed, reqNum, len(callbacks), colNon,
+			)
+			tb.FailNow()
+		}
+		response := callbacks[reqNum-1](tb, w, r)
+		if response != "" {
+			writeJSONResponsefromFile(tb, w, response, reqNum-1)
+		}
+	}))
+	RootURL = server.URL
+	finish := func() {
+		server.CloseClientConnections() // Closing without waiting for finishing requests.
+		if reqNum != len(callbacks) {
+			fmt.Printf(
+				"%s:%d: %sServer was requested %d times but expected to be %d times%s\n\n",
+				file, line, colRed, reqNum, len(callbacks), colNon,
+			)
+			tb.Error("server failed")
+		}
+	}
+	return finish, newTestClient()
+}
+
+func checkMethodAndPath(r *http.Request, method, path string) error {
+	var result *multierror.Error
+	if err := checkHeader(r.Header, "x-pm-appversion", "GoPMAPI_1.0.14"); err != nil {
+		result = multierror.Append(result, err)
+	}
+	if err := checkHeader(r.Header, "x-pm-apiversion", "3"); err != nil {
+		result = multierror.Append(result, err)
+	}
+	if r.Method != method {
+		err := fmt.Errorf("Invalid request method expected %v, got %v", method, r.Method)
+		result = multierror.Append(result, err)
+	}
+	if r.URL.RequestURI() != path {
+		err := fmt.Errorf("Invalid request path expected %v, got %v", path, r.URL.RequestURI())
+		result = multierror.Append(result, err)
+	}
+	return result.ErrorOrNil()
+}
+
+func httpResponse(code int) string {
+	return fmt.Sprintf("HTTP_%d.json", code)
+}
+
+func writeJSONResponsefromFile(tb testing.TB, w http.ResponseWriter, response string, reqNum int) {
+	if match := reHTTPCode.FindAllSubmatch([]byte(response), -1); len(match) != 0 {
+		httpCode, err := strconv.Atoi(string(match[0][len(match[0])-1]))
+		Ok(tb, err)
+		w.WriteHeader(httpCode)
+	}
+	f, err := os.Open("./testdata/routes/" + response)
+	Ok(tb, err)
+	w.Header().Set("content-type", "application/json;charset=utf-8")
+	w.Header().Set("x-test-pmapi-response", fmt.Sprintf("%s:%d", tb.Name(), reqNum))
+	_, err = io.Copy(w, f)
+	Ok(tb, err)
+}
+
+func checkHeader(h http.Header, field, exp string) error {
+	val := h.Get(field)
+	if val != exp {
+		msg := "wrong field %s expected %q but have %q"
+		return fmt.Errorf(msg, field, exp, val)
+	}
+	return nil
+}
+
+func isAuthReq(r *http.Request, uid, token string) error { // nolint[unparam]
+	if err := checkHeader(r.Header, "x-pm-uid", uid); err != nil {
+		return err
+	}
+	if err := checkHeader(r.Header, "authorization", "Bearer "+token); err != nil {
+		return err
+	}
+	return nil
+}

pkg/pmapi/settings.go

@@ -0,0 +1,118 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package pmapi
+
+type UserSettings struct {
+	PasswordMode int
+	Email        struct {
+		Value  string
+		Status int
+		Notify int
+		Reset  int
+	}
+	Phone struct {
+		Value  string
+		Status int
+		Notify int
+		Reset  int
+	}
+	News        int
+	Locale      string
+	LogAuth     string
+	InvoiceText string
+	TOTP        int
+	U2FKeys     []struct {
+		Label       string
+		KeyHandle   string
+		Compromised int
+	}
+}
+
+// GetUserSettings gets general settings.
+func (c *Client) GetUserSettings() (settings UserSettings, err error) {
+	req, err := NewRequest("GET", "/settings", nil)
+
+	if err != nil {
+		return
+	}
+
+	var res struct {
+		Res
+		UserSettings UserSettings
+	}
+
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+
+	return res.UserSettings, res.Err()
+}
+
+type MailSettings struct {
+	DisplayName        string
+	Signature          string `json:",omitempty"`
+	Theme              string `json:",omitempty"`
+	AutoSaveContacts   int
+	AutoWildcardSearch int
+	ComposerMode       int
+	MessageButtons     int
+	ShowImages         int
+	ShowMoved          int
+	ViewMode           int
+	ViewLayout         int
+	SwipeLeft          int
+	SwipeRight         int
+	AlsoArchive        int
+	Hotkeys            int
+	PMSignature        int
+	ImageProxy         int
+	TLS                int
+	RightToLeft        int
+	AttachPublicKey    int
+	Sign               int
+	PGPScheme          int
+	PromptPin          int
+	Autocrypt          int
+	NumMessagePerPage  int
+	DraftMIMEType      string
+	ReceiveMIMEType    string
+	ShowMIMEType       string
+
+	// Undocumented -- there's only `null` in example:
+	// AutoResponder string
+}
+
+// GetMailSettings gets contact details specified by contact ID.
+func (c *Client) GetMailSettings() (settings MailSettings, err error) {
+	req, err := NewRequest("GET", "/settings/mail", nil)
+
+	if err != nil {
+		return
+	}
+
+	var res struct {
+		Res
+		MailSettings MailSettings
+	}
+
+	if err = c.DoJSON(req, &res); err != nil {
+		return
+	}
+
+	return res.MailSettings, res.Err()
+}

pkg/pmapi/testdata/keyring_addressKeysPrimaryHasToken_JSON

@@ -0,0 +1,22 @@
+[
+  {
+	  "ID": "hKRtZeTDhvzfAaycb5BOVx6Y3hc3gs4QvET8H_YZBTwAQBPp3h6FI4nnkJePCYuM9CG0zf7TQzOJeB2rPi0YmQ==",
+	  "Primary": 1,
+	  "Flags": 3,
+	  "Version": 3,
+	  "Activation": null,
+	  "PrivateKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\nVersion: ProtonMail\nComment: https://protonmail.com\n\nxcMGBFzGzj4BCADh9uuhG3mDhlbFMgQpvP6AReyJYa6KSV9a6AJsNI0/DkVz\nFenIodav/r4RBgSoblqxCa/QH6galbX0eB8Vtxx4xTXCcHHZzTM9oAb4s8r6\ndGchNsSkoLspVwmc2fYfDRCTcapmeza7Jym8gD9wFUSWSwpC49jDeT6VIcOd\n+qNXgIJX1V36BOAB53lKenMYnxfzzAyJDeL/sQ/SWE9h/X+wGdkVmk7gk3pa\nQ9URoxAf7MO808hHyQYcjTV4tOHD/9k+AZC6sqmr2KAefYCxCMoCDj1wQp3J\nN6MfK2Zvc1r5KFQM2YzUXrMNrycAjo7gGeZnsvz758Q+8ouSP7iXGRl/ABEB\nAAH+CQMIQt3QQIJZYjRgTVgIQNSEkhMn4GJ5lQQPBXD0/YyK3xP0bm/bsX8S\nz9dAm1nkP0l8Q9z3lR6OcAX2O/KsqJdQGmJEZAaadz2je+EEWo1FDYaL76E6\n6z+AuteP8UtP61jBt3pTfIlhW8o7o2SlM1lvytpvc5FQplZ/iUuwfsIltCiD\n9VDkprNOVrRsjoh8BV4TrrIvU8qga58Aeg7SjOmm+3oMZ7yPTYirr11Tx/m3\nj1uzdEDfiPk4LmvlzSsWwKZuy6fSul+n92+9qN81wmdts/I2ucuKvOINZim4\nlk/p2AOsPjWpAgkefpTVLZnu2IH+VAyaXt1Fl84badXx4N921nPs7ova1Ud5\n2RddBc7b/01DtOyBSWDoNskLGpsc2mqz9kdkwwQKNjChzZc9nmY9M+AIfgT3\n+2DSQIuoJYPX69DKi/bZDwRzoHmiwiHT6Us7qxd6kD1dzCIHTptxwZQp4Tow\nnN6lmtK4S6O2B47+ROn8s0N+EH9GR8F6mvOTayNLH5yicpR3M4Of0ClvFa0G\n+JUKBXIQXUvF03G3nTPU17nLibC81UmbK3zobfbrLfuU2gU+sY+OfE1E/+GO\nSFpZcrkoRqRr39CfLkLk+GjU7RCLNddb5LxgaurVZo5h0Y7Rr8VvOQMWjjl/\nvTAG7gU/HtXi24TijNC0fP6j9w43K4b6t1SZYn5us7RRlFlWGKMSPf5Q9j6/\nryo3xULUQv0lTCQPtfQQ5UWE5ZpQ4Kjt/k5+/YtfuOcMbrDU4qa+H+rrisBu\nko7f4Wn0iYjRwRIuWh1NfUM3rIbNhq7/wonasEFOeFdPwprzMaawx3rL0Pq5\nGs/4LONqG61c9rBekbkGf7Jlkuq/5yo5RBgPnKwvJKsHqf1evD6kHC3aOfeO\n30UoMwe7g763pOXZrsOpZfPzxmraJJSYzS0iYXJvbjIxLTRAc2FkZW1iZS5v\ncmciIDxhcm9uMjEtNEBzYWRlbWJlLm9yZz7CwHUEEAEIAB8FAlzGzj4GCwkH\nCAMCBBUICgIDFgIBAhkBAhsDAh4BAAoJEC3c5LbRFpy6lOgH/RKA4QaTnCi8\nc7HHhDZncqwBSUNhTjCAvoiireX9gGm9JugxaPxHVH4RzznY6R7/Ui8Ak7S8\n+k/xhHbsOGc2exyWwUN1X3WJY3jSX2HNqDU4qw6hSwBGReYbOWJeKGhJWild\nPS4V6u6manGWXxQmW1XET3B0P72VJVSX1kUbslPBAhKbW3JCnDmEdV/sU6Ds\nXdh543Yph1OpO2Fq7b/+YYUDAzmHf/+k4ijVcvqjrjUCJJwv+2J9woi4ToW6\n3BQVG5gpAYzCfgoJjlaigInhoFrBjP25Oe6/ssDTssGJrHXhtyc8e+b7nm19\nSHOpWGcUn2F1+tU+E4O8SLCLGJxefJvHwwYEXMbOPgEIAKEkpHRBhTWOIeYB\ngPXh/Ng77x3Bs6EKwTQM/BePYC2uS+15+nIpiYHhb/sQ9aEbQgqmyfbkbfIf\n9Qahx4N9RFyqWcmSjfk0Bmo9xOziRQm1tfYkbUwkeI6NIr2ENUWVf7tt+UKz\n5dFmvSKsyrdPEtt+Ken17JoihhJ/9saLMkLn5Y3HrSGVXIniX1cQuarXGX9S\nyt7jIPeGZW2suuxlnlB6Sa/rCkaqR3C3a8knxiH8CDwAm+E8a1d/UbQ0np0k\nqTVVrc2fmxPMgZxGWrwIsO9D1Fs7dgw5rac7ijHvPXeWrzbMd1+rX5mmLF4+\nH2PDOUKN0Uu8WdVilCIEOMoVfY0AEQEAAf4JAwiPkDWd2zmFhmBFMZgWR+X2\nhS4q8bIohwFb8MDzrHAvtI8LOaxC32j3/tUwJKDwLNDeiiOGYfHUOmqgzVRi\nMBdkoVNT3OyFLbw6k/70spe5OcecZsM+OAQybX77Kv3H/VW/40TtUh52Zvvk\nqCoqtG86C4R5kma0zM+yvNyprkCIRwYuJ+OkdmMvem3fRqU49GNzJChBDAmZ\nfGc7W7r2JFIo1k88bh+kaGlDBA9p8GnA7KNXAlBdq6owJNJA0j2z6Rmngpip\nFuUEjdW8Kcs2ben4BZOWGgYmOmD1CAHRYm/xp+Us1kJDTTFxW7P7BNp1kVOP\n9s9AcV/t37bCHLVB2IjAb/tkOxyAexMTS/lFJrpJ89MpYiUt70uB+SDwRls9\noud9CQYS+6OdddgFUjtnOkpcR1Y24v1eF13JwUXmnPggvt6Do3gS6lGdq0Nt\nqNDg8+yPyMA8365Q8IbJDuzm3vfc560Szc/Kx4+1zr57Uaw/qYqhZHSkjRtZ\nfTP0v0ZNwxyXhGF3J31neJ5KgIO+zpSWSP8RwtUVBb8Tsyn8e/DSe/8fAs3V\ntmyxSj9mPQLxW8JCQMpocExEJV7PnxjhB0d1TlYu/wRUl4sxpS4pYa/TDhb9\n37oUYtG28TufT7AZE+XcU8Y1Xl7DVi5jgadUAI5rY71G31JeBuIcGRlVLBnT\nSHnu1v2iz1xqZKQFynH6DUrySn8nP8NM9TaVEnSBlVCjHcyvrs8crx358YFV\n4TG44XQ3n5GLjPetKGD/ccMYOUZm2jkLNzY9l4YqKJq7xsm2c5VhwUEB51Pi\n8Ey+x7hO3EjWUgJrtI/3/hPoKlltPUvnhJjNrR8ivocvd2v+2U0BHQOwTUt3\njSDssFDI9hfOMPp4yy/GHM+p3USC+3TMS8HjFbm6b2bIWzqEv83AaauZqiE1\nfJfCwF8EGAEIAAkFAlzGzj4CGwwACgkQLdzkttEWnLqI8Af/Uimov/qMfFdi\nSp1qYTzR9V84ZkfxBAcr9nEyNGtMlvCwrs3EtZPBe7ou57Qt9aI8UjJfsiov\naHUC1Gt8266T1+GGj6RCPSm6Sp7cZxGtURZDWAPb5u02+VtrqrrXQgkCxQ8/\nzWfg4vLyl052x+3F9SGy7SvH0qi8bcHzVjcwK9VyoSdBq/vhkPHQ33wLQ9ND\nuQCM8fxW+VOqMDlUCHfdcaYRYU7GEm9C37ZijpOLZRuAm6ojjCGtrOrGQ+y0\ni3q7Zn1yDradMlMGG9GjDlqOCmYhZbuT4uS578GzPg0L8zk/1rFOF+YY6Mfq\nV0GB3IX8qBHjAfPmqN9JPxBIn3/6ag==\n=U1yD\n-----END PGP PRIVATE KEY BLOCK-----\n",
+	  "Token": "-----BEGIN PGP MESSAGE-----\nVersion: ProtonMail\nComment: https://protonmail.com\n\nwcBMAxW/uQhm1PKxAQf8D4BAoXqnYgoTr7jEiabP9NkeuD6Z88fZDgUzXte5\n6aYMmCtPr0Vef1eDl2x0S0q/YJgR+A5Icmjxk+jG9nMSnRPlAozJfTCXu/Oy\nfkI4CcClGQv7U1EzfknBVNEKmuPO8XlEkGZ5lW/TzWk8ZtQqJsVuqZlZ5qQE\n/p8FNYEZ5RHPhhKlotA+T6XKh23z9mcN/JNEsSNcx+guERbivtTwnd5JpxQJ\nMNWzUYh+K0W7LgxoAo2jevDd4CZ/0sAAViIl+QrShodseV9agbldGUUoIMzn\nHUuIV6VYi7X82eKQXDInrtPc9IHekbDDFoncLnsGGrEqD/8O/qOHHOPwNwU+\n+cHATAOUTDv/ccWq1QEIALl8SNOQBmCuanAceUTSwCkM1fC3Ddqa8ZmnMMyG\nnDWDI7XkFU31CO10lN20/kAWjdN2073B/NT+17cp8fCqbQ1pAFJHpabdqmbI\ntm3pCC5M6otTN+MhjgFYcBuxo0rq9qtuEzz5j4Ub9MIIJTurUHMEPMI462Dc\ndK/d8BvDkU7q67Lkp65vpe9e/pv0lMMrQjdohnTHNgbZhbI/Z5LU0ApD//Ye\ncSC26BRUMJITiGKb9pKGAi0/ig0jJfzykgEarzOsY/v0W7016AMka+NsHuNc\n7Qfyg2LApF5s9Z6aK+Uy851haUk+p+abjBdcWACziAmimVGjlRYm49ra4UJ7\nc6LSeAElM584e4Z3VnqajAJWbmWt2atgBQmcwEBBsdNAtzIagNydMbBBIELA\n5wGUbMXfQxrMOo/Mdac/5EMTHT0/kVQNIBHtpX5SknWvbc5DjIHoH3+BbL87\neaxhKW94hMubKNKT3dbm4PKHtsMiS3TGkFZ3GjvA0Cy/Kw==\n=b1N8\n-----END PGP MESSAGE-----\n",
+	  "Signature": "-----BEGIN PGP SIGNATURE-----\nVersion: ProtonMail\nComment: https://protonmail.com\n\nwsBcBAEBCAAGBQJcxtSkAAoJEGfweL9m9sQefz8IAJhpEOvG7d+PVgq3bEL+\nXzY8yTUKB1ZXEMbcR/uHUWSfu0F8zy4CNtSG5HUVjNxx1xzzJVJcxWB/ljO+\n/bJSFOFexTyNh8i/xU+CiBfm5RhAFTYF9xFwfD3LKp5gaalJAhWhArk1/Wuh\nWTDpFpk39uzBRKNwcnSgiJYPxOjAZxj+w/hhHPwmco5cUwMiMR5MNrfzKf+x\nwX3Cfs9fsiiCDzohBzbK0FFsMnJ8aXNVsDBjEA1KrB7sdyaf8FnaM0RsFRDb\njxtIgfcFBbyNJh4414Unt4AYTIrIWhK4OOXI3AfsJy8p6KRBKQUcUkKcDxab\nPTXOPZsZ+UgQ5MevyVnP1zfCwFwEAQEIAAYFAlzG1KQACgkQm1DMMBzxetx/\nPwf9F04uHtix0zDpP1IvG4VYlor4rjYTdfXqxxiFXHO6MZXJoigS1E71E8r4\nsqZ6PoQ5/xCj2A01KRhuF1Bon3mEZEwaIUuBqTV91sLsVWfoxgyPAYpr6gK/\n1W9JhVNNrVRMGox42LQUjyiq0ESrfWmqC8SuMfZMoUoBZycHicA50RbyOUnT\nLO57ArL3JIVmYtyosaXM3idzmNHmaXSkcGt4cvTVysJZQrneaxmikfm5CH2O\n1z2goLBNnzsbRionoV6gCukZOiM/d14yiyeYtsFJ7u/vodkI5y0M0sF8VnN/\njL9keP4ZpHiJ4MBd6tyIH0pLueDRlurFL3fcHsEzD5SrOA==\n=wGCk\n-----END PGP SIGNATURE-----\n"
+  },
+  {
+    "ID": "MhF8dxtN5Lz_GruskN3L9kTKWusZHBdhWaxc7w1tgze2qB6uM9AyC6mWRQg1B7WGZ_r-9gn-XC95-IkpNJr0jg==",
+    "Primary": 0,
+    "Flags": 3,
+    "Version": 3,
+    "Activation": null,
+    "PrivateKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\n\nlQPGBF2IcLUBCACl7vans5JKYAEuOZfKNDLZKmLO675DhLjbno8Pw4Q8O6xQETUZ\nTFsIB6eTigCqaZasCeHIx4r7CI4TY2uZvV3+JNaAHN0omHh3iwVXQiE3FU5Zc6TO\niVswOoWPbVYFTuItP3FQHuossMIralIzG0/AyDx0NJawwpSxxz9+C0UoYeZD1A95\nfKveRVhDW0MgO1peDHrpr2HiWv06KkIop39CD/MBnBtDpsarGd16wvX+r8F3MrNE\nD1/e6szzi7oR5H0hPs+HSWZexXhiGSWoEyMiklX/sG/9+LxnHqPhCj7lrGfzmL7j\nCFuHZIqpIO8wVggOQYx1mtmMu7T9PZm66WMbABEBAAH+BwMCYpZ0ni+oe4DxIByk\nseTvdCIjQ3ieiTVvg9FsIVrupmIxDK9EVsGw1/sFjBtc5OOP8N9Vd8IyPh6WjBMu\n5pI5dsYKCLy6LMDmYTYQzXgWlpq2cpH+yZX21fZ+0ijEu+TTSJss3emCULrtCS5M\nUTzt813cIPJu8bYPdQ7taVRqK0Bo+S8YTj2a+lxHQRkbRclueuTm7l5ypU/9X4Xf\n4zwUKnq0OYu3U8IkDEOA+4Zv4SZssFrPr8plUzMHUahVV7NZCXOG11ju1CC9CkF2\n3lpkCZw96TwCGQnT4iTPix1NEJZo6ihRbMlnLo/neS3TWgdCpK5Qr8x4B82ykX6J\ngUeNozW/YFOwFwah22TeSN2dbaGbzJOSqMXHVzK9mz/oTajY/51pS/ZxGoxgy7F2\nD8E/m0I7dS3fw/A/6ZtQFa1bW8L/dBnN71gfr6PSVxn3oczPCXtzNeqHPaDIDGV1\nhgGbGFgmjJy9mnPuxQcScIJloLOVerAaqMcgFuo2MhGKLzQ7IGiSzinaJZ7k602f\nJPfpqBg2aMO9xxNC0S6p6KzMRZCjM1cOJYF2jMo2UWqM0XA1+aEqPksBTmyKSd9+\n1sc8coVl3iesji6GZoT5s0+sNEiQZuNOFUzmGKdgqZhDVfPTLmfV1Uv/tlZCGKmC\ntC51VGjqf8RE3ycdn0Hby8Q2DLsaTnC2vMUde2iJwGQU41TTlt4TKSp6YtqSxbp+\nzUpF4QuS2zVARtx3Z87vw6rHmrNCXHzxl9zWjF3M42FhEESrUL0dciM3c1pB5twd\nIx1TSI0+96zneGS4AkN5Tqua+RdvWXX8ljRgLFa8DqraumQyOjyMXRlJoiey1dt3\n3sRfeF8eI2QF9DooI+4Za/C/1VaEURSQ4p04Ng/xJJ9mwpxjKvxmujC9n2yZ1u4o\nvQh9cKQ9nQAItDdqYW1lcy10ZXN0QHByb3Rvbm1haWwuYmx1ZSA8amFtZXMtdGVz\ndEBwcm90b25tYWlsLmJsdWU+iQE1BBABCAAfBQJdiHC1BgsJBwgDAgQVCAoCAxYC\nAQIZAQIbAwIeAQAKCRC/qkLbur13GVuWB/410LkZxZltVA1odI+yL3utIhoNWj8R\n3AhlVPLfAymkJOFv2KMZ2pRkS8PTmq4sYjAlti60cpWwrNdU4eSfNxoWDEg2lILW\nFULm3TwygDFogSsqyKlGXGr0MqnegImDeQg2kMLPvudTNvEbnRDpaCxWUHenEH5+\nTb8+UyG1e3V2su5J6p+Ghh88Y+zJyU4DWta3wAymLPW+gUEFXDAw5KdSg/8iD5kH\n6aMbd9l1Wp5WiErMsJOwoxhFVhwUK7MCXAXxHMWTuyu78RYYNhafPpd+Rq6dqy1d\nm5M7BlDU4Y2Y921xOSHGbNXbWbh5WHBT0QkV7IJunwiPYc5apJbee69DnQPGBF2I\ncLUBCAC9RsknEX9jPP4AB2/DxWdqTKb1/AmDpzNstEUtSy+zUKmErSNQeNUuso+n\nH2lFtb/A+s3nC2UPqbgz/SWN37Z9ip6hvsQX0etFvzYiXFfVDbknqmpi8AGt1Bkr\nFP9jGvF7gIT436icfjd8vQHHMIdEbEF+S46LfW8xibZEYUHLr5m8VO7dqqsIIwvr\nTJWEGjtRfaacqV3aWDzHbXodmYC3kQH9UjpdXE/9/5VxWtxDAFwsCaXiE9+EL69Z\ncJDdPfM0iVL3XgP22EFwC8C+Xha1PiSNOGFTIbv4aT5yLgPb7wwvgaw3semLF1vM\nzni3z+FNPREt+9itIhGPpxmIyAQ/ABEBAAH+BwMCl4RZZJs4k1bxqOmzniBxB2y8\nX4sDxyy60IOO3Ip2cOeyJ/qwEBdXxL2EjS6y5Vt3GTnQDz1IJqYiz9eRbZQnXUh5\nSk9ND9+mgxKYHDV52flh2FBTaOgqsKLnwLRUE12D1BQ9Ou5cYcgOIbCncftQTnWV\nk6hg9PDFZD/LD3IuKS9/DVYxKkYIJZ1KH446bQTYto+fnU0ajkLfED7wN4JquHq4\nYS+L7h8GetOCyRsh8zDIOPF5jZYCSzDe8Q03ktozOE2+YV/A3Z/0Uub8/Qecl5u3\n1qURXeIC6oGRDVXegZu3491IG+UTzjaBDPu+Xk4zad7mGjBQ9vNuKZfNNg/nHOWL\nNHRCXbkQVpXCA+v+dNYov+bqKe1QpF/2jvU0Bi32HLG8gfmRwD0WaAnbvOLTYXPl\ndRNDn7ixfOC0uHT6epPlZxsRja5TXlFl5qSuAX2zQfFdb+pRPztNKer6iMNz3itz\nh2B/gcTwqXloy4Zq9wqxcXLV1qmoMJ5YzRcd4otRM7FhZD0LxBAOANdmPSmRHbf6\nSXyXgvK4KDwRB52uIYJzWyIi7LiU3qTbV0GgptgHeVLwgbA9ioPwIu85+XQEh/eE\nEjNhz0CdtCqmYY3+SjNOZ8K59LgWCuv/YkRkmwDimEz5gUlpayoOYib4J4wIx6OY\n+fmI+gByS+UN3v+ml7l909c2W8w0dwR8+7P1XGBkaiVROTM2hdnHMflWm2548WBx\na6tbdCQPUDra/5O/pCdefc44hwD08liutq9kF29sMwONZy0P0dlyFZq4LTLmQqs0\nq93GZRjoI+uit9oTLGCtJ99aiSIx97KfkKx5wLnE4DruQlqPWnxV5Yfn9RZ6j8V2\nRQjL8f5RbiBtkkCl3r+nEdQIeMRy+8xxO9dTYTlaCNgfd92LdQrwzKe+GneUgKYg\nWo+AiQEfBBgBCAAJBQJdiHC1AhsMAAoJEL+qQtu6vXcZskIH+we2sHxPnWOfkXTu\npmZBU1ba8JLcrx+CMwTPkt7GMYpDplVOybn+AjMxB9JxST71mBw6rbwd2XLM1AmJ\nbvXVfmCtmC2TmrORqcoxEql+9tdgnSTfZfrltnYeVuEAvAxrMmD4S2Mb5zVWFl+q\ni69A0rdD8a36nOlJnZxfs57W1zfvl/rh+/RdybPwx9y0hSnRPSypis2dcwSyBD9+\nnlbn3QoybaUJxWvL+9MphCXZ4CuuhG0VPcmdH+LzOytDTtJnDNm+Ru4sokroDJXe\n5XlrK3+wtSNL4rfa/MQYyK8XsWRVR/a0BPeqZA4cPlHqCOSfUfaFbORVbPaQdq2a\nYYsMDlA=\n=ZwJf\n-----END PGP PRIVATE KEY BLOCK-----",
+    "Token": null,
+    "Signature": null
+  }
+]

pkg/pmapi/testdata/keyring_addressKeysSecondaryHasToken_JSON

@@ -0,0 +1,22 @@
+[
+  {
+    "ID": "MhF8dxtN5Lz_GruskN3L9kTKWusZHBdhWaxc7w1tgze2qB6uM9AyC6mWRQg1B7WGZ_r-9gn-XC95-IkpNJr0jg==",
+    "Primary": 1,
+    "Flags": 3,
+    "Version": 3,
+    "Activation": null,
+    "PrivateKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\n\nlQPGBF2IcLUBCACl7vans5JKYAEuOZfKNDLZKmLO675DhLjbno8Pw4Q8O6xQETUZ\nTFsIB6eTigCqaZasCeHIx4r7CI4TY2uZvV3+JNaAHN0omHh3iwVXQiE3FU5Zc6TO\niVswOoWPbVYFTuItP3FQHuossMIralIzG0/AyDx0NJawwpSxxz9+C0UoYeZD1A95\nfKveRVhDW0MgO1peDHrpr2HiWv06KkIop39CD/MBnBtDpsarGd16wvX+r8F3MrNE\nD1/e6szzi7oR5H0hPs+HSWZexXhiGSWoEyMiklX/sG/9+LxnHqPhCj7lrGfzmL7j\nCFuHZIqpIO8wVggOQYx1mtmMu7T9PZm66WMbABEBAAH+BwMCYpZ0ni+oe4DxIByk\nseTvdCIjQ3ieiTVvg9FsIVrupmIxDK9EVsGw1/sFjBtc5OOP8N9Vd8IyPh6WjBMu\n5pI5dsYKCLy6LMDmYTYQzXgWlpq2cpH+yZX21fZ+0ijEu+TTSJss3emCULrtCS5M\nUTzt813cIPJu8bYPdQ7taVRqK0Bo+S8YTj2a+lxHQRkbRclueuTm7l5ypU/9X4Xf\n4zwUKnq0OYu3U8IkDEOA+4Zv4SZssFrPr8plUzMHUahVV7NZCXOG11ju1CC9CkF2\n3lpkCZw96TwCGQnT4iTPix1NEJZo6ihRbMlnLo/neS3TWgdCpK5Qr8x4B82ykX6J\ngUeNozW/YFOwFwah22TeSN2dbaGbzJOSqMXHVzK9mz/oTajY/51pS/ZxGoxgy7F2\nD8E/m0I7dS3fw/A/6ZtQFa1bW8L/dBnN71gfr6PSVxn3oczPCXtzNeqHPaDIDGV1\nhgGbGFgmjJy9mnPuxQcScIJloLOVerAaqMcgFuo2MhGKLzQ7IGiSzinaJZ7k602f\nJPfpqBg2aMO9xxNC0S6p6KzMRZCjM1cOJYF2jMo2UWqM0XA1+aEqPksBTmyKSd9+\n1sc8coVl3iesji6GZoT5s0+sNEiQZuNOFUzmGKdgqZhDVfPTLmfV1Uv/tlZCGKmC\ntC51VGjqf8RE3ycdn0Hby8Q2DLsaTnC2vMUde2iJwGQU41TTlt4TKSp6YtqSxbp+\nzUpF4QuS2zVARtx3Z87vw6rHmrNCXHzxl9zWjF3M42FhEESrUL0dciM3c1pB5twd\nIx1TSI0+96zneGS4AkN5Tqua+RdvWXX8ljRgLFa8DqraumQyOjyMXRlJoiey1dt3\n3sRfeF8eI2QF9DooI+4Za/C/1VaEURSQ4p04Ng/xJJ9mwpxjKvxmujC9n2yZ1u4o\nvQh9cKQ9nQAItDdqYW1lcy10ZXN0QHByb3Rvbm1haWwuYmx1ZSA8amFtZXMtdGVz\ndEBwcm90b25tYWlsLmJsdWU+iQE1BBABCAAfBQJdiHC1BgsJBwgDAgQVCAoCAxYC\nAQIZAQIbAwIeAQAKCRC/qkLbur13GVuWB/410LkZxZltVA1odI+yL3utIhoNWj8R\n3AhlVPLfAymkJOFv2KMZ2pRkS8PTmq4sYjAlti60cpWwrNdU4eSfNxoWDEg2lILW\nFULm3TwygDFogSsqyKlGXGr0MqnegImDeQg2kMLPvudTNvEbnRDpaCxWUHenEH5+\nTb8+UyG1e3V2su5J6p+Ghh88Y+zJyU4DWta3wAymLPW+gUEFXDAw5KdSg/8iD5kH\n6aMbd9l1Wp5WiErMsJOwoxhFVhwUK7MCXAXxHMWTuyu78RYYNhafPpd+Rq6dqy1d\nm5M7BlDU4Y2Y921xOSHGbNXbWbh5WHBT0QkV7IJunwiPYc5apJbee69DnQPGBF2I\ncLUBCAC9RsknEX9jPP4AB2/DxWdqTKb1/AmDpzNstEUtSy+zUKmErSNQeNUuso+n\nH2lFtb/A+s3nC2UPqbgz/SWN37Z9ip6hvsQX0etFvzYiXFfVDbknqmpi8AGt1Bkr\nFP9jGvF7gIT436icfjd8vQHHMIdEbEF+S46LfW8xibZEYUHLr5m8VO7dqqsIIwvr\nTJWEGjtRfaacqV3aWDzHbXodmYC3kQH9UjpdXE/9/5VxWtxDAFwsCaXiE9+EL69Z\ncJDdPfM0iVL3XgP22EFwC8C+Xha1PiSNOGFTIbv4aT5yLgPb7wwvgaw3semLF1vM\nzni3z+FNPREt+9itIhGPpxmIyAQ/ABEBAAH+BwMCl4RZZJs4k1bxqOmzniBxB2y8\nX4sDxyy60IOO3Ip2cOeyJ/qwEBdXxL2EjS6y5Vt3GTnQDz1IJqYiz9eRbZQnXUh5\nSk9ND9+mgxKYHDV52flh2FBTaOgqsKLnwLRUE12D1BQ9Ou5cYcgOIbCncftQTnWV\nk6hg9PDFZD/LD3IuKS9/DVYxKkYIJZ1KH446bQTYto+fnU0ajkLfED7wN4JquHq4\nYS+L7h8GetOCyRsh8zDIOPF5jZYCSzDe8Q03ktozOE2+YV/A3Z/0Uub8/Qecl5u3\n1qURXeIC6oGRDVXegZu3491IG+UTzjaBDPu+Xk4zad7mGjBQ9vNuKZfNNg/nHOWL\nNHRCXbkQVpXCA+v+dNYov+bqKe1QpF/2jvU0Bi32HLG8gfmRwD0WaAnbvOLTYXPl\ndRNDn7ixfOC0uHT6epPlZxsRja5TXlFl5qSuAX2zQfFdb+pRPztNKer6iMNz3itz\nh2B/gcTwqXloy4Zq9wqxcXLV1qmoMJ5YzRcd4otRM7FhZD0LxBAOANdmPSmRHbf6\nSXyXgvK4KDwRB52uIYJzWyIi7LiU3qTbV0GgptgHeVLwgbA9ioPwIu85+XQEh/eE\nEjNhz0CdtCqmYY3+SjNOZ8K59LgWCuv/YkRkmwDimEz5gUlpayoOYib4J4wIx6OY\n+fmI+gByS+UN3v+ml7l909c2W8w0dwR8+7P1XGBkaiVROTM2hdnHMflWm2548WBx\na6tbdCQPUDra/5O/pCdefc44hwD08liutq9kF29sMwONZy0P0dlyFZq4LTLmQqs0\nq93GZRjoI+uit9oTLGCtJ99aiSIx97KfkKx5wLnE4DruQlqPWnxV5Yfn9RZ6j8V2\nRQjL8f5RbiBtkkCl3r+nEdQIeMRy+8xxO9dTYTlaCNgfd92LdQrwzKe+GneUgKYg\nWo+AiQEfBBgBCAAJBQJdiHC1AhsMAAoJEL+qQtu6vXcZskIH+we2sHxPnWOfkXTu\npmZBU1ba8JLcrx+CMwTPkt7GMYpDplVOybn+AjMxB9JxST71mBw6rbwd2XLM1AmJ\nbvXVfmCtmC2TmrORqcoxEql+9tdgnSTfZfrltnYeVuEAvAxrMmD4S2Mb5zVWFl+q\ni69A0rdD8a36nOlJnZxfs57W1zfvl/rh+/RdybPwx9y0hSnRPSypis2dcwSyBD9+\nnlbn3QoybaUJxWvL+9MphCXZ4CuuhG0VPcmdH+LzOytDTtJnDNm+Ru4sokroDJXe\n5XlrK3+wtSNL4rfa/MQYyK8XsWRVR/a0BPeqZA4cPlHqCOSfUfaFbORVbPaQdq2a\nYYsMDlA=\n=ZwJf\n-----END PGP PRIVATE KEY BLOCK-----",
+    "Token": null,
+    "Signature": null
+  },
+  {
+	  "ID": "hKRtZeTDhvzfAaycb5BOVx6Y3hc3gs4QvET8H_YZBTwAQBPp3h6FI4nnkJePCYuM9CG0zf7TQzOJeB2rPi0YmQ==",
+	  "Primary": 0,
+	  "Flags": 3,
+	  "Version": 3,
+	  "Activation": null,
+	  "PrivateKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\nVersion: ProtonMail\nComment: https://protonmail.com\n\nxcMGBFzGzj4BCADh9uuhG3mDhlbFMgQpvP6AReyJYa6KSV9a6AJsNI0/DkVz\nFenIodav/r4RBgSoblqxCa/QH6galbX0eB8Vtxx4xTXCcHHZzTM9oAb4s8r6\ndGchNsSkoLspVwmc2fYfDRCTcapmeza7Jym8gD9wFUSWSwpC49jDeT6VIcOd\n+qNXgIJX1V36BOAB53lKenMYnxfzzAyJDeL/sQ/SWE9h/X+wGdkVmk7gk3pa\nQ9URoxAf7MO808hHyQYcjTV4tOHD/9k+AZC6sqmr2KAefYCxCMoCDj1wQp3J\nN6MfK2Zvc1r5KFQM2YzUXrMNrycAjo7gGeZnsvz758Q+8ouSP7iXGRl/ABEB\nAAH+CQMIQt3QQIJZYjRgTVgIQNSEkhMn4GJ5lQQPBXD0/YyK3xP0bm/bsX8S\nz9dAm1nkP0l8Q9z3lR6OcAX2O/KsqJdQGmJEZAaadz2je+EEWo1FDYaL76E6\n6z+AuteP8UtP61jBt3pTfIlhW8o7o2SlM1lvytpvc5FQplZ/iUuwfsIltCiD\n9VDkprNOVrRsjoh8BV4TrrIvU8qga58Aeg7SjOmm+3oMZ7yPTYirr11Tx/m3\nj1uzdEDfiPk4LmvlzSsWwKZuy6fSul+n92+9qN81wmdts/I2ucuKvOINZim4\nlk/p2AOsPjWpAgkefpTVLZnu2IH+VAyaXt1Fl84badXx4N921nPs7ova1Ud5\n2RddBc7b/01DtOyBSWDoNskLGpsc2mqz9kdkwwQKNjChzZc9nmY9M+AIfgT3\n+2DSQIuoJYPX69DKi/bZDwRzoHmiwiHT6Us7qxd6kD1dzCIHTptxwZQp4Tow\nnN6lmtK4S6O2B47+ROn8s0N+EH9GR8F6mvOTayNLH5yicpR3M4Of0ClvFa0G\n+JUKBXIQXUvF03G3nTPU17nLibC81UmbK3zobfbrLfuU2gU+sY+OfE1E/+GO\nSFpZcrkoRqRr39CfLkLk+GjU7RCLNddb5LxgaurVZo5h0Y7Rr8VvOQMWjjl/\nvTAG7gU/HtXi24TijNC0fP6j9w43K4b6t1SZYn5us7RRlFlWGKMSPf5Q9j6/\nryo3xULUQv0lTCQPtfQQ5UWE5ZpQ4Kjt/k5+/YtfuOcMbrDU4qa+H+rrisBu\nko7f4Wn0iYjRwRIuWh1NfUM3rIbNhq7/wonasEFOeFdPwprzMaawx3rL0Pq5\nGs/4LONqG61c9rBekbkGf7Jlkuq/5yo5RBgPnKwvJKsHqf1evD6kHC3aOfeO\n30UoMwe7g763pOXZrsOpZfPzxmraJJSYzS0iYXJvbjIxLTRAc2FkZW1iZS5v\ncmciIDxhcm9uMjEtNEBzYWRlbWJlLm9yZz7CwHUEEAEIAB8FAlzGzj4GCwkH\nCAMCBBUICgIDFgIBAhkBAhsDAh4BAAoJEC3c5LbRFpy6lOgH/RKA4QaTnCi8\nc7HHhDZncqwBSUNhTjCAvoiireX9gGm9JugxaPxHVH4RzznY6R7/Ui8Ak7S8\n+k/xhHbsOGc2exyWwUN1X3WJY3jSX2HNqDU4qw6hSwBGReYbOWJeKGhJWild\nPS4V6u6manGWXxQmW1XET3B0P72VJVSX1kUbslPBAhKbW3JCnDmEdV/sU6Ds\nXdh543Yph1OpO2Fq7b/+YYUDAzmHf/+k4ijVcvqjrjUCJJwv+2J9woi4ToW6\n3BQVG5gpAYzCfgoJjlaigInhoFrBjP25Oe6/ssDTssGJrHXhtyc8e+b7nm19\nSHOpWGcUn2F1+tU+E4O8SLCLGJxefJvHwwYEXMbOPgEIAKEkpHRBhTWOIeYB\ngPXh/Ng77x3Bs6EKwTQM/BePYC2uS+15+nIpiYHhb/sQ9aEbQgqmyfbkbfIf\n9Qahx4N9RFyqWcmSjfk0Bmo9xOziRQm1tfYkbUwkeI6NIr2ENUWVf7tt+UKz\n5dFmvSKsyrdPEtt+Ken17JoihhJ/9saLMkLn5Y3HrSGVXIniX1cQuarXGX9S\nyt7jIPeGZW2suuxlnlB6Sa/rCkaqR3C3a8knxiH8CDwAm+E8a1d/UbQ0np0k\nqTVVrc2fmxPMgZxGWrwIsO9D1Fs7dgw5rac7ijHvPXeWrzbMd1+rX5mmLF4+\nH2PDOUKN0Uu8WdVilCIEOMoVfY0AEQEAAf4JAwiPkDWd2zmFhmBFMZgWR+X2\nhS4q8bIohwFb8MDzrHAvtI8LOaxC32j3/tUwJKDwLNDeiiOGYfHUOmqgzVRi\nMBdkoVNT3OyFLbw6k/70spe5OcecZsM+OAQybX77Kv3H/VW/40TtUh52Zvvk\nqCoqtG86C4R5kma0zM+yvNyprkCIRwYuJ+OkdmMvem3fRqU49GNzJChBDAmZ\nfGc7W7r2JFIo1k88bh+kaGlDBA9p8GnA7KNXAlBdq6owJNJA0j2z6Rmngpip\nFuUEjdW8Kcs2ben4BZOWGgYmOmD1CAHRYm/xp+Us1kJDTTFxW7P7BNp1kVOP\n9s9AcV/t37bCHLVB2IjAb/tkOxyAexMTS/lFJrpJ89MpYiUt70uB+SDwRls9\noud9CQYS+6OdddgFUjtnOkpcR1Y24v1eF13JwUXmnPggvt6Do3gS6lGdq0Nt\nqNDg8+yPyMA8365Q8IbJDuzm3vfc560Szc/Kx4+1zr57Uaw/qYqhZHSkjRtZ\nfTP0v0ZNwxyXhGF3J31neJ5KgIO+zpSWSP8RwtUVBb8Tsyn8e/DSe/8fAs3V\ntmyxSj9mPQLxW8JCQMpocExEJV7PnxjhB0d1TlYu/wRUl4sxpS4pYa/TDhb9\n37oUYtG28TufT7AZE+XcU8Y1Xl7DVi5jgadUAI5rY71G31JeBuIcGRlVLBnT\nSHnu1v2iz1xqZKQFynH6DUrySn8nP8NM9TaVEnSBlVCjHcyvrs8crx358YFV\n4TG44XQ3n5GLjPetKGD/ccMYOUZm2jkLNzY9l4YqKJq7xsm2c5VhwUEB51Pi\n8Ey+x7hO3EjWUgJrtI/3/hPoKlltPUvnhJjNrR8ivocvd2v+2U0BHQOwTUt3\njSDssFDI9hfOMPp4yy/GHM+p3USC+3TMS8HjFbm6b2bIWzqEv83AaauZqiE1\nfJfCwF8EGAEIAAkFAlzGzj4CGwwACgkQLdzkttEWnLqI8Af/Uimov/qMfFdi\nSp1qYTzR9V84ZkfxBAcr9nEyNGtMlvCwrs3EtZPBe7ou57Qt9aI8UjJfsiov\naHUC1Gt8266T1+GGj6RCPSm6Sp7cZxGtURZDWAPb5u02+VtrqrrXQgkCxQ8/\nzWfg4vLyl052x+3F9SGy7SvH0qi8bcHzVjcwK9VyoSdBq/vhkPHQ33wLQ9ND\nuQCM8fxW+VOqMDlUCHfdcaYRYU7GEm9C37ZijpOLZRuAm6ojjCGtrOrGQ+y0\ni3q7Zn1yDradMlMGG9GjDlqOCmYhZbuT4uS578GzPg0L8zk/1rFOF+YY6Mfq\nV0GB3IX8qBHjAfPmqN9JPxBIn3/6ag==\n=U1yD\n-----END PGP PRIVATE KEY BLOCK-----\n",
+	  "Token": "-----BEGIN PGP MESSAGE-----\nVersion: ProtonMail\nComment: https://protonmail.com\n\nwcBMAxW/uQhm1PKxAQf8D4BAoXqnYgoTr7jEiabP9NkeuD6Z88fZDgUzXte5\n6aYMmCtPr0Vef1eDl2x0S0q/YJgR+A5Icmjxk+jG9nMSnRPlAozJfTCXu/Oy\nfkI4CcClGQv7U1EzfknBVNEKmuPO8XlEkGZ5lW/TzWk8ZtQqJsVuqZlZ5qQE\n/p8FNYEZ5RHPhhKlotA+T6XKh23z9mcN/JNEsSNcx+guERbivtTwnd5JpxQJ\nMNWzUYh+K0W7LgxoAo2jevDd4CZ/0sAAViIl+QrShodseV9agbldGUUoIMzn\nHUuIV6VYi7X82eKQXDInrtPc9IHekbDDFoncLnsGGrEqD/8O/qOHHOPwNwU+\n+cHATAOUTDv/ccWq1QEIALl8SNOQBmCuanAceUTSwCkM1fC3Ddqa8ZmnMMyG\nnDWDI7XkFU31CO10lN20/kAWjdN2073B/NT+17cp8fCqbQ1pAFJHpabdqmbI\ntm3pCC5M6otTN+MhjgFYcBuxo0rq9qtuEzz5j4Ub9MIIJTurUHMEPMI462Dc\ndK/d8BvDkU7q67Lkp65vpe9e/pv0lMMrQjdohnTHNgbZhbI/Z5LU0ApD//Ye\ncSC26BRUMJITiGKb9pKGAi0/ig0jJfzykgEarzOsY/v0W7016AMka+NsHuNc\n7Qfyg2LApF5s9Z6aK+Uy851haUk+p+abjBdcWACziAmimVGjlRYm49ra4UJ7\nc6LSeAElM584e4Z3VnqajAJWbmWt2atgBQmcwEBBsdNAtzIagNydMbBBIELA\n5wGUbMXfQxrMOo/Mdac/5EMTHT0/kVQNIBHtpX5SknWvbc5DjIHoH3+BbL87\neaxhKW94hMubKNKT3dbm4PKHtsMiS3TGkFZ3GjvA0Cy/Kw==\n=b1N8\n-----END PGP MESSAGE-----\n",
+	  "Signature": "-----BEGIN PGP SIGNATURE-----\nVersion: ProtonMail\nComment: https://protonmail.com\n\nwsBcBAEBCAAGBQJcxtSkAAoJEGfweL9m9sQefz8IAJhpEOvG7d+PVgq3bEL+\nXzY8yTUKB1ZXEMbcR/uHUWSfu0F8zy4CNtSG5HUVjNxx1xzzJVJcxWB/ljO+\n/bJSFOFexTyNh8i/xU+CiBfm5RhAFTYF9xFwfD3LKp5gaalJAhWhArk1/Wuh\nWTDpFpk39uzBRKNwcnSgiJYPxOjAZxj+w/hhHPwmco5cUwMiMR5MNrfzKf+x\nwX3Cfs9fsiiCDzohBzbK0FFsMnJ8aXNVsDBjEA1KrB7sdyaf8FnaM0RsFRDb\njxtIgfcFBbyNJh4414Unt4AYTIrIWhK4OOXI3AfsJy8p6KRBKQUcUkKcDxab\nPTXOPZsZ+UgQ5MevyVnP1zfCwFwEAQEIAAYFAlzG1KQACgkQm1DMMBzxetx/\nPwf9F04uHtix0zDpP1IvG4VYlor4rjYTdfXqxxiFXHO6MZXJoigS1E71E8r4\nsqZ6PoQ5/xCj2A01KRhuF1Bon3mEZEwaIUuBqTV91sLsVWfoxgyPAYpr6gK/\n1W9JhVNNrVRMGox42LQUjyiq0ESrfWmqC8SuMfZMoUoBZycHicA50RbyOUnT\nLO57ArL3JIVmYtyosaXM3idzmNHmaXSkcGt4cvTVysJZQrneaxmikfm5CH2O\n1z2goLBNnzsbRionoV6gCukZOiM/d14yiyeYtsFJ7u/vodkI5y0M0sF8VnN/\njL9keP4ZpHiJ4MBd6tyIH0pLueDRlurFL3fcHsEzD5SrOA==\n=wGCk\n-----END PGP SIGNATURE-----\n"
+  }
+]

pkg/pmapi/testdata/keyring_addressKeysWithTokens_JSON

@@ -0,0 +1,12 @@
+[
+  {
+    "ID": "hKRtZeTDhvzfAaycb5BOVx6Y3hc3gs4QvET8H_YZBTwAQBPp3h6FI4nnkJePCYuM9CG0zf7TQzOJeB2rPi0YmQ==",
+    "Primary": 1,
+    "Flags": 3,
+    "Version": 3,
+    "Activation": null,
+    "PrivateKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\nVersion: ProtonMail\nComment: https://protonmail.com\n\nxcMGBFzGzj4BCADh9uuhG3mDhlbFMgQpvP6AReyJYa6KSV9a6AJsNI0/DkVz\nFenIodav/r4RBgSoblqxCa/QH6galbX0eB8Vtxx4xTXCcHHZzTM9oAb4s8r6\ndGchNsSkoLspVwmc2fYfDRCTcapmeza7Jym8gD9wFUSWSwpC49jDeT6VIcOd\n+qNXgIJX1V36BOAB53lKenMYnxfzzAyJDeL/sQ/SWE9h/X+wGdkVmk7gk3pa\nQ9URoxAf7MO808hHyQYcjTV4tOHD/9k+AZC6sqmr2KAefYCxCMoCDj1wQp3J\nN6MfK2Zvc1r5KFQM2YzUXrMNrycAjo7gGeZnsvz758Q+8ouSP7iXGRl/ABEB\nAAH+CQMIQt3QQIJZYjRgTVgIQNSEkhMn4GJ5lQQPBXD0/YyK3xP0bm/bsX8S\nz9dAm1nkP0l8Q9z3lR6OcAX2O/KsqJdQGmJEZAaadz2je+EEWo1FDYaL76E6\n6z+AuteP8UtP61jBt3pTfIlhW8o7o2SlM1lvytpvc5FQplZ/iUuwfsIltCiD\n9VDkprNOVrRsjoh8BV4TrrIvU8qga58Aeg7SjOmm+3oMZ7yPTYirr11Tx/m3\nj1uzdEDfiPk4LmvlzSsWwKZuy6fSul+n92+9qN81wmdts/I2ucuKvOINZim4\nlk/p2AOsPjWpAgkefpTVLZnu2IH+VAyaXt1Fl84badXx4N921nPs7ova1Ud5\n2RddBc7b/01DtOyBSWDoNskLGpsc2mqz9kdkwwQKNjChzZc9nmY9M+AIfgT3\n+2DSQIuoJYPX69DKi/bZDwRzoHmiwiHT6Us7qxd6kD1dzCIHTptxwZQp4Tow\nnN6lmtK4S6O2B47+ROn8s0N+EH9GR8F6mvOTayNLH5yicpR3M4Of0ClvFa0G\n+JUKBXIQXUvF03G3nTPU17nLibC81UmbK3zobfbrLfuU2gU+sY+OfE1E/+GO\nSFpZcrkoRqRr39CfLkLk+GjU7RCLNddb5LxgaurVZo5h0Y7Rr8VvOQMWjjl/\nvTAG7gU/HtXi24TijNC0fP6j9w43K4b6t1SZYn5us7RRlFlWGKMSPf5Q9j6/\nryo3xULUQv0lTCQPtfQQ5UWE5ZpQ4Kjt/k5+/YtfuOcMbrDU4qa+H+rrisBu\nko7f4Wn0iYjRwRIuWh1NfUM3rIbNhq7/wonasEFOeFdPwprzMaawx3rL0Pq5\nGs/4LONqG61c9rBekbkGf7Jlkuq/5yo5RBgPnKwvJKsHqf1evD6kHC3aOfeO\n30UoMwe7g763pOXZrsOpZfPzxmraJJSYzS0iYXJvbjIxLTRAc2FkZW1iZS5v\ncmciIDxhcm9uMjEtNEBzYWRlbWJlLm9yZz7CwHUEEAEIAB8FAlzGzj4GCwkH\nCAMCBBUICgIDFgIBAhkBAhsDAh4BAAoJEC3c5LbRFpy6lOgH/RKA4QaTnCi8\nc7HHhDZncqwBSUNhTjCAvoiireX9gGm9JugxaPxHVH4RzznY6R7/Ui8Ak7S8\n+k/xhHbsOGc2exyWwUN1X3WJY3jSX2HNqDU4qw6hSwBGReYbOWJeKGhJWild\nPS4V6u6manGWXxQmW1XET3B0P72VJVSX1kUbslPBAhKbW3JCnDmEdV/sU6Ds\nXdh543Yph1OpO2Fq7b/+YYUDAzmHf/+k4ijVcvqjrjUCJJwv+2J9woi4ToW6\n3BQVG5gpAYzCfgoJjlaigInhoFrBjP25Oe6/ssDTssGJrHXhtyc8e+b7nm19\nSHOpWGcUn2F1+tU+E4O8SLCLGJxefJvHwwYEXMbOPgEIAKEkpHRBhTWOIeYB\ngPXh/Ng77x3Bs6EKwTQM/BePYC2uS+15+nIpiYHhb/sQ9aEbQgqmyfbkbfIf\n9Qahx4N9RFyqWcmSjfk0Bmo9xOziRQm1tfYkbUwkeI6NIr2ENUWVf7tt+UKz\n5dFmvSKsyrdPEtt+Ken17JoihhJ/9saLMkLn5Y3HrSGVXIniX1cQuarXGX9S\nyt7jIPeGZW2suuxlnlB6Sa/rCkaqR3C3a8knxiH8CDwAm+E8a1d/UbQ0np0k\nqTVVrc2fmxPMgZxGWrwIsO9D1Fs7dgw5rac7ijHvPXeWrzbMd1+rX5mmLF4+\nH2PDOUKN0Uu8WdVilCIEOMoVfY0AEQEAAf4JAwiPkDWd2zmFhmBFMZgWR+X2\nhS4q8bIohwFb8MDzrHAvtI8LOaxC32j3/tUwJKDwLNDeiiOGYfHUOmqgzVRi\nMBdkoVNT3OyFLbw6k/70spe5OcecZsM+OAQybX77Kv3H/VW/40TtUh52Zvvk\nqCoqtG86C4R5kma0zM+yvNyprkCIRwYuJ+OkdmMvem3fRqU49GNzJChBDAmZ\nfGc7W7r2JFIo1k88bh+kaGlDBA9p8GnA7KNXAlBdq6owJNJA0j2z6Rmngpip\nFuUEjdW8Kcs2ben4BZOWGgYmOmD1CAHRYm/xp+Us1kJDTTFxW7P7BNp1kVOP\n9s9AcV/t37bCHLVB2IjAb/tkOxyAexMTS/lFJrpJ89MpYiUt70uB+SDwRls9\noud9CQYS+6OdddgFUjtnOkpcR1Y24v1eF13JwUXmnPggvt6Do3gS6lGdq0Nt\nqNDg8+yPyMA8365Q8IbJDuzm3vfc560Szc/Kx4+1zr57Uaw/qYqhZHSkjRtZ\nfTP0v0ZNwxyXhGF3J31neJ5KgIO+zpSWSP8RwtUVBb8Tsyn8e/DSe/8fAs3V\ntmyxSj9mPQLxW8JCQMpocExEJV7PnxjhB0d1TlYu/wRUl4sxpS4pYa/TDhb9\n37oUYtG28TufT7AZE+XcU8Y1Xl7DVi5jgadUAI5rY71G31JeBuIcGRlVLBnT\nSHnu1v2iz1xqZKQFynH6DUrySn8nP8NM9TaVEnSBlVCjHcyvrs8crx358YFV\n4TG44XQ3n5GLjPetKGD/ccMYOUZm2jkLNzY9l4YqKJq7xsm2c5VhwUEB51Pi\n8Ey+x7hO3EjWUgJrtI/3/hPoKlltPUvnhJjNrR8ivocvd2v+2U0BHQOwTUt3\njSDssFDI9hfOMPp4yy/GHM+p3USC+3TMS8HjFbm6b2bIWzqEv83AaauZqiE1\nfJfCwF8EGAEIAAkFAlzGzj4CGwwACgkQLdzkttEWnLqI8Af/Uimov/qMfFdi\nSp1qYTzR9V84ZkfxBAcr9nEyNGtMlvCwrs3EtZPBe7ou57Qt9aI8UjJfsiov\naHUC1Gt8266T1+GGj6RCPSm6Sp7cZxGtURZDWAPb5u02+VtrqrrXQgkCxQ8/\nzWfg4vLyl052x+3F9SGy7SvH0qi8bcHzVjcwK9VyoSdBq/vhkPHQ33wLQ9ND\nuQCM8fxW+VOqMDlUCHfdcaYRYU7GEm9C37ZijpOLZRuAm6ojjCGtrOrGQ+y0\ni3q7Zn1yDradMlMGG9GjDlqOCmYhZbuT4uS578GzPg0L8zk/1rFOF+YY6Mfq\nV0GB3IX8qBHjAfPmqN9JPxBIn3/6ag==\n=U1yD\n-----END PGP PRIVATE KEY BLOCK-----\n",
+    "Token": "-----BEGIN PGP MESSAGE-----\nVersion: ProtonMail\nComment: https://protonmail.com\n\nwcBMAxW/uQhm1PKxAQf8D4BAoXqnYgoTr7jEiabP9NkeuD6Z88fZDgUzXte5\n6aYMmCtPr0Vef1eDl2x0S0q/YJgR+A5Icmjxk+jG9nMSnRPlAozJfTCXu/Oy\nfkI4CcClGQv7U1EzfknBVNEKmuPO8XlEkGZ5lW/TzWk8ZtQqJsVuqZlZ5qQE\n/p8FNYEZ5RHPhhKlotA+T6XKh23z9mcN/JNEsSNcx+guERbivtTwnd5JpxQJ\nMNWzUYh+K0W7LgxoAo2jevDd4CZ/0sAAViIl+QrShodseV9agbldGUUoIMzn\nHUuIV6VYi7X82eKQXDInrtPc9IHekbDDFoncLnsGGrEqD/8O/qOHHOPwNwU+\n+cHATAOUTDv/ccWq1QEIALl8SNOQBmCuanAceUTSwCkM1fC3Ddqa8ZmnMMyG\nnDWDI7XkFU31CO10lN20/kAWjdN2073B/NT+17cp8fCqbQ1pAFJHpabdqmbI\ntm3pCC5M6otTN+MhjgFYcBuxo0rq9qtuEzz5j4Ub9MIIJTurUHMEPMI462Dc\ndK/d8BvDkU7q67Lkp65vpe9e/pv0lMMrQjdohnTHNgbZhbI/Z5LU0ApD//Ye\ncSC26BRUMJITiGKb9pKGAi0/ig0jJfzykgEarzOsY/v0W7016AMka+NsHuNc\n7Qfyg2LApF5s9Z6aK+Uy851haUk+p+abjBdcWACziAmimVGjlRYm49ra4UJ7\nc6LSeAElM584e4Z3VnqajAJWbmWt2atgBQmcwEBBsdNAtzIagNydMbBBIELA\n5wGUbMXfQxrMOo/Mdac/5EMTHT0/kVQNIBHtpX5SknWvbc5DjIHoH3+BbL87\neaxhKW94hMubKNKT3dbm4PKHtsMiS3TGkFZ3GjvA0Cy/Kw==\n=b1N8\n-----END PGP MESSAGE-----\n",
+    "Signature": "-----BEGIN PGP SIGNATURE-----\nVersion: ProtonMail\nComment: https://protonmail.com\n\nwsBcBAEBCAAGBQJcxtSkAAoJEGfweL9m9sQefz8IAJhpEOvG7d+PVgq3bEL+\nXzY8yTUKB1ZXEMbcR/uHUWSfu0F8zy4CNtSG5HUVjNxx1xzzJVJcxWB/ljO+\n/bJSFOFexTyNh8i/xU+CiBfm5RhAFTYF9xFwfD3LKp5gaalJAhWhArk1/Wuh\nWTDpFpk39uzBRKNwcnSgiJYPxOjAZxj+w/hhHPwmco5cUwMiMR5MNrfzKf+x\nwX3Cfs9fsiiCDzohBzbK0FFsMnJ8aXNVsDBjEA1KrB7sdyaf8FnaM0RsFRDb\njxtIgfcFBbyNJh4414Unt4AYTIrIWhK4OOXI3AfsJy8p6KRBKQUcUkKcDxab\nPTXOPZsZ+UgQ5MevyVnP1zfCwFwEAQEIAAYFAlzG1KQACgkQm1DMMBzxetx/\nPwf9F04uHtix0zDpP1IvG4VYlor4rjYTdfXqxxiFXHO6MZXJoigS1E71E8r4\nsqZ6PoQ5/xCj2A01KRhuF1Bon3mEZEwaIUuBqTV91sLsVWfoxgyPAYpr6gK/\n1W9JhVNNrVRMGox42LQUjyiq0ESrfWmqC8SuMfZMoUoBZycHicA50RbyOUnT\nLO57ArL3JIVmYtyosaXM3idzmNHmaXSkcGt4cvTVysJZQrneaxmikfm5CH2O\n1z2goLBNnzsbRionoV6gCukZOiM/d14yiyeYtsFJ7u/vodkI5y0M0sF8VnN/\njL9keP4ZpHiJ4MBd6tyIH0pLueDRlurFL3fcHsEzD5SrOA==\n=wGCk\n-----END PGP SIGNATURE-----\n"
+  }
+]

pkg/pmapi/testdata/keyring_addressKeysWithoutTokens_JSON

@@ -0,0 +1,12 @@
+[
+	{
+	  "ID": "MhF8dxtN5Lz_GruskN3L9kTKWusZHBdhWaxc7w1tgze2qB6uM9AyC6mWRQg1B7WGZ_r-9gn-XC95-IkpNJr0jg==",
+	  "Primary": 1,
+	  "Flags": 3,
+	  "Version": 3,
+	  "Activation": null,
+	  "PrivateKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\n\nlQPGBF2IcLUBCACl7vans5JKYAEuOZfKNDLZKmLO675DhLjbno8Pw4Q8O6xQETUZ\nTFsIB6eTigCqaZasCeHIx4r7CI4TY2uZvV3+JNaAHN0omHh3iwVXQiE3FU5Zc6TO\niVswOoWPbVYFTuItP3FQHuossMIralIzG0/AyDx0NJawwpSxxz9+C0UoYeZD1A95\nfKveRVhDW0MgO1peDHrpr2HiWv06KkIop39CD/MBnBtDpsarGd16wvX+r8F3MrNE\nD1/e6szzi7oR5H0hPs+HSWZexXhiGSWoEyMiklX/sG/9+LxnHqPhCj7lrGfzmL7j\nCFuHZIqpIO8wVggOQYx1mtmMu7T9PZm66WMbABEBAAH+BwMCYpZ0ni+oe4DxIByk\nseTvdCIjQ3ieiTVvg9FsIVrupmIxDK9EVsGw1/sFjBtc5OOP8N9Vd8IyPh6WjBMu\n5pI5dsYKCLy6LMDmYTYQzXgWlpq2cpH+yZX21fZ+0ijEu+TTSJss3emCULrtCS5M\nUTzt813cIPJu8bYPdQ7taVRqK0Bo+S8YTj2a+lxHQRkbRclueuTm7l5ypU/9X4Xf\n4zwUKnq0OYu3U8IkDEOA+4Zv4SZssFrPr8plUzMHUahVV7NZCXOG11ju1CC9CkF2\n3lpkCZw96TwCGQnT4iTPix1NEJZo6ihRbMlnLo/neS3TWgdCpK5Qr8x4B82ykX6J\ngUeNozW/YFOwFwah22TeSN2dbaGbzJOSqMXHVzK9mz/oTajY/51pS/ZxGoxgy7F2\nD8E/m0I7dS3fw/A/6ZtQFa1bW8L/dBnN71gfr6PSVxn3oczPCXtzNeqHPaDIDGV1\nhgGbGFgmjJy9mnPuxQcScIJloLOVerAaqMcgFuo2MhGKLzQ7IGiSzinaJZ7k602f\nJPfpqBg2aMO9xxNC0S6p6KzMRZCjM1cOJYF2jMo2UWqM0XA1+aEqPksBTmyKSd9+\n1sc8coVl3iesji6GZoT5s0+sNEiQZuNOFUzmGKdgqZhDVfPTLmfV1Uv/tlZCGKmC\ntC51VGjqf8RE3ycdn0Hby8Q2DLsaTnC2vMUde2iJwGQU41TTlt4TKSp6YtqSxbp+\nzUpF4QuS2zVARtx3Z87vw6rHmrNCXHzxl9zWjF3M42FhEESrUL0dciM3c1pB5twd\nIx1TSI0+96zneGS4AkN5Tqua+RdvWXX8ljRgLFa8DqraumQyOjyMXRlJoiey1dt3\n3sRfeF8eI2QF9DooI+4Za/C/1VaEURSQ4p04Ng/xJJ9mwpxjKvxmujC9n2yZ1u4o\nvQh9cKQ9nQAItDdqYW1lcy10ZXN0QHByb3Rvbm1haWwuYmx1ZSA8amFtZXMtdGVz\ndEBwcm90b25tYWlsLmJsdWU+iQE1BBABCAAfBQJdiHC1BgsJBwgDAgQVCAoCAxYC\nAQIZAQIbAwIeAQAKCRC/qkLbur13GVuWB/410LkZxZltVA1odI+yL3utIhoNWj8R\n3AhlVPLfAymkJOFv2KMZ2pRkS8PTmq4sYjAlti60cpWwrNdU4eSfNxoWDEg2lILW\nFULm3TwygDFogSsqyKlGXGr0MqnegImDeQg2kMLPvudTNvEbnRDpaCxWUHenEH5+\nTb8+UyG1e3V2su5J6p+Ghh88Y+zJyU4DWta3wAymLPW+gUEFXDAw5KdSg/8iD5kH\n6aMbd9l1Wp5WiErMsJOwoxhFVhwUK7MCXAXxHMWTuyu78RYYNhafPpd+Rq6dqy1d\nm5M7BlDU4Y2Y921xOSHGbNXbWbh5WHBT0QkV7IJunwiPYc5apJbee69DnQPGBF2I\ncLUBCAC9RsknEX9jPP4AB2/DxWdqTKb1/AmDpzNstEUtSy+zUKmErSNQeNUuso+n\nH2lFtb/A+s3nC2UPqbgz/SWN37Z9ip6hvsQX0etFvzYiXFfVDbknqmpi8AGt1Bkr\nFP9jGvF7gIT436icfjd8vQHHMIdEbEF+S46LfW8xibZEYUHLr5m8VO7dqqsIIwvr\nTJWEGjtRfaacqV3aWDzHbXodmYC3kQH9UjpdXE/9/5VxWtxDAFwsCaXiE9+EL69Z\ncJDdPfM0iVL3XgP22EFwC8C+Xha1PiSNOGFTIbv4aT5yLgPb7wwvgaw3semLF1vM\nzni3z+FNPREt+9itIhGPpxmIyAQ/ABEBAAH+BwMCl4RZZJs4k1bxqOmzniBxB2y8\nX4sDxyy60IOO3Ip2cOeyJ/qwEBdXxL2EjS6y5Vt3GTnQDz1IJqYiz9eRbZQnXUh5\nSk9ND9+mgxKYHDV52flh2FBTaOgqsKLnwLRUE12D1BQ9Ou5cYcgOIbCncftQTnWV\nk6hg9PDFZD/LD3IuKS9/DVYxKkYIJZ1KH446bQTYto+fnU0ajkLfED7wN4JquHq4\nYS+L7h8GetOCyRsh8zDIOPF5jZYCSzDe8Q03ktozOE2+YV/A3Z/0Uub8/Qecl5u3\n1qURXeIC6oGRDVXegZu3491IG+UTzjaBDPu+Xk4zad7mGjBQ9vNuKZfNNg/nHOWL\nNHRCXbkQVpXCA+v+dNYov+bqKe1QpF/2jvU0Bi32HLG8gfmRwD0WaAnbvOLTYXPl\ndRNDn7ixfOC0uHT6epPlZxsRja5TXlFl5qSuAX2zQfFdb+pRPztNKer6iMNz3itz\nh2B/gcTwqXloy4Zq9wqxcXLV1qmoMJ5YzRcd4otRM7FhZD0LxBAOANdmPSmRHbf6\nSXyXgvK4KDwRB52uIYJzWyIi7LiU3qTbV0GgptgHeVLwgbA9ioPwIu85+XQEh/eE\nEjNhz0CdtCqmYY3+SjNOZ8K59LgWCuv/YkRkmwDimEz5gUlpayoOYib4J4wIx6OY\n+fmI+gByS+UN3v+ml7l909c2W8w0dwR8+7P1XGBkaiVROTM2hdnHMflWm2548WBx\na6tbdCQPUDra/5O/pCdefc44hwD08liutq9kF29sMwONZy0P0dlyFZq4LTLmQqs0\nq93GZRjoI+uit9oTLGCtJ99aiSIx97KfkKx5wLnE4DruQlqPWnxV5Yfn9RZ6j8V2\nRQjL8f5RbiBtkkCl3r+nEdQIeMRy+8xxO9dTYTlaCNgfd92LdQrwzKe+GneUgKYg\nWo+AiQEfBBgBCAAJBQJdiHC1AhsMAAoJEL+qQtu6vXcZskIH+we2sHxPnWOfkXTu\npmZBU1ba8JLcrx+CMwTPkt7GMYpDplVOybn+AjMxB9JxST71mBw6rbwd2XLM1AmJ\nbvXVfmCtmC2TmrORqcoxEql+9tdgnSTfZfrltnYeVuEAvAxrMmD4S2Mb5zVWFl+q\ni69A0rdD8a36nOlJnZxfs57W1zfvl/rh+/RdybPwx9y0hSnRPSypis2dcwSyBD9+\nnlbn3QoybaUJxWvL+9MphCXZ4CuuhG0VPcmdH+LzOytDTtJnDNm+Ru4sokroDJXe\n5XlrK3+wtSNL4rfa/MQYyK8XsWRVR/a0BPeqZA4cPlHqCOSfUfaFbORVbPaQdq2a\nYYsMDlA=\n=ZwJf\n-----END PGP PRIVATE KEY BLOCK-----",
+	  "Token": null,
+	  "Signature": null
+	}
+]

pkg/pmapi/testdata/keyring_userKey

@@ -0,0 +1,62 @@
+-----BEGIN PGP PRIVATE KEY BLOCK-----
+Version: OpenPGP.js v4.4.5
+Comment: testpassphrase
+
+xcLYBFzGzhEBCADBxfqTFMqfQzT77A5tuuhPFwPq8dfC2evs8u1OvTqFbztY
+5FOuSxzduyeDqQ1Fx6dKEOKgcYE8t1Uh4VSS7z6bTdY8j9yrL81kCVB46sE1
+OzStzyx/5l7OdH/pM4F+aKslnLvqlw0UeJr+UNizVtOCEUaNfVjPK3cc1ocx
+v+36K4RnnyfEtjUW9gDZbhgaF02G5ILHmWmbgM7I+77gCd2wI0EdY9s/JZQ+
+VmkMFqoMdY9PyBchoOIPUkkGQi1SaF4IEzMaAUSbnCYkHHY/SbfDTcR46VGq
+cXlkB1rq5xskaUQ9r+giCC/K4pc7bBkI1lQ7ADVuWvdrWnWapK0FO6CfABEB
+AAEAB/0YPhPJ0phA/EWviN+16bmGVOZNaVapjt2zMMybWmrtEQv3OeWgO3nP
+4cohRi/zaCBCphcm+dxbLhftW7AFi/9PVcR09436MB+oTCQFugpUWw+4TmA5
+BidxTpDxf4X2vH3rquQLBufWL6U7JlPeKAGL1xZ2aCq0DIeOk5D+xTjZizV2
+GIyQRVCLWb+LfDmvvcp3Y94X60KXdBAMuS1ZMKcY3Sl8VAXNB4KQsC/kByzf
+6FCB097XZRYV7lvJJQ7+6Wisb3yVi8sEQx2sFm5fAp+0qi3a6zRTEp49r6Hr
+gyWViH5zOOpA7DcNwx1Bwhi7GG0tak6EUnnKUNLfOupglcphBADmpXCgT4nc
+uSBYTiZSVcB/ICCkTxVsHL1WcXtPK2Ikzussx2n9kb0rapvuC0YLipX9lUkQ
+fyeC3jQJeCyN79AkDGkOfWaESueT2hM0Po+RwDgMibKn6yJ1zebz4Lc2J3C9
+oVFcAnql+9KyGsAPn03fyQzDnvhNnJvHJi4Hx8AWoQQA1xLoXeVBjRi0IjjU
+E6Mqaq5RLEog4kXRp86VSSEGHBwyIYnDiM//gjseo/CXuVyHwL7UXitp8s1B
+D1uE3APrhqUS66fD5pkF+z+RcSqiIv7I76NJ24Cdg38L6seGSjOHrq7/dEeG
+K6WqfQUCEjta3yNSg7pXb2wn2WZqKIK+rz8EALZRuMXeql/FtO3Cjb0sv7oT
+9dLP4cn1bskGRJ+Vok9lfCERbfXGccoAk3V+qSfpHgKxsebkRbUhf+trOGnw
+tW+kBWo/5hYGQuN+A9JogSJViT+nuZyE+x1/rKswDFmlMSdf2GIDARWIV0gc
+b1yOEwUmNBSthPcnFXvBr4BG3XTtNPTNLSJhcm9uMjEtM0BzYWRlbWJlLm9y
+ZyIgPGFyb24yMS0zQHNhZGVtYmUub3JnPsLAdQQQAQgAHwUCXMbOEQYLCQcI
+AwIEFQgKAgMWAgECGQECGwMCHgEACgkQZ/B4v2b2xB6XUgf/dHGRHimyMR78
+QYbEm2cuaEvOtq4a+J6Zv3P4VOWAbvkGWS9LDKSvVi60vq4oYOmF54HgPzur
+nA4OtZDf0HKwQK45VZ7CYD693o70jkKPrAAJG3yTsbesfiS7RbFyGKzKJ7EL
+nsUIJkfgm/SlKmXU/u8MOBO5Wg7/TcsS33sRWHl90j+9jbhqdl92R+vY/CwC
+ieFkQA7/TDv1u+NAalH+Lpkd8AIuEcki+TAogZ7oi/SnofwnoB7BxRm+mIkp
+ZZhIDSCaPOzLG8CSZ81d3HVHhqbf8dh0DFKFoUYyKdbOqIkNWWASf+c/ZEme
+IWcekY8hqwf/raZ56tGM/bRwYPcotMfC1wRcxs4RAQgAsMb5/ELWmrfPy3ba
+5qif+RXhGSbjitATNgHpoPUHrfTC7cn4JWHqehoXLAQpFAoKd+O/ZNpZozK9
+ilpqGUx05yMw06jNQEhYIbgIF4wzPpz02Lp6YeMwdF5LF+Rw83PHdHrA/wRV
+/QjL04+kZnN+G5HmzMlhFY+oZSpL+Gp1bTXgtAVDkhCnMB5tP2VwULMGyJ+X
+vRYxwTK2CrLjIVZv5n1VYY+caCowU6j/XFqvlCJj+G5oV+UhFOWffaMRXhOh
+a64RrhqT1Np7wCLvLMP2wpys9xlMcLQJLqDNxqOTp504V7dm67ncC0fKUsT4
+m4oTktnxKPd6MU+4VYveaLCquwARAQABAAf4u9s7gpGErs1USxmDO9TlyGZK
+aBlri8nMf3s+hOJCOo3cRaRHJBfdY6pu/baG6H6JTsWzeY4MHwr6N+dhVIEh
+FPMa9EZAjagyc4GugxWGiMVTfU+2AEfdrdynhQKMgXSctnnNCdkRuX0nwqb3
+nlupm1hsz2ze4+Wg0BKSLS0FQdoUbITdJUR69OHr4dNJVHWYI0JSBx4SdhV3
+y9163dDvmc+lW9AEaD53vyZWfzCHZxsR/gI32VmT0z5gn1t8w9AOdXo2lA1H
+bf7wh4/qCyujGu64ToZtiEny/GCyM6PofLtiZuJNLw3s/y+B2tKv22aTJ760
++Gib1xB9WcWjKyrxBADoeCyq+nHGrl0CwOkmjanlFymgo7mnBOXuiFOvGrKk
+M1meMU1TI4TEBWkVnDVMcSejgjAf/bX1dtouba1tMAMu7DlaV/0EwbSADRel
+RSqEbIzIOys+y9TY/BMI/uCKNyEKHvu1KUXADb+CBpdBpCfMBWDANFlo9xLz
+Ajcmu2dyawQAwquwC0VXQcvzfs+Hd5au5XvHdm1KidOiAdu6PH1SrOgenIN4
+lkEjHrJD9jmloO2/GVcxDBB2pmf0B4HEg7DuY9LXBrksP5eSbbRc5+UH1HUv
+u82AqQnfNKTd/jae+lLwaOS++ohtwMkkD6W0LdWnHPjyyXg4Oi9zPID3asRu
+3PED/3CYyjl6S8GTMY4FNH7Yxu9+NV2xpKE92Hf8K/hnYlmSSVKDCEeOJtLt
+BkkcSqY6liCNSMmJdVyAF2GrR+zmDac7UQRssf57oOWtSsGozt0aqJXuspMT
+6aB+P1UhZ8Ly9rWZNiJ0jwyfnQNOLCYDaqjFmiSpqrNnJ2Q1Xge3+k80P9DC
+wF8EGAEIAAkFAlzGzhECGwwACgkQZ/B4v2b2xB5wlwgAjZA1zdv5irFjyWVo
+4/itONtyO1NbdpyYpcct7vD0oV+a4wahQP0J3Kk1GhZ5tvAoZF/jakQQOM5o
+GjUYpXAGnr09Mv9EiQ2pDwXc2yq0WfXnGxNrpzOqdtV+IqY9NYkl55Tme7x+
+WRvrkPSUeUsyEGvxwR1stdv8eg9jUmxdl8Io3PYoFJJlrM/6aXeC1r3KOj7q
+XAnR0XHJ+QBSNKCWLlQv5hui9BKfcLiVKFK/dNhs82nRyhPr4sWFw6MTqdAK
+4zkn7l0jmy6Evi1AiiGPiHPnxeNErnofOIEh4REQj00deZADHrixTLtx2FuR
+uaSC3IcBmBsj1fNb4eYXElILjQ==
+=fMOl
+-----END PGP PRIVATE KEY BLOCK-----

pkg/pmapi/testdata/keyring_userKey_JSON

@@ -0,0 +1,10 @@
+[
+    {
+        "ID": "IlnTbqicN-2HfUGIn-ki8bqZfLqNj5ErUB0z24Qx5g-4NvrrIc6GLvEpj2EPfwGDv28aKYVRRrSgEFhR_zhlkA==",
+        "Version": 3,
+        "PrivateKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\r\nVersion: OpenPGP.js v0.7.1\r\nComment: http://openpgpjs.org\r\n\r\nxcMGBFRJbc0BCAC0mMLZPDBbtSCWvxwmOfXfJkE2+ssM3ux21LhD/bPiWefE\nWSHlCjJ8PqPHy7snSiUuxuj3f9AvXPvg+mjGLBwu1/QsnSP24sl3qD2onl39\nvPiLJXUqZs20ZRgnvX70gjkgEzMFBxINiy2MTIG+4RU8QA7y8KzWev0btqKi\nMeVa+GLEHhgZ2KPOn4Jv1q4bI9hV0C9NUe2tTXS6/Vv3vbCY7lRR0kbJ65T5\nc8CmpqJuASIJNrSXM/Q3NnnsY4kBYH0s5d2FgbASQvzrjuC2rngUg0EoPsrb\nDEVRA2/BCJonw7aASiNCrSP92lkZdtYlax/pcoE/mQ4WSwySFmcFT7yFABEB\nAAH+CQMIvzcDReuJkc9gnxAkfgmnkBFwRQrqT/4UAPOF8WGVo0uNvDo7Snlk\nqWsJS+54+/Xx6Jur/PdBWeEu+6+6GnppYuvsaT0D0nFdFhF6pjng+02IOxfG\nqlYXYcW4hRru3BfvJlSvU2LL/Z/ooBnw3T5vqd0eFHKrvabUuwf0x3+K/sru\nFp24rl2PU+bzQlUgKpWzKDmO+0RdKQ6KVCyCDMIXaAkALwNffAvYxI0wnb2y\nWAV/bGn1ODnszOYPk3pEMR6kKSxLLaO69kYx4eTERFyJ+1puAxEPCk3Cfeif\nyDWi4rU03YB16XH7hQLSFl61SKeIYlkKmkO5Hk1ybi/BhvOGBPVeGGbxWnwI\n46G8DfBHW0+uvD5cAQtk2d/q3Ge1I+DIyvuRCcSu0XSBNv/Bkpp4IbAUPBaW\nTIvf5p9oxw+AjrMtTtcdSiee1S6CvMMaHhVD7SI6qGA8GqwaXueeLuEXa0Ok\nBWlehx8wibMi4a9fLcQZtzJkmGhR1WzXcJfiEg32srILwIzPQYxuFdZZ2elb\ngYp/bMEIp4LKhi43IyM6peCDHDzEba8NuOSd0heEqFIm0vlXujMhkyMUvDBv\nH0V5On4aMuw/aSEKcAdbazppOru/W1ndyFa5ZHQIC19g72ZaDVyYjPyvNgOV\nAFqO4o3IbC5z31zMlTtMbAq2RG9svwUVejn0tmF6UPluTe0U1NuXFpLK6TCH\nwqocLz4ecptfJQulpYjClVLgzaYGDuKwQpIwPWg5G/DtKSCGNtEkfqB3aemH\nV5xmoYm1v5CQZAEvvsrLA6jxCk9lzqYV8QMivWNXUG+mneIEM35G0HOPzXca\nLLyB+N8Zxioc9DPGfdbcxXuVgOKRepbkq4xv1pUpMQ4BUmlkejDRSP+5SIR3\niEthg+FU6GRSQbORE6nhrKjGBk8fpNpozQZVc2VySUTCwHIEEAEIACYFAlRJ\nbc8GCwkIBwMCCRA+tiWe3yHfJAQVCAIKAxYCAQIbAwIeAQAA9J0H/RLR/Uwt\nCakrPKtfeGaNuOI45SRTNxM8TklC6tM28sJSzkX8qKPzvI1PxyLhs/i0/fCQ\n7Z5bU6n41oLuqUt2S9vy+ABlChKAeziOqCHUcMzHOtbKiPkKW88aO687nx+A\nol2XOnMTkVIC+edMUgnKp6tKtZnbO4ea6Cg88TFuli4hLHNXTfCECswuxHOc\nAO1OKDRrCd08iPI5CLNCIV60QnduitE1vF6ehgrH25Vl6LEdd8vPVlTYAvsa\n6ySk2RIrHNLUZZ3iII3MBFL8HyINp/XA1BQP+QbH801uSLq8agxM4iFT9C+O\nD147SawUGhjD5RG7T+YtqItzgA1V9l277EXHwwYEVEltzwEIAJD57uX6bOc4\nTgf3utfL/4hdyoqIMVHkYQOvE27wPsZxX08QsdlaNeGji9Ap2ifIDuckUqn6\nJi9jtZDKtOzdTBm6rnG5nPmkn6BJXPhnecQRP8N0XBISnAGmE4t+bxtts5Wb\nqeMdxJYqMiGqzrLBRJEIDTcg3+QF2Y3RywOqlcXqgG/xX++PsvR1Jiz0rEVP\nTcBc7ytyb/Av7mx1S802HRYGJHOFtVLoPTrtPCvv+DRDK8JzxQW2XSQLlI0M\n9s1tmYhCogYIIqKx9qOTd5mFJ1hJlL6i9xDkvE21qPFASFtww5tiYmUfFaxI\nLwbXPZlQ1I/8fuaUdOxctQ+g40ZgHPcAEQEAAf4JAwgdUg8ubE2BT2DITBD+\nXFgjrnUlQBilbN8/do/36KHuImSPO/GGLzKh4+oXxrvLc5fQLjeO+bzeen4u\nCOCBRO0hG7KpJPhQ6+T02uEF6LegE1sEz5hp6BpKUdPZ1+8799Rylb5kubC5\nIKnLqqpGDbH3hIsmSV3CG/ESkaGMLc/K0ZPt1JRWtUQ9GesXT0v6fdM5GB/L\ncZWFdDoYgZAw5BtymE44knIodfDAYJ4DHnPCh/oilWe1qVTQcNMdtkpBgkuo\nTHecqEmiODQz5EX8pVmS596XsnPO299Lo3TbaHUQo7EC6Au1Au9+b5hC1pDa\nFVCLcproi/Cgch0B/NOCFkVLYmp6BEljRj2dSZRWbO0vgl9kFmJEeiiH41+k\nEAI6PASSKZs3BYLFc2I8mBkcvt90kg4MTBjreuk0uWf1hdH2Rv8zprH4h5Uh\ngjx5nUDX8WXyeLxTU5EBKry+A2DIe0Gm0/waxp6lBlUl+7ra28KYEoHm8Nq/\nN9FCuEhFkFgw6EwUp7jsrFcqBKvmni6jyplm+mJXi3CK+IiNcqub4XPnBI97\nlR19fupB/Y6M7yEaxIM8fTQXmP+x/fe8zRphdo+7o+pJQ3hk5LrrNPK8GEZ6\nDLDOHjZzROhOgBvWtbxRktHk+f5YpuQL+xWd33IV1xYSSHuoAm0Zwt0QJxBs\noFBwJEq1NWM4FxXJBogvzV7KFhl/hXgtvx+GaMv3y8gucj+gE89xVv0XBXjl\n5dy5/PgCI0Id+KAFHyKpJA0N0h8O4xdJoNyIBAwDZ8LHt0vlnLGwcJFR9X7/\nPfWe0PFtC3d7cYY3RopDhnRP7MZs1Wo9nZ4IvlXoEsE2nPkWcns+Wv5Yaewr\ns2ra9ZIK7IIJhqKKgmQtCeiXyFwTq+kfunDnxeCavuWL3HuLKIOZf7P9vXXt\nXgEir9rCwF8EGAEIABMFAlRJbdIJED62JZ7fId8kAhsMAAD+LAf+KT1EpkwH\n0ivTHmYako+6qG6DCtzd3TibWw51cmbY20Ph13NIS/MfBo828S9SXm/sVUzN\n/r7qZgZYfI0/j57tG3BguVGm53qya4bINKyi1RjK6aKo/rrzRkh5ZVD5rVNO\nE2zzvyYAnLUWG9AV1OYDxcgLrXqEMWlqZAo+Wmg7VrTBmdCGs/BPvscNgQRr\n6Gpjgmv9ru6LjRL7vFhEcov/tkBLj+CtaWWFTd1s2vBLOs4rCsD9TT/23vfw\nCnokvvVjKYN5oviy61yhpqF1rWlOsxZ4+2sKW3Pq7JLBtmzsZegTONfcQAf7\nqqGRQm3MxoTdgQUShAwbNwNNQR9cInfMnA==\r\n=2wIY\r\n-----END PGP PRIVATE KEY BLOCK-----\r\n",
+        "Fingerprint": "c93f767df53b0ca8395cfde90483475164ec6353",
+        "Activation": null,
+        "Primary": 1
+    }
+]

pkg/pmapi/testdata/routes/HTTP_200.json

@@ -0,0 +1,3 @@
+{
+    "Code": 1000
+}

pkg/pmapi/testdata/routes/HTTP_401.json

@@ -0,0 +1,4 @@
+{
+    "Code": 5000,
+    "Error": "Status unauthorized"
+}

pkg/pmapi/testdata/routes/HTTP_402.json

@@ -0,0 +1,4 @@
+{
+    "Code": 5000,
+    "Error": "Status payment required"
+}