We build too many walls and not enough bridges

test/liveapi/calls.go

@@ -0,0 +1,75 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package liveapi
+
+import (
+	"fmt"
+
+	"github.com/nsf/jsondiff"
+)
+
+type fakeCall struct {
+	method  string
+	path    string
+	request []byte
+}
+
+func (cntrl *Controller) recordCall(method, path string, request []byte) {
+	cntrl.lock.Lock()
+	defer cntrl.lock.Unlock()
+
+	cntrl.calls = append(cntrl.calls, &fakeCall{
+		method:  method,
+		path:    path,
+		request: request,
+	})
+}
+
+func (cntrl *Controller) PrintCalls() {
+	fmt.Println("API calls:")
+	for idx, call := range cntrl.calls {
+		fmt.Printf("%02d: [%s] %s\n", idx+1, call.method, call.path)
+		if call.request != nil && string(call.request) != "null" {
+			fmt.Printf("\t%s\n", call.request)
+		}
+	}
+}
+
+func (cntrl *Controller) WasCalled(method, path string, expectedRequest []byte) bool {
+	for _, call := range cntrl.calls {
+		if call.method != method && call.path != path {
+			continue
+		}
+		diff, _ := jsondiff.Compare(call.request, expectedRequest, &jsondiff.Options{})
+		isSuperset := diff == jsondiff.FullMatch || diff == jsondiff.SupersetMatch
+		if isSuperset {
+			return true
+		}
+	}
+	return false
+}
+
+func (cntrl *Controller) GetCalls(method, path string) [][]byte {
+	requests := [][]byte{}
+	for _, call := range cntrl.calls {
+		if call.method == method && call.path == path {
+			requests = append(requests, call.request)
+		}
+	}
+	return requests
+}

test/liveapi/cleanup.go

@@ -0,0 +1,132 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package liveapi
+
+import (
+	"time"
+
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/pkg/errors"
+)
+
+func cleanup(client *pmapi.Client) error {
+	if err := cleanSystemFolders(client); err != nil {
+		return errors.Wrap(err, "failed to clean system folders")
+	}
+	if err := cleanCustomLables(client); err != nil {
+		return errors.Wrap(err, "failed to clean cusotm labels")
+	}
+	if err := cleanTrash(client); err != nil {
+		return errors.Wrap(err, "failed to clean trash")
+	}
+	return nil
+}
+
+func cleanSystemFolders(client *pmapi.Client) error {
+	for _, labelID := range []string{pmapi.InboxLabel, pmapi.SentLabel, pmapi.ArchiveLabel} {
+		for {
+			messages, total, err := client.ListMessages(&pmapi.MessagesFilter{
+				PageSize: 150,
+				LabelID:  labelID,
+			})
+			if err != nil {
+				return errors.Wrap(err, "failed to list messages")
+			}
+
+			if total == 0 {
+				break
+			}
+
+			messageIDs := []string{}
+			for _, message := range messages {
+				messageIDs = append(messageIDs, message.ID)
+			}
+
+			if err := client.DeleteMessages(messageIDs); err != nil {
+				return errors.Wrap(err, "failed to delete messages")
+			}
+
+			if total == len(messages) {
+				break
+			}
+		}
+	}
+	return nil
+}
+
+func cleanCustomLables(client *pmapi.Client) error {
+	labels, err := client.ListLabels()
+	if err != nil {
+		return errors.Wrap(err, "failed to list labels")
+	}
+
+	for _, label := range labels {
+		if err := emptyFolder(client, label.ID); err != nil {
+			return errors.Wrap(err, "failed to empty label")
+		}
+		if err := client.DeleteLabel(label.ID); err != nil {
+			return errors.Wrap(err, "failed to delete label")
+		}
+	}
+
+	return nil
+}
+
+func cleanTrash(client *pmapi.Client) error {
+	for {
+		_, total, err := client.ListMessages(&pmapi.MessagesFilter{
+			PageSize: 1,
+			LabelID:  pmapi.TrashLabel,
+		})
+		if err == nil && total == 0 {
+			break
+		}
+
+		err = emptyFolder(client, pmapi.TrashLabel)
+		if err == nil {
+			break
+		}
+		if err.Error() == "Folder or label is currently being emptied" {
+			time.Sleep(500 * time.Millisecond)
+			continue
+		}
+		return errors.Wrap(err, "failed to empty trash")
+	}
+	return nil
+}
+
+func emptyFolder(client *pmapi.Client, labelID string) error {
+	err := client.EmptyFolder(labelID, "")
+	if err != nil {
+		return err
+	}
+	for {
+		_, total, err := client.ListMessages(&pmapi.MessagesFilter{
+			PageSize: 1,
+			LabelID:  labelID,
+		})
+		if err != nil {
+			return err
+		}
+		if total == 0 {
+			break
+		}
+		time.Sleep(1 * time.Second)
+	}
+	return nil
+}

test/liveapi/controller.go

@@ -0,0 +1,62 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package liveapi
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+	"sync"
+
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+)
+
+type Controller struct {
+	// Internal states.
+	lock                 *sync.RWMutex
+	calls                []*fakeCall
+	pmapiByUsername      map[string]*pmapi.Client
+	messageIDsByUsername map[string][]string
+
+	// State controlled by test.
+	noInternetConnection bool
+}
+
+func NewController() *Controller {
+	return &Controller{
+		lock:                 &sync.RWMutex{},
+		calls:                []*fakeCall{},
+		pmapiByUsername:      map[string]*pmapi.Client{},
+		messageIDsByUsername: map[string][]string{},
+
+		noInternetConnection: false,
+	}
+}
+
+func (cntrl *Controller) GetClient(userID string) *pmapi.Client {
+	cfg := &pmapi.ClientConfig{
+		AppVersion: fmt.Sprintf("Bridge_%s", os.Getenv("VERSION")),
+		ClientID:   "bridge-test",
+		Transport: &fakeTransport{
+			cntrl:     cntrl,
+			transport: http.DefaultTransport,
+		},
+		TokenManager: pmapi.NewTokenManager(),
+	}
+	return pmapi.NewClient(cfg, userID)
+}

test/liveapi/labels.go

@@ -0,0 +1,105 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package liveapi
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/pkg/errors"
+)
+
+var systemLabelNameToID = map[string]string{ //nolint[gochecknoglobals]
+	"INBOX":    pmapi.InboxLabel,
+	"Trash":    pmapi.TrashLabel,
+	"Spam":     pmapi.SpamLabel,
+	"All Mail": pmapi.AllMailLabel,
+	"Archive":  pmapi.ArchiveLabel,
+	"Sent":     pmapi.SentLabel,
+	"Drafts":   pmapi.DraftLabel,
+}
+
+func (cntrl *Controller) AddUserLabel(username string, label *pmapi.Label) error {
+	client, ok := cntrl.pmapiByUsername[username]
+	if !ok {
+		return fmt.Errorf("user %s does not exist", username)
+	}
+
+	label.Exclusive = getLabelExclusive(label.Name)
+	label.Name = getLabelNameWithoutPrefix(label.Name)
+	label.Color = pmapi.LabelColors[0]
+	if _, err := client.CreateLabel(label); err != nil {
+		return errors.Wrap(err, "failed to create label")
+	}
+	return nil
+}
+
+func (cntrl *Controller) GetLabelIDs(username string, labelNames []string) ([]string, error) {
+	labelIDs := []string{}
+	for _, labelName := range labelNames {
+		labelID, err := cntrl.getLabelID(username, labelName)
+		if err != nil {
+			return nil, err
+		}
+		labelIDs = append(labelIDs, labelID)
+	}
+	return labelIDs, nil
+}
+
+func (cntrl *Controller) getLabelID(username, labelName string) (string, error) {
+	if labelID, ok := systemLabelNameToID[labelName]; ok {
+		return labelID, nil
+	}
+
+	client, ok := cntrl.pmapiByUsername[username]
+	if !ok {
+		return "", fmt.Errorf("user %s does not exist", username)
+	}
+
+	labels, err := client.ListLabels()
+	if err != nil {
+		return "", errors.Wrap(err, "failed to list labels")
+	}
+
+	exclusive := getLabelExclusive(labelName)
+	labelName = getLabelNameWithoutPrefix(labelName)
+	for _, label := range labels {
+		if label.Exclusive == exclusive && label.Name == labelName {
+			return label.ID, nil
+		}
+	}
+	return "", fmt.Errorf("label %s:%s does not exist", username, labelName)
+}
+
+func getLabelNameWithoutPrefix(name string) string {
+	if strings.HasPrefix(name, "Folders/") {
+		return strings.TrimPrefix(name, "Folders/")
+	}
+	if strings.HasPrefix(name, "Labels/") {
+		return strings.TrimPrefix(name, "Labels/")
+	}
+	return name
+}
+
+func getLabelExclusive(name string) int {
+	if strings.HasPrefix(name, "Folders/") {
+		return 1
+	}
+	return 0
+}

test/liveapi/messages.go

@@ -0,0 +1,134 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package liveapi
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"strconv"
+
+	messageUtils "github.com/ProtonMail/proton-bridge/pkg/message"
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/pkg/errors"
+)
+
+func (cntrl *Controller) AddUserMessage(username string, message *pmapi.Message) error {
+	client, ok := cntrl.pmapiByUsername[username]
+	if !ok {
+		return fmt.Errorf("user %s does not exist", username)
+	}
+
+	body, err := buildMessage(client, message)
+	if err != nil {
+		return errors.Wrap(err, "failed to build message")
+	}
+
+	req := &pmapi.ImportMsgReq{
+		AddressID: message.AddressID,
+		Body:      body.Bytes(),
+		Unread:    message.Unread,
+		Time:      message.Time,
+		Flags:     message.Flags,
+		LabelIDs:  message.LabelIDs,
+	}
+
+	results, err := client.Import([]*pmapi.ImportMsgReq{req})
+	if err != nil {
+		return errors.Wrap(err, "failed to make an import")
+	}
+	for _, result := range results {
+		if result.Error != nil {
+			return errors.Wrap(result.Error, "failed to import message")
+		}
+		cntrl.messageIDsByUsername[username] = append(cntrl.messageIDsByUsername[username], result.MessageID)
+	}
+
+	return nil
+}
+
+func buildMessage(client *pmapi.Client, message *pmapi.Message) (*bytes.Buffer, error) {
+	if err := encryptMessage(client, message); err != nil {
+		return nil, errors.Wrap(err, "failed to encrypt message")
+	}
+
+	body := &bytes.Buffer{}
+	if err := buildMessageHeader(message, body); err != nil {
+		return nil, errors.Wrap(err, "failed to build message header")
+	}
+	if err := buildMessageBody(message, body); err != nil {
+		return nil, errors.Wrap(err, "failed to build message body")
+	}
+	return body, nil
+}
+
+func encryptMessage(client *pmapi.Client, message *pmapi.Message) error {
+	addresses, err := client.GetAddresses()
+	if err != nil {
+		return errors.Wrap(err, "failed to get address")
+	}
+	kr := addresses.ByID(message.AddressID).KeyRing()
+
+	if err = message.Encrypt(kr, nil); err != nil {
+		return errors.Wrap(err, "failed to encrypt message body")
+	}
+	return nil
+}
+
+func buildMessageHeader(message *pmapi.Message, body *bytes.Buffer) error {
+	header := messageUtils.GetHeader(message)
+	header.Set("Content-Type", "multipart/mixed; boundary="+messageUtils.GetBoundary(message))
+	header.Del("Content-Disposition")
+	header.Del("Content-Transfer-Encoding")
+
+	if err := http.Header(header).Write(body); err != nil {
+		return errors.Wrap(err, "failed to write header")
+	}
+	_, _ = body.WriteString("\r\n")
+	return nil
+}
+
+func buildMessageBody(message *pmapi.Message, body *bytes.Buffer) error {
+	mw := multipart.NewWriter(body)
+	if err := mw.SetBoundary(messageUtils.GetBoundary(message)); err != nil {
+		return errors.Wrap(err, "failed to set boundary")
+	}
+
+	bodyHeader := messageUtils.GetBodyHeader(message)
+	bodyHeader.Set("Content-Transfer-Encoding", "7bit")
+
+	part, err := mw.CreatePart(bodyHeader)
+	if err != nil {
+		return errors.Wrap(err, "failed to create message body part")
+	}
+	if _, err := io.WriteString(part, message.Body); err != nil {
+		return errors.Wrap(err, "failed to write message body")
+	}
+	_ = mw.Close()
+	return nil
+}
+
+func (cntrl *Controller) GetMessageID(username, messageIndex string) string {
+	idx, err := strconv.Atoi(messageIndex)
+	if err != nil {
+		panic(fmt.Sprintf("message index %s not found", messageIndex))
+	}
+	return cntrl.messageIDsByUsername[username][idx-1]
+}

test/liveapi/transport.go

@@ -0,0 +1,59 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package liveapi
+
+import (
+	"io/ioutil"
+	"net/http"
+
+	"github.com/pkg/errors"
+)
+
+func (cntrl *Controller) TurnInternetConnectionOff() {
+	cntrl.noInternetConnection = true
+}
+
+func (cntrl *Controller) TurnInternetConnectionOn() {
+	cntrl.noInternetConnection = false
+}
+
+type fakeTransport struct {
+	cntrl     *Controller
+	transport http.RoundTripper
+}
+
+func (t *fakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	if t.cntrl.noInternetConnection {
+		return nil, errors.New("no route to host")
+	}
+
+	body := []byte{}
+	if req.GetBody != nil {
+		bodyReader, err := req.GetBody()
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to get body")
+		}
+		body, err = ioutil.ReadAll(bodyReader)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to read body")
+		}
+	}
+	t.cntrl.recordCall(req.Method, req.URL.Path, body)
+
+	return t.transport.RoundTrip(req)
+}

test/liveapi/users.go

@@ -0,0 +1,66 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package liveapi
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/cucumber/godog"
+	"github.com/pkg/errors"
+)
+
+func (cntrl *Controller) AddUser(user *pmapi.User, addresses *pmapi.AddressList, password string, twoFAEnabled bool) error {
+	if twoFAEnabled {
+		return godog.ErrPending
+	}
+
+	client := pmapi.NewClient(&pmapi.ClientConfig{
+		AppVersion:   fmt.Sprintf("Bridge_%s", os.Getenv("VERSION")),
+		ClientID:     "bridge-cntrl",
+		TokenManager: pmapi.NewTokenManager(),
+	}, user.ID)
+
+	authInfo, err := client.AuthInfo(user.Name)
+	if err != nil {
+		return errors.Wrap(err, "failed to get auth info")
+	}
+	auth, err := client.Auth(user.Name, password, authInfo)
+	if err != nil {
+		return errors.Wrap(err, "failed to auth user")
+	}
+
+	mailboxPassword, err := pmapi.HashMailboxPassword(password, auth.KeySalt)
+	if err != nil {
+		return errors.Wrap(err, "failed to hash mailbox password")
+	}
+	if _, err := client.Unlock(mailboxPassword); err != nil {
+		return errors.Wrap(err, "failed to unlock user")
+	}
+	if err := client.UnlockAddresses([]byte(mailboxPassword)); err != nil {
+		return errors.Wrap(err, "failed to unlock addresses")
+	}
+
+	if err := cleanup(client); err != nil {
+		return errors.Wrap(err, "failed to clean user")
+	}
+
+	cntrl.pmapiByUsername[user.Name] = client
+	return nil
+}