GODT-1044: lite parser

internal/imap/mailbox_append.go

@@ -18,7 +18,9 @@
 package imap
 
 import (
-	"io"
+	"bufio"
+	"bytes"
+	"io/ioutil"
 	"net/mail"
 	"strings"
 	"time"
@@ -28,6 +30,7 @@ import (
 	"github.com/ProtonMail/proton-bridge/pkg/message"
 	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
 	"github.com/emersion/go-imap"
+	"github.com/emersion/go-message/textproto"
 	"github.com/pkg/errors"
 )
 
@@ -43,11 +46,15 @@ func (im *imapMailbox) CreateMessage(flags []string, date time.Time, body imap.L
 	}, "APPEND", flags, date)
 }
 
-func (im *imapMailbox) createMessage(flags []string, date time.Time, body imap.Literal) error { //nolint[funlen]
+func (im *imapMailbox) createMessage(imapFlags []string, date time.Time, r imap.Literal) error { //nolint[funlen]
 	// Called from go-imap in goroutines - we need to handle panics for each function.
 	defer im.panicHandler.HandlePanic()
 
-	m, _, _, readers, err := message.Parse(body)
+	// NOTE: Is this lock meant to be here?
+	im.user.appendExpungeLock.Lock()
+	defer im.user.appendExpungeLock.Unlock()
+
+	body, err := ioutil.ReadAll(r)
 	if err != nil {
 		return err
 	}
@@ -56,112 +63,87 @@ func (im *imapMailbox) createMessage(flags []string, date time.Time, body imap.L
 	if addr == nil {
 		return errors.New("no available address for encryption")
 	}
-	m.AddressID = addr.ID
 
 	kr, err := im.user.client().KeyRingForAddressID(addr.ID)
 	if err != nil {
 		return err
 	}
 
-	// Handle imported messages which have no "Sender" address.
+	if im.storeMailbox.LabelID() == pmapi.DraftLabel {
-	// This sometimes occurs with outlook which reports errors as imported emails or for drafts.
+		return im.createDraftMessage(kr, addr.Email, body)
+	}
+
+	if im.storeMailbox.LabelID() == pmapi.SentLabel {
+		m, _, _, _, err := message.Parse(bytes.NewReader(body))
+		if err != nil {
+			return err
+		}
+
 		if m.Sender == nil {
-		im.log.Warning("Append: Missing email sender. Will use main address")
+			m.Sender = &mail.Address{Address: addr.Email}
-		m.Sender = &mail.Address{
+		}
-			Name:    "",
+
-			Address: addr.Email,
+		if user, err := im.user.backend.bridge.GetUser(pmapi.SanitizeEmail(m.Sender.Address)); err == nil && user.ID() == im.storeUser.UserID() {
+			logEntry := im.log.WithField("sender", m.Sender).WithField("extID", m.Header.Get("Message-Id")).WithField("date", date)
+
+			if foundUID := im.storeMailbox.GetUIDByHeader(&m.Header); foundUID != uint32(0) {
+				logEntry.Info("Ignoring APPEND of duplicate to Sent folder")
+				return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), &uidplus.OrderedSeq{foundUID})
+			}
+
+			logEntry.Info("No matching UID, continuing APPEND to Sent")
 		}
 	}
 
-	// "Drafts" needs to call special API routes.
+	hdr, err := textproto.ReadHeader(bufio.NewReader(bytes.NewReader(body)))
-	// Clients always append the whole message again and remove the old one.
+	if err != nil {
-	if im.storeMailbox.LabelID() == pmapi.DraftLabel {
+		return err
-		// Sender address needs to be sanitised (drafts need to match cases exactly).
+	}
-		m.Sender.Address = pmapi.ConstructAddress(m.Sender.Address, addr.Email)
+
+	// Avoid appending a message which is already on the server. Apply the new label instead.
+	// This always happens with Outlook because it uses APPEND instead of COPY.
+	internalID := hdr.Get("X-Pm-Internal-Id")
+
+	// In case there is a mail client which corrupts headers, try "References" too.
+	if internalID == "" {
+		if references := strings.Fields(hdr.Get("References")); len(references) > 0 {
+			if match := pmapi.RxInternalReferenceFormat.FindStringSubmatch(references[len(references)-1]); len(match) == 2 {
+				internalID = match[1]
+			}
+		}
+	}
+
+	if internalID != "" {
+		if msg, err := im.storeMailbox.GetMessage(internalID); err == nil {
+			if im.user.user.IsCombinedAddressMode() || im.storeAddress.AddressID() == msg.Message().AddressID {
+				return im.labelExistingMessage(msg.ID(), msg.IsMarkedDeleted())
+			}
+		}
+	}
+
+	return im.importMessage(kr, hdr, body, imapFlags, date)
+}
+
+func (im *imapMailbox) createDraftMessage(kr *crypto.KeyRing, email string, body []byte) error {
+	im.log.Info("Creating draft message")
+
+	m, _, _, readers, err := message.Parse(bytes.NewReader(body))
+	if err != nil {
+		return err
+	}
+
+	m.Sender.Address = pmapi.ConstructAddress(m.Sender.Address, email)
 
 	draft, _, err := im.user.storeUser.CreateDraft(kr, m, readers, "", "", "")
 	if err != nil {
 		return errors.Wrap(err, "failed to create draft")
 	}
 
-		targetSeq := im.storeMailbox.GetUIDList([]string{draft.ID})
+	return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), im.storeMailbox.GetUIDList([]string{draft.ID}))
-		return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), targetSeq)
+}
-	}
 
-	// We need to make sure this is an import, and not a sent message from this account
+func (im *imapMailbox) labelExistingMessage(messageID string, isDeleted bool) error {
-	// (sent messages from the account will be added by the event loop).
+	im.log.Info("Labelling existing message")
-	if im.storeMailbox.LabelID() == pmapi.SentLabel {
-		sanitizedSender := pmapi.SanitizeEmail(m.Sender.Address)
-
-		// Check whether this message was sent by a bridge user.
-		user, err := im.user.backend.bridge.GetUser(sanitizedSender)
-		if err == nil && user.ID() == im.storeUser.UserID() {
-			logEntry := im.log.WithField("addr", sanitizedSender).WithField("extID", m.Header.Get("Message-Id"))
-
-			// If we find the message in the store already, we can skip importing it.
-			if foundUID := im.storeMailbox.GetUIDByHeader(&m.Header); foundUID != uint32(0) {
-				logEntry.Info("Ignoring APPEND of duplicate to Sent folder")
-				return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), &uidplus.OrderedSeq{foundUID})
-			}
-
-			// We didn't find the message in the store, so we are currently sending it.
-			logEntry.WithField("time", date).Info("No matching UID, continuing APPEND to Sent")
-		}
-	}
-
-	message.ParseFlags(m, flags)
-	if !date.IsZero() {
-		m.Time = date.Unix()
-	}
-
-	internalID := m.Header.Get("X-Pm-Internal-Id")
-	references := m.Header.Get("References")
-	referenceList := strings.Fields(references)
-
-	// In case there is a mail client which corrupts headers, try
-	// "References" too.
-	if internalID == "" && len(referenceList) > 0 {
-		lastReference := referenceList[len(referenceList)-1]
-		match := pmapi.RxInternalReferenceFormat.FindStringSubmatch(lastReference)
-		if len(match) == 2 {
-			internalID = match[1]
-		}
-	}
-
-	im.user.appendExpungeLock.Lock()
-	defer im.user.appendExpungeLock.Unlock()
-
-	// Avoid appending a message which is already on the server. Apply the
-	// new label instead. This always happens with Outlook (it uses APPEND
-	// instead of COPY).
-	if internalID != "" {
-		// Check to see if this belongs to a different address in split mode or another ProtonMail account.
-		msg, err := im.storeMailbox.GetMessage(internalID)
-		if err == nil && (im.user.user.IsCombinedAddressMode() || (im.storeAddress.AddressID() == msg.Message().AddressID)) {
-			IDs := []string{internalID}
-
-			// See the comment bellow.
-			if msg.IsMarkedDeleted() {
-				if err := im.storeMailbox.MarkMessagesUndeleted(IDs); err != nil {
-					log.WithError(err).Error("Failed to undelete re-imported internal message")
-				}
-			}
-
-			err = im.storeMailbox.LabelMessages(IDs)
-			if err != nil {
-				return err
-			}
-
-			targetSeq := im.storeMailbox.GetUIDList(IDs)
-			return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), targetSeq)
-		}
-	}
-
-	im.log.Info("Importing external message")
-	if err := im.importMessage(m, readers, kr); err != nil {
-		im.log.Error("Import failed: ", err)
-		return err
-	}
 
 	// IMAP clients can move message to local folder (setting \Deleted flag)
 	// and then move it back (IMAP client does not remember the message,
@@ -170,29 +152,76 @@ func (im *imapMailbox) createMessage(flags []string, date time.Time, body imap.L
 	// not delete the message (EXPUNGE would delete the original message and
 	// the new duplicate one would stay). API detects duplicates; therefore
 	// we need to remove \Deleted flag if IMAP client re-imports.
-	msg, err := im.storeMailbox.GetMessage(m.ID)
+	if isDeleted {
-	if err == nil && msg.IsMarkedDeleted() {
+		if err := im.storeMailbox.MarkMessagesUndeleted([]string{messageID}); err != nil {
-		if err := im.storeMailbox.MarkMessagesUndeleted([]string{m.ID}); err != nil {
 			log.WithError(err).Error("Failed to undelete re-imported message")
 		}
 	}
 
-	targetSeq := im.storeMailbox.GetUIDList([]string{m.ID})
+	if err := im.storeMailbox.LabelMessages([]string{messageID}); err != nil {
-	return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), targetSeq)
+		return err
+	}
+
+	return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), im.storeMailbox.GetUIDList([]string{messageID}))
 }
 
-func (im *imapMailbox) importMessage(m *pmapi.Message, readers []io.Reader, kr *crypto.KeyRing) (err error) {
+func (im *imapMailbox) importMessage(kr *crypto.KeyRing, hdr textproto.Header, body []byte, imapFlags []string, date time.Time) error {
-	body, err := message.BuildEncrypted(m, readers, kr)
+	im.log.Info("Importing external message")
+
+	var (
+		seen     bool
+		flags    int64
+		labelIDs []string
+		time     int64
+	)
+
+	if hdr.Get("received") == "" {
+		flags = pmapi.FlagSent
+	} else {
+		flags = pmapi.FlagReceived
+	}
+
+	for _, flag := range imapFlags {
+		switch flag {
+		case imap.DraftFlag:
+			flags &= ^pmapi.FlagSent
+			flags &= ^pmapi.FlagReceived
+
+		case imap.SeenFlag:
+			seen = true
+
+		case imap.FlaggedFlag:
+			labelIDs = append(labelIDs, pmapi.StarredLabel)
+
+		case imap.AnsweredFlag:
+			flags |= pmapi.FlagReplied
+		}
+	}
+
+	if !date.IsZero() {
+		time = date.Unix()
+	}
+
+	enc, err := message.EncryptRFC822(kr, bytes.NewReader(body))
 	if err != nil {
 		return err
 	}
 
-	labels := []string{}
+	messageID, err := im.storeMailbox.ImportMessage(enc, seen, labelIDs, flags, time)
-	for _, l := range m.LabelIDs {
+	if err != nil {
-		if l == pmapi.StarredLabel {
+		return err
-			labels = append(labels, pmapi.StarredLabel)
+	}
+
+	msg, err := im.storeMailbox.GetMessage(messageID)
+	if err != nil {
+		return err
+	}
+
+	if msg.IsMarkedDeleted() {
+		if err := im.storeMailbox.MarkMessagesUndeleted([]string{messageID}); err != nil {
+			log.WithError(err).Error("Failed to undelete re-imported message")
 		}
 	}
 
-	return im.storeMailbox.ImportMessage(m, body, labels)
+	return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), im.storeMailbox.GetUIDList([]string{messageID}))
 }

internal/imap/mailbox_header.go

@@ -18,13 +18,11 @@
 package imap
 
 import (
-	"bufio"
 	"bytes"
-	"io"
 	"strings"
 
+	"github.com/ProtonMail/proton-bridge/pkg/message"
 	"github.com/emersion/go-imap"
-	"github.com/pkg/errors"
 )
 
 func filterHeader(header []byte, section *imap.BodySectionName) []byte {
@@ -53,7 +51,7 @@ func filterHeader(header []byte, section *imap.BodySectionName) []byte {
 func filterHeaderLines(header []byte, wantField func(string) bool) []byte {
 	var res []byte
 
-	for _, line := range headerLines(header) {
+	for _, line := range message.HeaderLines(header) {
 		if len(bytes.TrimSpace(line)) == 0 {
 			res = append(res, line...)
 		} else {
@@ -71,34 +69,3 @@ func filterHeaderLines(header []byte, wantField func(string) bool) []byte {
 
 	return res
 }
-
-// NOTE: This sucks because we trim and split stuff here already, only to do it again when we use this function!
-func headerLines(header []byte) [][]byte {
-	var lines [][]byte
-
-	r := bufio.NewReader(bytes.NewReader(header))
-
-	for {
-		b, err := r.ReadBytes('\n')
-		if err != nil {
-			if err != io.EOF {
-				panic(errors.Wrap(err, "failed to read header line"))
-			}
-
-			break
-		}
-
-		switch {
-		case len(bytes.TrimSpace(b)) == 0:
-			lines = append(lines, b)
-
-		case len(bytes.SplitN(b, []byte(": "), 2)) != 2:
-			lines[len(lines)-1] = append(lines[len(lines)-1], b...)
-
-		default:
-			lines = append(lines, b)
-		}
-	}
-
-	return lines
-}

internal/imap/store.go

@@ -89,7 +89,7 @@ type storeMailboxProvider interface {
 	MarkMessagesUnstarred(apiID []string) error
 	MarkMessagesDeleted(apiID []string) error
 	MarkMessagesUndeleted(apiID []string) error
-	ImportMessage(msg *pmapi.Message, body []byte, labelIDs []string) error
+	ImportMessage(enc []byte, seen bool, labelIDs []string, flags, time int64) (string, error)
 	RemoveDeleted(apiIDs []string) error
 }
 

internal/store/mailbox_message.go

@@ -48,9 +48,7 @@ func (storeMailbox *Mailbox) FetchMessage(apiID string) (*Message, error) {
 	return newStoreMessage(storeMailbox, msg), nil
 }
 
-// ImportMessage imports the message by calling an API.
+func (storeMailbox *Mailbox) ImportMessage(enc []byte, seen bool, labelIDs []string, flags, time int64) (string, error) {
-// It has to be propagated to all mailboxes which is done by the event loop.
-func (storeMailbox *Mailbox) ImportMessage(msg *pmapi.Message, body []byte, labelIDs []string) error {
 	defer storeMailbox.pollNow()
 
 	if storeMailbox.labelID != pmapi.AllMailLabel {
@@ -59,24 +57,25 @@ func (storeMailbox *Mailbox) ImportMessage(msg *pmapi.Message, body []byte, labe
 
 	importReqs := &pmapi.ImportMsgReq{
 		Metadata: &pmapi.ImportMetadata{
-			AddressID: msg.AddressID,
+			AddressID: storeMailbox.storeAddress.addressID,
-			Unread:    msg.Unread,
+			Unread:    pmapi.Boolean(!seen),
-			Flags:     msg.Flags,
+			Flags:     flags,
-			Time:      msg.Time,
+			Time:      time,
 			LabelIDs:  labelIDs,
 		},
-		Message: body,
+		Message: append(enc, "\r\n"...),
 	}
 
 	res, err := storeMailbox.client().Import(exposeContextForIMAP(), pmapi.ImportMsgReqs{importReqs})
 	if err != nil {
-		return err
+		return "", err
 	}
+
 	if len(res) == 0 {
-		return errors.New("no import response")
+		return "", errors.New("no import response")
 	}
-	msg.ID = res[0].MessageID
+
-	return res[0].Error
+	return res[0].MessageID, res[0].Error
 }
 
 // LabelMessages adds the label by calling an API.

pkg/message/encrypt.go

@@ -0,0 +1,244 @@
+// Copyright (c) 2021 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package message
+
+import (
+	"bytes"
+	"encoding/base64"
+	"io"
+	"io/ioutil"
+	"mime"
+	"mime/quotedprintable"
+	"strings"
+
+	"github.com/ProtonMail/gopenpgp/v2/crypto"
+	pmmime "github.com/ProtonMail/proton-bridge/pkg/mime"
+	"github.com/emersion/go-message/textproto"
+)
+
+func EncryptRFC822(kr *crypto.KeyRing, r io.Reader) ([]byte, error) {
+	b, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+
+	header, body, err := readHeaderBody(b)
+	if err != nil {
+		return nil, err
+	}
+
+	buf := new(bytes.Buffer)
+
+	result, err := writeEncryptedPart(kr, header, bytes.NewReader(body))
+	if err != nil {
+		return nil, err
+	}
+
+	if err := textproto.WriteHeader(buf, *header); err != nil {
+		return nil, err
+	}
+
+	if _, err := result.WriteTo(buf); err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
+
+func writeEncryptedPart(kr *crypto.KeyRing, header *textproto.Header, r io.Reader) (io.WriterTo, error) {
+	decoder := getTransferDecoder(r, header.Get("Content-Transfer-Encoding"))
+	encoded := new(bytes.Buffer)
+
+	contentType, contentParams, err := parseContentType(header.Get("Content-Type"))
+	if err != nil {
+		return nil, err
+	}
+
+	switch {
+	case contentType == "", strings.HasPrefix(contentType, "text/"), strings.HasPrefix(contentType, "message/"):
+		header.Del("Content-Transfer-Encoding")
+
+		if charset, ok := contentParams["charset"]; ok {
+			if reader, err := pmmime.CharsetReader(charset, decoder); err == nil {
+				decoder = reader
+
+				// We can decode the charset to utf-8 so let's set that as the content type charset parameter.
+				contentParams["charset"] = "utf-8"
+
+				header.Set("Content-Type", mime.FormatMediaType(contentType, contentParams))
+			}
+		}
+
+		if err := encode(&writeCloser{encoded}, func(w io.Writer) error {
+			return writeEncryptedTextPart(w, decoder, kr)
+		}); err != nil {
+			return nil, err
+		}
+
+	case contentType == "multipart/encrypted":
+		if _, err := encoded.ReadFrom(decoder); err != nil {
+			return nil, err
+		}
+
+	case strings.HasPrefix(contentType, "multipart/"):
+		if err := encode(&writeCloser{encoded}, func(w io.Writer) error {
+			return writeEncryptedMultiPart(kr, w, header, decoder)
+		}); err != nil {
+			return nil, err
+		}
+
+	default:
+		header.Set("Content-Transfer-Encoding", "base64")
+
+		if err := encode(base64.NewEncoder(base64.StdEncoding, encoded), func(w io.Writer) error {
+			return writeEncryptedAttachmentPart(w, decoder, kr)
+		}); err != nil {
+			return nil, err
+		}
+	}
+
+	return encoded, nil
+}
+
+func writeEncryptedTextPart(w io.Writer, r io.Reader, kr *crypto.KeyRing) error {
+	dec, err := ioutil.ReadAll(r)
+	if err != nil {
+		return err
+	}
+
+	var arm string
+
+	if msg, err := crypto.NewPGPMessageFromArmored(string(dec)); err != nil {
+		enc, err := kr.Encrypt(crypto.NewPlainMessage(dec), kr)
+		if err != nil {
+			return err
+		}
+
+		if arm, err = enc.GetArmored(); err != nil {
+			return err
+		}
+	} else if arm, err = msg.GetArmored(); err != nil {
+		return err
+	}
+
+	if _, err := io.WriteString(w, arm); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func writeEncryptedAttachmentPart(w io.Writer, r io.Reader, kr *crypto.KeyRing) error {
+	dec, err := ioutil.ReadAll(r)
+	if err != nil {
+		return err
+	}
+
+	enc, err := kr.Encrypt(crypto.NewPlainMessage(dec), kr)
+	if err != nil {
+		return err
+	}
+
+	if _, err := w.Write(enc.GetBinary()); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func writeEncryptedMultiPart(kr *crypto.KeyRing, w io.Writer, header *textproto.Header, r io.Reader) error {
+	_, contentParams, err := parseContentType(header.Get("Content-Type"))
+	if err != nil {
+		return err
+	}
+
+	scanner, err := newPartScanner(r, contentParams["boundary"])
+	if err != nil {
+		return err
+	}
+
+	parts, err := scanner.scanAll()
+	if err != nil {
+		return err
+	}
+
+	writer := newPartWriter(w, contentParams["boundary"])
+
+	for _, part := range parts {
+		header, body, err := readHeaderBody(part.b)
+		if err != nil {
+			return err
+		}
+
+		result, err := writeEncryptedPart(kr, header, bytes.NewReader(body))
+		if err != nil {
+			return err
+		}
+
+		if err := writer.createPart(func(w io.Writer) error {
+			if err := textproto.WriteHeader(w, *header); err != nil {
+				return err
+			}
+
+			if _, err := result.WriteTo(w); err != nil {
+				return err
+			}
+
+			return nil
+		}); err != nil {
+			return err
+		}
+	}
+
+	return writer.done()
+}
+
+func getTransferDecoder(r io.Reader, encoding string) io.Reader {
+	switch strings.ToLower(encoding) {
+	case "base64":
+		return base64.NewDecoder(base64.StdEncoding, r)
+
+	case "quoted-printable":
+		return quotedprintable.NewReader(r)
+
+	default:
+		return r
+	}
+}
+
+func encode(wc io.WriteCloser, fn func(io.Writer) error) error {
+	if err := fn(wc); err != nil {
+		return err
+	}
+
+	return wc.Close()
+}
+
+type writeCloser struct {
+	io.Writer
+}
+
+func (writeCloser) Close() error { return nil }
+
+func parseContentType(val string) (string, map[string]string, error) {
+	if val == "" {
+		val = "text/plain"
+	}
+
+	return pmmime.ParseMediaType(val)
+}

pkg/message/encrypt_test.go

@@ -0,0 +1,101 @@
+// Copyright (c) 2021 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package message
+
+import (
+	"bytes"
+	"io/ioutil"
+	"testing"
+
+	"github.com/ProtonMail/gopenpgp/v2/crypto"
+	"github.com/stretchr/testify/require"
+)
+
+func TestEncryptRFC822(t *testing.T) {
+	literal, err := ioutil.ReadFile("testdata/text_plain_latin1.eml")
+	require.NoError(t, err)
+
+	key, err := crypto.GenerateKey("name", "email", "rsa", 2048)
+	require.NoError(t, err)
+
+	kr, err := crypto.NewKeyRing(key)
+	require.NoError(t, err)
+
+	enc, err := EncryptRFC822(kr, bytes.NewReader(literal))
+	require.NoError(t, err)
+
+	section(t, enc).
+		expectContentType(is(`text/plain`)).
+		expectContentTypeParam(`charset`, is(`utf-8`)).
+		expectBody(decryptsTo(kr, `ééééééé`))
+}
+
+func TestEncryptRFC822Multipart(t *testing.T) {
+	literal, err := ioutil.ReadFile("testdata/multipart_alternative_nested.eml")
+	require.NoError(t, err)
+
+	key, err := crypto.GenerateKey("name", "email", "rsa", 2048)
+	require.NoError(t, err)
+
+	kr, err := crypto.NewKeyRing(key)
+	require.NoError(t, err)
+
+	enc, err := EncryptRFC822(kr, bytes.NewReader(literal))
+	require.NoError(t, err)
+
+	section(t, enc).
+		expectContentType(is(`multipart/alternative`))
+
+	section(t, enc, 1).
+		expectContentType(is(`multipart/alternative`))
+
+	section(t, enc, 1, 1).
+		expectContentType(is(`text/plain`)).
+		expectBody(decryptsTo(kr, "*multipart 1.1*\n\n"))
+
+	section(t, enc, 1, 2).
+		expectContentType(is(`text/html`)).
+		expectBody(decryptsTo(kr, `<html>
+  <head>
+    <meta http-equiv="content-type" content="text/html; charset=UTF-8">
+  </head>
+  <body>
+    <b>multipart 1.2</b>
+  </body>
+</html>
+`))
+
+	section(t, enc, 2).
+		expectContentType(is(`multipart/alternative`))
+
+	section(t, enc, 2, 1).
+		expectContentType(is(`text/plain`)).
+		expectBody(decryptsTo(kr, "*multipart 2.1*\n\n"))
+
+	section(t, enc, 2, 2).
+		expectContentType(is(`text/html`)).
+		expectBody(decryptsTo(kr, `<html>
+  <head>
+    <meta http-equiv="content-type" content="text/html; charset=UTF-8">
+  </head>
+  <body>
+    <b>multipart 2.2</b>
+  </body>
+</html>
+`))
+}

pkg/message/flags.go

@@ -59,30 +59,3 @@ func GetFlags(m *pmapi.Message) (flags []string) {
 
 	return
 }
-
-// ParseFlags sets attributes to pmapi messages based on imap flags.
-func ParseFlags(m *pmapi.Message, flags []string) {
-	if m.Header.Get("received") == "" {
-		m.Flags = pmapi.FlagSent
-	} else {
-		m.Flags = pmapi.FlagReceived
-	}
-
-	m.Unread = true
-	for _, f := range flags {
-		switch f {
-		case imap.SeenFlag:
-			m.Unread = false
-		case imap.DraftFlag:
-			m.Flags &= ^pmapi.FlagSent
-			m.Flags &= ^pmapi.FlagReceived
-			m.LabelIDs = append(m.LabelIDs, pmapi.DraftLabel)
-		case imap.FlaggedFlag:
-			m.LabelIDs = append(m.LabelIDs, pmapi.StarredLabel)
-		case imap.AnsweredFlag:
-			m.Flags |= pmapi.FlagReplied
-		case AppleMailJunkFlag, ThunderbirdJunkFlag:
-			m.LabelIDs = append(m.LabelIDs, pmapi.SpamLabel)
-		}
-	}
-}

pkg/message/header.go

@@ -0,0 +1,123 @@
+// Copyright (c) 2021 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package message
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"io/ioutil"
+
+	"github.com/emersion/go-message/textproto"
+	"github.com/pkg/errors"
+)
+
+// HeaderLines returns each line in the given header.
+func HeaderLines(header []byte) [][]byte {
+	var (
+		lines [][]byte
+		quote int
+	)
+
+	forEachLine(bufio.NewReader(bytes.NewReader(header)), func(line []byte) {
+		switch {
+		case len(bytes.TrimSpace(line)) == 0:
+			lines = append(lines, line)
+
+		case quote%2 != 0, len(bytes.SplitN(line, []byte(`: `), 2)) != 2:
+			if len(lines) > 0 {
+				lines[len(lines)-1] = append(lines[len(lines)-1], line...)
+			} else {
+				lines = append(lines, line)
+			}
+
+		default:
+			lines = append(lines, line)
+		}
+
+		quote += bytes.Count(line, []byte(`"`))
+	})
+
+	return lines
+}
+
+func forEachLine(br *bufio.Reader, fn func([]byte)) {
+	for {
+		b, err := br.ReadBytes('\n')
+		if err != nil {
+			if !errors.Is(err, io.EOF) {
+				panic(err)
+			}
+
+			if len(b) > 0 {
+				fn(b)
+			}
+
+			return
+		}
+
+		fn(b)
+	}
+}
+
+func readHeaderBody(b []byte) (*textproto.Header, []byte, error) {
+	rawHeader, body, err := splitHeaderBody(b)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var header textproto.Header
+
+	for _, line := range HeaderLines(rawHeader) {
+		if len(bytes.TrimSpace(line)) > 0 {
+			header.AddRaw(line)
+		}
+	}
+
+	return &header, body, nil
+}
+
+func splitHeaderBody(b []byte) ([]byte, []byte, error) {
+	br := bufio.NewReader(bytes.NewReader(b))
+
+	var header []byte
+
+	for {
+		b, err := br.ReadBytes('\n')
+		if err != nil {
+			if !errors.Is(err, io.EOF) {
+				panic(err)
+			}
+
+			break
+		}
+
+		header = append(header, b...)
+
+		if len(bytes.TrimSpace(b)) == 0 {
+			break
+		}
+	}
+
+	body, err := ioutil.ReadAll(br)
+	if err != nil && !errors.Is(err, io.EOF) {
+		return nil, nil, err
+	}
+
+	return header, body, nil
+}

pkg/message/header_test.go

@@ -0,0 +1,76 @@
+// Copyright (c) 2021 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package message
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHeaderLines(t *testing.T) {
+	const header = "To: somebody\r\nFrom: somebody else\r\nSubject: this is\r\n\ta multiline field\r\n\r\n"
+
+	assert.Equal(t, [][]byte{
+		[]byte("To: somebody\r\n"),
+		[]byte("From: somebody else\r\n"),
+		[]byte("Subject: this is\r\n\ta multiline field\r\n"),
+		[]byte("\r\n"),
+	}, HeaderLines([]byte(header)))
+}
+
+func TestHeaderLinesMultilineFilename(t *testing.T) {
+	const header = "Content-Type: application/msword; name=\"this is a very long\nfilename.doc\""
+
+	assert.Equal(t, [][]byte{
+		[]byte("Content-Type: application/msword; name=\"this is a very long\nfilename.doc\""),
+	}, HeaderLines([]byte(header)))
+}
+
+func TestHeaderLinesMultilineFilenameWithColon(t *testing.T) {
+	const header = "Content-Type: application/msword; name=\"this is a very long\nfilename: too long.doc\""
+
+	assert.Equal(t, [][]byte{
+		[]byte("Content-Type: application/msword; name=\"this is a very long\nfilename: too long.doc\""),
+	}, HeaderLines([]byte(header)))
+}
+
+func TestHeaderLinesMultilineFilenameWithColonAndNewline(t *testing.T) {
+	const header = "Content-Type: application/msword; name=\"this is a very long\nfilename: too long.doc\"\n"
+
+	assert.Equal(t, [][]byte{
+		[]byte("Content-Type: application/msword; name=\"this is a very long\nfilename: too long.doc\"\n"),
+	}, HeaderLines([]byte(header)))
+}
+
+func TestHeaderLinesMultipleMultilineFilenames(t *testing.T) {
+	const header = `Content-Type: application/msword; name="=E5=B8=B6=E6=9C=89=E5=A4=96=E5=9C=8B=E5=AD=97=E7=AC=A6=E7=9A=84=E9=99=84=E4=
+=BB=B6.DOC"
+Content-Transfer-Encoding: base64
+Content-Disposition: attachment; filename="=E5=B8=B6=E6=9C=89=E5=A4=96=E5=9C=8B=E5=AD=97=E7=AC=A6=E7=9A=84=E9=99=84=E4=
+=BB=B6.DOC"
+Content-ID: <>
+`
+
+	assert.Equal(t, [][]byte{
+		[]byte("Content-Type: application/msword; name=\"=E5=B8=B6=E6=9C=89=E5=A4=96=E5=9C=8B=E5=AD=97=E7=AC=A6=E7=9A=84=E9=99=84=E4=\n=BB=B6.DOC\"\n"),
+		[]byte("Content-Transfer-Encoding: base64\n"),
+		[]byte("Content-Disposition: attachment; filename=\"=E5=B8=B6=E6=9C=89=E5=A4=96=E5=9C=8B=E5=AD=97=E7=AC=A6=E7=9A=84=E9=99=84=E4=\n=BB=B6.DOC\"\n"),
+		[]byte("Content-ID: <>\n"),
+	}, HeaderLines([]byte(header)))
+}

pkg/message/scanner.go

@@ -0,0 +1,96 @@
+// Copyright (c) 2021 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package message
+
+import (
+	"bufio"
+	"bytes"
+	"errors"
+	"io"
+)
+
+type partScanner struct {
+	r *bufio.Reader
+
+	boundary string
+	progress int
+}
+
+type part struct {
+	b      []byte
+	offset int
+}
+
+func newPartScanner(r io.Reader, boundary string) (*partScanner, error) {
+	scanner := &partScanner{r: bufio.NewReader(r), boundary: boundary}
+
+	if _, _, err := scanner.readToBoundary(); err != nil {
+		return nil, err
+	}
+
+	return scanner, nil
+}
+
+func (s *partScanner) scanAll() ([]part, error) {
+	var parts []part
+
+	for {
+		offset := s.progress
+
+		b, more, err := s.readToBoundary()
+		if err != nil {
+			return nil, err
+		}
+
+		if !more {
+			return parts, nil
+		}
+
+		parts = append(parts, part{b: b, offset: offset})
+	}
+}
+
+func (s *partScanner) readToBoundary() ([]byte, bool, error) {
+	var res []byte
+
+	for {
+		line, err := s.r.ReadBytes('\n')
+		if err != nil {
+			if !errors.Is(err, io.EOF) {
+				return nil, false, err
+			}
+
+			if len(line) == 0 {
+				return nil, false, nil
+			}
+		}
+
+		s.progress += len(line)
+
+		switch {
+		case bytes.HasPrefix(bytes.TrimSpace(line), []byte("--"+s.boundary)):
+			return bytes.TrimSuffix(bytes.TrimSuffix(res, []byte("\n")), []byte("\r")), true, nil
+
+		case bytes.HasSuffix(bytes.TrimSpace(line), []byte(s.boundary+"--")):
+			return bytes.TrimSuffix(bytes.TrimSuffix(res, []byte("\n")), []byte("\r")), false, nil
+
+		default:
+			res = append(res, line...)
+		}
+	}
+}

pkg/message/scanner_test.go

@@ -0,0 +1,136 @@
+// Copyright (c) 2021 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package message
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestScanner(t *testing.T) {
+	const literal = `this part of the text should be ignored
+
+--longrandomstring
+
+body1
+
+--longrandomstring
+
+body2
+
+--longrandomstring--
+`
+
+	scanner, err := newPartScanner(strings.NewReader(literal), "longrandomstring")
+	require.NoError(t, err)
+
+	parts, err := scanner.scanAll()
+	require.NoError(t, err)
+
+	assert.Equal(t, "\nbody1\n", string(parts[0].b))
+	assert.Equal(t, "\nbody2\n", string(parts[1].b))
+
+	assert.Equal(t, "\nbody1\n", literal[parts[0].offset:parts[0].offset+len(parts[0].b)])
+	assert.Equal(t, "\nbody2\n", literal[parts[1].offset:parts[1].offset+len(parts[1].b)])
+}
+
+func TestScannerNested(t *testing.T) {
+	const literal = `This is the preamble.  It is to be ignored, though it 
+is a handy place for mail composers to include an 
+explanatory note to non-MIME compliant readers. 
+--simple boundary 
+Content-type: multipart/mixed; boundary="nested boundary" 
+
+This is the preamble.  It is to be ignored, though it 
+is a handy place for mail composers to include an 
+explanatory note to non-MIME compliant readers. 
+--nested boundary 
+Content-type: text/plain; charset=us-ascii
+
+This part does not end with a linebreak.
+--nested boundary 
+Content-type: text/plain; charset=us-ascii
+
+This part does end with a linebreak.
+
+--nested boundary--
+--simple boundary 
+Content-type: text/plain; charset=us-ascii
+
+This part does end with a linebreak.
+
+--simple boundary-- 
+This is the epilogue.  It is also to be ignored.
+`
+
+	scanner, err := newPartScanner(strings.NewReader(literal), "simple boundary")
+	require.NoError(t, err)
+
+	parts, err := scanner.scanAll()
+	require.NoError(t, err)
+
+	assert.Equal(t, `Content-type: multipart/mixed; boundary="nested boundary" 
+
+This is the preamble.  It is to be ignored, though it 
+is a handy place for mail composers to include an 
+explanatory note to non-MIME compliant readers. 
+--nested boundary 
+Content-type: text/plain; charset=us-ascii
+
+This part does not end with a linebreak.
+--nested boundary 
+Content-type: text/plain; charset=us-ascii
+
+This part does end with a linebreak.
+
+--nested boundary--`, string(parts[0].b))
+	assert.Equal(t, `Content-type: text/plain; charset=us-ascii
+
+This part does end with a linebreak.
+`, string(parts[1].b))
+}
+
+func TestScannerNoFinalLinebreak(t *testing.T) {
+	const literal = `--nested boundary 
+Content-type: text/plain; charset=us-ascii
+
+This part does not end with a linebreak.
+--nested boundary 
+Content-type: text/plain; charset=us-ascii
+
+This part does end with a linebreak.
+
+--nested boundary--`
+
+	scanner, err := newPartScanner(strings.NewReader(literal), "nested boundary")
+	require.NoError(t, err)
+
+	parts, err := scanner.scanAll()
+	require.NoError(t, err)
+
+	assert.Equal(t, `Content-type: text/plain; charset=us-ascii
+
+This part does not end with a linebreak.`, string(parts[0].b))
+	assert.Equal(t, `Content-type: text/plain; charset=us-ascii
+
+This part does end with a linebreak.
+`, string(parts[1].b))
+}

pkg/message/writer.go

@@ -0,0 +1,48 @@
+// Copyright (c) 2021 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package message
+
+import (
+	"fmt"
+	"io"
+)
+
+type partWriter struct {
+	w        io.Writer
+	boundary string
+}
+
+func newPartWriter(w io.Writer, boundary string) *partWriter {
+	return &partWriter{w: w, boundary: boundary}
+}
+
+func (w *partWriter) createPart(fn func(io.Writer) error) error {
+	if _, err := fmt.Fprintf(w.w, "\r\n--%v\r\n", w.boundary); err != nil {
+		return err
+	}
+
+	return fn(w.w)
+}
+
+func (w *partWriter) done() error {
+	if _, err := fmt.Fprintf(w.w, "\r\n--%v--\r\n", w.boundary); err != nil {
+		return err
+	}
+
+	return nil
+}