From e01dc77a61dd2136bed69ce2f978be08911a27d2 Mon Sep 17 00:00:00 2001 From: James Houlahan Date: Mon, 17 May 2021 15:56:22 +0200 Subject: [PATCH] GODT-1044: lite parser --- internal/imap/mailbox_append.go | 227 +++++++++++++++------------ internal/imap/mailbox_header.go | 37 +---- internal/imap/store.go | 2 +- internal/store/mailbox_message.go | 23 ++- pkg/message/encrypt.go | 244 ++++++++++++++++++++++++++++++ pkg/message/encrypt_test.go | 101 +++++++++++++ pkg/message/flags.go | 27 ---- pkg/message/header.go | 123 +++++++++++++++ pkg/message/header_test.go | 76 ++++++++++ pkg/message/scanner.go | 96 ++++++++++++ pkg/message/scanner_test.go | 136 +++++++++++++++++ pkg/message/writer.go | 48 ++++++ 12 files changed, 966 insertions(+), 174 deletions(-) create mode 100644 pkg/message/encrypt.go create mode 100644 pkg/message/encrypt_test.go create mode 100644 pkg/message/header.go create mode 100644 pkg/message/header_test.go create mode 100644 pkg/message/scanner.go create mode 100644 pkg/message/scanner_test.go create mode 100644 pkg/message/writer.go diff --git a/internal/imap/mailbox_append.go b/internal/imap/mailbox_append.go index a4f9f6d8..6cbb68fe 100644 --- a/internal/imap/mailbox_append.go +++ b/internal/imap/mailbox_append.go @@ -18,7 +18,9 @@ package imap import ( - "io" + "bufio" + "bytes" + "io/ioutil" "net/mail" "strings" "time" @@ -28,6 +30,7 @@ import ( "github.com/ProtonMail/proton-bridge/pkg/message" "github.com/ProtonMail/proton-bridge/pkg/pmapi" "github.com/emersion/go-imap" + "github.com/emersion/go-message/textproto" "github.com/pkg/errors" ) @@ -43,11 +46,15 @@ func (im *imapMailbox) CreateMessage(flags []string, date time.Time, body imap.L }, "APPEND", flags, date) } -func (im *imapMailbox) createMessage(flags []string, date time.Time, body imap.Literal) error { //nolint[funlen] +func (im *imapMailbox) createMessage(imapFlags []string, date time.Time, r imap.Literal) error { //nolint[funlen] // Called from go-imap in goroutines - we need to handle panics for each function. defer im.panicHandler.HandlePanic() - m, _, _, readers, err := message.Parse(body) + // NOTE: Is this lock meant to be here? + im.user.appendExpungeLock.Lock() + defer im.user.appendExpungeLock.Unlock() + + body, err := ioutil.ReadAll(r) if err != nil { return err } @@ -56,113 +63,88 @@ func (im *imapMailbox) createMessage(flags []string, date time.Time, body imap.L if addr == nil { return errors.New("no available address for encryption") } - m.AddressID = addr.ID kr, err := im.user.client().KeyRingForAddressID(addr.ID) if err != nil { return err } - // Handle imported messages which have no "Sender" address. - // This sometimes occurs with outlook which reports errors as imported emails or for drafts. - if m.Sender == nil { - im.log.Warning("Append: Missing email sender. Will use main address") - m.Sender = &mail.Address{ - Name: "", - Address: addr.Email, - } - } - - // "Drafts" needs to call special API routes. - // Clients always append the whole message again and remove the old one. if im.storeMailbox.LabelID() == pmapi.DraftLabel { - // Sender address needs to be sanitised (drafts need to match cases exactly). - m.Sender.Address = pmapi.ConstructAddress(m.Sender.Address, addr.Email) - - draft, _, err := im.user.storeUser.CreateDraft(kr, m, readers, "", "", "") - if err != nil { - return errors.Wrap(err, "failed to create draft") - } - - targetSeq := im.storeMailbox.GetUIDList([]string{draft.ID}) - return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), targetSeq) + return im.createDraftMessage(kr, addr.Email, body) } - // We need to make sure this is an import, and not a sent message from this account - // (sent messages from the account will be added by the event loop). if im.storeMailbox.LabelID() == pmapi.SentLabel { - sanitizedSender := pmapi.SanitizeEmail(m.Sender.Address) + m, _, _, _, err := message.Parse(bytes.NewReader(body)) + if err != nil { + return err + } - // Check whether this message was sent by a bridge user. - user, err := im.user.backend.bridge.GetUser(sanitizedSender) - if err == nil && user.ID() == im.storeUser.UserID() { - logEntry := im.log.WithField("addr", sanitizedSender).WithField("extID", m.Header.Get("Message-Id")) + if m.Sender == nil { + m.Sender = &mail.Address{Address: addr.Email} + } + + if user, err := im.user.backend.bridge.GetUser(pmapi.SanitizeEmail(m.Sender.Address)); err == nil && user.ID() == im.storeUser.UserID() { + logEntry := im.log.WithField("sender", m.Sender).WithField("extID", m.Header.Get("Message-Id")).WithField("date", date) - // If we find the message in the store already, we can skip importing it. if foundUID := im.storeMailbox.GetUIDByHeader(&m.Header); foundUID != uint32(0) { logEntry.Info("Ignoring APPEND of duplicate to Sent folder") return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), &uidplus.OrderedSeq{foundUID}) } - // We didn't find the message in the store, so we are currently sending it. - logEntry.WithField("time", date).Info("No matching UID, continuing APPEND to Sent") + logEntry.Info("No matching UID, continuing APPEND to Sent") } } - message.ParseFlags(m, flags) - if !date.IsZero() { - m.Time = date.Unix() - } - - internalID := m.Header.Get("X-Pm-Internal-Id") - references := m.Header.Get("References") - referenceList := strings.Fields(references) - - // In case there is a mail client which corrupts headers, try - // "References" too. - if internalID == "" && len(referenceList) > 0 { - lastReference := referenceList[len(referenceList)-1] - match := pmapi.RxInternalReferenceFormat.FindStringSubmatch(lastReference) - if len(match) == 2 { - internalID = match[1] - } - } - - im.user.appendExpungeLock.Lock() - defer im.user.appendExpungeLock.Unlock() - - // Avoid appending a message which is already on the server. Apply the - // new label instead. This always happens with Outlook (it uses APPEND - // instead of COPY). - if internalID != "" { - // Check to see if this belongs to a different address in split mode or another ProtonMail account. - msg, err := im.storeMailbox.GetMessage(internalID) - if err == nil && (im.user.user.IsCombinedAddressMode() || (im.storeAddress.AddressID() == msg.Message().AddressID)) { - IDs := []string{internalID} - - // See the comment bellow. - if msg.IsMarkedDeleted() { - if err := im.storeMailbox.MarkMessagesUndeleted(IDs); err != nil { - log.WithError(err).Error("Failed to undelete re-imported internal message") - } - } - - err = im.storeMailbox.LabelMessages(IDs) - if err != nil { - return err - } - - targetSeq := im.storeMailbox.GetUIDList(IDs) - return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), targetSeq) - } - } - - im.log.Info("Importing external message") - if err := im.importMessage(m, readers, kr); err != nil { - im.log.Error("Import failed: ", err) + hdr, err := textproto.ReadHeader(bufio.NewReader(bytes.NewReader(body))) + if err != nil { return err } + // Avoid appending a message which is already on the server. Apply the new label instead. + // This always happens with Outlook because it uses APPEND instead of COPY. + internalID := hdr.Get("X-Pm-Internal-Id") + + // In case there is a mail client which corrupts headers, try "References" too. + if internalID == "" { + if references := strings.Fields(hdr.Get("References")); len(references) > 0 { + if match := pmapi.RxInternalReferenceFormat.FindStringSubmatch(references[len(references)-1]); len(match) == 2 { + internalID = match[1] + } + } + } + + if internalID != "" { + if msg, err := im.storeMailbox.GetMessage(internalID); err == nil { + if im.user.user.IsCombinedAddressMode() || im.storeAddress.AddressID() == msg.Message().AddressID { + return im.labelExistingMessage(msg.ID(), msg.IsMarkedDeleted()) + } + } + } + + return im.importMessage(kr, hdr, body, imapFlags, date) +} + +func (im *imapMailbox) createDraftMessage(kr *crypto.KeyRing, email string, body []byte) error { + im.log.Info("Creating draft message") + + m, _, _, readers, err := message.Parse(bytes.NewReader(body)) + if err != nil { + return err + } + + m.Sender.Address = pmapi.ConstructAddress(m.Sender.Address, email) + + draft, _, err := im.user.storeUser.CreateDraft(kr, m, readers, "", "", "") + if err != nil { + return errors.Wrap(err, "failed to create draft") + } + + return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), im.storeMailbox.GetUIDList([]string{draft.ID})) +} + +func (im *imapMailbox) labelExistingMessage(messageID string, isDeleted bool) error { + im.log.Info("Labelling existing message") + // IMAP clients can move message to local folder (setting \Deleted flag) // and then move it back (IMAP client does not remember the message, // so instead removing the flag it imports duplicate message). @@ -170,29 +152,76 @@ func (im *imapMailbox) createMessage(flags []string, date time.Time, body imap.L // not delete the message (EXPUNGE would delete the original message and // the new duplicate one would stay). API detects duplicates; therefore // we need to remove \Deleted flag if IMAP client re-imports. - msg, err := im.storeMailbox.GetMessage(m.ID) - if err == nil && msg.IsMarkedDeleted() { - if err := im.storeMailbox.MarkMessagesUndeleted([]string{m.ID}); err != nil { + if isDeleted { + if err := im.storeMailbox.MarkMessagesUndeleted([]string{messageID}); err != nil { log.WithError(err).Error("Failed to undelete re-imported message") } } - targetSeq := im.storeMailbox.GetUIDList([]string{m.ID}) - return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), targetSeq) + if err := im.storeMailbox.LabelMessages([]string{messageID}); err != nil { + return err + } + + return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), im.storeMailbox.GetUIDList([]string{messageID})) } -func (im *imapMailbox) importMessage(m *pmapi.Message, readers []io.Reader, kr *crypto.KeyRing) (err error) { - body, err := message.BuildEncrypted(m, readers, kr) +func (im *imapMailbox) importMessage(kr *crypto.KeyRing, hdr textproto.Header, body []byte, imapFlags []string, date time.Time) error { + im.log.Info("Importing external message") + + var ( + seen bool + flags int64 + labelIDs []string + time int64 + ) + + if hdr.Get("received") == "" { + flags = pmapi.FlagSent + } else { + flags = pmapi.FlagReceived + } + + for _, flag := range imapFlags { + switch flag { + case imap.DraftFlag: + flags &= ^pmapi.FlagSent + flags &= ^pmapi.FlagReceived + + case imap.SeenFlag: + seen = true + + case imap.FlaggedFlag: + labelIDs = append(labelIDs, pmapi.StarredLabel) + + case imap.AnsweredFlag: + flags |= pmapi.FlagReplied + } + } + + if !date.IsZero() { + time = date.Unix() + } + + enc, err := message.EncryptRFC822(kr, bytes.NewReader(body)) if err != nil { return err } - labels := []string{} - for _, l := range m.LabelIDs { - if l == pmapi.StarredLabel { - labels = append(labels, pmapi.StarredLabel) + messageID, err := im.storeMailbox.ImportMessage(enc, seen, labelIDs, flags, time) + if err != nil { + return err + } + + msg, err := im.storeMailbox.GetMessage(messageID) + if err != nil { + return err + } + + if msg.IsMarkedDeleted() { + if err := im.storeMailbox.MarkMessagesUndeleted([]string{messageID}); err != nil { + log.WithError(err).Error("Failed to undelete re-imported message") } } - return im.storeMailbox.ImportMessage(m, body, labels) + return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), im.storeMailbox.GetUIDList([]string{messageID})) } diff --git a/internal/imap/mailbox_header.go b/internal/imap/mailbox_header.go index fd5d3fe0..a5978ae7 100644 --- a/internal/imap/mailbox_header.go +++ b/internal/imap/mailbox_header.go @@ -18,13 +18,11 @@ package imap import ( - "bufio" "bytes" - "io" "strings" + "github.com/ProtonMail/proton-bridge/pkg/message" "github.com/emersion/go-imap" - "github.com/pkg/errors" ) func filterHeader(header []byte, section *imap.BodySectionName) []byte { @@ -53,7 +51,7 @@ func filterHeader(header []byte, section *imap.BodySectionName) []byte { func filterHeaderLines(header []byte, wantField func(string) bool) []byte { var res []byte - for _, line := range headerLines(header) { + for _, line := range message.HeaderLines(header) { if len(bytes.TrimSpace(line)) == 0 { res = append(res, line...) } else { @@ -71,34 +69,3 @@ func filterHeaderLines(header []byte, wantField func(string) bool) []byte { return res } - -// NOTE: This sucks because we trim and split stuff here already, only to do it again when we use this function! -func headerLines(header []byte) [][]byte { - var lines [][]byte - - r := bufio.NewReader(bytes.NewReader(header)) - - for { - b, err := r.ReadBytes('\n') - if err != nil { - if err != io.EOF { - panic(errors.Wrap(err, "failed to read header line")) - } - - break - } - - switch { - case len(bytes.TrimSpace(b)) == 0: - lines = append(lines, b) - - case len(bytes.SplitN(b, []byte(": "), 2)) != 2: - lines[len(lines)-1] = append(lines[len(lines)-1], b...) - - default: - lines = append(lines, b) - } - } - - return lines -} diff --git a/internal/imap/store.go b/internal/imap/store.go index 7088cd87..918976be 100644 --- a/internal/imap/store.go +++ b/internal/imap/store.go @@ -89,7 +89,7 @@ type storeMailboxProvider interface { MarkMessagesUnstarred(apiID []string) error MarkMessagesDeleted(apiID []string) error MarkMessagesUndeleted(apiID []string) error - ImportMessage(msg *pmapi.Message, body []byte, labelIDs []string) error + ImportMessage(enc []byte, seen bool, labelIDs []string, flags, time int64) (string, error) RemoveDeleted(apiIDs []string) error } diff --git a/internal/store/mailbox_message.go b/internal/store/mailbox_message.go index 914e720a..cbdda7c3 100644 --- a/internal/store/mailbox_message.go +++ b/internal/store/mailbox_message.go @@ -48,9 +48,7 @@ func (storeMailbox *Mailbox) FetchMessage(apiID string) (*Message, error) { return newStoreMessage(storeMailbox, msg), nil } -// ImportMessage imports the message by calling an API. -// It has to be propagated to all mailboxes which is done by the event loop. -func (storeMailbox *Mailbox) ImportMessage(msg *pmapi.Message, body []byte, labelIDs []string) error { +func (storeMailbox *Mailbox) ImportMessage(enc []byte, seen bool, labelIDs []string, flags, time int64) (string, error) { defer storeMailbox.pollNow() if storeMailbox.labelID != pmapi.AllMailLabel { @@ -59,24 +57,25 @@ func (storeMailbox *Mailbox) ImportMessage(msg *pmapi.Message, body []byte, labe importReqs := &pmapi.ImportMsgReq{ Metadata: &pmapi.ImportMetadata{ - AddressID: msg.AddressID, - Unread: msg.Unread, - Flags: msg.Flags, - Time: msg.Time, + AddressID: storeMailbox.storeAddress.addressID, + Unread: pmapi.Boolean(!seen), + Flags: flags, + Time: time, LabelIDs: labelIDs, }, - Message: body, + Message: append(enc, "\r\n"...), } res, err := storeMailbox.client().Import(exposeContextForIMAP(), pmapi.ImportMsgReqs{importReqs}) if err != nil { - return err + return "", err } + if len(res) == 0 { - return errors.New("no import response") + return "", errors.New("no import response") } - msg.ID = res[0].MessageID - return res[0].Error + + return res[0].MessageID, res[0].Error } // LabelMessages adds the label by calling an API. diff --git a/pkg/message/encrypt.go b/pkg/message/encrypt.go new file mode 100644 index 00000000..7324d629 --- /dev/null +++ b/pkg/message/encrypt.go @@ -0,0 +1,244 @@ +// Copyright (c) 2021 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package message + +import ( + "bytes" + "encoding/base64" + "io" + "io/ioutil" + "mime" + "mime/quotedprintable" + "strings" + + "github.com/ProtonMail/gopenpgp/v2/crypto" + pmmime "github.com/ProtonMail/proton-bridge/pkg/mime" + "github.com/emersion/go-message/textproto" +) + +func EncryptRFC822(kr *crypto.KeyRing, r io.Reader) ([]byte, error) { + b, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + + header, body, err := readHeaderBody(b) + if err != nil { + return nil, err + } + + buf := new(bytes.Buffer) + + result, err := writeEncryptedPart(kr, header, bytes.NewReader(body)) + if err != nil { + return nil, err + } + + if err := textproto.WriteHeader(buf, *header); err != nil { + return nil, err + } + + if _, err := result.WriteTo(buf); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func writeEncryptedPart(kr *crypto.KeyRing, header *textproto.Header, r io.Reader) (io.WriterTo, error) { + decoder := getTransferDecoder(r, header.Get("Content-Transfer-Encoding")) + encoded := new(bytes.Buffer) + + contentType, contentParams, err := parseContentType(header.Get("Content-Type")) + if err != nil { + return nil, err + } + + switch { + case contentType == "", strings.HasPrefix(contentType, "text/"), strings.HasPrefix(contentType, "message/"): + header.Del("Content-Transfer-Encoding") + + if charset, ok := contentParams["charset"]; ok { + if reader, err := pmmime.CharsetReader(charset, decoder); err == nil { + decoder = reader + + // We can decode the charset to utf-8 so let's set that as the content type charset parameter. + contentParams["charset"] = "utf-8" + + header.Set("Content-Type", mime.FormatMediaType(contentType, contentParams)) + } + } + + if err := encode(&writeCloser{encoded}, func(w io.Writer) error { + return writeEncryptedTextPart(w, decoder, kr) + }); err != nil { + return nil, err + } + + case contentType == "multipart/encrypted": + if _, err := encoded.ReadFrom(decoder); err != nil { + return nil, err + } + + case strings.HasPrefix(contentType, "multipart/"): + if err := encode(&writeCloser{encoded}, func(w io.Writer) error { + return writeEncryptedMultiPart(kr, w, header, decoder) + }); err != nil { + return nil, err + } + + default: + header.Set("Content-Transfer-Encoding", "base64") + + if err := encode(base64.NewEncoder(base64.StdEncoding, encoded), func(w io.Writer) error { + return writeEncryptedAttachmentPart(w, decoder, kr) + }); err != nil { + return nil, err + } + } + + return encoded, nil +} + +func writeEncryptedTextPart(w io.Writer, r io.Reader, kr *crypto.KeyRing) error { + dec, err := ioutil.ReadAll(r) + if err != nil { + return err + } + + var arm string + + if msg, err := crypto.NewPGPMessageFromArmored(string(dec)); err != nil { + enc, err := kr.Encrypt(crypto.NewPlainMessage(dec), kr) + if err != nil { + return err + } + + if arm, err = enc.GetArmored(); err != nil { + return err + } + } else if arm, err = msg.GetArmored(); err != nil { + return err + } + + if _, err := io.WriteString(w, arm); err != nil { + return err + } + + return nil +} + +func writeEncryptedAttachmentPart(w io.Writer, r io.Reader, kr *crypto.KeyRing) error { + dec, err := ioutil.ReadAll(r) + if err != nil { + return err + } + + enc, err := kr.Encrypt(crypto.NewPlainMessage(dec), kr) + if err != nil { + return err + } + + if _, err := w.Write(enc.GetBinary()); err != nil { + return err + } + + return nil +} + +func writeEncryptedMultiPart(kr *crypto.KeyRing, w io.Writer, header *textproto.Header, r io.Reader) error { + _, contentParams, err := parseContentType(header.Get("Content-Type")) + if err != nil { + return err + } + + scanner, err := newPartScanner(r, contentParams["boundary"]) + if err != nil { + return err + } + + parts, err := scanner.scanAll() + if err != nil { + return err + } + + writer := newPartWriter(w, contentParams["boundary"]) + + for _, part := range parts { + header, body, err := readHeaderBody(part.b) + if err != nil { + return err + } + + result, err := writeEncryptedPart(kr, header, bytes.NewReader(body)) + if err != nil { + return err + } + + if err := writer.createPart(func(w io.Writer) error { + if err := textproto.WriteHeader(w, *header); err != nil { + return err + } + + if _, err := result.WriteTo(w); err != nil { + return err + } + + return nil + }); err != nil { + return err + } + } + + return writer.done() +} + +func getTransferDecoder(r io.Reader, encoding string) io.Reader { + switch strings.ToLower(encoding) { + case "base64": + return base64.NewDecoder(base64.StdEncoding, r) + + case "quoted-printable": + return quotedprintable.NewReader(r) + + default: + return r + } +} + +func encode(wc io.WriteCloser, fn func(io.Writer) error) error { + if err := fn(wc); err != nil { + return err + } + + return wc.Close() +} + +type writeCloser struct { + io.Writer +} + +func (writeCloser) Close() error { return nil } + +func parseContentType(val string) (string, map[string]string, error) { + if val == "" { + val = "text/plain" + } + + return pmmime.ParseMediaType(val) +} diff --git a/pkg/message/encrypt_test.go b/pkg/message/encrypt_test.go new file mode 100644 index 00000000..df256350 --- /dev/null +++ b/pkg/message/encrypt_test.go @@ -0,0 +1,101 @@ +// Copyright (c) 2021 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package message + +import ( + "bytes" + "io/ioutil" + "testing" + + "github.com/ProtonMail/gopenpgp/v2/crypto" + "github.com/stretchr/testify/require" +) + +func TestEncryptRFC822(t *testing.T) { + literal, err := ioutil.ReadFile("testdata/text_plain_latin1.eml") + require.NoError(t, err) + + key, err := crypto.GenerateKey("name", "email", "rsa", 2048) + require.NoError(t, err) + + kr, err := crypto.NewKeyRing(key) + require.NoError(t, err) + + enc, err := EncryptRFC822(kr, bytes.NewReader(literal)) + require.NoError(t, err) + + section(t, enc). + expectContentType(is(`text/plain`)). + expectContentTypeParam(`charset`, is(`utf-8`)). + expectBody(decryptsTo(kr, `ééééééé`)) +} + +func TestEncryptRFC822Multipart(t *testing.T) { + literal, err := ioutil.ReadFile("testdata/multipart_alternative_nested.eml") + require.NoError(t, err) + + key, err := crypto.GenerateKey("name", "email", "rsa", 2048) + require.NoError(t, err) + + kr, err := crypto.NewKeyRing(key) + require.NoError(t, err) + + enc, err := EncryptRFC822(kr, bytes.NewReader(literal)) + require.NoError(t, err) + + section(t, enc). + expectContentType(is(`multipart/alternative`)) + + section(t, enc, 1). + expectContentType(is(`multipart/alternative`)) + + section(t, enc, 1, 1). + expectContentType(is(`text/plain`)). + expectBody(decryptsTo(kr, "*multipart 1.1*\n\n")) + + section(t, enc, 1, 2). + expectContentType(is(`text/html`)). + expectBody(decryptsTo(kr, ` + + + + + multipart 1.2 + + +`)) + + section(t, enc, 2). + expectContentType(is(`multipart/alternative`)) + + section(t, enc, 2, 1). + expectContentType(is(`text/plain`)). + expectBody(decryptsTo(kr, "*multipart 2.1*\n\n")) + + section(t, enc, 2, 2). + expectContentType(is(`text/html`)). + expectBody(decryptsTo(kr, ` + + + + + multipart 2.2 + + +`)) +} diff --git a/pkg/message/flags.go b/pkg/message/flags.go index 5ed2c2c6..9b4cf1b0 100644 --- a/pkg/message/flags.go +++ b/pkg/message/flags.go @@ -59,30 +59,3 @@ func GetFlags(m *pmapi.Message) (flags []string) { return } - -// ParseFlags sets attributes to pmapi messages based on imap flags. -func ParseFlags(m *pmapi.Message, flags []string) { - if m.Header.Get("received") == "" { - m.Flags = pmapi.FlagSent - } else { - m.Flags = pmapi.FlagReceived - } - - m.Unread = true - for _, f := range flags { - switch f { - case imap.SeenFlag: - m.Unread = false - case imap.DraftFlag: - m.Flags &= ^pmapi.FlagSent - m.Flags &= ^pmapi.FlagReceived - m.LabelIDs = append(m.LabelIDs, pmapi.DraftLabel) - case imap.FlaggedFlag: - m.LabelIDs = append(m.LabelIDs, pmapi.StarredLabel) - case imap.AnsweredFlag: - m.Flags |= pmapi.FlagReplied - case AppleMailJunkFlag, ThunderbirdJunkFlag: - m.LabelIDs = append(m.LabelIDs, pmapi.SpamLabel) - } - } -} diff --git a/pkg/message/header.go b/pkg/message/header.go new file mode 100644 index 00000000..c4cf9659 --- /dev/null +++ b/pkg/message/header.go @@ -0,0 +1,123 @@ +// Copyright (c) 2021 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package message + +import ( + "bufio" + "bytes" + "io" + "io/ioutil" + + "github.com/emersion/go-message/textproto" + "github.com/pkg/errors" +) + +// HeaderLines returns each line in the given header. +func HeaderLines(header []byte) [][]byte { + var ( + lines [][]byte + quote int + ) + + forEachLine(bufio.NewReader(bytes.NewReader(header)), func(line []byte) { + switch { + case len(bytes.TrimSpace(line)) == 0: + lines = append(lines, line) + + case quote%2 != 0, len(bytes.SplitN(line, []byte(`: `), 2)) != 2: + if len(lines) > 0 { + lines[len(lines)-1] = append(lines[len(lines)-1], line...) + } else { + lines = append(lines, line) + } + + default: + lines = append(lines, line) + } + + quote += bytes.Count(line, []byte(`"`)) + }) + + return lines +} + +func forEachLine(br *bufio.Reader, fn func([]byte)) { + for { + b, err := br.ReadBytes('\n') + if err != nil { + if !errors.Is(err, io.EOF) { + panic(err) + } + + if len(b) > 0 { + fn(b) + } + + return + } + + fn(b) + } +} + +func readHeaderBody(b []byte) (*textproto.Header, []byte, error) { + rawHeader, body, err := splitHeaderBody(b) + if err != nil { + return nil, nil, err + } + + var header textproto.Header + + for _, line := range HeaderLines(rawHeader) { + if len(bytes.TrimSpace(line)) > 0 { + header.AddRaw(line) + } + } + + return &header, body, nil +} + +func splitHeaderBody(b []byte) ([]byte, []byte, error) { + br := bufio.NewReader(bytes.NewReader(b)) + + var header []byte + + for { + b, err := br.ReadBytes('\n') + if err != nil { + if !errors.Is(err, io.EOF) { + panic(err) + } + + break + } + + header = append(header, b...) + + if len(bytes.TrimSpace(b)) == 0 { + break + } + } + + body, err := ioutil.ReadAll(br) + if err != nil && !errors.Is(err, io.EOF) { + return nil, nil, err + } + + return header, body, nil +} diff --git a/pkg/message/header_test.go b/pkg/message/header_test.go new file mode 100644 index 00000000..89968740 --- /dev/null +++ b/pkg/message/header_test.go @@ -0,0 +1,76 @@ +// Copyright (c) 2021 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package message + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestHeaderLines(t *testing.T) { + const header = "To: somebody\r\nFrom: somebody else\r\nSubject: this is\r\n\ta multiline field\r\n\r\n" + + assert.Equal(t, [][]byte{ + []byte("To: somebody\r\n"), + []byte("From: somebody else\r\n"), + []byte("Subject: this is\r\n\ta multiline field\r\n"), + []byte("\r\n"), + }, HeaderLines([]byte(header))) +} + +func TestHeaderLinesMultilineFilename(t *testing.T) { + const header = "Content-Type: application/msword; name=\"this is a very long\nfilename.doc\"" + + assert.Equal(t, [][]byte{ + []byte("Content-Type: application/msword; name=\"this is a very long\nfilename.doc\""), + }, HeaderLines([]byte(header))) +} + +func TestHeaderLinesMultilineFilenameWithColon(t *testing.T) { + const header = "Content-Type: application/msword; name=\"this is a very long\nfilename: too long.doc\"" + + assert.Equal(t, [][]byte{ + []byte("Content-Type: application/msword; name=\"this is a very long\nfilename: too long.doc\""), + }, HeaderLines([]byte(header))) +} + +func TestHeaderLinesMultilineFilenameWithColonAndNewline(t *testing.T) { + const header = "Content-Type: application/msword; name=\"this is a very long\nfilename: too long.doc\"\n" + + assert.Equal(t, [][]byte{ + []byte("Content-Type: application/msword; name=\"this is a very long\nfilename: too long.doc\"\n"), + }, HeaderLines([]byte(header))) +} + +func TestHeaderLinesMultipleMultilineFilenames(t *testing.T) { + const header = `Content-Type: application/msword; name="=E5=B8=B6=E6=9C=89=E5=A4=96=E5=9C=8B=E5=AD=97=E7=AC=A6=E7=9A=84=E9=99=84=E4= +=BB=B6.DOC" +Content-Transfer-Encoding: base64 +Content-Disposition: attachment; filename="=E5=B8=B6=E6=9C=89=E5=A4=96=E5=9C=8B=E5=AD=97=E7=AC=A6=E7=9A=84=E9=99=84=E4= +=BB=B6.DOC" +Content-ID: <> +` + + assert.Equal(t, [][]byte{ + []byte("Content-Type: application/msword; name=\"=E5=B8=B6=E6=9C=89=E5=A4=96=E5=9C=8B=E5=AD=97=E7=AC=A6=E7=9A=84=E9=99=84=E4=\n=BB=B6.DOC\"\n"), + []byte("Content-Transfer-Encoding: base64\n"), + []byte("Content-Disposition: attachment; filename=\"=E5=B8=B6=E6=9C=89=E5=A4=96=E5=9C=8B=E5=AD=97=E7=AC=A6=E7=9A=84=E9=99=84=E4=\n=BB=B6.DOC\"\n"), + []byte("Content-ID: <>\n"), + }, HeaderLines([]byte(header))) +} diff --git a/pkg/message/scanner.go b/pkg/message/scanner.go new file mode 100644 index 00000000..657c3fe5 --- /dev/null +++ b/pkg/message/scanner.go @@ -0,0 +1,96 @@ +// Copyright (c) 2021 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package message + +import ( + "bufio" + "bytes" + "errors" + "io" +) + +type partScanner struct { + r *bufio.Reader + + boundary string + progress int +} + +type part struct { + b []byte + offset int +} + +func newPartScanner(r io.Reader, boundary string) (*partScanner, error) { + scanner := &partScanner{r: bufio.NewReader(r), boundary: boundary} + + if _, _, err := scanner.readToBoundary(); err != nil { + return nil, err + } + + return scanner, nil +} + +func (s *partScanner) scanAll() ([]part, error) { + var parts []part + + for { + offset := s.progress + + b, more, err := s.readToBoundary() + if err != nil { + return nil, err + } + + if !more { + return parts, nil + } + + parts = append(parts, part{b: b, offset: offset}) + } +} + +func (s *partScanner) readToBoundary() ([]byte, bool, error) { + var res []byte + + for { + line, err := s.r.ReadBytes('\n') + if err != nil { + if !errors.Is(err, io.EOF) { + return nil, false, err + } + + if len(line) == 0 { + return nil, false, nil + } + } + + s.progress += len(line) + + switch { + case bytes.HasPrefix(bytes.TrimSpace(line), []byte("--"+s.boundary)): + return bytes.TrimSuffix(bytes.TrimSuffix(res, []byte("\n")), []byte("\r")), true, nil + + case bytes.HasSuffix(bytes.TrimSpace(line), []byte(s.boundary+"--")): + return bytes.TrimSuffix(bytes.TrimSuffix(res, []byte("\n")), []byte("\r")), false, nil + + default: + res = append(res, line...) + } + } +} diff --git a/pkg/message/scanner_test.go b/pkg/message/scanner_test.go new file mode 100644 index 00000000..918e91cd --- /dev/null +++ b/pkg/message/scanner_test.go @@ -0,0 +1,136 @@ +// Copyright (c) 2021 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package message + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestScanner(t *testing.T) { + const literal = `this part of the text should be ignored + +--longrandomstring + +body1 + +--longrandomstring + +body2 + +--longrandomstring-- +` + + scanner, err := newPartScanner(strings.NewReader(literal), "longrandomstring") + require.NoError(t, err) + + parts, err := scanner.scanAll() + require.NoError(t, err) + + assert.Equal(t, "\nbody1\n", string(parts[0].b)) + assert.Equal(t, "\nbody2\n", string(parts[1].b)) + + assert.Equal(t, "\nbody1\n", literal[parts[0].offset:parts[0].offset+len(parts[0].b)]) + assert.Equal(t, "\nbody2\n", literal[parts[1].offset:parts[1].offset+len(parts[1].b)]) +} + +func TestScannerNested(t *testing.T) { + const literal = `This is the preamble. It is to be ignored, though it +is a handy place for mail composers to include an +explanatory note to non-MIME compliant readers. +--simple boundary +Content-type: multipart/mixed; boundary="nested boundary" + +This is the preamble. It is to be ignored, though it +is a handy place for mail composers to include an +explanatory note to non-MIME compliant readers. +--nested boundary +Content-type: text/plain; charset=us-ascii + +This part does not end with a linebreak. +--nested boundary +Content-type: text/plain; charset=us-ascii + +This part does end with a linebreak. + +--nested boundary-- +--simple boundary +Content-type: text/plain; charset=us-ascii + +This part does end with a linebreak. + +--simple boundary-- +This is the epilogue. It is also to be ignored. +` + + scanner, err := newPartScanner(strings.NewReader(literal), "simple boundary") + require.NoError(t, err) + + parts, err := scanner.scanAll() + require.NoError(t, err) + + assert.Equal(t, `Content-type: multipart/mixed; boundary="nested boundary" + +This is the preamble. It is to be ignored, though it +is a handy place for mail composers to include an +explanatory note to non-MIME compliant readers. +--nested boundary +Content-type: text/plain; charset=us-ascii + +This part does not end with a linebreak. +--nested boundary +Content-type: text/plain; charset=us-ascii + +This part does end with a linebreak. + +--nested boundary--`, string(parts[0].b)) + assert.Equal(t, `Content-type: text/plain; charset=us-ascii + +This part does end with a linebreak. +`, string(parts[1].b)) +} + +func TestScannerNoFinalLinebreak(t *testing.T) { + const literal = `--nested boundary +Content-type: text/plain; charset=us-ascii + +This part does not end with a linebreak. +--nested boundary +Content-type: text/plain; charset=us-ascii + +This part does end with a linebreak. + +--nested boundary--` + + scanner, err := newPartScanner(strings.NewReader(literal), "nested boundary") + require.NoError(t, err) + + parts, err := scanner.scanAll() + require.NoError(t, err) + + assert.Equal(t, `Content-type: text/plain; charset=us-ascii + +This part does not end with a linebreak.`, string(parts[0].b)) + assert.Equal(t, `Content-type: text/plain; charset=us-ascii + +This part does end with a linebreak. +`, string(parts[1].b)) +} diff --git a/pkg/message/writer.go b/pkg/message/writer.go new file mode 100644 index 00000000..ed910519 --- /dev/null +++ b/pkg/message/writer.go @@ -0,0 +1,48 @@ +// Copyright (c) 2021 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package message + +import ( + "fmt" + "io" +) + +type partWriter struct { + w io.Writer + boundary string +} + +func newPartWriter(w io.Writer, boundary string) *partWriter { + return &partWriter{w: w, boundary: boundary} +} + +func (w *partWriter) createPart(fn func(io.Writer) error) error { + if _, err := fmt.Fprintf(w.w, "\r\n--%v\r\n", w.boundary); err != nil { + return err + } + + return fn(w.w) +} + +func (w *partWriter) done() error { + if _, err := fmt.Fprintf(w.w, "\r\n--%v--\r\n", w.boundary); err != nil { + return err + } + + return nil +}