We build too many walls and not enough bridges

This commit is contained in:
Jakub
2020-04-08 12:59:16 +02:00
commit 17f4d6097a
494 changed files with 62753 additions and 0 deletions

56
pkg/message/address.go Normal file
View File

@ -0,0 +1,56 @@
// Copyright (c) 2020 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package message
import (
"net/mail"
"strings"
"github.com/emersion/go-imap"
)
func getAddresses(addrs []*mail.Address) (imapAddrs []*imap.Address) {
for _, a := range addrs {
if a == nil {
continue
}
parts := strings.SplitN(a.Address, "@", 2)
if len(parts) != 2 {
continue
}
imapAddrs = append(imapAddrs, &imap.Address{
PersonalName: a.Name,
MailboxName: parts[0],
HostName: parts[1],
})
}
return
}
func formatAddressList(addrs []*mail.Address) (s string) {
for i, addr := range addrs {
if i > 0 {
s += ", "
}
s += addr.String()
}
return
}

75
pkg/message/body.go Normal file
View File

@ -0,0 +1,75 @@
// Copyright (c) 2020 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package message
import (
"encoding/base64"
"fmt"
"io"
"mime/quotedprintable"
pmcrypto "github.com/ProtonMail/gopenpgp/crypto"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/emersion/go-textwrapper"
openpgperrors "golang.org/x/crypto/openpgp/errors"
)
func WriteBody(w io.Writer, kr *pmcrypto.KeyRing, m *pmapi.Message) error {
// Decrypt body.
if err := m.Decrypt(kr); err != nil && err != openpgperrors.ErrSignatureExpired {
return err
}
if m.MIMEType != pmapi.ContentTypeMultipartMixed {
// Encode it.
qp := quotedprintable.NewWriter(w)
if _, err := io.WriteString(qp, m.Body); err != nil {
return err
}
return qp.Close()
}
_, err := io.WriteString(w, m.Body)
return err
}
func WriteAttachmentBody(w io.Writer, kr *pmcrypto.KeyRing, m *pmapi.Message, att *pmapi.Attachment, r io.Reader) (err error) {
// Decrypt it
var dr io.Reader
dr, err = att.Decrypt(r, kr)
if err == openpgperrors.ErrKeyIncorrect {
// Do not fail if attachment is encrypted with a different key.
dr = r
err = nil
att.Name += ".gpg"
att.MIMEType = "application/pgp-encrypted"
} else if err != nil && err != openpgperrors.ErrSignatureExpired {
err = fmt.Errorf("cannot decrypt attachment: %v", err)
return
}
// Encode it.
ww := textwrapper.NewRFC822(w)
bw := base64.NewEncoder(base64.StdEncoding, ww)
var n int64
if n, err = io.Copy(bw, dr); err != nil {
err = fmt.Errorf("cannot write attachment: %v (wrote %v bytes)", err, n)
}
_ = bw.Close()
return
}

48
pkg/message/envelope.go Normal file
View File

@ -0,0 +1,48 @@
// Copyright (c) 2020 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package message
import (
"net/mail"
"time"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/emersion/go-imap"
)
func GetEnvelope(m *pmapi.Message) *imap.Envelope {
messageID := m.ExternalID
if messageID == "" {
messageID = m.Header.Get("Message-Id")
} else {
messageID = "<" + messageID + ">"
}
return &imap.Envelope{
Date: time.Unix(m.Time, 0),
Subject: m.Subject,
From: getAddresses([]*mail.Address{m.Sender}),
Sender: getAddresses([]*mail.Address{m.Sender}),
ReplyTo: getAddresses(m.ReplyTos),
To: getAddresses(m.ToList),
Cc: getAddresses(m.CCList),
Bcc: getAddresses(m.BCCList),
InReplyTo: m.Header.Get("In-Reply-To"),
MessageId: messageID,
}
}

83
pkg/message/flags.go Normal file
View File

@ -0,0 +1,83 @@
// Copyright (c) 2020 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package message
import (
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/emersion/go-imap"
)
//nolint[gochecknoglobals]
var (
AppleMailJunkFlag = imap.CanonicalFlag("$Junk")
ThunderbirdJunkFlag = imap.CanonicalFlag("Junk")
ThunderbirdNonJunkFlag = imap.CanonicalFlag("NonJunk")
)
func GetFlags(m *pmapi.Message) (flags []string) {
if m.Unread == 0 {
flags = append(flags, imap.SeenFlag)
}
if !m.Has(pmapi.FlagSent) && !m.Has(pmapi.FlagReceived) {
flags = append(flags, imap.DraftFlag)
}
if m.Has(pmapi.FlagReplied) || m.Has(pmapi.FlagRepliedAll) {
flags = append(flags, imap.AnsweredFlag)
}
hasSpam := false
for _, l := range m.LabelIDs {
if l == pmapi.StarredLabel {
flags = append(flags, imap.FlaggedFlag)
}
if l == pmapi.SpamLabel {
flags = append(flags, AppleMailJunkFlag, ThunderbirdJunkFlag)
hasSpam = true
}
}
if !hasSpam {
flags = append(flags, ThunderbirdNonJunkFlag)
}
return
}
func ParseFlags(m *pmapi.Message, flags []string) {
if (m.Flags & pmapi.FlagSent) == 0 {
m.Flags |= pmapi.FlagReceived
}
m.Unread = 1
for _, f := range flags {
switch f {
case imap.SeenFlag:
m.Unread = 0
case imap.DraftFlag:
m.Flags &= ^pmapi.FlagSent
m.Flags &= ^pmapi.FlagReceived
m.LabelIDs = append(m.LabelIDs, pmapi.DraftLabel)
case imap.FlaggedFlag:
m.LabelIDs = append(m.LabelIDs, pmapi.StarredLabel)
case imap.AnsweredFlag:
m.Flags |= pmapi.FlagReplied
case AppleMailJunkFlag, ThunderbirdJunkFlag:
m.LabelIDs = append(m.LabelIDs, pmapi.SpamLabel)
}
}
}

214
pkg/message/header.go Normal file
View File

@ -0,0 +1,214 @@
// Copyright (c) 2020 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package message
import (
"mime"
"net/mail"
"net/textproto"
"strings"
"time"
pmmime "github.com/ProtonMail/proton-bridge/pkg/mime"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
)
// GetHeader builds the header for the message.
func GetHeader(msg *pmapi.Message) textproto.MIMEHeader { //nolint[funlen]
h := make(textproto.MIMEHeader)
// Copy the custom header fields if there are some.
if msg.Header != nil {
h = textproto.MIMEHeader(msg.Header)
}
// Add or rewrite fields.
h.Set("Subject", pmmime.EncodeHeader(msg.Subject))
if msg.Sender != nil {
h.Set("From", pmmime.EncodeHeader(msg.Sender.String()))
}
if len(msg.ReplyTos) > 0 {
h.Set("Reply-To", pmmime.EncodeHeader(formatAddressList(msg.ReplyTos)))
}
if len(msg.ToList) > 0 {
h.Set("To", pmmime.EncodeHeader(formatAddressList(msg.ToList)))
}
if len(msg.CCList) > 0 {
h.Set("Cc", pmmime.EncodeHeader(formatAddressList(msg.CCList)))
}
if len(msg.BCCList) > 0 {
h.Set("Bcc", pmmime.EncodeHeader(formatAddressList(msg.BCCList)))
}
// Add or rewrite date related fields.
if msg.Time > 0 {
h.Set("X-Pm-Date", time.Unix(msg.Time, 0).Format(time.RFC1123Z))
if d, err := msg.Header.Date(); err != nil || d.IsZero() { // Fix date if needed.
h.Set("Date", time.Unix(msg.Time, 0).Format(time.RFC1123Z))
}
}
// Use External-Id if available to ensure email clients:
// * build the conversations threads correctly (Thunderbird, Mac Outlook, Apple Mail)
// * do not think the message is lost (Apple Mail)
if msg.ExternalID != "" {
h.Set("X-Pm-External-Id", "<"+msg.ExternalID+">")
if h.Get("Message-Id") == "" {
h.Set("Message-Id", "<"+msg.ExternalID+">")
}
}
if msg.ID != "" {
if h.Get("Message-Id") == "" {
h.Set("Message-Id", "<"+msg.ID+"@protonmail.internalid>")
}
h.Set("X-Pm-Internal-Id", msg.ID)
// Forward References, and include the message ID here (to improve outlook support).
if references := h.Get("References"); !strings.Contains(references, msg.ID) {
references += " <" + msg.ID + "@protonmail.internalid>"
h.Set("References", references)
}
}
if msg.ConversationID != "" {
h.Set("X-Pm-ConversationID-Id", msg.ConversationID)
if references := h.Get("References"); !strings.Contains(references, msg.ConversationID) {
references += " <" + msg.ConversationID + "@protonmail.conversationid>"
h.Set("References", references)
}
}
return h
}
func SetBodyContentFields(h *textproto.MIMEHeader, m *pmapi.Message) {
h.Set("Content-Type", m.MIMEType+"; charset=utf-8")
h.Set("Content-Disposition", "inline")
h.Set("Content-Transfer-Encoding", "quoted-printable")
}
func GetBodyHeader(m *pmapi.Message) textproto.MIMEHeader {
h := make(textproto.MIMEHeader)
SetBodyContentFields(&h, m)
return h
}
func GetRelatedHeader(m *pmapi.Message) textproto.MIMEHeader {
h := make(textproto.MIMEHeader)
h.Set("Content-Type", "multipart/related; boundary="+GetRelatedBoundary(m))
return h
}
func GetAttachmentHeader(att *pmapi.Attachment) textproto.MIMEHeader {
mediaType := att.MIMEType
if mediaType == "application/pgp-encrypted" {
mediaType = "application/octet-stream"
}
encodedName := pmmime.EncodeHeader(att.Name)
disposition := "attachment" //nolint[goconst]
if strings.Contains(att.Header.Get("Content-Disposition"), "inline") {
disposition = "inline"
}
h := make(textproto.MIMEHeader)
h.Set("Content-Type", mime.FormatMediaType(mediaType, map[string]string{"name": encodedName}))
h.Set("Content-Transfer-Encoding", "base64")
h.Set("Content-Disposition", mime.FormatMediaType(disposition, map[string]string{"filename": encodedName}))
// Forward some original header lines.
forward := []string{"Content-Id", "Content-Description", "Content-Location"}
for _, k := range forward {
v := att.Header.Get(k)
if v != "" {
h.Set(k, v)
}
}
return h
}
// ========= Header parsing and sanitizing functions =========
func parseHeader(h mail.Header) (m *pmapi.Message, err error) { //nolint[unparam]
m = pmapi.NewMessage()
if subject, err := pmmime.DecodeHeader(h.Get("Subject")); err == nil {
m.Subject = subject
}
if addrs, err := sanitizeAddressList(h, "From"); err == nil && len(addrs) > 0 {
m.Sender = addrs[0]
}
if addrs, err := sanitizeAddressList(h, "Reply-To"); err == nil && len(addrs) > 0 {
m.ReplyTos = addrs
}
if addrs, err := sanitizeAddressList(h, "To"); err == nil {
m.ToList = addrs
}
if addrs, err := sanitizeAddressList(h, "Cc"); err == nil {
m.CCList = addrs
}
if addrs, err := sanitizeAddressList(h, "Bcc"); err == nil {
m.BCCList = addrs
}
m.Time = 0
if t, err := h.Date(); err == nil && !t.IsZero() {
m.Time = t.Unix()
}
m.Header = h
return
}
func sanitizeAddressList(h mail.Header, field string) (addrs []*mail.Address, err error) {
raw := h.Get(field)
if raw == "" {
err = mail.ErrHeaderNotPresent
return
}
var decoded string
decoded, err = pmmime.DecodeHeader(raw)
if err != nil {
return
}
addrs, err = mail.ParseAddressList(parseAddressComment(decoded))
if err == nil {
if addrs == nil {
addrs = []*mail.Address{}
}
return
}
// Probably missing encoding error -- try to at least parse addresses in brackets.
addrStr := h.Get(field)
first := strings.Index(addrStr, "<")
last := strings.LastIndex(addrStr, ">")
if first < 0 || last < 0 || first >= last {
return
}
var addrList []string
open := first
for open < last && 0 <= open {
addrStr = addrStr[open:]
close := strings.Index(addrStr, ">")
addrList = append(addrList, addrStr[:close+1])
addrStr = addrStr[close:]
open = strings.Index(addrStr, "<")
last = strings.LastIndex(addrStr, ">")
}
addrStr = strings.Join(addrList, ", ")
//
return mail.ParseAddressList(addrStr)
}

71
pkg/message/html.go Normal file
View File

@ -0,0 +1,71 @@
// Copyright (c) 2020 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package message
import (
"bytes"
escape "html"
"strings"
"github.com/andybalholm/cascadia"
"golang.org/x/net/html"
)
func plaintextToHTML(text string) (output string) {
text = escape.EscapeString(text)
text = strings.Replace(text, "\n\r", "<br>", -1)
text = strings.Replace(text, "\r\n", "<br>", -1)
text = strings.Replace(text, "\n", "<br>", -1)
text = strings.Replace(text, "\r", "<br>", -1)
return "<div>" + text + "</div>"
}
func stripHTML(input string) (stripped string, err error) {
reader := strings.NewReader(input)
doc, _ := html.Parse(reader)
body := cascadia.MustCompile("body").MatchFirst(doc)
var buf1 bytes.Buffer
if err = html.Render(&buf1, body); err != nil {
stripped = input
return
}
stripped = buf1.String()
// Handle double body tags edge case.
if strings.Index(stripped, "<body") == 0 {
startIndex := strings.Index(stripped, ">")
if startIndex < 5 {
return
}
stripped = stripped[startIndex+1:]
// Closing body tag is optional.
closingIndex := strings.Index(stripped, "</body>")
if closingIndex > -1 {
stripped = stripped[:closingIndex]
}
}
return
}
func addOuterHTMLTags(input string) (output string) {
return "<html><head></head><body>" + input + "</body></html>"
}
func makeEmbeddedImageHTML(cid, name string) (output string) {
return "<img class=\"proton-embedded\" alt=\"" + name + "\" src=\"cid:" + cid + "\">"
}

188
pkg/message/message.go Normal file
View File

@ -0,0 +1,188 @@
// Copyright (c) 2020 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package message
import (
"crypto/sha512"
"fmt"
"strings"
pmmime "github.com/ProtonMail/proton-bridge/pkg/mime"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/emersion/go-imap"
"github.com/jhillyerd/enmime"
log "github.com/sirupsen/logrus"
)
const textPlain = "text/plain"
func GetBoundary(m *pmapi.Message) string {
// The boundary needs to be deterministic because messages are not supposed to
// change.
return fmt.Sprintf("%x", sha512.Sum512_256([]byte(m.ID)))
}
func GetRelatedBoundary(m *pmapi.Message) string {
// The boundary needs to be deterministic because messages are not supposed to
// change.
return fmt.Sprintf("%x", sha512.Sum512_256([]byte(m.ID+m.ID)))
}
func GetBodyStructure(m *pmapi.Message) (bs *imap.BodyStructure) { //nolint[funlen]
bs = &imap.BodyStructure{
MimeType: "multipart",
MimeSubType: "mixed",
Params: map[string]string{"boundary": GetBoundary(m)},
}
var inlineParts []*imap.BodyStructure
var attParts []*imap.BodyStructure
for _, att := range m.Attachments {
typeParts := strings.SplitN(att.MIMEType, "/", 2)
if len(typeParts) != 2 {
continue
}
if typeParts[0] == "application" && typeParts[1] == "pgp-encrypted" {
typeParts[1] = "octet-stream"
}
part := &imap.BodyStructure{
MimeType: typeParts[0],
MimeSubType: typeParts[1],
Params: map[string]string{"name": att.Name},
Encoding: "base64",
}
if strings.Contains(att.Header.Get("Content-Disposition"), "inline") {
part.Disposition = "inline"
inlineParts = append(inlineParts, part)
} else {
part.Disposition = "attachment"
attParts = append(attParts, part)
}
}
if len(inlineParts) > 0 {
// Set to multipart-related for inline attachments.
relatedPart := &imap.BodyStructure{
MimeType: "multipart",
MimeSubType: "related",
Params: map[string]string{"boundary": GetRelatedBoundary(m)},
}
subType := "html"
if m.MIMEType == textPlain {
subType = "plain"
}
relatedPart.Parts = append(relatedPart.Parts, &imap.BodyStructure{
MimeType: "text",
MimeSubType: subType,
Params: map[string]string{"charset": "utf-8"},
Encoding: "quoted-printable",
Disposition: "inline",
})
bs.Parts = append(bs.Parts, relatedPart)
} else {
subType := "html"
if m.MIMEType == textPlain {
subType = "plain"
}
bs.Parts = append(bs.Parts, &imap.BodyStructure{
MimeType: "text",
MimeSubType: subType,
Params: map[string]string{"charset": "utf-8"},
Encoding: "quoted-printable",
Disposition: "inline",
})
}
bs.Parts = append(bs.Parts, attParts...)
return bs
}
func SeparateInlineAttachments(m *pmapi.Message) (atts, inlines []*pmapi.Attachment) {
for _, att := range m.Attachments {
if strings.Contains(att.Header.Get("Content-Disposition"), "inline") {
inlines = append(inlines, att)
} else {
atts = append(atts, att)
}
}
return
}
func GetMIMEBodyStructure(m *pmapi.Message, parsedMsg *enmime.Envelope) (bs *imap.BodyStructure, err error) {
// We recursively look through the MIME structure.
root := parsedMsg.Root
if root == nil {
return GetBodyStructure(m), nil
}
mediaType, params, err := pmmime.ParseMediaType(root.ContentType)
if err != nil {
log.Warnf("Cannot parse Content-Type '%v': %v", root.ContentType, err)
err = nil
mediaType = textPlain
}
typeParts := strings.SplitN(mediaType, "/", 2)
bs = &imap.BodyStructure{
MimeType: typeParts[0],
Params: params,
}
if len(typeParts) > 1 {
bs.MimeSubType = typeParts[1]
}
bs.Parts = getChildrenParts(root)
return
}
func getChildrenParts(root *enmime.Part) (parts []*imap.BodyStructure) {
for child := root.FirstChild; child != nil; child = child.NextSibling {
mediaType, params, err := pmmime.ParseMediaType(child.ContentType)
if err != nil {
log.Warnf("Cannot parse Content-Type '%v': %v", child.ContentType, err)
mediaType = textPlain
}
typeParts := strings.SplitN(mediaType, "/", 2)
childrenParts := getChildrenParts(child)
part := &imap.BodyStructure{
MimeType: typeParts[0],
Params: params,
Encoding: child.Charset,
Disposition: child.Disposition,
Parts: childrenParts,
}
if len(typeParts) > 1 {
part.MimeSubType = typeParts[1]
}
parts = append(parts, part)
}
return
}

468
pkg/message/parser.go Normal file
View File

@ -0,0 +1,468 @@
// Copyright (c) 2020 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package message
import (
"bytes"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"fmt"
"io"
"io/ioutil"
"mime"
"mime/quotedprintable"
"net/mail"
"net/textproto"
"regexp"
"strconv"
"strings"
pmmime "github.com/ProtonMail/proton-bridge/pkg/mime"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/jaytaylor/html2text"
log "github.com/sirupsen/logrus"
)
func parseAttachment(filename string, mediaType string, h textproto.MIMEHeader) (att *pmapi.Attachment) {
if decoded, err := pmmime.DecodeHeader(filename); err == nil {
filename = decoded
}
if filename == "" {
ext, err := mime.ExtensionsByType(mediaType)
if err == nil && len(ext) > 0 {
filename = "attachment" + ext[0]
}
}
att = &pmapi.Attachment{
Name: filename,
MIMEType: mediaType,
Header: h,
}
headerContentID := strings.Trim(h.Get("Content-Id"), " <>")
if headerContentID != "" {
att.ContentID = headerContentID
}
return
}
var reEmailComment = regexp.MustCompile("[(][^)]*[)]") //nolint[gochecknoglobals]
// parseAddressComment removes the comments completely even though they should be allowed
// http://tools.wordtothewise.com/rfc/822
// NOTE: This should be supported in go>1.10 but it seems it's not ¯\_(ツ)_/¯
func parseAddressComment(raw string) string {
return reEmailComment.ReplaceAllString(raw, "")
}
// Some clients incorrectly format messages with embedded attachments to have a format like
// I. text/plain II. attachment III. text/plain
// which we need to convert to a single HTML part with an embedded attachment.
func combineParts(m *pmapi.Message, parts []io.Reader, headers []textproto.MIMEHeader, convertPlainToHTML bool, atts *[]io.Reader) (isHTML bool, err error) { //nolint[funlen]
isHTML = true
foundText := false
for i := len(parts) - 1; i >= 0; i-- {
part := parts[i]
h := headers[i]
disp, dispParams, _ := pmmime.ParseMediaType(h.Get("Content-Disposition"))
d := pmmime.DecodeContentEncoding(part, h.Get("Content-Transfer-Encoding"))
if d == nil {
log.Warnf("Unsupported Content-Transfer-Encoding '%v'", h.Get("Content-Transfer-Encoding"))
d = part
}
contentType := h.Get("Content-Type")
if contentType == "" {
contentType = "text/plain"
}
mediaType, params, _ := pmmime.ParseMediaType(contentType)
if strings.HasPrefix(mediaType, "text/") && mediaType != "text/calendar" && disp != "attachment" {
// This is text.
var b []byte
if b, err = ioutil.ReadAll(d); err != nil {
continue
}
b, err = pmmime.DecodeCharset(b, params)
if err != nil {
log.Warn("Decode charset error: ", err)
return false, err
}
contents := string(b)
if strings.Contains(mediaType, "text/plain") && len(contents) > 0 {
if !convertPlainToHTML {
isHTML = false
} else {
contents = plaintextToHTML(contents)
}
} else if strings.Contains(mediaType, "text/html") && len(contents) > 0 {
contents, err = stripHTML(contents)
if err != nil {
return isHTML, err
}
}
m.Body = contents + m.Body
foundText = true
} else {
// This is an attachment.
filename := dispParams["filename"]
if filename == "" {
// Using "name" in Content-Type is discouraged.
filename = params["name"]
}
if filename == "" && mediaType == "text/calendar" {
filename = "event.ics"
}
att := parseAttachment(filename, mediaType, h)
b := &bytes.Buffer{}
if _, err = io.Copy(b, d); err != nil {
continue
}
if foundText && att.ContentID == "" && strings.Contains(mediaType, "image") {
// Treat this as an inline attachment even though it is not marked as one.
hasher := sha256.New()
_, _ = hasher.Write([]byte(att.Name + strconv.Itoa(b.Len())))
bytes := hasher.Sum(nil)
cid := hex.EncodeToString(bytes) + "@protonmail.com"
att.ContentID = cid
embeddedHTML := makeEmbeddedImageHTML(cid, att.Name)
m.Body = embeddedHTML + m.Body
}
m.Attachments = append(m.Attachments, att)
*atts = append(*atts, b)
}
}
if isHTML {
m.Body = addOuterHTMLTags(m.Body)
}
return isHTML, nil
}
func checkHeaders(headers []textproto.MIMEHeader) bool {
foundAttachment := false
for i := 0; i < len(headers); i++ {
h := headers[i]
mediaType, _, _ := pmmime.ParseMediaType(h.Get("Content-Type"))
if !strings.HasPrefix(mediaType, "text/") {
foundAttachment = true
} else if foundAttachment {
// This means that there is a text part after the first attachment,
// so we will have to convert the body from plain->HTML.
return true
}
}
return false
}
// ============================== 7bit Filter ==========================
// For every MIME part in the tree that has "8bit" or "binary" content
// transfer encoding: transcode it to "quoted-printable".
type SevenBitFilter struct {
target pmmime.VisitAcceptor
}
func NewSevenBitFilter(targetAccepter pmmime.VisitAcceptor) *SevenBitFilter {
return &SevenBitFilter{
target: targetAccepter,
}
}
func decodePart(partReader io.Reader, header textproto.MIMEHeader) (decodedPart io.Reader) {
decodedPart = pmmime.DecodeContentEncoding(partReader, header.Get("Content-Transfer-Encoding"))
if decodedPart == nil {
log.Warnf("Unsupported Content-Transfer-Encoding '%v'", header.Get("Content-Transfer-Encoding"))
decodedPart = partReader
}
return
}
func (sd SevenBitFilter) Accept(partReader io.Reader, header textproto.MIMEHeader, hasPlainSibling bool, isFirst, isLast bool) error {
cte := strings.ToLower(header.Get("Content-Transfer-Encoding"))
if isFirst && pmmime.IsLeaf(header) && cte != "quoted-printable" && cte != "base64" && cte != "7bit" {
decodedPart := decodePart(partReader, header)
filteredHeader := textproto.MIMEHeader{}
for k, v := range header {
filteredHeader[k] = v
}
filteredHeader.Set("Content-Transfer-Encoding", "quoted-printable")
//filteredHeader.Set("Content-Transfer-Encoding", "base64")
filteredBuffer := &bytes.Buffer{}
decodedSlice, _ := ioutil.ReadAll(decodedPart)
w := quotedprintable.NewWriter(filteredBuffer)
//w := base64.NewEncoder(base64.StdEncoding, filteredBuffer)
if _, err := w.Write(decodedSlice); err != nil {
log.Errorf("cannot write quotedprintable from %q: %v", cte, err)
}
if err := w.Close(); err != nil {
log.Errorf("cannot close quotedprintable from %q: %v", cte, err)
}
_ = sd.target.Accept(filteredBuffer, filteredHeader, hasPlainSibling, true, isLast)
} else {
_ = sd.target.Accept(partReader, header, hasPlainSibling, isFirst, isLast)
}
return nil
}
// =================== HTML Only convertor ==================================
// In any part of MIME tree structure, replace standalone text/html with
// multipart/alternative containing both text/html and text/plain.
type HTMLOnlyConvertor struct {
target pmmime.VisitAcceptor
}
func NewHTMLOnlyConvertor(targetAccepter pmmime.VisitAcceptor) *HTMLOnlyConvertor {
return &HTMLOnlyConvertor{
target: targetAccepter,
}
}
func randomBoundary() string {
var buf [30]byte
_, err := io.ReadFull(rand.Reader, buf[:])
if err != nil {
panic(err)
}
return fmt.Sprintf("%x", buf[:])
}
func (hoc HTMLOnlyConvertor) Accept(partReader io.Reader, header textproto.MIMEHeader, hasPlainSiblings bool, isFirst, isLast bool) error {
mediaType, _, err := pmmime.ParseMediaType(header.Get("Content-Type"))
if isFirst && err == nil && mediaType == "text/html" && !hasPlainSiblings {
multiPartHeaders := make(textproto.MIMEHeader)
for k, v := range header {
multiPartHeaders[k] = v
}
boundary := randomBoundary()
multiPartHeaders.Set("Content-Type", "multipart/alternative; boundary=\""+boundary+"\"")
childCte := header.Get("Content-Transfer-Encoding")
_ = hoc.target.Accept(partReader, multiPartHeaders, false, true, false)
partData, _ := ioutil.ReadAll(partReader)
htmlChildHeaders := make(textproto.MIMEHeader)
htmlChildHeaders.Set("Content-Transfer-Encoding", childCte)
htmlChildHeaders.Set("Content-Type", "text/html")
htmlReader := bytes.NewReader(partData)
_ = hoc.target.Accept(htmlReader, htmlChildHeaders, false, true, false)
_ = hoc.target.Accept(partReader, multiPartHeaders, hasPlainSiblings, false, false)
plainChildHeaders := make(textproto.MIMEHeader)
plainChildHeaders.Set("Content-Transfer-Encoding", childCte)
plainChildHeaders.Set("Content-Type", "text/plain")
unHtmlized, err := html2text.FromReader(bytes.NewReader(partData))
if err != nil {
unHtmlized = string(partData)
}
plainReader := strings.NewReader(unHtmlized)
_ = hoc.target.Accept(plainReader, plainChildHeaders, false, true, true)
_ = hoc.target.Accept(partReader, multiPartHeaders, hasPlainSiblings, false, true)
} else {
_ = hoc.target.Accept(partReader, header, hasPlainSiblings, isFirst, isLast)
}
return nil
}
// ======= Public Key Attacher ========
type PublicKeyAttacher struct {
target pmmime.VisitAcceptor
attachedPublicKey string
attachedPublicKeyName string
appendToMultipart bool
depth int
}
func NewPublicKeyAttacher(targetAccepter pmmime.VisitAcceptor, attachedPublicKey, attachedPublicKeyName string) *PublicKeyAttacher {
return &PublicKeyAttacher{
target: targetAccepter,
attachedPublicKey: attachedPublicKey,
attachedPublicKeyName: attachedPublicKeyName,
appendToMultipart: false,
depth: 0,
}
}
func min(a, b int) int {
if a < b {
return a
}
return b
}
func split(input string, sliceLength int) string {
processed := input
result := ""
for len(processed) > 0 {
cutPoint := min(sliceLength, len(processed))
part := processed[0:cutPoint]
result = result + part + "\n"
processed = processed[cutPoint:]
}
return result
}
func createKeyAttachment(publicKey, publicKeyName string) (headers textproto.MIMEHeader, contents io.Reader) {
attachmentHeaders := make(textproto.MIMEHeader)
attachmentHeaders.Set("Content-Type", "application/pgp-key; name=\""+publicKeyName+"\"")
attachmentHeaders.Set("Content-Transfer-Encoding", "base64")
attachmentHeaders.Set("Content-Disposition", "attachment; filename=\""+publicKeyName+".asc.pgp\"")
buffer := &bytes.Buffer{}
w := base64.NewEncoder(base64.StdEncoding, buffer)
_, _ = w.Write([]byte(publicKey))
_ = w.Close()
return attachmentHeaders, strings.NewReader(split(buffer.String(), 73))
}
func (pka *PublicKeyAttacher) Accept(partReader io.Reader, header textproto.MIMEHeader, hasPlainSiblings bool, isFirst, isLast bool) error {
if isFirst && !pmmime.IsLeaf(header) {
pka.depth++
}
if isLast && !pmmime.IsLeaf(header) {
defer func() {
pka.depth--
}()
}
isRoot := (header.Get("From") != "")
mediaType, _, err := pmmime.ParseMediaType(header.Get("Content-Type"))
if isRoot && isFirst && err == nil && pka.attachedPublicKey != "" { //nolint[gocritic]
if strings.HasPrefix(mediaType, "multipart/mixed") {
pka.appendToMultipart = true
_ = pka.target.Accept(partReader, header, hasPlainSiblings, isFirst, isLast)
} else {
// Create two siblings with attachment in the case toplevel is not multipart/mixed.
multiPartHeaders := make(textproto.MIMEHeader)
for k, v := range header {
multiPartHeaders[k] = v
}
boundary := randomBoundary()
multiPartHeaders.Set("Content-Type", "multipart/mixed; boundary=\""+boundary+"\"")
multiPartHeaders.Del("Content-Transfer-Encoding")
_ = pka.target.Accept(partReader, multiPartHeaders, false, true, false)
originalHeader := make(textproto.MIMEHeader)
originalHeader.Set("Content-Type", header.Get("Content-Type"))
if header.Get("Content-Transfer-Encoding") != "" {
originalHeader.Set("Content-Transfer-Encoding", header.Get("Content-Transfer-Encoding"))
}
_ = pka.target.Accept(partReader, originalHeader, false, true, false)
_ = pka.target.Accept(partReader, multiPartHeaders, hasPlainSiblings, false, false)
attachmentHeaders, attachmentReader := createKeyAttachment(pka.attachedPublicKey, pka.attachedPublicKeyName)
_ = pka.target.Accept(attachmentReader, attachmentHeaders, false, true, true)
_ = pka.target.Accept(partReader, multiPartHeaders, hasPlainSiblings, false, true)
}
} else if isLast && pka.depth == 1 && pka.attachedPublicKey != "" {
_ = pka.target.Accept(partReader, header, hasPlainSiblings, isFirst, false)
attachmentHeaders, attachmentReader := createKeyAttachment(pka.attachedPublicKey, pka.attachedPublicKeyName)
_ = pka.target.Accept(attachmentReader, attachmentHeaders, hasPlainSiblings, true, true)
_ = pka.target.Accept(partReader, header, hasPlainSiblings, isFirst, true)
} else {
_ = pka.target.Accept(partReader, header, hasPlainSiblings, isFirst, isLast)
}
return nil
}
// ======= Parser ==========
func Parse(r io.Reader, attachedPublicKey, attachedPublicKeyName string) (m *pmapi.Message, mimeBody string, plainContents string, atts []io.Reader, err error) {
secondReader := new(bytes.Buffer)
_, _ = secondReader.ReadFrom(r)
mimeBody = secondReader.String()
mm, err := mail.ReadMessage(secondReader)
if err != nil {
return
}
if m, err = parseHeader(mm.Header); err != nil {
return
}
h := textproto.MIMEHeader(m.Header)
mmBodyData, err := ioutil.ReadAll(mm.Body)
if err != nil {
return
}
printAccepter := pmmime.NewMIMEPrinter()
publicKeyAttacher := NewPublicKeyAttacher(printAccepter, attachedPublicKey, attachedPublicKeyName)
sevenBitFilter := NewSevenBitFilter(publicKeyAttacher)
plainTextCollector := pmmime.NewPlainTextCollector(sevenBitFilter)
htmlOnlyConvertor := NewHTMLOnlyConvertor(plainTextCollector)
visitor := pmmime.NewMimeVisitor(htmlOnlyConvertor)
err = pmmime.VisitAll(bytes.NewReader(mmBodyData), h, visitor)
/*
err = visitor.VisitAll(h, bytes.NewReader(mmBodyData))
*/
if err != nil {
return
}
mimeBody = printAccepter.String()
plainContents = plainTextCollector.GetPlainText()
parts, headers, err := pmmime.GetAllChildParts(bytes.NewReader(mmBodyData), h)
if err != nil {
return
}
convertPlainToHTML := checkHeaders(headers)
isHTML, err := combineParts(m, parts, headers, convertPlainToHTML, &atts)
if isHTML {
m.MIMEType = "text/html"
} else {
m.MIMEType = "text/plain"
}
return m, mimeBody, plainContents, atts, err
}

107
pkg/message/parser_test.go Normal file
View File

@ -0,0 +1,107 @@
// Copyright (c) 2020 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package message
import (
"net/mail"
"testing"
)
func TestRFC822AddressFormat(t *testing.T) { //nolint[funlen]
tests := []struct {
address string
expected []string
}{
{
" normal name <username@server.com>",
[]string{
"\"normal name\" <username@server.com>",
},
},
{
" \"comma, name\" <username@server.com>",
[]string{
"\"comma, name\" <username@server.com>",
},
},
{
" name <username@server.com> (ignore comment)",
[]string{
"\"name\" <username@server.com>",
},
},
{
" name (ignore comment) <username@server.com>, (Comment as name) username2@server.com",
[]string{
"\"name\" <username@server.com>",
"<username2@server.com>",
},
},
{
" normal name <username@server.com>, (comment)All.(around)address@(the)server.com",
[]string{
"\"normal name\" <username@server.com>",
"<All.address@server.com>",
},
},
{
" normal name <username@server.com>, All.(\"comma, in comment\")address@(the)server.com",
[]string{
"\"normal name\" <username@server.com>",
"<All.address@server.com>",
},
},
{
" \"normal name\" <username@server.com>, \"comma, name\" <address@server.com>",
[]string{
"\"normal name\" <username@server.com>",
"\"comma, name\" <address@server.com>",
},
},
{
" \"comma, one\" <username@server.com>, \"comma, two\" <address@server.com>",
[]string{
"\"comma, one\" <username@server.com>",
"\"comma, two\" <address@server.com>",
},
},
{
" \"comma, name\" <username@server.com>, another, name <address@server.com>",
[]string{
"\"comma, name\" <username@server.com>",
"\"another, name\" <address@server.com>",
},
},
}
for _, data := range tests {
uncommented := parseAddressComment(data.address)
result, err := mail.ParseAddressList(uncommented)
if err != nil {
t.Errorf("Can not parse '%s' created from '%s': %v", uncommented, data.address, err)
}
if len(result) != len(data.expected) {
t.Errorf("Wrong parsing of '%s' created from '%s': expected '%s' but have '%+v'", uncommented, data.address, data.expected, result)
}
for i, result := range result {
if data.expected[i] != result.String() {
t.Errorf("Wrong parsing\nof %q\ncreated from %q:\nexpected %q\nbut have %q", uncommented, data.address, data.expected, result.String())
}
}
}
}

413
pkg/message/section.go Normal file
View File

@ -0,0 +1,413 @@
// Copyright (c) 2020 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package message
import (
"bufio"
"bytes"
"errors"
"io"
"io/ioutil"
"net/textproto"
"strconv"
"strings"
pmmime "github.com/ProtonMail/proton-bridge/pkg/mime"
"github.com/emersion/go-imap"
)
type sectionInfo struct {
header textproto.MIMEHeader
start, bsize, size, lines int
reader io.Reader
}
// Count and read.
func (si *sectionInfo) Read(p []byte) (n int, err error) {
n, err = si.reader.Read(p)
si.size += n
si.lines += bytes.Count(p, []byte("\n"))
return
}
type boundaryReader struct {
reader *bufio.Reader
closed, first bool
skipped int
nl []byte // "\r\n" or "\n" (set after seeing first boundary line)
nlDashBoundary []byte // nl + "--boundary"
dashBoundaryDash []byte // "--boundary--"
dashBoundary []byte // "--boundary"
}
func newBoundaryReader(r *bufio.Reader, boundary string) (br *boundaryReader, err error) {
b := []byte("\r\n--" + boundary + "--")
br = &boundaryReader{
reader: r,
closed: false,
first: true,
nl: b[:2],
nlDashBoundary: b[:len(b)-2],
dashBoundaryDash: b[2:],
dashBoundary: b[2 : len(b)-2],
}
err = br.WriteNextPartTo(nil)
return
}
func skipLWSPChar(b []byte) []byte {
for len(b) > 0 && (b[0] == ' ' || b[0] == '\t') {
b = b[1:]
}
return b
}
func (br *boundaryReader) isFinalBoundary(line []byte) bool {
if !bytes.HasPrefix(line, br.dashBoundaryDash) {
return false
}
rest := line[len(br.dashBoundaryDash):]
rest = skipLWSPChar(rest)
return len(rest) == 0 || bytes.Equal(rest, br.nl)
}
func (br *boundaryReader) isBoundaryDelimiterLine(line []byte) (ret bool) {
if !bytes.HasPrefix(line, br.dashBoundary) {
return false
}
rest := line[len(br.dashBoundary):]
rest = skipLWSPChar(rest)
if br.first && len(rest) == 1 && rest[0] == '\n' {
br.nl = br.nl[1:]
br.nlDashBoundary = br.nlDashBoundary[1:]
}
return bytes.Equal(rest, br.nl)
}
func (br *boundaryReader) WriteNextPartTo(part io.Writer) (err error) {
if br.closed {
return io.EOF
}
var line, slice []byte
br.skipped = 0
for {
slice, err = br.reader.ReadSlice('\n')
line = append(line, slice...)
if err == bufio.ErrBufferFull {
continue
}
br.skipped += len(line)
if err == io.EOF && br.isFinalBoundary(line) {
err = nil
br.closed = true
return
}
if err != nil {
return
}
if br.isBoundaryDelimiterLine(line) {
br.first = false
return
}
if br.isFinalBoundary(line) {
br.closed = true
return
}
if part != nil {
if _, err = part.Write(line); err != nil {
return
}
}
line = []byte{}
}
}
type BodyStructure map[string]*sectionInfo
func NewBodyStructure(reader io.Reader) (structure *BodyStructure, err error) {
structure = &BodyStructure{}
err = structure.Parse(reader)
return
}
func (bs *BodyStructure) Parse(r io.Reader) error {
return bs.parseAllChildSections(r, []int{}, 0)
}
func (bs *BodyStructure) parseAllChildSections(r io.Reader, currentPath []int, start int) (err error) { //nolint[funlen]
info := &sectionInfo{
start: start,
size: 0,
bsize: 0,
lines: 0,
reader: r,
}
bufInfo := bufio.NewReader(info)
tp := textproto.NewReader(bufInfo)
if info.header, err = tp.ReadMIMEHeader(); err != nil {
return
}
bodyInfo := &sectionInfo{reader: tp.R}
bodyReader := bufio.NewReader(bodyInfo)
mediaType, params, _ := pmmime.ParseMediaType(info.header.Get("Content-Type"))
// If multipart, call getAllParts, else read to count lines.
if (strings.HasPrefix(mediaType, "multipart/") || mediaType == "message/rfc822") && params["boundary"] != "" {
newPath := append(currentPath, 1)
var br *boundaryReader
br, err = newBoundaryReader(bodyReader, params["boundary"])
// New reader seeks first boundary.
if err != nil {
// Return also EOF.
return
}
for err == nil {
start += br.skipped
part := &bytes.Buffer{}
err = br.WriteNextPartTo(part)
if err != nil {
break
}
err = bs.parseAllChildSections(part, newPath, start)
part.Reset()
newPath[len(newPath)-1]++
}
br.reader = nil
if err == io.EOF {
err = nil
}
if err != nil {
return
}
} else {
// Count length.
_, _ = bodyReader.WriteTo(ioutil.Discard)
}
// Clear all buffers.
bodyReader = nil
bodyInfo.reader = nil
tp.R = nil
tp = nil
bufInfo = nil // nolint
info.reader = nil
// Store boundaries.
info.bsize = bodyInfo.size
path := stringPathFromInts(currentPath)
(*bs)[path] = info
// Fix start of subsections.
newPath := append(currentPath, 1)
shift := info.size - info.bsize
subInfo, err := bs.getInfo(newPath)
// If it has subparts.
for err == nil {
subInfo.start += shift
// Level down.
subInfo, err = bs.getInfo(append(newPath, 1))
if err == nil {
newPath = append(newPath, 1)
continue
}
// Next.
newPath[len(newPath)-1]++
subInfo, err = bs.getInfo(newPath)
if err == nil {
continue
}
// Level up.
for {
newPath = newPath[:len(newPath)-1]
if len(newPath) > 0 {
newPath[len(newPath)-1]++
subInfo, err = bs.getInfo(newPath)
if err != nil {
err = nil
continue
}
}
break
}
// The end.
if len(newPath) == 0 {
break
}
}
return nil
}
func stringPathFromInts(ints []int) (ret string) {
for i, n := range ints {
if i != 0 {
ret += "."
}
ret += strconv.Itoa(n)
}
return
}
func (bs *BodyStructure) getInfo(sectionPath []int) (sectionInfo *sectionInfo, err error) {
path := stringPathFromInts(sectionPath)
sectionInfo, ok := (*bs)[path]
if !ok {
err = errors.New("wrong section " + path)
}
return
}
func (bs *BodyStructure) GetSection(wholeMail io.ReadSeeker, sectionPath []int) (section []byte, err error) {
info, err := bs.getInfo(sectionPath)
if err != nil {
return
}
if _, err = wholeMail.Seek(int64(info.start), io.SeekStart); err != nil {
return
}
section = make([]byte, info.size)
_, err = wholeMail.Read(section)
return
}
func (bs *BodyStructure) GetSectionContent(wholeMail io.ReadSeeker, sectionPath []int) (section []byte, err error) {
info, err := bs.getInfo(sectionPath)
if err != nil {
return
}
if _, err = wholeMail.Seek(int64(info.start+info.size-info.bsize), io.SeekStart); err != nil {
return
}
section = make([]byte, info.bsize)
_, err = wholeMail.Read(section)
return
/* This is slow:
sectionBuf, err := bs.GetSection(wholeMail, sectionPath)
if err != nil {
return
}
tp := textproto.NewReader(bufio.NewReader(buf))
if _, err = tp.ReadMIMEHeader(); err != nil {
return err
}
sectionBuf = &bytes.Buffer{}
_, err = io.Copy(sectionBuf, tp.R)
return
*/
}
func (bs *BodyStructure) GetSectionHeader(sectionPath []int) (header textproto.MIMEHeader, err error) {
info, err := bs.getInfo(sectionPath)
if err != nil {
return
}
header = info.header
return
}
func (bs *BodyStructure) Size() uint32 {
info, err := bs.getInfo([]int{})
if err != nil {
return uint32(0)
}
return uint32(info.size)
}
func (bs *BodyStructure) IMAPBodyStructure(currentPart []int) (imapBS *imap.BodyStructure, err error) {
var info *sectionInfo
if info, err = bs.getInfo(currentPart); err != nil {
return
}
mediaType, params, _ := pmmime.ParseMediaType(info.header.Get("Content-Type"))
mediaTypeSep := strings.Split(mediaType, "/")
// If it is empty or missing it will not crash.
mediaTypeSep = append(mediaTypeSep, "")
imapBS = &imap.BodyStructure{
MimeType: mediaTypeSep[0],
MimeSubType: mediaTypeSep[1],
Params: params,
Size: uint32(info.bsize),
Lines: uint32(info.lines),
}
if val := info.header.Get("Content-ID"); val != "" {
imapBS.Id = val
}
if val := info.header.Get("Content-Transfer-Encoding"); val != "" {
imapBS.Encoding = val
}
if val := info.header.Get("Content-Description"); val != "" {
imapBS.Description = val
}
if val := info.header.Get("Content-Disposition"); val != "" {
imapBS.Disposition = val
}
nextPart := append(currentPart, 1)
for {
if _, err := bs.getInfo(nextPart); err != nil {
break
}
var subStruct *imap.BodyStructure
subStruct, err = bs.IMAPBodyStructure(nextPart)
if err != nil {
return
}
if imapBS.Parts == nil {
imapBS.Parts = []*imap.BodyStructure{}
}
imapBS.Parts = append(imapBS.Parts, subStruct)
nextPart[len(nextPart)-1]++
}
return imapBS, nil
}

414
pkg/message/section_test.go Normal file
View File

@ -0,0 +1,414 @@
// Copyright (c) 2020 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package message
import (
"fmt"
"path/filepath"
"runtime"
"sort"
"strings"
"testing"
"github.com/stretchr/testify/require"
)
func debug(msg string, v ...interface{}) {
_, file, line, _ := runtime.Caller(1)
fmt.Printf("%s:%d: \033[2;33m"+msg+"\033[0;39m\n", append([]interface{}{filepath.Base(file), line}, v...)...)
}
func TestParseBodyStructure(t *testing.T) {
expectedStructure := map[string]string{
"": "multipart/mixed; boundary=\"0000MAIN\"",
"1": "text/plain",
"2": "application/octet-stream",
"3": "message/rfc822; boundary=\"0003MSG\"",
"3.1": "text/plain",
"3.2": "application/octet-stream",
"4": "multipart/mixed; boundary=\"0004ATTACH\"",
"4.1": "image/gif",
"4.2": "message/rfc822; boundary=\"0042MSG\"",
"4.2.1": "text/plain",
"4.2.2": "multipart/alternative; boundary=\"0422ALTER\"",
"4.2.2.1": "text/plain",
"4.2.2.2": "text/html",
}
mailReader := strings.NewReader(sampleMail)
bs, err := NewBodyStructure(mailReader)
require.NoError(t, err)
paths := []string{}
for path := range *bs {
paths = append(paths, path)
}
sort.Strings(paths)
debug("%10s: %-50s %5s %5s %5s %5s", "section", "type", "start", "size", "bsize", "lines")
for _, path := range paths {
sec := (*bs)[path]
contentType := sec.header.Get("Content-Type")
debug("%10s: %-50s %5d %5d %5d %5d", path, contentType, sec.start, sec.size, sec.bsize, sec.lines)
require.Equal(t, expectedStructure[path], contentType)
}
require.True(t, len(*bs) == len(expectedStructure), "Wrong number of sections expected %d but have %d", len(expectedStructure), len(*bs))
}
func TestGetSection(t *testing.T) {
structReader := strings.NewReader(sampleMail)
bs, err := NewBodyStructure(structReader)
require.NoError(t, err)
// Whole section.
for _, try := range testPaths {
mailReader := strings.NewReader(sampleMail)
info, err := bs.getInfo(try.path)
require.NoError(t, err)
section, err := bs.GetSection(mailReader, try.path)
require.NoError(t, err)
debug("section %v: %d %d\n___\n%s\n‾‾‾\n", try.path, info.start, info.size, string(section))
require.True(t, string(section) == try.expectedSection, "not same as expected:\n___\n%s\n‾‾‾", try.expectedSection)
}
// Body content.
for _, try := range testPaths {
mailReader := strings.NewReader(sampleMail)
info, err := bs.getInfo(try.path)
require.NoError(t, err)
section, err := bs.GetSectionContent(mailReader, try.path)
require.NoError(t, err)
debug("content %v: %d %d\n___\n%s\n‾‾‾\n", try.path, info.start+info.size-info.bsize, info.bsize, string(section))
require.True(t, string(section) == try.expectedBody, "not same as expected:\n___\n%s\n‾‾‾", try.expectedBody)
}
}
/* Structure example:
HEADER ([RFC-2822] header of the message)
TEXT ([RFC-2822] text body of the message) MULTIPART/MIXED
1 TEXT/PLAIN
2 APPLICATION/OCTET-STREAM
3 MESSAGE/RFC822
3.HEADER ([RFC-2822] header of the message)
3.TEXT ([RFC-2822] text body of the message) MULTIPART/MIXED
3.1 TEXT/PLAIN
3.2 APPLICATION/OCTET-STREAM
4 MULTIPART/MIXED
4.1 IMAGE/GIF
4.1.MIME ([MIME-IMB] header for the IMAGE/GIF)
4.2 MESSAGE/RFC822
4.2.HEADER ([RFC-2822] header of the message)
4.2.TEXT ([RFC-2822] text body of the message) MULTIPART/MIXED
4.2.1 TEXT/PLAIN
4.2.2 MULTIPART/ALTERNATIVE
4.2.2.1 TEXT/PLAIN
4.2.2.2 TEXT/RICHTEXT
*/
var sampleMail = `Subject: Sample mail
From: John Doe <jdoe@machine.example>
To: Mary Smith <mary@example.net>
Date: Fri, 21 Nov 1997 09:55:06 -0600
Content-Type: multipart/mixed; boundary="0000MAIN"
main summary
--0000MAIN
Content-Type: text/plain
1. main message
--0000MAIN
Content-Type: application/octet-stream
Content-Disposition: inline; filename="main_signature.sig"
Content-Transfer-Encoding: base64
2/MainOctetStream
--0000MAIN
Subject: Inside mail 3
From: Mary Smith <mary@example.net>
To: John Doe <jdoe@machine.example>
Date: Fri, 20 Nov 1997 09:55:06 -0600
Content-Type: message/rfc822; boundary="0003MSG"
3. message summary
--0003MSG
Content-Type: text/plain
3.1 message text
--0003MSG
Content-Type: application/octet-stream
Content-Disposition: attachment; filename="msg_3_signature.sig"
Content-Transfer-Encoding: base64
3/2/MessageOctestStream/==
--0003MSG--
--0000MAIN
Content-Type: multipart/mixed; boundary="0004ATTACH"
4 attach summary
--0004ATTACH
Content-Type: image/gif
Content-Disposition: attachment; filename="att4.1_gif.sig"
Content-Transfer-Encoding: base64
4/1/Gif=
--0004ATTACH
Subject: Inside mail 4.2
From: Mary Smith <mary@example.net>
To: John Doe <jdoe@machine.example>
Date: Fri, 10 Nov 1997 09:55:06 -0600
Content-Type: message/rfc822; boundary="0042MSG"
4.2 message summary
--0042MSG
Content-Type: text/plain
4.2.1 message text
--0042MSG
Content-Type: multipart/alternative; boundary="0422ALTER"
4.2.2 alternative summary
--0422ALTER
Content-Type: text/plain
4.2.2.1 plain text
--0422ALTER
Content-Type: text/html
<h1>4.2.2.2 html text</h1>
--0422ALTER--
--0042MSG--
--0004ATTACH--
--0000MAIN--
`
var testPaths = []struct {
path []int
expectedSection, expectedBody string
}{
{[]int{},
sampleMail,
`main summary
--0000MAIN
Content-Type: text/plain
1. main message
--0000MAIN
Content-Type: application/octet-stream
Content-Disposition: inline; filename="main_signature.sig"
Content-Transfer-Encoding: base64
2/MainOctetStream
--0000MAIN
Subject: Inside mail 3
From: Mary Smith <mary@example.net>
To: John Doe <jdoe@machine.example>
Date: Fri, 20 Nov 1997 09:55:06 -0600
Content-Type: message/rfc822; boundary="0003MSG"
3. message summary
--0003MSG
Content-Type: text/plain
3.1 message text
--0003MSG
Content-Type: application/octet-stream
Content-Disposition: attachment; filename="msg_3_signature.sig"
Content-Transfer-Encoding: base64
3/2/MessageOctestStream/==
--0003MSG--
--0000MAIN
Content-Type: multipart/mixed; boundary="0004ATTACH"
4 attach summary
--0004ATTACH
Content-Type: image/gif
Content-Disposition: attachment; filename="att4.1_gif.sig"
Content-Transfer-Encoding: base64
4/1/Gif=
--0004ATTACH
Subject: Inside mail 4.2
From: Mary Smith <mary@example.net>
To: John Doe <jdoe@machine.example>
Date: Fri, 10 Nov 1997 09:55:06 -0600
Content-Type: message/rfc822; boundary="0042MSG"
4.2 message summary
--0042MSG
Content-Type: text/plain
4.2.1 message text
--0042MSG
Content-Type: multipart/alternative; boundary="0422ALTER"
4.2.2 alternative summary
--0422ALTER
Content-Type: text/plain
4.2.2.1 plain text
--0422ALTER
Content-Type: text/html
<h1>4.2.2.2 html text</h1>
--0422ALTER--
--0042MSG--
--0004ATTACH--
--0000MAIN--
`,
},
{[]int{1},
`Content-Type: text/plain
1. main message
`,
`1. main message
`,
},
{[]int{3},
`Subject: Inside mail 3
From: Mary Smith <mary@example.net>
To: John Doe <jdoe@machine.example>
Date: Fri, 20 Nov 1997 09:55:06 -0600
Content-Type: message/rfc822; boundary="0003MSG"
3. message summary
--0003MSG
Content-Type: text/plain
3.1 message text
--0003MSG
Content-Type: application/octet-stream
Content-Disposition: attachment; filename="msg_3_signature.sig"
Content-Transfer-Encoding: base64
3/2/MessageOctestStream/==
--0003MSG--
`,
`3. message summary
--0003MSG
Content-Type: text/plain
3.1 message text
--0003MSG
Content-Type: application/octet-stream
Content-Disposition: attachment; filename="msg_3_signature.sig"
Content-Transfer-Encoding: base64
3/2/MessageOctestStream/==
--0003MSG--
`,
},
{[]int{3, 1},
`Content-Type: text/plain
3.1 message text
`,
`3.1 message text
`,
},
{[]int{3, 2},
`Content-Type: application/octet-stream
Content-Disposition: attachment; filename="msg_3_signature.sig"
Content-Transfer-Encoding: base64
3/2/MessageOctestStream/==
`,
`3/2/MessageOctestStream/==
`,
},
{[]int{4, 2, 2, 1},
`Content-Type: text/plain
4.2.2.1 plain text
`,
`4.2.2.1 plain text
`,
},
{[]int{4, 2, 2, 2},
`Content-Type: text/html
<h1>4.2.2.2 html text</h1>
`,
`<h1>4.2.2.2 html text</h1>
`,
},
}