yesBad/proton-bridge
1
0
Fork 0
forked from Silverfish/proton-bridge
Code Actions Activity

Other refactor: clean old builder

Browse Source
This commit is contained in:
Jakub Cuth
2021-04-27 08:12:50 +00:00
parent 286f51a4e7
commit 94b5799ba7
14 changed files with 790 additions and 935 deletions
Download Patch File Download Diff File Expand all files Collapse all files

198
internal/imap/mailbox_append.go Normal file
View File

@ -0,0 +1,198 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package imap
import (
"io"
"net/mail"
"strings"
"time"
"github.com/ProtonMail/gopenpgp/v2/crypto"
"github.com/ProtonMail/proton-bridge/internal/imap/uidplus"
"github.com/ProtonMail/proton-bridge/pkg/message"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/emersion/go-imap"
"github.com/pkg/errors"
)
// CreateMessage appends a new message to this mailbox. The \Recent flag will
// be added regardless of whether flags is empty or not. If date is nil, the
// current time will be used.
//
// If the Backend implements Updater, it must notify the client immediately
// via a mailbox update.
func (im *imapMailbox) CreateMessage(flags []string, date time.Time, body imap.Literal) error {
return im.logCommand(func() error {
return im.createMessage(flags, date, body)
}, "APPEND", flags, date)
}
func (im *imapMailbox) createMessage(flags []string, date time.Time, body imap.Literal) error { //nolint[funlen]
// Called from go-imap in goroutines - we need to handle panics for each function.
defer im.panicHandler.HandlePanic()
m, _, _, readers, err := message.Parse(body)
if err != nil {
return err
}
addr := im.storeAddress.APIAddress()
if addr == nil {
return errors.New("no available address for encryption")
}
m.AddressID = addr.ID
kr, err := im.user.client().KeyRingForAddressID(addr.ID)
if err != nil {
return err
}
// Handle imported messages which have no "Sender" address.
// This sometimes occurs with outlook which reports errors as imported emails or for drafts.
if m.Sender == nil {
im.log.Warning("Append: Missing email sender. Will use main address")
m.Sender = &mail.Address{
Name: "",
Address: addr.Email,
}
}
// "Drafts" needs to call special API routes.
// Clients always append the whole message again and remove the old one.
if im.storeMailbox.LabelID() == pmapi.DraftLabel {
// Sender address needs to be sanitised (drafts need to match cases exactly).
m.Sender.Address = pmapi.ConstructAddress(m.Sender.Address, addr.Email)
draft, _, err := im.user.storeUser.CreateDraft(kr, m, readers, "", "", "")
if err != nil {
return errors.Wrap(err, "failed to create draft")
}
targetSeq := im.storeMailbox.GetUIDList([]string{draft.ID})
return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), targetSeq)
}
// We need to make sure this is an import, and not a sent message from this account
// (sent messages from the account will be added by the event loop).
if im.storeMailbox.LabelID() == pmapi.SentLabel {
sanitizedSender := pmapi.SanitizeEmail(m.Sender.Address)
// Check whether this message was sent by a bridge user.
user, err := im.user.backend.bridge.GetUser(sanitizedSender)
if err == nil && user.ID() == im.storeUser.UserID() {
logEntry := im.log.WithField("addr", sanitizedSender).WithField("extID", m.Header.Get("Message-Id"))
// If we find the message in the store already, we can skip importing it.
if foundUID := im.storeMailbox.GetUIDByHeader(&m.Header); foundUID != uint32(0) {
logEntry.Info("Ignoring APPEND of duplicate to Sent folder")
return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), &uidplus.OrderedSeq{foundUID})
}
// We didn't find the message in the store, so we are currently sending it.
logEntry.WithField("time", date).Info("No matching UID, continuing APPEND to Sent")
}
}
message.ParseFlags(m, flags)
if !date.IsZero() {
m.Time = date.Unix()
}
internalID := m.Header.Get("X-Pm-Internal-Id")
references := m.Header.Get("References")
referenceList := strings.Fields(references)
// In case there is a mail client which corrupts headers, try
// "References" too.
if internalID == "" && len(referenceList) > 0 {
lastReference := referenceList[len(referenceList)-1]
match := pmapi.RxInternalReferenceFormat.FindStringSubmatch(lastReference)
if len(match) == 2 {
internalID = match[1]
}
}
im.user.appendExpungeLock.Lock()
defer im.user.appendExpungeLock.Unlock()
// Avoid appending a message which is already on the server. Apply the
// new label instead. This always happens with Outlook (it uses APPEND
// instead of COPY).
if internalID != "" {
// Check to see if this belongs to a different address in split mode or another ProtonMail account.
msg, err := im.storeMailbox.GetMessage(internalID)
if err == nil && (im.user.user.IsCombinedAddressMode() || (im.storeAddress.AddressID() == msg.Message().AddressID)) {
IDs := []string{internalID}
// See the comment bellow.
if msg.IsMarkedDeleted() {
if err := im.storeMailbox.MarkMessagesUndeleted(IDs); err != nil {
log.WithError(err).Error("Failed to undelete re-imported internal message")
}
}
err = im.storeMailbox.LabelMessages(IDs)
if err != nil {
return err
}
targetSeq := im.storeMailbox.GetUIDList(IDs)
return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), targetSeq)
}
}
im.log.Info("Importing external message")
if err := im.importMessage(m, readers, kr); err != nil {
im.log.Error("Import failed: ", err)
return err
}
// IMAP clients can move message to local folder (setting \Deleted flag)
// and then move it back (IMAP client does not remember the message,
// so instead removing the flag it imports duplicate message).
// Regular IMAP server would keep the message twice and later EXPUNGE would
// not delete the message (EXPUNGE would delete the original message and
// the new duplicate one would stay). API detects duplicates; therefore
// we need to remove \Deleted flag if IMAP client re-imports.
msg, err := im.storeMailbox.GetMessage(m.ID)
if err == nil && msg.IsMarkedDeleted() {
if err := im.storeMailbox.MarkMessagesUndeleted([]string{m.ID}); err != nil {
log.WithError(err).Error("Failed to undelete re-imported message")
}
}
targetSeq := im.storeMailbox.GetUIDList([]string{m.ID})
return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), targetSeq)
}
func (im *imapMailbox) importMessage(m *pmapi.Message, readers []io.Reader, kr *crypto.KeyRing) (err error) {
body, err := message.BuildEncrypted(m, readers, kr)
if err != nil {
return err
}
labels := []string{}
for _, l := range m.LabelIDs {
if l == pmapi.StarredLabel {
labels = append(labels, pmapi.StarredLabel)
}
}
return im.storeMailbox.ImportMessage(m, body, labels)
}

368
internal/imap/mailbox_fetch.go Normal file
View File

@ -0,0 +1,368 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package imap
import (
"bytes"
"context"
"fmt"
"net/textproto"
"sort"
"github.com/ProtonMail/proton-bridge/internal/imap/cache"
"github.com/ProtonMail/proton-bridge/pkg/message"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/emersion/go-imap"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
func (im *imapMailbox) getMessage(
storeMessage storeMessageProvider,
items []imap.FetchItem,
msgBuildCountHistogram *msgBuildCountHistogram,
) (msg *imap.Message, err error) {
msglog := im.log.WithField("msgID", storeMessage.ID())
msglog.Trace("Getting message")
seqNum, err := storeMessage.SequenceNumber()
if err != nil {
return
}
m := storeMessage.Message()
msg = imap.NewMessage(seqNum, items)
for _, item := range items {
switch item {
case imap.FetchEnvelope:
// No need to check IsFullHeaderCached here. API header
// contain enough information to build the envelope.
msg.Envelope = message.GetEnvelope(m, storeMessage.GetHeader())
case imap.FetchBody, imap.FetchBodyStructure:
structure, err := im.getBodyStructure(storeMessage)
if err != nil {
return nil, err
}
if msg.BodyStructure, err = structure.IMAPBodyStructure([]int{}); err != nil {
return nil, err
}
case imap.FetchFlags:
msg.Flags = message.GetFlags(m)
if storeMessage.IsMarkedDeleted() {
msg.Flags = append(msg.Flags, imap.DeletedFlag)
}
case imap.FetchInternalDate:
// Apple Mail crashes fetching messages with date older than 1970.
// There is no point having message older than RFC itself, it's not possible.
msg.InternalDate = message.SanitizeMessageDate(m.Time)
case imap.FetchRFC822Size:
if msg.Size, err = im.getSize(storeMessage); err != nil {
return nil, err
}
case imap.FetchUid:
if msg.Uid, err = storeMessage.UID(); err != nil {
return nil, err
}
case imap.FetchAll, imap.FetchFast, imap.FetchFull, imap.FetchRFC822, imap.FetchRFC822Header, imap.FetchRFC822Text:
fallthrough // this is list of defined items by go-imap, but items can be also sections generated from requests
default:
if err = im.getLiteralForSection(item, msg, storeMessage, msgBuildCountHistogram); err != nil {
return
}
}
}
return msg, err
}
// getSize returns cached size or it will build the message, save the size in
// DB and then returns the size after build.
//
// We are storing size in DB as part of pmapi messages metada. The size
// attribute on the server represents size of encrypted body. The value is
// cleared in Bridge and the final decrypted size (including header, attachment
// and MIME structure) is computed after building the message.
func (im *imapMailbox) getSize(storeMessage storeMessageProvider) (uint32, error) {
m := storeMessage.Message()
if m.Size <= 0 {
im.log.WithField("msgID", m.ID).Debug("Size unknown - downloading body")
// We are sure the size is not a problem right now. Clients
// might not first check sizes of all messages so we couldn't
// be sure if seeing 1st or 2nd sync is all right or not.
// Therefore, it's better to exclude getting size from the
// counting and see build count as real message build.
if _, _, err := im.getBodyAndStructure(storeMessage, nil); err != nil {
return 0, err
}
}
return uint32(m.Size), nil
}
func (im *imapMailbox) getLiteralForSection(
itemSection imap.FetchItem,
msg *imap.Message,
storeMessage storeMessageProvider,
msgBuildCountHistogram *msgBuildCountHistogram,
) error {
section, err := imap.ParseBodySectionName(itemSection)
if err != nil {
log.WithError(err).Warn("Failed to parse body section name; part will be skipped")
return nil //nolint[nilerr] ignore error
}
var literal imap.Literal
if literal, err = im.getMessageBodySection(storeMessage, section, msgBuildCountHistogram); err != nil {
return err
}
msg.Body[section] = literal
return nil
}
// getBodyStructure returns the cached body structure or it will build the message,
// save the structure in DB and then returns the structure after build.
//
// Apple Mail requests body structure for all messages irregularly. We cache
// bodystructure in local database in order to not re-download all messages
// from server.
func (im *imapMailbox) getBodyStructure(storeMessage storeMessageProvider) (bs *message.BodyStructure, err error) {
bs, err = storeMessage.GetBodyStructure()
if err != nil {
im.log.WithError(err).Debug("Fail to retrieve bodystructure from database")
}
if bs == nil {
// We are sure the body structure is not a problem right now.
// Clients might do first fetch body structure so we couldn't
// be sure if seeing 1st or 2nd sync is all right or not.
// Therefore, it's better to exclude first body structure fetch
// from the counting and see build count as real message build.
if bs, _, err = im.getBodyAndStructure(storeMessage, nil); err != nil {
return
}
}
return
}
func (im *imapMailbox) getBodyAndStructure(
storeMessage storeMessageProvider, msgBuildCountHistogram *msgBuildCountHistogram,
) (
structure *message.BodyStructure, bodyReader *bytes.Reader, err error,
) {
m := storeMessage.Message()
id := im.storeUser.UserID() + m.ID
cache.BuildLock(id)
defer cache.BuildUnlock(id)
bodyReader, structure = cache.LoadMail(id)
// return the message which was found in cache
if bodyReader.Len() != 0 && structure != nil {
return structure, bodyReader, nil
}
structure, body, err := im.buildMessage(m)
bodyReader = bytes.NewReader(body)
size := int64(len(body))
l := im.log.WithField("newSize", size).WithField("msgID", m.ID)
if err != nil || structure == nil || size == 0 {
l.WithField("hasStructure", structure != nil).Warn("Failed to build message")
return structure, bodyReader, err
}
// Save the size, body structure and header even for messages which
// were unable to decrypt. Hence they doesn't have to be computed every
// time.
m.Size = size
cacheMessageInStore(storeMessage, structure, body, l)
if msgBuildCountHistogram != nil {
times, errCount := storeMessage.IncreaseBuildCount()
if errCount != nil {
l.WithError(errCount).Warn("Cannot increase build count")
}
msgBuildCountHistogram.add(times)
}
// Drafts can change therefore we don't want to cache them.
if !isMessageInDraftFolder(m) {
cache.SaveMail(id, body, structure)
}
return structure, bodyReader, err
}
func cacheMessageInStore(storeMessage storeMessageProvider, structure *message.BodyStructure, body []byte, l *logrus.Entry) {
m := storeMessage.Message()
if errSize := storeMessage.SetSize(m.Size); errSize != nil {
l.WithError(errSize).Warn("Cannot update size while building")
}
if structure != nil && !isMessageInDraftFolder(m) {
if errStruct := storeMessage.SetBodyStructure(structure); errStruct != nil {
l.WithError(errStruct).Warn("Cannot update bodystructure while building")
}
}
header, errHead := structure.GetMailHeaderBytes(bytes.NewReader(body))
if errHead == nil && len(header) != 0 {
if errStore := storeMessage.SetHeader(header); errStore != nil {
l.WithError(errStore).Warn("Cannot update header in store")
}
} else {
l.WithError(errHead).Warn("Cannot get header bytes from structure")
}
}
func isMessageInDraftFolder(m *pmapi.Message) bool {
for _, labelID := range m.LabelIDs {
if labelID == pmapi.DraftLabel {
return true
}
}
return false
}
// This will download message (or read from cache) and pick up the section,
// extract data (header,body, both) and trim the output if needed.
func (im *imapMailbox) getMessageBodySection( //nolint[funlen]
storeMessage storeMessageProvider,
section *imap.BodySectionName,
msgBuildCountHistogram *msgBuildCountHistogram,
) (imap.Literal, error) {
var header textproto.MIMEHeader
var extraNewlineAfterHeader bool
var response []byte
im.log.WithField("msgID", storeMessage.ID()).Trace("Getting message body")
isMainHeaderRequested := len(section.Path) == 0 && section.Specifier == imap.HeaderSpecifier
if isMainHeaderRequested && storeMessage.IsFullHeaderCached() {
// In order to speed up (avoid download and decryptions) we
// cache the header. If a mail header was requested and DB
// contains full header (it means it was already built once)
// the DB header can be used without downloading and decrypting.
// Otherwise header is incomplete and clients would have issues
// e.g. AppleMail expects `text/plain` in HTML mails.
header = storeMessage.GetHeader()
} else {
// For all other cases it is necessary to download and decrypt the message
// and drop the header which was obtained from cache. The header will
// will be stored in DB once successfully built. Check `getBodyAndStructure`.
structure, bodyReader, err := im.getBodyAndStructure(storeMessage, msgBuildCountHistogram)
if err != nil {
return nil, err
}
switch {
case section.Specifier == imap.EntireSpecifier && len(section.Path) == 0:
// An empty section specification refers to the entire message, including the header.
response, err = structure.GetSection(bodyReader, section.Path)
case section.Specifier == imap.TextSpecifier || (section.Specifier == imap.EntireSpecifier && len(section.Path) != 0):
// The TEXT specifier refers to the content of the message (or section), omitting the [RFC-2822] header.
// Non-empty section with no specifier (imap.EntireSpecifier) refers to section content without header.
response, err = structure.GetSectionContent(bodyReader, section.Path)
case section.Specifier == imap.MIMESpecifier: // The MIME part specifier refers to the [MIME-IMB] header for this part.
fallthrough
case section.Specifier == imap.HeaderSpecifier:
if content, err := structure.GetSectionContent(bodyReader, section.Path); err == nil && content != nil {
extraNewlineAfterHeader = true
}
header, err = structure.GetSectionHeader(section.Path)
default:
err = errors.New("Unknown specifier " + string(section.Specifier))
}
if err != nil {
return nil, err
}
}
if header != nil {
response = filteredHeaderAsBytes(header, section)
// The blank line is included in all header fetches,
// except in the case of a message which has no body.
if extraNewlineAfterHeader {
response = append(response, []byte("\r\n")...)
}
}
// Trim any output if requested.
return bytes.NewBuffer(section.ExtractPartial(response)), nil
}
// filteredHeaderAsBytes filters the header fields by section fields and it
// returns the filtered fields as bytes.
// Options are: all fields, only selected fields, all fields except selected.
func filteredHeaderAsBytes(header textproto.MIMEHeader, section *imap.BodySectionName) []byte {
// remove fields
if len(section.Fields) != 0 && section.NotFields {
for _, field := range section.Fields {
header.Del(field)
}
}
fields := make([]string, 0, len(header))
if len(section.Fields) == 0 || section.NotFields { // add all and sort
for f := range header {
fields = append(fields, f)
}
sort.Strings(fields)
} else { // add only requested (in requested order)
for _, f := range section.Fields {
fields = append(fields, textproto.CanonicalMIMEHeaderKey(f))
}
}
headerBuf := &bytes.Buffer{}
for _, canonical := range fields {
if values, ok := header[canonical]; !ok {
continue
} else {
for _, val := range values {
fmt.Fprintf(headerBuf, "%s: %s\r\n", canonical, val)
}
}
}
return headerBuf.Bytes()
}
// buildMessage from PM to IMAP.
func (im *imapMailbox) buildMessage(m *pmapi.Message) (*message.BodyStructure, []byte, error) {
body, err := im.builder.NewJobWithOptions(
context.Background(),
im.user.client(),
m.ID,
message.JobOptions{
IgnoreDecryptionErrors: true, // Whether to ignore decryption errors and create a "custom message" instead.
SanitizeDate: true, // Whether to replace all dates before 1970 with RFC822's birthdate.
AddInternalID: true, // Whether to include MessageID as X-Pm-Internal-Id.
AddExternalID: true, // Whether to include ExternalID as X-Pm-External-Id.
AddMessageDate: true, // Whether to include message time as X-Pm-Date.
AddMessageIDReference: true, // Whether to include the MessageID in References.
},
).GetResult()
if err != nil {
return nil, nil, err
}
structure, err := message.NewBodyStructure(bytes.NewReader(body))
if err != nil {
return nil, nil, err
}
return structure, body, nil
}

540
internal/imap/mailbox_message.go
View File

@ -1,540 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package imap
import (
"bytes"
"context"
"fmt"
"io"
"net/mail"
"net/textproto"
"sort"
"strings"
"time"
"github.com/ProtonMail/gopenpgp/v2/crypto"
"github.com/ProtonMail/proton-bridge/internal/imap/cache"
"github.com/ProtonMail/proton-bridge/internal/imap/uidplus"
"github.com/ProtonMail/proton-bridge/pkg/message"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/emersion/go-imap"
"github.com/hashicorp/go-multierror"
"github.com/pkg/errors"
)
var (
rfc822Birthday = time.Date(1982, 8, 13, 0, 0, 0, 0, time.UTC) //nolint[gochecknoglobals]
)
type doNotCacheError struct{ e error }
func (dnc *doNotCacheError) Error() string { return dnc.e.Error() }
func (dnc *doNotCacheError) add(err error) { dnc.e = multierror.Append(dnc.e, err) }
func (dnc *doNotCacheError) errorOrNil() error {
if dnc == nil {
return nil
}
if dnc.e != nil {
return dnc
}
return nil
}
// CreateMessage appends a new message to this mailbox. The \Recent flag will
// be added regardless of whether flags is empty or not. If date is nil, the
// current time will be used.
//
// If the Backend implements Updater, it must notify the client immediately
// via a mailbox update.
func (im *imapMailbox) CreateMessage(flags []string, date time.Time, body imap.Literal) error {
return im.logCommand(func() error {
return im.createMessage(flags, date, body)
}, "APPEND", flags, date)
}
func (im *imapMailbox) createMessage(flags []string, date time.Time, body imap.Literal) error { // nolint[funlen]
// Called from go-imap in goroutines - we need to handle panics for each function.
defer im.panicHandler.HandlePanic()
m, _, _, readers, err := message.Parse(body)
if err != nil {
return err
}
addr := im.storeAddress.APIAddress()
if addr == nil {
return errors.New("no available address for encryption")
}
m.AddressID = addr.ID
kr, err := im.user.client().KeyRingForAddressID(addr.ID)
if err != nil {
return err
}
// Handle imported messages which have no "Sender" address.
// This sometimes occurs with outlook which reports errors as imported emails or for drafts.
if m.Sender == nil {
im.log.Warning("Append: Missing email sender. Will use main address")
m.Sender = &mail.Address{
Name: "",
Address: addr.Email,
}
}
// "Drafts" needs to call special API routes.
// Clients always append the whole message again and remove the old one.
if im.storeMailbox.LabelID() == pmapi.DraftLabel {
// Sender address needs to be sanitised (drafts need to match cases exactly).
m.Sender.Address = pmapi.ConstructAddress(m.Sender.Address, addr.Email)
draft, _, err := im.user.storeUser.CreateDraft(kr, m, readers, "", "", "")
if err != nil {
return errors.Wrap(err, "failed to create draft")
}
targetSeq := im.storeMailbox.GetUIDList([]string{draft.ID})
return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), targetSeq)
}
// We need to make sure this is an import, and not a sent message from this account
// (sent messages from the account will be added by the event loop).
if im.storeMailbox.LabelID() == pmapi.SentLabel {
sanitizedSender := pmapi.SanitizeEmail(m.Sender.Address)
// Check whether this message was sent by a bridge user.
user, err := im.user.backend.bridge.GetUser(sanitizedSender)
if err == nil && user.ID() == im.storeUser.UserID() {
logEntry := im.log.WithField("addr", sanitizedSender).WithField("extID", m.Header.Get("Message-Id"))
// If we find the message in the store already, we can skip importing it.
if foundUID := im.storeMailbox.GetUIDByHeader(&m.Header); foundUID != uint32(0) {
logEntry.Info("Ignoring APPEND of duplicate to Sent folder")
return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), &uidplus.OrderedSeq{foundUID})
}
// We didn't find the message in the store, so we are currently sending it.
logEntry.WithField("time", date).Info("No matching UID, continuing APPEND to Sent")
}
}
message.ParseFlags(m, flags)
if !date.IsZero() {
m.Time = date.Unix()
}
internalID := m.Header.Get("X-Pm-Internal-Id")
references := m.Header.Get("References")
referenceList := strings.Fields(references)
// In case there is a mail client which corrupts headers, try
// "References" too.
if internalID == "" && len(referenceList) > 0 {
lastReference := referenceList[len(referenceList)-1]
match := pmapi.RxInternalReferenceFormat.FindStringSubmatch(lastReference)
if len(match) == 2 {
internalID = match[1]
}
}
im.user.appendExpungeLock.Lock()
defer im.user.appendExpungeLock.Unlock()
// Avoid appending a message which is already on the server. Apply the
// new label instead. This always happens with Outlook (it uses APPEND
// instead of COPY).
if internalID != "" {
// Check to see if this belongs to a different address in split mode or another ProtonMail account.
msg, err := im.storeMailbox.GetMessage(internalID)
if err == nil && (im.user.user.IsCombinedAddressMode() || (im.storeAddress.AddressID() == msg.Message().AddressID)) {
IDs := []string{internalID}
// See the comment bellow.
if msg.IsMarkedDeleted() {
if err := im.storeMailbox.MarkMessagesUndeleted(IDs); err != nil {
log.WithError(err).Error("Failed to undelete re-imported internal message")
}
}
err = im.storeMailbox.LabelMessages(IDs)
if err != nil {
return err
}
targetSeq := im.storeMailbox.GetUIDList(IDs)
return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), targetSeq)
}
}
im.log.Info("Importing external message")
if err := im.importMessage(m, readers, kr); err != nil {
im.log.Error("Import failed: ", err)
return err
}
// IMAP clients can move message to local folder (setting \Deleted flag)
// and then move it back (IMAP client does not remember the message,
// so instead removing the flag it imports duplicate message).
// Regular IMAP server would keep the message twice and later EXPUNGE would
// not delete the message (EXPUNGE would delete the original message and
// the new duplicate one would stay). API detects duplicates; therefore
// we need to remove \Deleted flag if IMAP client re-imports.
msg, err := im.storeMailbox.GetMessage(m.ID)
if err == nil && msg.IsMarkedDeleted() {
if err := im.storeMailbox.MarkMessagesUndeleted([]string{m.ID}); err != nil {
log.WithError(err).Error("Failed to undelete re-imported message")
}
}
targetSeq := im.storeMailbox.GetUIDList([]string{m.ID})
return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), targetSeq)
}
func (im *imapMailbox) importMessage(m *pmapi.Message, readers []io.Reader, kr *crypto.KeyRing) (err error) { // nolint[funlen]
body, err := message.BuildEncrypted(m, readers, kr)
if err != nil {
return err
}
labels := []string{}
for _, l := range m.LabelIDs {
if l == pmapi.StarredLabel {
labels = append(labels, pmapi.StarredLabel)
}
}
return im.storeMailbox.ImportMessage(m, body, labels)
}
func (im *imapMailbox) getMessage(storeMessage storeMessageProvider, items []imap.FetchItem, msgBuildCountHistogram *msgBuildCountHistogram) (msg *imap.Message, err error) { //nolint[funlen]
msglog := im.log.WithField("msgID", storeMessage.ID())
msglog.Trace("Getting message")
seqNum, err := storeMessage.SequenceNumber()
if err != nil {
return
}
m := storeMessage.Message()
msg = imap.NewMessage(seqNum, items)
for _, item := range items {
switch item {
case imap.FetchEnvelope:
// No need to check IsFullHeaderCached here. API header
// contain enough information to build the envelope.
msg.Envelope = message.GetEnvelope(m, storeMessage.GetHeader())
case imap.FetchBody, imap.FetchBodyStructure:
var structure *message.BodyStructure
structure, err = im.getBodyStructure(storeMessage)
if err != nil {
return
}
if msg.BodyStructure, err = structure.IMAPBodyStructure([]int{}); err != nil {
return
}
case imap.FetchFlags:
msg.Flags = message.GetFlags(m)
if storeMessage.IsMarkedDeleted() {
msg.Flags = append(msg.Flags, imap.DeletedFlag)
}
case imap.FetchInternalDate:
msg.InternalDate = time.Unix(m.Time, 0)
// Apple Mail crashes fetching messages with date older than 1970.
// There is no point having message older than RFC itself, it's not possible.
if msg.InternalDate.Before(rfc822Birthday) {
msg.InternalDate = rfc822Birthday
}
case imap.FetchRFC822Size:
// Size attribute on the server counts encrypted data. The value is cleared
// on our part and we need to compute "real" size of decrypted data.
if m.Size <= 0 {
msglog.Debug("Size unknown - downloading body")
// We are sure the size is not a problem right now. Clients
// might not first check sizes of all messages so we couldn't
// be sure if seeing 1st or 2nd sync is all right or not.
// Therefore, it's better to exclude getting size from the
// counting and see build count as real message build.
if _, _, err = im.getBodyAndStructure(storeMessage, nil); err != nil {
return
}
}
msg.Size = uint32(m.Size)
case imap.FetchUid:
msg.Uid, err = storeMessage.UID()
if err != nil {
return nil, err
}
case imap.FetchAll, imap.FetchFast, imap.FetchFull, imap.FetchRFC822, imap.FetchRFC822Header, imap.FetchRFC822Text:
fallthrough // this is list of defined items by go-imap, but items can be also sections generated from requests
default:
if err = im.getLiteralForSection(item, msg, storeMessage, msgBuildCountHistogram); err != nil {
return
}
}
}
return msg, err
}
func (im *imapMailbox) getLiteralForSection(itemSection imap.FetchItem, msg *imap.Message, storeMessage storeMessageProvider, msgBuildCountHistogram *msgBuildCountHistogram) error {
section, err := imap.ParseBodySectionName(itemSection)
if err != nil {
log.WithError(err).Warn("Failed to parse body section name; part will be skipped")
return nil //nolint[nilerr] ignore error
}
var literal imap.Literal
if literal, err = im.getMessageBodySection(storeMessage, section, msgBuildCountHistogram); err != nil {
return err
}
msg.Body[section] = literal
return nil
}
func (im *imapMailbox) getBodyStructure(storeMessage storeMessageProvider) (bs *message.BodyStructure, err error) {
// Apple Mail requests body structure for all
// messages irregularly. We cache bodystructure in
// local database in order to not re-download all
// messages from server.
bs, err = storeMessage.GetBodyStructure()
if err != nil {
im.log.WithError(err).Debug("Fail to retrieve bodystructure from database")
}
if bs == nil {
// We are sure the body structure is not a problem right now.
// Clients might do first fetch body structure so we couldn't
// be sure if seeing 1st or 2nd sync is all right or not.
// Therefore, it's better to exclude first body structure fetch
// from the counting and see build count as real message build.
if bs, _, err = im.getBodyAndStructure(storeMessage, nil); err != nil {
return
}
}
return
}
//nolint[funlen] Jakub will fix in refactor
func (im *imapMailbox) getBodyAndStructure(storeMessage storeMessageProvider, msgBuildCountHistogram *msgBuildCountHistogram) (
structure *message.BodyStructure,
bodyReader *bytes.Reader, err error,
) {
m := storeMessage.Message()
id := im.storeUser.UserID() + m.ID
cache.BuildLock(id)
if bodyReader, structure = cache.LoadMail(id); bodyReader.Len() == 0 || structure == nil {
var body []byte
structure, body, err = im.buildMessage(m)
m.Size = int64(len(body))
// Save size and body structure even for messages unable to decrypt
// so the size or body structure doesn't have to be computed every time.
if err := storeMessage.SetSize(m.Size); err != nil {
im.log.WithError(err).
WithField("newSize", m.Size).
WithField("msgID", m.ID).
Warn("Cannot update size while building")
}
if structure != nil && !isMessageInDraftFolder(m) {
if err := storeMessage.SetBodyStructure(structure); err != nil {
im.log.WithError(err).
WithField("msgID", m.ID).
Warn("Cannot update bodystructure while building")
}
}
if err == nil && structure != nil && len(body) > 0 {
header, errHead := structure.GetMailHeaderBytes(bytes.NewReader(body))
if errHead == nil {
if errHead := storeMessage.SetHeader(header); errHead != nil {
im.log.WithError(errHead).
WithField("msgID", m.ID).
Warn("Cannot update header after building")
}
} else {
im.log.WithError(errHead).
WithField("msgID", m.ID).
Warn("Cannot get header bytes after building")
}
if msgBuildCountHistogram != nil {
times, err := storeMessage.IncreaseBuildCount()
if err != nil {
im.log.WithError(err).
WithField("msgID", m.ID).
Warn("Cannot increase build count")
}
msgBuildCountHistogram.add(times)
}
// Drafts can change and we don't want to cache them.
if !isMessageInDraftFolder(m) {
cache.SaveMail(id, body, structure)
}
bodyReader = bytes.NewReader(body)
}
if _, ok := err.(*doNotCacheError); ok {
im.log.WithField("msgID", m.ID).Errorf("do not cache message: %v", err)
err = nil
bodyReader = bytes.NewReader(body)
}
}
cache.BuildUnlock(id)
return structure, bodyReader, err
}
func isMessageInDraftFolder(m *pmapi.Message) bool {
for _, labelID := range m.LabelIDs {
if labelID == pmapi.DraftLabel {
return true
}
}
return false
}
// This will download message (or read from cache) and pick up the section,
// extract data (header,body, both) and trim the output if needed.
func (im *imapMailbox) getMessageBodySection( //nolint[funlen]
storeMessage storeMessageProvider,
section *imap.BodySectionName,
msgBuildCountHistogram *msgBuildCountHistogram,
) (imap.Literal, error) {
var header textproto.MIMEHeader
var extraNewlineAfterHeader bool
var response []byte
im.log.WithField("msgID", storeMessage.ID()).Trace("Getting message body")
isMainHeaderRequested := len(section.Path) == 0 && section.Specifier == imap.HeaderSpecifier
if isMainHeaderRequested && storeMessage.IsFullHeaderCached() {
// In order to speed up (avoid download and decryptions) we
// cache the header. If a mail header was requested and DB
// contains full header (it means it was already built once)
// the DB header can be used without downloading and decrypting.
// Otherwise header is incomplete and clients would have issues
// e.g. AppleMail expects `text/plain` in HTML mails.
header = storeMessage.GetHeader()
} else {
// For all other cases it is necessary to download and decrypt the message
// and drop the header which was obtained from cache. The header will
// will be stored in DB once successfully built. Check `getBodyAndStructure`.
structure, bodyReader, err := im.getBodyAndStructure(storeMessage, msgBuildCountHistogram)
if err != nil {
return nil, err
}
switch {
case section.Specifier == imap.EntireSpecifier && len(section.Path) == 0:
// An empty section specification refers to the entire message, including the header.
response, err = structure.GetSection(bodyReader, section.Path)
case section.Specifier == imap.TextSpecifier || (section.Specifier == imap.EntireSpecifier && len(section.Path) != 0):
// The TEXT specifier refers to the content of the message (or section), omitting the [RFC-2822] header.
// Non-empty section with no specifier (imap.EntireSpecifier) refers to section content without header.
response, err = structure.GetSectionContent(bodyReader, section.Path)
case section.Specifier == imap.MIMESpecifier: // The MIME part specifier refers to the [MIME-IMB] header for this part.
fallthrough
case section.Specifier == imap.HeaderSpecifier:
if content, err := structure.GetSectionContent(bodyReader, section.Path); err == nil && content != nil {
extraNewlineAfterHeader = true
}
header, err = structure.GetSectionHeader(section.Path)
default:
err = errors.New("Unknown specifier " + string(section.Specifier))
}
if err != nil {
return nil, err
}
}
if header != nil {
response = filteredHeaderAsBytes(header, section)
// The blank line is included in all header fetches,
// except in the case of a message which has no body.
if extraNewlineAfterHeader {
response = append(response, []byte("\r\n")...)
}
}
// Trim any output if requested.
return bytes.NewBuffer(section.ExtractPartial(response)), nil
}
// filteredHeaderAsBytes filters the header fields by section fields and it
// returns the filtered fields as bytes.
// Options are: all fields, only selected fields, all fields except selected.
func filteredHeaderAsBytes(header textproto.MIMEHeader, section *imap.BodySectionName) []byte {
// remove fields
if len(section.Fields) != 0 && section.NotFields {
for _, field := range section.Fields {
header.Del(field)
}
}
fields := make([]string, 0, len(header))
if len(section.Fields) == 0 || section.NotFields { // add all and sort
for f := range header {
fields = append(fields, f)
}
sort.Strings(fields)
} else { // add only requested (in requested order)
for _, f := range section.Fields {
fields = append(fields, textproto.CanonicalMIMEHeaderKey(f))
}
}
headerBuf := &bytes.Buffer{}
for _, canonical := range fields {
if values, ok := header[canonical]; !ok {
continue
} else {
for _, val := range values {
fmt.Fprintf(headerBuf, "%s: %s\r\n", canonical, val)
}
}
}
return headerBuf.Bytes()
}
// buildMessage from PM to IMAP.
func (im *imapMailbox) buildMessage(m *pmapi.Message) (*message.BodyStructure, []byte, error) {
body, err := im.builder.NewJobWithOptions(
context.Background(),
im.user.client(),
m.ID,
message.JobOptions{
IgnoreDecryptionErrors: true, // Whether to ignore decryption errors and create a "custom message" instead.
SanitizeDate: true, // Whether to replace all dates before 1970 with RFC822's birthdate.
AddInternalID: true, // Whether to include MessageID as X-Pm-Internal-Id.
AddExternalID: true, // Whether to include ExternalID as X-Pm-External-Id.
AddMessageDate: true, // Whether to include message time as X-Pm-Date.
AddMessageIDReference: true, // Whether to include the MessageID in References.
},
).GetResult()
if err != nil {
return nil, nil, err
}
structure, err := message.NewBodyStructure(bytes.NewReader(body))
if err != nil {
return nil, nil, err
}
return structure, body, nil
}

41
internal/imap/mailbox_message_test.go
View File

@ -1,41 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package imap
import (
"errors"
"testing"
"github.com/stretchr/testify/require"
)
func TestDoNotCache(t *testing.T) {
var dnc doNotCacheError
require.NoError(t, dnc.errorOrNil())
_, ok := dnc.errorOrNil().(*doNotCacheError)
require.True(t, !ok, "should not be type doNotCacheError")
dnc.add(errors.New("first"))
require.True(t, dnc.errorOrNil() != nil, "should be error")
_, ok = dnc.errorOrNil().(*doNotCacheError)
require.True(t, ok, "should be type doNotCacheError")
dnc.add(errors.New("second"))
dnc.add(errors.New("third"))
t.Log(dnc.errorOrNil())
}

130
pkg/message/boundary_reader.go Normal file
View File

@ -0,0 +1,130 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package message
import (
"bufio"
"bytes"
"io"
)
type boundaryReader struct {
reader *bufio.Reader
closed, first bool
skipped int
nl []byte // "\r\n" or "\n" (set after seeing first boundary line)
nlDashBoundary []byte // nl + "--boundary"
dashBoundaryDash []byte // "--boundary--"
dashBoundary []byte // "--boundary"
}
func newBoundaryReader(r *bufio.Reader, boundary string) (br *boundaryReader, err error) {
b := []byte("\r\n--" + boundary + "--")
br = &boundaryReader{
reader: r,
closed: false,
first: true,
nl: b[:2],
nlDashBoundary: b[:len(b)-2],
dashBoundaryDash: b[2:],
dashBoundary: b[2 : len(b)-2],
}
err = br.writeNextPartTo(nil)
return
}
// writeNextPartTo will copy the the bytes of next part and write them to
// writer. Will return EOF if the underlying reader is empty.
func (br *boundaryReader) writeNextPartTo(part io.Writer) (err error) {
if br.closed {
return io.EOF
}
var line, slice []byte
br.skipped = 0
for {
slice, err = br.reader.ReadSlice('\n')
line = append(line, slice...)
if err == bufio.ErrBufferFull {
continue
}
br.skipped += len(line)
if err == io.EOF && br.isFinalBoundary(line) {
err = nil
br.closed = true
return
}
if err != nil {
return
}
if br.isBoundaryDelimiterLine(line) {
br.first = false
return
}
if br.isFinalBoundary(line) {
br.closed = true
return
}
if part != nil {
if _, err = part.Write(line); err != nil {
return
}
}
line = []byte{}
}
}
func (br *boundaryReader) isFinalBoundary(line []byte) bool {
if !bytes.HasPrefix(line, br.dashBoundaryDash) {
return false
}
rest := line[len(br.dashBoundaryDash):]
rest = skipLWSPChar(rest)
return len(rest) == 0 || bytes.Equal(rest, br.nl)
}
func (br *boundaryReader) isBoundaryDelimiterLine(line []byte) (ret bool) {
if !bytes.HasPrefix(line, br.dashBoundary) {
return false
}
rest := line[len(br.dashBoundary):]
rest = skipLWSPChar(rest)
if br.first && len(rest) == 1 && rest[0] == '\n' {
br.nl = br.nl[1:]
br.nlDashBoundary = br.nlDashBoundary[1:]
}
return bytes.Equal(rest, br.nl)
}
func skipLWSPChar(b []byte) []byte {
for len(b) > 0 && (b[0] == ' ' || b[0] == '\t') {
b = b[1:]
}
return b
}

63
pkg/message/build_encrypted.go
View File

@ -22,30 +22,36 @@ import (
"encoding/base64" "encoding/base64"
"io" "io"
"io/ioutil" "io/ioutil"
"mime"
"mime/multipart" "mime/multipart"
"net/http" "net/http"
"net/textproto" "net/textproto"
"strings"
"github.com/ProtonMail/gopenpgp/v2/crypto" "github.com/ProtonMail/gopenpgp/v2/crypto"
pmmime "github.com/ProtonMail/proton-bridge/pkg/mime"
"github.com/ProtonMail/proton-bridge/pkg/pmapi" "github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/emersion/go-message"
"github.com/emersion/go-textwrapper" "github.com/emersion/go-textwrapper"
) )
// BuildEncrypted is used for importing encrypted message.
func BuildEncrypted(m *pmapi.Message, readers []io.Reader, kr *crypto.KeyRing) ([]byte, error) { //nolint[funlen] func BuildEncrypted(m *pmapi.Message, readers []io.Reader, kr *crypto.KeyRing) ([]byte, error) { //nolint[funlen]
b := &bytes.Buffer{} b := &bytes.Buffer{}
boundary := newBoundary(m.ID).gen()
// Overwrite content for main header for import. // Overwrite content for main header for import.
// Even if message has just simple body we should upload as multipart/mixed. // Even if message has just simple body we should upload as multipart/mixed.
// Each part has encrypted body and header reflects the original header. // Each part has encrypted body and header reflects the original header.
mainHeader := GetHeader(m) mainHeader := convertGoMessageToTextprotoHeader(getMessageHeader(m, JobOptions{}))
mainHeader.Set("Content-Type", "multipart/mixed; boundary="+GetBoundary(m)) mainHeader.Set("Content-Type", "multipart/mixed; boundary="+boundary)
mainHeader.Del("Content-Disposition") mainHeader.Del("Content-Disposition")
mainHeader.Del("Content-Transfer-Encoding") mainHeader.Del("Content-Transfer-Encoding")
if err := WriteHeader(b, mainHeader); err != nil { if err := WriteHeader(b, mainHeader); err != nil {
return nil, err return nil, err
} }
mw := multipart.NewWriter(b) mw := multipart.NewWriter(b)
if err := mw.SetBoundary(GetBoundary(m)); err != nil { if err := mw.SetBoundary(boundary); err != nil {
return nil, err return nil, err
} }
@ -71,7 +77,7 @@ func BuildEncrypted(m *pmapi.Message, readers []io.Reader, kr *crypto.KeyRing) (
for i := 0; i < len(m.Attachments); i++ { for i := 0; i < len(m.Attachments); i++ {
att := m.Attachments[i] att := m.Attachments[i]
r := readers[i] r := readers[i]
h := GetAttachmentHeader(att, false) h := getAttachmentHeader(att, false)
p, err := mw.CreatePart(h) p, err := mw.CreatePart(h)
if err != nil { if err != nil {
return nil, err return nil, err
@ -105,6 +111,55 @@ func BuildEncrypted(m *pmapi.Message, readers []io.Reader, kr *crypto.KeyRing) (
return b.Bytes(), nil return b.Bytes(), nil
} }
func convertGoMessageToTextprotoHeader(h message.Header) textproto.MIMEHeader {
out := make(textproto.MIMEHeader)
hf := h.Fields()
for hf.Next() {
// go-message fields are in the reverse order.
// textproto.MIMEHeader is not ordered except for the values of
// the same key which are ordered
key := textproto.CanonicalMIMEHeaderKey(hf.Key())
out[key] = append([]string{hf.Value()}, out[key]...)
}
return out
}
func getAttachmentHeader(att *pmapi.Attachment, buildForIMAP bool) textproto.MIMEHeader {
mediaType := att.MIMEType
if mediaType == "application/pgp-encrypted" {
mediaType = "application/octet-stream"
}
transferEncoding := "base64"
if mediaType == rfc822Message && buildForIMAP {
transferEncoding = "8bit"
}
encodedName := pmmime.EncodeHeader(att.Name)
disposition := "attachment" //nolint[goconst]
if strings.Contains(att.Header.Get("Content-Disposition"), pmapi.DispositionInline) {
disposition = pmapi.DispositionInline
}
h := make(textproto.MIMEHeader)
h.Set("Content-Type", mime.FormatMediaType(mediaType, map[string]string{"name": encodedName}))
if transferEncoding != "" {
h.Set("Content-Transfer-Encoding", transferEncoding)
}
h.Set("Content-Disposition", mime.FormatMediaType(disposition, map[string]string{"filename": encodedName}))
// Forward some original header lines.
forward := []string{"Content-Id", "Content-Description", "Content-Location"}
for _, k := range forward {
v := att.Header.Get(k)
if v != "" {
h.Set(k, v)
}
}
return h
}
func WriteHeader(w io.Writer, h textproto.MIMEHeader) (err error) { func WriteHeader(w io.Writer, h textproto.MIMEHeader) (err error) {
if err = http.Header(h).Write(w); err != nil { if err = http.Header(h).Write(w); err != nil {
return return

6
pkg/message/build_rfc822.go
View File

@ -329,7 +329,7 @@ func getMessageHeader(msg *pmapi.Message, opts JobOptions) message.Header { // n
// Sanitize the date; it needs to have a valid unix timestamp. // Sanitize the date; it needs to have a valid unix timestamp.
if opts.SanitizeDate { if opts.SanitizeDate {
if date, err := rfc5322.ParseDateTime(hdr.Get("Date")); err != nil || date.Before(time.Unix(0, 0)) { if date, err := rfc5322.ParseDateTime(hdr.Get("Date")); err != nil || date.Before(time.Unix(0, 0)) {
msgDate := sanitizeMessageDate(msg.Time) msgDate := SanitizeMessageDate(msg.Time)
hdr.Set("Date", msgDate.In(time.UTC).Format(time.RFC1123Z)) hdr.Set("Date", msgDate.In(time.UTC).Format(time.RFC1123Z))
// We clobbered the date so we save it under X-Original-Date. // We clobbered the date so we save it under X-Original-Date.
hdr.Set("X-Original-Date", date.In(time.UTC).Format(time.RFC1123Z)) hdr.Set("X-Original-Date", date.In(time.UTC).Format(time.RFC1123Z))
@ -364,10 +364,10 @@ func getMessageHeader(msg *pmapi.Message, opts JobOptions) message.Header { // n
return hdr return hdr
} }
// sanitizeMessageDate will return time from msgTime timestamp. If timestamp is // SanitizeMessageDate will return time from msgTime timestamp. If timestamp is
// not after epoch the RFC822 publish day will be used. No message should // not after epoch the RFC822 publish day will be used. No message should
// realistically be older than RFC822 itself. // realistically be older than RFC822 itself.
func sanitizeMessageDate(msgTime int64) time.Time { func SanitizeMessageDate(msgTime int64) time.Time {
if msgTime := time.Unix(msgTime, 0); msgTime.After(time.Unix(0, 0)) { if msgTime := time.Unix(msgTime, 0); msgTime.After(time.Unix(0, 0)) {
return msgTime return msgTime
} }

2
pkg/message/envelope.go
View File

@ -32,7 +32,7 @@ func GetEnvelope(msg *pmapi.Message, header textproto.MIMEHeader) *imap.Envelope
setMessageIDIfNeeded(msg, &hdr) setMessageIDIfNeeded(msg, &hdr)
return &imap.Envelope{ return &imap.Envelope{
Date: sanitizeMessageDate(msg.Time), Date: SanitizeMessageDate(msg.Time),
Subject: msg.Subject, Subject: msg.Subject,
From: getAddresses([]*mail.Address{msg.Sender}), From: getAddresses([]*mail.Address{msg.Sender}),
Sender: getAddresses([]*mail.Address{msg.Sender}), Sender: getAddresses([]*mail.Address{msg.Sender}),

12
pkg/message/flags.go
View File

@ -22,13 +22,14 @@ import (
"github.com/emersion/go-imap" "github.com/emersion/go-imap"
) )
//nolint[gochecknoglobals] // Various client specific flags.
var ( const (
AppleMailJunkFlag = imap.CanonicalFlag("$Junk") AppleMailJunkFlag = "$Junk"
ThunderbirdJunkFlag = imap.CanonicalFlag("Junk") ThunderbirdJunkFlag = "Junk"
ThunderbirdNonJunkFlag = imap.CanonicalFlag("NonJunk") ThunderbirdNonJunkFlag = "NonJunk"
) )
// GetFlags returns imap flags from pmapi message attributes.
func GetFlags(m *pmapi.Message) (flags []string) { func GetFlags(m *pmapi.Message) (flags []string) {
if m.Unread == 0 { if m.Unread == 0 {
flags = append(flags, imap.SeenFlag) flags = append(flags, imap.SeenFlag)
@ -59,6 +60,7 @@ func GetFlags(m *pmapi.Message) (flags []string) {
return return
} }
// ParseFlags sets attributes to pmapi messages based on imap flags.
func ParseFlags(m *pmapi.Message, flags []string) { func ParseFlags(m *pmapi.Message, flags []string) {
if m.Header.Get("received") == "" { if m.Header.Get("received") == "" {
m.Flags = pmapi.FlagSent m.Flags = pmapi.FlagSent

138
pkg/message/header.go
View File

@ -1,138 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package message
import (
"mime"
"net/textproto"
"strings"
"time"
pmmime "github.com/ProtonMail/proton-bridge/pkg/mime"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
)
// GetHeader builds the header for the message.
func GetHeader(msg *pmapi.Message) textproto.MIMEHeader { //nolint[funlen]
h := make(textproto.MIMEHeader)
// Copy the custom header fields if there are some.
if msg.Header != nil {
h = textproto.MIMEHeader(msg.Header)
}
// Add or rewrite fields.
h.Set("Subject", pmmime.EncodeHeader(msg.Subject))
if msg.Sender != nil {
h.Set("From", pmmime.EncodeHeader(msg.Sender.String()))
}
if len(msg.ReplyTos) > 0 {
h.Set("Reply-To", pmmime.EncodeHeader(toAddressList(msg.ReplyTos)))
}
if len(msg.ToList) > 0 {
h.Set("To", pmmime.EncodeHeader(toAddressList(msg.ToList)))
}
if len(msg.CCList) > 0 {
h.Set("Cc", pmmime.EncodeHeader(toAddressList(msg.CCList)))
}
if len(msg.BCCList) > 0 {
h.Set("Bcc", pmmime.EncodeHeader(toAddressList(msg.BCCList)))
}
// Add or rewrite date related fields.
if msg.Time > 0 {
h.Set("X-Pm-Date", time.Unix(msg.Time, 0).Format(time.RFC1123Z))
if d, err := msg.Header.Date(); err != nil || d.IsZero() { // Fix date if needed.
h.Set("Date", time.Unix(msg.Time, 0).Format(time.RFC1123Z))
}
}
// Use External-Id if available to ensure email clients:
// * build the conversations threads correctly (Thunderbird, Mac Outlook, Apple Mail)
// * do not think the message is lost (Apple Mail)
if msg.ExternalID != "" {
h.Set("X-Pm-External-Id", "<"+msg.ExternalID+">")
if h.Get("Message-Id") == "" {
h.Set("Message-Id", "<"+msg.ExternalID+">")
}
}
if msg.ID != "" {
if h.Get("Message-Id") == "" {
h.Set("Message-Id", "<"+msg.ID+"@"+pmapi.InternalIDDomain+">")
}
h.Set("X-Pm-Internal-Id", msg.ID)
// Forward References, and include the message ID here (to improve outlook support).
if references := h.Get("References"); !strings.Contains(references, msg.ID) {
references += " <" + msg.ID + "@" + pmapi.InternalIDDomain + ">"
h.Set("References", references)
}
}
if msg.ConversationID != "" {
h.Set("X-Pm-ConversationID-Id", msg.ConversationID)
}
return h
}
func SetBodyContentFields(h *textproto.MIMEHeader, m *pmapi.Message) {
h.Set("Content-Type", m.MIMEType+"; charset=utf-8")
h.Set("Content-Disposition", pmapi.DispositionInline)
h.Set("Content-Transfer-Encoding", "quoted-printable")
}
func GetBodyHeader(m *pmapi.Message) textproto.MIMEHeader {
h := make(textproto.MIMEHeader)
SetBodyContentFields(&h, m)
return h
}
func GetAttachmentHeader(att *pmapi.Attachment, buildForIMAP bool) textproto.MIMEHeader {
mediaType := att.MIMEType
if mediaType == "application/pgp-encrypted" {
mediaType = "application/octet-stream"
}
transferEncoding := "base64"
if mediaType == rfc822Message && buildForIMAP {
transferEncoding = "8bit"
}
encodedName := pmmime.EncodeHeader(att.Name)
disposition := "attachment" //nolint[goconst]
if strings.Contains(att.Header.Get("Content-Disposition"), pmapi.DispositionInline) {
disposition = pmapi.DispositionInline
}
h := make(textproto.MIMEHeader)
h.Set("Content-Type", mime.FormatMediaType(mediaType, map[string]string{"name": encodedName}))
if transferEncoding != "" {
h.Set("Content-Transfer-Encoding", transferEncoding)
}
h.Set("Content-Disposition", mime.FormatMediaType(disposition, map[string]string{"filename": encodedName}))
// Forward some original header lines.
forward := []string{"Content-Id", "Content-Description", "Content-Location"}
for _, k := range forward {
v := att.Header.Get(k)
if v != "" {
h.Set(k, v)
}
}
return h
}

22
pkg/message/message.go
View File

@ -15,12 +15,11 @@
// You should have received a copy of the GNU General Public License // You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>. // along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
// Package message contains set of tools to convert message between Proton API
// and IMAP format.
package message package message
import ( import (
"strings"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
) )
@ -29,20 +28,3 @@ const (
) )
var log = logrus.WithField("pkg", "pkg/message") //nolint[gochecknoglobals] var log = logrus.WithField("pkg", "pkg/message") //nolint[gochecknoglobals]
func GetBoundary(m *pmapi.Message) string {
// The boundary needs to be deterministic because messages are not supposed to
// change.
return newBoundary(m.ID).gen()
}
func SeparateInlineAttachments(m *pmapi.Message) (atts, inlines []*pmapi.Attachment) {
for _, att := range m.Attachments {
if strings.Contains(att.Header.Get("Content-Disposition"), pmapi.DispositionInline) {
inlines = append(inlines, att)
} else {
atts = append(atts, att)
}
}
return
}

129
pkg/message/section.go
View File

@ -32,13 +32,18 @@ import (
"github.com/vmihailenco/msgpack/v5" "github.com/vmihailenco/msgpack/v5"
) )
// BodyStructure is used to parse an email into MIME sections and then generate
// body structure for IMAP server.
type BodyStructure map[string]*SectionInfo
// SectionInfo is used to hold data about parts of each section.
type SectionInfo struct { type SectionInfo struct {
Header textproto.MIMEHeader Header textproto.MIMEHeader
Start, BSize, Size, Lines int Start, BSize, Size, Lines int
reader io.Reader reader io.Reader
} }
// Read and count. // Read will also count the final size of section.
func (si *SectionInfo) Read(p []byte) (n int, err error) { func (si *SectionInfo) Read(p []byte) (n int, err error) {
n, err = si.reader.Read(p) n, err = si.reader.Read(p)
si.Size += n si.Size += n
@ -46,118 +51,13 @@ func (si *SectionInfo) Read(p []byte) (n int, err error) {
return return
} }
type boundaryReader struct {
reader *bufio.Reader
closed, first bool
skipped int
nl []byte // "\r\n" or "\n" (set after seeing first boundary line)
nlDashBoundary []byte // nl + "--boundary"
dashBoundaryDash []byte // "--boundary--"
dashBoundary []byte // "--boundary"
}
func newBoundaryReader(r *bufio.Reader, boundary string) (br *boundaryReader, err error) {
b := []byte("\r\n--" + boundary + "--")
br = &boundaryReader{
reader: r,
closed: false,
first: true,
nl: b[:2],
nlDashBoundary: b[:len(b)-2],
dashBoundaryDash: b[2:],
dashBoundary: b[2 : len(b)-2],
}
err = br.WriteNextPartTo(nil)
return
}
func skipLWSPChar(b []byte) []byte {
for len(b) > 0 && (b[0] == ' ' || b[0] == '\t') {
b = b[1:]
}
return b
}
func (br *boundaryReader) isFinalBoundary(line []byte) bool {
if !bytes.HasPrefix(line, br.dashBoundaryDash) {
return false
}
rest := line[len(br.dashBoundaryDash):]
rest = skipLWSPChar(rest)
return len(rest) == 0 || bytes.Equal(rest, br.nl)
}
func (br *boundaryReader) isBoundaryDelimiterLine(line []byte) (ret bool) {
if !bytes.HasPrefix(line, br.dashBoundary) {
return false
}
rest := line[len(br.dashBoundary):]
rest = skipLWSPChar(rest)
if br.first && len(rest) == 1 && rest[0] == '\n' {
br.nl = br.nl[1:]
br.nlDashBoundary = br.nlDashBoundary[1:]
}
return bytes.Equal(rest, br.nl)
}
func (br *boundaryReader) WriteNextPartTo(part io.Writer) (err error) {
if br.closed {
return io.EOF
}
var line, slice []byte
br.skipped = 0
for {
slice, err = br.reader.ReadSlice('\n')
line = append(line, slice...)
if err == bufio.ErrBufferFull {
continue
}
br.skipped += len(line)
if err == io.EOF && br.isFinalBoundary(line) {
err = nil
br.closed = true
return
}
if err != nil {
return
}
if br.isBoundaryDelimiterLine(line) {
br.first = false
return
}
if br.isFinalBoundary(line) {
br.closed = true
return
}
if part != nil {
if _, err = part.Write(line); err != nil {
return
}
}
line = []byte{}
}
}
type BodyStructure map[string]*SectionInfo
func NewBodyStructure(reader io.Reader) (structure *BodyStructure, err error) { func NewBodyStructure(reader io.Reader) (structure *BodyStructure, err error) {
structure = &BodyStructure{} structure = &BodyStructure{}
err = structure.Parse(reader) err = structure.Parse(reader)
return return
} }
// DeserializeBodyStructure will create new structure from msgpack bytes.
func DeserializeBodyStructure(raw []byte) (*BodyStructure, error) { func DeserializeBodyStructure(raw []byte) (*BodyStructure, error) {
bs := &BodyStructure{} bs := &BodyStructure{}
err := msgpack.Unmarshal(raw, bs) err := msgpack.Unmarshal(raw, bs)
@ -167,6 +67,7 @@ func DeserializeBodyStructure(raw []byte) (*BodyStructure, error) {
return bs, err return bs, err
} }
// Serialize will write msgpack bytes.
func (bs *BodyStructure) Serialize() ([]byte, error) { func (bs *BodyStructure) Serialize() ([]byte, error) {
data, err := msgpack.Marshal(bs) data, err := msgpack.Marshal(bs)
if err != nil { if err != nil {
@ -175,6 +76,7 @@ func (bs *BodyStructure) Serialize() ([]byte, error) {
return data, nil return data, nil
} }
// Parse will read the mail and create all body structures.
func (bs *BodyStructure) Parse(r io.Reader) error { func (bs *BodyStructure) Parse(r io.Reader) error {
return bs.parseAllChildSections(r, []int{}, 0) return bs.parseAllChildSections(r, []int{}, 0)
} }
@ -215,7 +117,7 @@ func (bs *BodyStructure) parseAllChildSections(r io.Reader, currentPath []int, s
for err == nil { for err == nil {
start += br.skipped start += br.skipped
part := &bytes.Buffer{} part := &bytes.Buffer{}
err = br.WriteNextPartTo(part) err = br.writeNextPartTo(part)
if err != nil { if err != nil {
break break
} }
@ -319,19 +221,16 @@ func (bs *BodyStructure) getInfo(sectionPath []int) (sectionInfo *SectionInfo, e
return return
} }
// GetSection returns bytes of section including MIME header.
func (bs *BodyStructure) GetSection(wholeMail io.ReadSeeker, sectionPath []int) (section []byte, err error) { func (bs *BodyStructure) GetSection(wholeMail io.ReadSeeker, sectionPath []int) (section []byte, err error) {
info, err := bs.getInfo(sectionPath) info, err := bs.getInfo(sectionPath)
if err != nil { if err != nil {
return return
} }
if _, err = wholeMail.Seek(int64(info.Start), io.SeekStart); err != nil { return goToOffsetAndReadNBytes(wholeMail, info.Start, info.Size)
return
}
section = make([]byte, info.Size)
_, err = wholeMail.Read(section)
return
} }
// GetSectionContent returns bytes of section content (excluding MIME header).
func (bs *BodyStructure) GetSectionContent(wholeMail io.ReadSeeker, sectionPath []int) (section []byte, err error) { func (bs *BodyStructure) GetSectionContent(wholeMail io.ReadSeeker, sectionPath []int) (section []byte, err error) {
info, err := bs.getInfo(sectionPath) info, err := bs.getInfo(sectionPath)
if err != nil { if err != nil {
@ -380,6 +279,8 @@ func (bs *BodyStructure) GetSectionHeader(sectionPath []int) (header textproto.M
return return
} }
// IMAPBodyStructure will prepare imap bodystructure recurently for given part.
// Use empty path to create whole email structure.
func (bs *BodyStructure) IMAPBodyStructure(currentPart []int) (imapBS *imap.BodyStructure, err error) { func (bs *BodyStructure) IMAPBodyStructure(currentPart []int) (imapBS *imap.BodyStructure, err error) {
var info *SectionInfo var info *SectionInfo
if info, err = bs.getInfo(currentPart); err != nil { if info, err = bs.getInfo(currentPart); err != nil {

3
test/fakeapi/controller_control.go
View File

@ -20,10 +20,8 @@ package fakeapi
import ( import (
"errors" "errors"
"fmt" "fmt"
"net/mail"
"strings" "strings"
messageUtils "github.com/ProtonMail/proton-bridge/pkg/message"
"github.com/ProtonMail/proton-bridge/pkg/pmapi" "github.com/ProtonMail/proton-bridge/pkg/pmapi"
) )
@ -141,7 +139,6 @@ func (ctl *Controller) AddUserMessage(username string, message *pmapi.Message) (
} }
message.ID = ctl.messageIDGenerator.next("") message.ID = ctl.messageIDGenerator.next("")
message.LabelIDs = append(message.LabelIDs, pmapi.AllMailLabel) message.LabelIDs = append(message.LabelIDs, pmapi.AllMailLabel)
message.Header = mail.Header(messageUtils.GetHeader(message))
for iAtt := 0; iAtt < message.NumAttachments; iAtt++ { for iAtt := 0; iAtt < message.NumAttachments; iAtt++ {
message.Attachments = append(message.Attachments, newTestAttachment(iAtt, message.ID)) message.Attachments = append(message.Attachments, newTestAttachment(iAtt, message.ID))

73
test/liveapi/messages.go
View File

@ -18,11 +18,7 @@
package liveapi package liveapi
import ( import (
"bytes"
"fmt" "fmt"
"io"
"mime/multipart"
"net/http"
messageUtils "github.com/ProtonMail/proton-bridge/pkg/message" messageUtils "github.com/ProtonMail/proton-bridge/pkg/message"
"github.com/ProtonMail/proton-bridge/pkg/pmapi" "github.com/ProtonMail/proton-bridge/pkg/pmapi"
@ -43,14 +39,19 @@ func (ctl *Controller) AddUserMessage(username string, message *pmapi.Message) (
message.Flags = pmapi.ComputeMessageFlagsByLabels(message.LabelIDs) message.Flags = pmapi.ComputeMessageFlagsByLabels(message.LabelIDs)
} }
body, err := buildMessage(client, message) kr, err := client.KeyRingForAddressID(message.AddressID)
if err != nil {
return "", errors.Wrap(err, "failed to get keyring while adding user message")
}
body, err := messageUtils.BuildEncrypted(message, nil, kr)
if err != nil { if err != nil {
return "", errors.Wrap(err, "failed to build message") return "", errors.Wrap(err, "failed to build message")
} }
req := &pmapi.ImportMsgReq{ req := &pmapi.ImportMsgReq{
AddressID: message.AddressID, AddressID: message.AddressID,
Body: body.Bytes(), Body: body,
Unread: message.Unread, Unread: message.Unread,
Time: message.Time, Time: message.Time,
Flags: message.Flags, Flags: message.Flags,
@ -70,66 +71,6 @@ func (ctl *Controller) AddUserMessage(username string, message *pmapi.Message) (
return result.MessageID, nil return result.MessageID, nil
} }
func buildMessage(client pmapi.Client, message *pmapi.Message) (*bytes.Buffer, error) {
if err := encryptMessage(client, message); err != nil {
return nil, errors.Wrap(err, "failed to encrypt message")
}
body := &bytes.Buffer{}
if err := buildMessageHeader(message, body); err != nil {
return nil, errors.Wrap(err, "failed to build message header")
}
if err := buildMessageBody(message, body); err != nil {
return nil, errors.Wrap(err, "failed to build message body")
}
return body, nil
}
func encryptMessage(client pmapi.Client, message *pmapi.Message) error {
kr, err := client.KeyRingForAddressID(message.AddressID)
if err != nil {
return errors.Wrap(err, "failed to get keyring for address")
}
if err = message.Encrypt(kr, nil); err != nil {
return errors.Wrap(err, "failed to encrypt message body")
}
return nil
}
func buildMessageHeader(message *pmapi.Message, body *bytes.Buffer) error {
header := messageUtils.GetHeader(message)
header.Set("Content-Type", "multipart/mixed; boundary="+messageUtils.GetBoundary(message))
header.Del("Content-Disposition")
header.Del("Content-Transfer-Encoding")
if err := http.Header(header).Write(body); err != nil {
return errors.Wrap(err, "failed to write header")
}
_, _ = body.WriteString("\r\n")
return nil
}
func buildMessageBody(message *pmapi.Message, body *bytes.Buffer) error {
mw := multipart.NewWriter(body)
if err := mw.SetBoundary(messageUtils.GetBoundary(message)); err != nil {
return errors.Wrap(err, "failed to set boundary")
}
bodyHeader := messageUtils.GetBodyHeader(message)
bodyHeader.Set("Content-Transfer-Encoding", "7bit")
part, err := mw.CreatePart(bodyHeader)
if err != nil {
return errors.Wrap(err, "failed to create message body part")
}
if _, err := io.WriteString(part, message.Body); err != nil {
return errors.Wrap(err, "failed to write message body")
}
_ = mw.Close()
return nil
}
func (ctl *Controller) GetMessages(username, labelID string) ([]*pmapi.Message, error) { func (ctl *Controller) GetMessages(username, labelID string) ([]*pmapi.Message, error) {
client, ok := ctl.pmapiByUsername[username] client, ok := ctl.pmapiByUsername[username]
if !ok { if !ok {