Silverfish/proton-bridge
1
0
Fork 1
mirror of https://github.com/ProtonMail/proton-bridge.git synced 2025-12-15 14:56:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

GODT-1044: lite parser

Browse Source
This commit is contained in:
James Houlahan
2021-05-17 15:56:22 +02:00
committed by Jakub Cuth
parent 509ba52ba2
commit e01dc77a61
12 changed files with 966 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

227
internal/imap/mailbox_append.go
View File

@ -18,7 +18,9 @@
package imap
import (
"io"
"bufio"
"bytes"
"io/ioutil"
"net/mail"
"strings"
"time"
@ -28,6 +30,7 @@ import (
"github.com/ProtonMail/proton-bridge/pkg/message"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/emersion/go-imap"
"github.com/emersion/go-message/textproto"
"github.com/pkg/errors"
)
@ -43,11 +46,15 @@ func (im *imapMailbox) CreateMessage(flags []string, date time.Time, body imap.L
}, "APPEND", flags, date)
}
func (im *imapMailbox) createMessage(flags []string, date time.Time, body imap.Literal) error { //nolint[funlen]
func (im *imapMailbox) createMessage(imapFlags []string, date time.Time, r imap.Literal) error { //nolint[funlen]
// Called from go-imap in goroutines - we need to handle panics for each function.
defer im.panicHandler.HandlePanic()
m, _, _, readers, err := message.Parse(body)
// NOTE: Is this lock meant to be here?
im.user.appendExpungeLock.Lock()
defer im.user.appendExpungeLock.Unlock()
body, err := ioutil.ReadAll(r)
if err != nil {
return err
}
@ -56,113 +63,88 @@ func (im *imapMailbox) createMessage(flags []string, date time.Time, body imap.L
if addr == nil {
return errors.New("no available address for encryption")
}
m.AddressID = addr.ID
kr, err := im.user.client().KeyRingForAddressID(addr.ID)
if err != nil {
return err
}
// Handle imported messages which have no "Sender" address.
// This sometimes occurs with outlook which reports errors as imported emails or for drafts.
if m.Sender == nil {
im.log.Warning("Append: Missing email sender. Will use main address")
m.Sender = &mail.Address{
Name: "",
Address: addr.Email,
}
}
// "Drafts" needs to call special API routes.
// Clients always append the whole message again and remove the old one.
if im.storeMailbox.LabelID() == pmapi.DraftLabel {
// Sender address needs to be sanitised (drafts need to match cases exactly).
m.Sender.Address = pmapi.ConstructAddress(m.Sender.Address, addr.Email)
draft, _, err := im.user.storeUser.CreateDraft(kr, m, readers, "", "", "")
if err != nil {
return errors.Wrap(err, "failed to create draft")
}
targetSeq := im.storeMailbox.GetUIDList([]string{draft.ID})
return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), targetSeq)
return im.createDraftMessage(kr, addr.Email, body)
}
// We need to make sure this is an import, and not a sent message from this account
// (sent messages from the account will be added by the event loop).
if im.storeMailbox.LabelID() == pmapi.SentLabel {
sanitizedSender := pmapi.SanitizeEmail(m.Sender.Address)
m, _, _, _, err := message.Parse(bytes.NewReader(body))
if err != nil {
return err
}
// Check whether this message was sent by a bridge user.
user, err := im.user.backend.bridge.GetUser(sanitizedSender)
if err == nil && user.ID() == im.storeUser.UserID() {
logEntry := im.log.WithField("addr", sanitizedSender).WithField("extID", m.Header.Get("Message-Id"))
if m.Sender == nil {
m.Sender = &mail.Address{Address: addr.Email}
}
if user, err := im.user.backend.bridge.GetUser(pmapi.SanitizeEmail(m.Sender.Address)); err == nil && user.ID() == im.storeUser.UserID() {
logEntry := im.log.WithField("sender", m.Sender).WithField("extID", m.Header.Get("Message-Id")).WithField("date", date)
// If we find the message in the store already, we can skip importing it.
if foundUID := im.storeMailbox.GetUIDByHeader(&m.Header); foundUID != uint32(0) {
logEntry.Info("Ignoring APPEND of duplicate to Sent folder")
return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), &uidplus.OrderedSeq{foundUID})
}
// We didn't find the message in the store, so we are currently sending it.
logEntry.WithField("time", date).Info("No matching UID, continuing APPEND to Sent")
logEntry.Info("No matching UID, continuing APPEND to Sent")
}
}
message.ParseFlags(m, flags)
if !date.IsZero() {
m.Time = date.Unix()
}
internalID := m.Header.Get("X-Pm-Internal-Id")
references := m.Header.Get("References")
referenceList := strings.Fields(references)
// In case there is a mail client which corrupts headers, try
// "References" too.
if internalID == "" && len(referenceList) > 0 {
lastReference := referenceList[len(referenceList)-1]
match := pmapi.RxInternalReferenceFormat.FindStringSubmatch(lastReference)
if len(match) == 2 {
internalID = match[1]
}
}
im.user.appendExpungeLock.Lock()
defer im.user.appendExpungeLock.Unlock()
// Avoid appending a message which is already on the server. Apply the
// new label instead. This always happens with Outlook (it uses APPEND
// instead of COPY).
if internalID != "" {
// Check to see if this belongs to a different address in split mode or another ProtonMail account.
msg, err := im.storeMailbox.GetMessage(internalID)
if err == nil && (im.user.user.IsCombinedAddressMode() || (im.storeAddress.AddressID() == msg.Message().AddressID)) {
IDs := []string{internalID}
// See the comment bellow.
if msg.IsMarkedDeleted() {
if err := im.storeMailbox.MarkMessagesUndeleted(IDs); err != nil {
log.WithError(err).Error("Failed to undelete re-imported internal message")
}
}
err = im.storeMailbox.LabelMessages(IDs)
if err != nil {
return err
}
targetSeq := im.storeMailbox.GetUIDList(IDs)
return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), targetSeq)
}
}
im.log.Info("Importing external message")
if err := im.importMessage(m, readers, kr); err != nil {
im.log.Error("Import failed: ", err)
hdr, err := textproto.ReadHeader(bufio.NewReader(bytes.NewReader(body)))
if err != nil {
return err
}
// Avoid appending a message which is already on the server. Apply the new label instead.
// This always happens with Outlook because it uses APPEND instead of COPY.
internalID := hdr.Get("X-Pm-Internal-Id")
// In case there is a mail client which corrupts headers, try "References" too.
if internalID == "" {
if references := strings.Fields(hdr.Get("References")); len(references) > 0 {
if match := pmapi.RxInternalReferenceFormat.FindStringSubmatch(references[len(references)-1]); len(match) == 2 {
internalID = match[1]
}
}
}
if internalID != "" {
if msg, err := im.storeMailbox.GetMessage(internalID); err == nil {
if im.user.user.IsCombinedAddressMode() || im.storeAddress.AddressID() == msg.Message().AddressID {
return im.labelExistingMessage(msg.ID(), msg.IsMarkedDeleted())
}
}
}
return im.importMessage(kr, hdr, body, imapFlags, date)
}
func (im *imapMailbox) createDraftMessage(kr *crypto.KeyRing, email string, body []byte) error {
im.log.Info("Creating draft message")
m, _, _, readers, err := message.Parse(bytes.NewReader(body))
if err != nil {
return err
}
m.Sender.Address = pmapi.ConstructAddress(m.Sender.Address, email)
draft, _, err := im.user.storeUser.CreateDraft(kr, m, readers, "", "", "")
if err != nil {
return errors.Wrap(err, "failed to create draft")
}
return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), im.storeMailbox.GetUIDList([]string{draft.ID}))
}
func (im *imapMailbox) labelExistingMessage(messageID string, isDeleted bool) error {
im.log.Info("Labelling existing message")
// IMAP clients can move message to local folder (setting \Deleted flag)
// and then move it back (IMAP client does not remember the message,
// so instead removing the flag it imports duplicate message).
@ -170,29 +152,76 @@ func (im *imapMailbox) createMessage(flags []string, date time.Time, body imap.L
// not delete the message (EXPUNGE would delete the original message and
// the new duplicate one would stay). API detects duplicates; therefore
// we need to remove \Deleted flag if IMAP client re-imports.
msg, err := im.storeMailbox.GetMessage(m.ID)
if err == nil && msg.IsMarkedDeleted() {
if err := im.storeMailbox.MarkMessagesUndeleted([]string{m.ID}); err != nil {
if isDeleted {
if err := im.storeMailbox.MarkMessagesUndeleted([]string{messageID}); err != nil {
log.WithError(err).Error("Failed to undelete re-imported message")
}
}
targetSeq := im.storeMailbox.GetUIDList([]string{m.ID})
return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), targetSeq)
if err := im.storeMailbox.LabelMessages([]string{messageID}); err != nil {
return err
}
return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), im.storeMailbox.GetUIDList([]string{messageID}))
}
func (im *imapMailbox) importMessage(m *pmapi.Message, readers []io.Reader, kr *crypto.KeyRing) (err error) {
body, err := message.BuildEncrypted(m, readers, kr)
func (im *imapMailbox) importMessage(kr *crypto.KeyRing, hdr textproto.Header, body []byte, imapFlags []string, date time.Time) error {
im.log.Info("Importing external message")
var (
seen bool
flags int64
labelIDs []string
time int64
)
if hdr.Get("received") == "" {
flags = pmapi.FlagSent
} else {
flags = pmapi.FlagReceived
}
for _, flag := range imapFlags {
switch flag {
case imap.DraftFlag:
flags &= ^pmapi.FlagSent
flags &= ^pmapi.FlagReceived
case imap.SeenFlag:
seen = true
case imap.FlaggedFlag:
labelIDs = append(labelIDs, pmapi.StarredLabel)
case imap.AnsweredFlag:
flags |= pmapi.FlagReplied
}
}
if !date.IsZero() {
time = date.Unix()
}
enc, err := message.EncryptRFC822(kr, bytes.NewReader(body))
if err != nil {
return err
}
labels := []string{}
for _, l := range m.LabelIDs {
if l == pmapi.StarredLabel {
labels = append(labels, pmapi.StarredLabel)
messageID, err := im.storeMailbox.ImportMessage(enc, seen, labelIDs, flags, time)
if err != nil {
return err
}
msg, err := im.storeMailbox.GetMessage(messageID)
if err != nil {
return err
}
if msg.IsMarkedDeleted() {
if err := im.storeMailbox.MarkMessagesUndeleted([]string{messageID}); err != nil {
log.WithError(err).Error("Failed to undelete re-imported message")
}
}
return im.storeMailbox.ImportMessage(m, body, labels)
return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), im.storeMailbox.GetUIDList([]string{messageID}))
}

37
internal/imap/mailbox_header.go
View File

@ -18,13 +18,11 @@
package imap
import (
"bufio"
"bytes"
"io"
"strings"
"github.com/ProtonMail/proton-bridge/pkg/message"
"github.com/emersion/go-imap"
"github.com/pkg/errors"
)
func filterHeader(header []byte, section *imap.BodySectionName) []byte {
@ -53,7 +51,7 @@ func filterHeader(header []byte, section *imap.BodySectionName) []byte {
func filterHeaderLines(header []byte, wantField func(string) bool) []byte {
var res []byte
for _, line := range headerLines(header) {
for _, line := range message.HeaderLines(header) {
if len(bytes.TrimSpace(line)) == 0 {
res = append(res, line...)
} else {
@ -71,34 +69,3 @@ func filterHeaderLines(header []byte, wantField func(string) bool) []byte {
return res
}
// NOTE: This sucks because we trim and split stuff here already, only to do it again when we use this function!
func headerLines(header []byte) [][]byte {
var lines [][]byte
r := bufio.NewReader(bytes.NewReader(header))
for {
b, err := r.ReadBytes('\n')
if err != nil {
if err != io.EOF {
panic(errors.Wrap(err, "failed to read header line"))
}
break
}
switch {
case len(bytes.TrimSpace(b)) == 0:
lines = append(lines, b)
case len(bytes.SplitN(b, []byte(": "), 2)) != 2:
lines[len(lines)-1] = append(lines[len(lines)-1], b...)
default:
lines = append(lines, b)
}
}
return lines
}

2
internal/imap/store.go
View File

@ -89,7 +89,7 @@ type storeMailboxProvider interface {
MarkMessagesUnstarred(apiID []string) error
MarkMessagesDeleted(apiID []string) error
MarkMessagesUndeleted(apiID []string) error
ImportMessage(msg *pmapi.Message, body []byte, labelIDs []string) error
ImportMessage(enc []byte, seen bool, labelIDs []string, flags, time int64) (string, error)
RemoveDeleted(apiIDs []string) error
}

23
internal/store/mailbox_message.go
View File

@ -48,9 +48,7 @@ func (storeMailbox *Mailbox) FetchMessage(apiID string) (*Message, error) {
return newStoreMessage(storeMailbox, msg), nil
}
// ImportMessage imports the message by calling an API.
// It has to be propagated to all mailboxes which is done by the event loop.
func (storeMailbox *Mailbox) ImportMessage(msg *pmapi.Message, body []byte, labelIDs []string) error {
func (storeMailbox *Mailbox) ImportMessage(enc []byte, seen bool, labelIDs []string, flags, time int64) (string, error) {
defer storeMailbox.pollNow()
if storeMailbox.labelID != pmapi.AllMailLabel {
@ -59,24 +57,25 @@ func (storeMailbox *Mailbox) ImportMessage(msg *pmapi.Message, body []byte, labe
importReqs := &pmapi.ImportMsgReq{
Metadata: &pmapi.ImportMetadata{
AddressID: msg.AddressID,
Unread: msg.Unread,
Flags: msg.Flags,
Time: msg.Time,
AddressID: storeMailbox.storeAddress.addressID,
Unread: pmapi.Boolean(!seen),
Flags: flags,
Time: time,
LabelIDs: labelIDs,
},
Message: body,
Message: append(enc, "\r\n"...),
}
res, err := storeMailbox.client().Import(exposeContextForIMAP(), pmapi.ImportMsgReqs{importReqs})
if err != nil {
return err
return "", err
}
if len(res) == 0 {
return errors.New("no import response")
return "", errors.New("no import response")
}
msg.ID = res[0].MessageID
return res[0].Error
return res[0].MessageID, res[0].Error
}
// LabelMessages adds the label by calling an API.