yesBad/proton-bridge
1
0
Fork 0
forked from Silverfish/proton-bridge
Code Actions Activity

GODT-213: Message Builder

Browse Source
This commit is contained in:
James Houlahan
2021-03-17 10:33:03 +01:00
parent 8db89a1a6c
commit 50550d42b4
42 changed files with 3041 additions and 816 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
internal/imap/backend.go
View File

@ -26,10 +26,17 @@ import (
"github.com/ProtonMail/proton-bridge/internal/bridge"
"github.com/ProtonMail/proton-bridge/internal/events"
"github.com/ProtonMail/proton-bridge/pkg/listener"
"github.com/ProtonMail/proton-bridge/pkg/message"
"github.com/emersion/go-imap"
goIMAPBackend "github.com/emersion/go-imap/backend"
)
const (
fetchWorkers = 20 // In how many workers to fetch message (group list on IMAP).
attachWorkers = 5 // In how many workers to fetch attachments (for one message).
buildWorkers = 20 // In how many workers to build messages.
)
type panicHandler interface {
HandlePanic()
}
@ -43,6 +50,8 @@ type imapBackend struct {
users map[string]*imapUser
usersLocker sync.Locker
builder *message.Builder
imapCache map[string]map[string]string
imapCachePath string
imapCacheLock *sync.RWMutex
@ -78,6 +87,8 @@ func newIMAPBackend(
users: map[string]*imapUser{},
usersLocker: &sync.Mutex{},
builder: message.NewBuilder(fetchWorkers, attachWorkers, buildWorkers),
imapCachePath: cache.GetIMAPCachePath(),
imapCacheLock: &sync.RWMutex{},
}

9
internal/imap/imap.go
View File

@ -19,11 +19,4 @@ package imap
import "github.com/sirupsen/logrus"
const (
fetchMessagesWorkers = 5 // In how many workers to fetch message (group list on IMAP).
fetchAttachmentsWorkers = 5 // In how many workers to fetch attachments (for one message).
)
var (
log = logrus.WithField("pkg", "imap") //nolint[gochecknoglobals]
)
var log = logrus.WithField("pkg", "imap") //nolint[gochecknoglobals]

6
internal/imap/mailbox.go
View File

@ -37,10 +37,12 @@ type imapMailbox struct {
storeUser storeUserProvider
storeAddress storeAddressProvider
storeMailbox storeMailboxProvider
builder *message.Builder
}
// newIMAPMailbox returns struct implementing go-imap/mailbox interface.
func newIMAPMailbox(panicHandler panicHandler, user *imapUser, storeMailbox storeMailboxProvider) *imapMailbox {
func newIMAPMailbox(panicHandler panicHandler, user *imapUser, storeMailbox storeMailboxProvider, builder *message.Builder) *imapMailbox {
return &imapMailbox{
panicHandler: panicHandler,
user: user,
@ -54,6 +56,8 @@ func newIMAPMailbox(panicHandler panicHandler, user *imapUser, storeMailbox stor
storeUser: user.storeUser,
storeAddress: user.storeAddress,
storeMailbox: storeMailbox,
builder: builder,
}
}

274
internal/imap/mailbox_message.go
View File

@ -19,9 +19,9 @@ package imap
import (
"bytes"
"context"
"fmt"
"io"
"mime/multipart"
"net/mail"
"net/textproto"
"sort"
@ -32,12 +32,10 @@ import (
"github.com/ProtonMail/proton-bridge/internal/imap/cache"
"github.com/ProtonMail/proton-bridge/internal/imap/uidplus"
"github.com/ProtonMail/proton-bridge/pkg/message"
"github.com/ProtonMail/proton-bridge/pkg/parallel"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/emersion/go-imap"
"github.com/hashicorp/go-multierror"
"github.com/pkg/errors"
openpgperrors "golang.org/x/crypto/openpgp/errors"
)
var (
@ -512,91 +510,6 @@ func (im *imapMailbox) fetchMessage(m *pmapi.Message) (err error) {
return
}
func (im *imapMailbox) writeMessageBody(w io.Writer, m *pmapi.Message) (err error) {
im.log.Trace("Writing message body")
if m.Body == "" {
im.log.Trace("While writing message body, noticed message body is null, need to fetch")
if err = im.fetchMessage(m); err != nil {
return
}
}
kr, err := im.user.client().KeyRingForAddressID(m.AddressID)
if err != nil {
return errors.Wrap(err, "failed to get keyring for address ID")
}
err = message.WriteBody(w, kr, m)
if err != nil {
if customMessageErr := message.CustomMessage(m, err, true); customMessageErr != nil {
im.log.WithError(customMessageErr).Warn("Failed to make custom message")
}
_, _ = io.WriteString(w, m.Body)
err = nil
}
return
}
func (im *imapMailbox) writeAttachmentBody(w io.Writer, m *pmapi.Message, att *pmapi.Attachment) (err error) {
// Retrieve encrypted attachment.
r, err := im.user.client().GetAttachment(att.ID)
if err != nil {
return
}
defer r.Close() //nolint[errcheck]
kr, err := im.user.client().KeyRingForAddressID(m.AddressID)
if err != nil {
return errors.Wrap(err, "failed to get keyring for address ID")
}
if err = message.WriteAttachmentBody(w, kr, m, att, r); err != nil {
// Returning an error here makes certain mail clients behave badly,
// trying to retrieve the message again and again.
im.log.Warn("Cannot write attachment body: ", err)
err = nil
}
return
}
func (im *imapMailbox) writeRelatedPart(p io.Writer, m *pmapi.Message, inlines []*pmapi.Attachment) (err error) {
related := multipart.NewWriter(p)
_ = related.SetBoundary(message.GetRelatedBoundary(m))
buf := &bytes.Buffer{}
if err = im.writeMessageBody(buf, m); err != nil {
return
}
// Write the body part.
h := message.GetBodyHeader(m)
if p, err = related.CreatePart(h); err != nil {
return
}
_, _ = buf.WriteTo(p)
for _, inline := range inlines {
buf = &bytes.Buffer{}
if err = im.writeAttachmentBody(buf, m, inline); err != nil {
return
}
h := message.GetAttachmentHeader(inline, true)
if p, err = related.CreatePart(h); err != nil {
return
}
_, _ = buf.WriteTo(p)
}
_ = related.Close()
return nil
}
const (
noMultipart = iota // only body
simpleMultipart // body + attachment or inline
@ -634,179 +547,28 @@ func (im *imapMailbox) setMessageContentType(m *pmapi.Message) (multipartType in
}
// buildMessage from PM to IMAP.
func (im *imapMailbox) buildMessage(m *pmapi.Message) (structure *message.BodyStructure, msgBody []byte, err error) {
im.log.Trace("Building message")
var errNoCache doNotCacheError
// If fetch or decryption fails we need to change the MIMEType (in customMessage).
err = im.fetchMessage(m)
func (im *imapMailbox) buildMessage(m *pmapi.Message) (*message.BodyStructure, []byte, error) {
body, err := im.builder.NewJobWithOptions(
context.Background(),
im.user.client(),
m.ID,
message.JobOptions{
IgnoreDecryptionErrors: true, // Whether to ignore decryption errors and create a "custom message" instead.
SanitizeDate: true, // Whether to replace all dates before 1970 with RFC822's birthdate.
AddInternalID: true, // Whether to include MessageID as X-Pm-Internal-Id.
AddExternalID: true, // Whether to include ExternalID as X-Pm-External-Id.
AddMessageDate: true, // Whether to include message time as X-Pm-Date.
AddMessageIDReference: true, // Whether to include the MessageID in References.
},
).GetResult()
if err != nil {
return
}
kr, err := im.user.client().KeyRingForAddressID(m.AddressID)
if err != nil {
err = errors.Wrap(err, "failed to get keyring for address ID")
return
}
errDecrypt := m.Decrypt(kr)
if errDecrypt != nil && errDecrypt != openpgperrors.ErrSignatureExpired {
errNoCache.add(errDecrypt)
if customMessageErr := message.CustomMessage(m, errDecrypt, true); customMessageErr != nil {
im.log.WithError(customMessageErr).Warn("Failed to make custom message")
}
}
// Inner function can fail even when message is decrypted.
// #1048 For example we have problem with double-encrypted messages
// which seems as still encrypted and we try them to decrypt again
// and that fails. For any building error is better to return custom
// message than error because it will not be fixed and users would
// get error message all the time and could not see some messages.
structure, msgBody, err = im.buildMessageInner(m, kr)
if err == pmapi.ErrAPINotReachable || err == pmapi.ErrInvalidToken || err == pmapi.ErrUpgradeApplication {
return nil, nil, err
} else if err != nil {
errNoCache.add(err)
if customMessageErr := message.CustomMessage(m, err, true); customMessageErr != nil {
im.log.WithError(customMessageErr).Warn("Failed to make custom message")
}
structure, msgBody, err = im.buildMessageInner(m, kr)
if err != nil {
return nil, nil, err
}
}
err = errNoCache.errorOrNil()
return structure, msgBody, err
}
func (im *imapMailbox) buildMessageInner(m *pmapi.Message, kr *crypto.KeyRing) (structure *message.BodyStructure, msgBody []byte, err error) { // nolint[funlen]
multipartType, err := im.setMessageContentType(m)
structure, err := message.NewBodyStructure(bytes.NewReader(body))
if err != nil {
return
return nil, nil, err
}
tmpBuf := &bytes.Buffer{}
mainHeader := buildHeader(m)
if err = writeHeader(tmpBuf, mainHeader); err != nil {
return
}
_, _ = io.WriteString(tmpBuf, "\r\n")
switch multipartType {
case noMultipart:
err = message.WriteBody(tmpBuf, kr, m)
if err != nil {
return
}
case complexMultipart:
_, _ = io.WriteString(tmpBuf, "\r\n--"+message.GetBoundary(m)+"\r\n")
err = message.WriteBody(tmpBuf, kr, m)
if err != nil {
return
}
_, _ = io.WriteString(tmpBuf, "\r\n--"+message.GetBoundary(m)+"--\r\n")
case simpleMultipart:
atts, inlines := message.SeparateInlineAttachments(m)
mw := multipart.NewWriter(tmpBuf)
_ = mw.SetBoundary(message.GetBoundary(m))
var partWriter io.Writer
if len(inlines) > 0 {
relatedHeader := message.GetRelatedHeader(m)
if partWriter, err = mw.CreatePart(relatedHeader); err != nil {
return
}
_ = im.writeRelatedPart(partWriter, m, inlines)
} else {
buf := &bytes.Buffer{}
if err = im.writeMessageBody(buf, m); err != nil {
return
}
// Write the body part.
bodyHeader := message.GetBodyHeader(m)
if partWriter, err = mw.CreatePart(bodyHeader); err != nil {
return
}
_, _ = buf.WriteTo(partWriter)
}
// Write the attachments parts.
input := make([]interface{}, len(atts))
for i, att := range atts {
input[i] = att
}
processCallback := func(value interface{}) (interface{}, error) {
att := value.(*pmapi.Attachment) //nolint[forcetypeassert] we want to panic here
buf := &bytes.Buffer{}
if err = im.writeAttachmentBody(buf, m, att); err != nil {
return nil, err
}
return buf, nil
}
collectCallback := func(idx int, value interface{}) error {
buf := value.(*bytes.Buffer) //nolint[forcetypeassert] we want to panic here
defer buf.Reset()
att := atts[idx]
attachmentHeader := message.GetAttachmentHeader(att, true)
if partWriter, err = mw.CreatePart(attachmentHeader); err != nil {
return err
}
_, _ = buf.WriteTo(partWriter)
return nil
}
err = parallel.RunParallel(fetchAttachmentsWorkers, input, processCallback, collectCallback)
if err != nil {
return
}
_ = mw.Close()
default:
fmt.Fprintf(tmpBuf, "\r\n\r\nUknown multipart type: %d\r\n\r\n", multipartType)
}
// We need to copy buffer before building body structure.
msgBody = tmpBuf.Bytes()
structure, err = message.NewBodyStructure(tmpBuf)
if err != nil {
// NOTE: We need to set structure if it fails and is empty.
if structure == nil {
structure = &message.BodyStructure{}
}
}
return structure, msgBody, err
}
func buildHeader(msg *pmapi.Message) textproto.MIMEHeader {
header := message.GetHeader(msg)
msgTime := time.Unix(msg.Time, 0)
// Apple Mail crashes fetching messages with date older than 1970.
// There is no point having message older than RFC itself, it's not possible.
d, err := msg.Header.Date()
if err != nil || d.Before(rfc822Birthday) || msgTime.Before(rfc822Birthday) {
if err != nil || d.IsZero() {
header.Set("X-Original-Date", msgTime.Format(time.RFC1123Z))
} else {
header.Set("X-Original-Date", d.Format(time.RFC1123Z))
}
header.Set("Date", rfc822Birthday.Format(time.RFC1123Z))
}
return header
return structure, body, nil
}

2
internal/imap/mailbox_messages.go
View File

@ -575,7 +575,7 @@ func (im *imapMailbox) listMessages(isUID bool, seqSet *imap.SeqSet, items []ima
return nil
}
err = parallel.RunParallel(fetchMessagesWorkers, input, processCallback, collectCallback)
err = parallel.RunParallel(fetchWorkers, input, processCallback, collectCallback)
if err != nil {
return err
}

4
internal/imap/user.go
View File

@ -135,7 +135,7 @@ func (iu *imapUser) ListMailboxes(showOnlySubcribed bool) ([]goIMAPBackend.Mailb
if showOnlySubcribed && !iu.isSubscribed(storeMailbox.LabelID()) {
continue
}
mailbox := newIMAPMailbox(iu.panicHandler, iu, storeMailbox)
mailbox := newIMAPMailbox(iu.panicHandler, iu, storeMailbox, iu.backend.builder)
mailboxes = append(mailboxes, mailbox)
}
@ -167,7 +167,7 @@ func (iu *imapUser) GetMailbox(name string) (mb goIMAPBackend.Mailbox, err error
return
}
return newIMAPMailbox(iu.panicHandler, iu, storeMailbox), nil
return newIMAPMailbox(iu.panicHandler, iu, storeMailbox, iu.backend.builder), nil
}
// CreateMailbox creates a new mailbox.

32
internal/imap/utils.go
View File

@ -1,32 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package imap
import (
"io"
"net/http"
"net/textproto"
)
func writeHeader(w io.Writer, h textproto.MIMEHeader) (err error) {
if err = http.Header(h).Write(w); err != nil {
return
}
_, err = io.WriteString(w, "\r\n")
return
}

9
internal/transfer/provider_pmapi.go
View File

@ -21,16 +21,24 @@ import (
"sort"
"github.com/ProtonMail/gopenpgp/v2/crypto"
"github.com/ProtonMail/proton-bridge/pkg/message"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/pkg/errors"
)
const (
fetchWorkers = 20 // In how many workers to fetch message (group list on IMAP).
attachWorkers = 5 // In how many workers to fetch attachments (for one message).
buildWorkers = 20 // In how many workers to build messages.
)
// PMAPIProvider implements import and export to/from ProtonMail server.
type PMAPIProvider struct {
clientManager ClientManager
userID string
addressID string
keyRing *crypto.KeyRing
builder *message.Builder
nextImportRequests map[string]*pmapi.ImportMsgReq // Key is msg transfer ID.
nextImportRequestsSize int
@ -44,6 +52,7 @@ func NewPMAPIProvider(clientManager ClientManager, userID, addressID string) (*P
clientManager: clientManager,
userID: userID,
addressID: addressID,
builder: message.NewBuilder(fetchWorkers, attachWorkers, buildWorkers),
nextImportRequests: map[string]*pmapi.ImportMsgReq{},
nextImportRequestsSize: 0,

27
internal/transfer/provider_pmapi_source.go
View File

@ -18,12 +18,13 @@
package transfer
import (
"context"
"errors"
"fmt"
"sync"
pkgMsg "github.com/ProtonMail/proton-bridge/pkg/message"
"github.com/ProtonMail/proton-bridge/pkg/message"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
@ -144,6 +145,7 @@ func (p *PMAPIProvider) transferTo(rule *Rule, progress *Progress, ch chan<- Mes
func (p *PMAPIProvider) exportMessage(rule *Rule, progress *Progress, pmapiMsgID, msgID string, skipEncryptedMessages bool) (Message, error) {
var msg *pmapi.Message
progress.callWrap(func() error {
var err error
msg, err = p.getMessage(pmapiMsgID)
@ -153,19 +155,18 @@ func (p *PMAPIProvider) exportMessage(rule *Rule, progress *Progress, pmapiMsgID
p.timeIt.start("build", msgID)
defer p.timeIt.stop("build", msgID)
msgBuilder := pkgMsg.NewBuilder(p.client(), msg)
msgBuilder.EncryptedToHTML = false
_, body, err := msgBuilder.BuildMessage()
body, err := p.builder.NewJobWithOptions(
context.Background(),
p.client(),
msg.ID,
message.JobOptions{IgnoreDecryptionErrors: !skipEncryptedMessages},
).GetResult()
if err != nil {
return Message{
Body: body, // Keep body to show details about the message to user.
}, errors.Wrap(err, "failed to build message")
}
if errors.Is(err, message.ErrDecryptionFailed) && skipEncryptedMessages {
err = errors.New("skipping encrypted message")
}
if !msgBuilder.SuccessfullyDecrypted() && skipEncryptedMessages {
return Message{
Body: body, // Keep body to show details about the message to user.
}, errors.New("skipping encrypted message")
return Message{Body: []byte(msg.Body)}, err
}
unread := false

17
internal/versioner/version_test.go
View File

@ -27,6 +27,7 @@ import (
"github.com/Masterminds/semver/v3"
"github.com/ProtonMail/gopenpgp/v2/crypto"
"github.com/ProtonMail/proton-bridge/pkg/sum"
tests "github.com/ProtonMail/proton-bridge/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -68,7 +69,7 @@ func TestVerifyWithBadFile(t *testing.T) {
filepath.Join("sub", "f5.tgz"),
)
badKeyRing := makeKeyRing(t)
badKeyRing := tests.MakeKeyRing(t)
signFile(t, filepath.Join(tempDir, "f3.bad"), badKeyRing)
assert.Error(t, version.VerifyFiles(kr))
@ -91,14 +92,14 @@ func TestVerifyWithBadSubFile(t *testing.T) {
filepath.Join("sub", "f5.bad"),
)
badKeyRing := makeKeyRing(t)
badKeyRing := tests.MakeKeyRing(t)
signFile(t, filepath.Join(tempDir, "sub", "f5.bad"), badKeyRing)
assert.Error(t, version.VerifyFiles(kr))
}
func createSignedFiles(t *testing.T, root string, paths ...string) *crypto.KeyRing {
kr := makeKeyRing(t)
kr := tests.MakeKeyRing(t)
for _, path := range paths {
makeFile(t, filepath.Join(root, path))
@ -118,16 +119,6 @@ func createSignedFiles(t *testing.T, root string, paths ...string) *crypto.KeyRi
return kr
}
func makeKeyRing(t *testing.T) *crypto.KeyRing {
key, err := crypto.GenerateKey("name", "email", "rsa", 2048)
require.NoError(t, err)
kr, err := crypto.NewKeyRing(key)
require.NoError(t, err)
return kr
}
func makeFile(t *testing.T, path string) {
require.NoError(t, os.MkdirAll(filepath.Dir(path), 0700))