yesBad/proton-bridge
1
0
Fork 0
forked from Silverfish/proton-bridge
Code Actions Activity

GODT-1136 DB Cache header from builder and test

Browse Source
This commit is contained in:
Jakub
2021-04-13 07:49:13 +02:00
committed by James Houlahan
parent 454d248819
commit 8ab05a000c
21 changed files with 471 additions and 306 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
pkg/message/address.go
View File

@ -1,46 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package message
import (
"net/mail"
"strings"
"github.com/emersion/go-imap"
)
func getAddresses(addrs []*mail.Address) (imapAddrs []*imap.Address) {
for _, a := range addrs {
if a == nil {
continue
}
parts := strings.SplitN(a.Address, "@", 2)
if len(parts) != 2 {
continue
}
imapAddrs = append(imapAddrs, &imap.Address{
PersonalName: a.Name,
MailboxName: parts[0],
HostName: parts[1],
})
}
return
}

5
pkg/message/build_build.go
View File

@ -73,9 +73,14 @@ func buildWorker(buildReqCh <-chan fetchRes, buildResCh chan<- buildRes, wg *syn
defer wg.Done()
for req := range buildReqCh {
l := log.
WithField("addrID", req.msg.AddressID).
WithField("msgID", req.msg.ID)
if kr, err := req.api.KeyRingForAddressID(req.msg.AddressID); err != nil {
l.WithError(err).Warn("Cannot find keyring for address")
buildResCh <- newBuildResFailure(req.msg.ID, errors.Wrap(ErrNoSuchKeyRing, err.Error()))
} else if literal, err := buildRFC822(kr, req.msg, req.atts, req.opts); err != nil {
l.WithError(err).Warn("Build failed")
buildResCh <- newBuildResFailure(req.msg.ID, err)
} else {
buildResCh <- newBuildResSuccess(req.msg.ID, literal)

46
pkg/message/build_rfc822.go
View File

@ -187,6 +187,12 @@ func writeAttachmentPart(
return errors.Wrap(ErrDecryptionFailed, err.Error())
}
log.
WithField("attID", att.ID).
WithField("msgID", att.MessageID).
WithError(err).
Warn("Attachment decryption failed")
return writeCustomAttachmentPart(w, att, msg, err)
}
@ -309,25 +315,13 @@ func getMessageHeader(msg *pmapi.Message, opts JobOptions) message.Header { // n
hdr.Set("Bcc", toAddressList(msg.BCCList))
}
// Set Message-Id from ExternalID or ID if it's not already set.
if hdr.Get("Message-Id") == "" {
if msg.ExternalID != "" {
hdr.Set("Message-Id", "<"+msg.ExternalID+">")
} else {
hdr.Set("Message-Id", "<"+msg.ID+"@"+pmapi.InternalIDDomain+">")
}
}
setMessageIDIfNeeded(msg, &hdr)
// Sanitize the date; it needs to have a valid unix timestamp.
if opts.SanitizeDate {
if date, err := rfc5322.ParseDateTime(hdr.Get("Date")); err != nil || date.Before(time.Unix(0, 0)) {
if msgTime := time.Unix(msg.Time, 0); msgTime.After(time.Unix(0, 0)) {
hdr.Set("Date", msgTime.In(time.UTC).Format(time.RFC1123Z))
} else {
// No message should realistically be older than RFC822 itself.
hdr.Set("Date", time.Date(1982, 8, 13, 0, 0, 0, 0, time.UTC).Format(time.RFC1123Z))
}
msgDate := sanitizeMessageDate(msg.Time)
hdr.Set("Date", msgDate.In(time.UTC).Format(time.RFC1123Z))
// We clobbered the date so we save it under X-Original-Date.
hdr.Set("X-Original-Date", date.In(time.UTC).Format(time.RFC1123Z))
}
@ -361,6 +355,28 @@ func getMessageHeader(msg *pmapi.Message, opts JobOptions) message.Header { // n
return hdr
}
// sanitizeMessageDate will return time from msgTime timestamp. If timestamp is
// not after epoch the RFC822 publish day will be used. No message should
// realistically be older than RFC822 itself.
func sanitizeMessageDate(msgTime int64) time.Time {
if msgTime := time.Unix(msgTime, 0); msgTime.After(time.Unix(0, 0)) {
return msgTime
}
return time.Date(1982, 8, 13, 0, 0, 0, 0, time.UTC)
}
// setMessageIDIfNeeded sets Message-Id from ExternalID or ID if it's not
// already set.
func setMessageIDIfNeeded(msg *pmapi.Message, hdr *message.Header) {
if hdr.Get("Message-Id") == "" {
if msg.ExternalID != "" {
hdr.Set("Message-Id", "<"+msg.ExternalID+">")
} else {
hdr.Set("Message-Id", "<"+msg.ID+"@"+pmapi.InternalIDDomain+">")
}
}
}
func getTextPartHeader(hdr message.Header, body []byte, mimeType string) message.Header {
params := make(map[string]string)

55
pkg/message/envelope.go
View File

@ -19,30 +19,49 @@ package message
import (
"net/mail"
"time"
"net/textproto"
"strings"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/emersion/go-imap"
)
func GetEnvelope(m *pmapi.Message) *imap.Envelope {
messageID := m.ExternalID
if messageID == "" {
messageID = m.Header.Get("Message-Id")
} else {
messageID = "<" + messageID + ">"
}
// GetEnvelope will prepare envelope from pmapi message and cached header.
func GetEnvelope(msg *pmapi.Message, header textproto.MIMEHeader) *imap.Envelope {
hdr := toMessageHeader(mail.Header(header))
setMessageIDIfNeeded(msg, &hdr)
return &imap.Envelope{
Date: time.Unix(m.Time, 0),
Subject: m.Subject,
From: getAddresses([]*mail.Address{m.Sender}),
Sender: getAddresses([]*mail.Address{m.Sender}),
ReplyTo: getAddresses(m.ReplyTos),
To: getAddresses(m.ToList),
Cc: getAddresses(m.CCList),
Bcc: getAddresses(m.BCCList),
InReplyTo: m.Header.Get("In-Reply-To"),
MessageId: messageID,
Date: sanitizeMessageDate(msg.Time),
Subject: msg.Subject,
From: getAddresses([]*mail.Address{msg.Sender}),
Sender: getAddresses([]*mail.Address{msg.Sender}),
ReplyTo: getAddresses(msg.ReplyTos),
To: getAddresses(msg.ToList),
Cc: getAddresses(msg.CCList),
Bcc: getAddresses(msg.BCCList),
InReplyTo: hdr.Get("In-Reply-To"),
MessageId: hdr.Get("Message-Id"),
}
}
func getAddresses(addrs []*mail.Address) (imapAddrs []*imap.Address) {
for _, a := range addrs {
if a == nil {
continue
}
parts := strings.SplitN(a.Address, "@", 2)
if len(parts) != 2 {
continue
}
imapAddrs = append(imapAddrs, &imap.Address{
PersonalName: a.Name,
MailboxName: parts[0],
HostName: parts[1],
})
}
return
}

4
pkg/message/message.go
View File

@ -18,8 +18,6 @@
package message
import (
"crypto/sha512"
"fmt"
"strings"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
@ -35,7 +33,7 @@ var log = logrus.WithField("pkg", "pkg/message") //nolint[gochecknoglobals]
func GetBoundary(m *pmapi.Message) string {
// The boundary needs to be deterministic because messages are not supposed to
// change.
return fmt.Sprintf("%x", sha512.Sum512_256([]byte(m.ID)))
return newBoundary(m.ID).gen()
}
func SeparateInlineAttachments(m *pmapi.Message) (atts, inlines []*pmapi.Attachment) {

46
pkg/message/section.go
View File

@ -337,30 +337,40 @@ func (bs *BodyStructure) GetSectionContent(wholeMail io.ReadSeeker, sectionPath
if err != nil {
return
}
if _, err = wholeMail.Seek(int64(info.Start+info.Size-info.BSize), io.SeekStart); err != nil {
return
}
section = make([]byte, info.BSize)
_, err = wholeMail.Read(section)
return
return goToOffsetAndReadNBytes(wholeMail, info.Start+info.Size-info.BSize, info.BSize)
}
/* This is slow:
sectionBuf, err := bs.GetSection(wholeMail, sectionPath)
// GetMailHeader returns the main header of mail.
func (bs *BodyStructure) GetMailHeader() (header textproto.MIMEHeader, err error) {
return bs.GetSectionHeader([]int{})
}
// GetMailHeaderBytes returns the bytes with main mail header.
// Warning: It can contain extra lines or multipart comment.
func (bs *BodyStructure) GetMailHeaderBytes(wholeMail io.ReadSeeker) (header []byte, err error) {
info, err := bs.getInfo([]int{})
if err != nil {
return
}
tp := textproto.NewReader(bufio.NewReader(buf))
if _, err = tp.ReadMIMEHeader(); err != nil {
return err
}
sectionBuf = &bytes.Buffer{}
_, err = io.Copy(sectionBuf, tp.R)
return
*/
headerLength := info.Size - info.BSize
return goToOffsetAndReadNBytes(wholeMail, 0, headerLength)
}
func goToOffsetAndReadNBytes(wholeMail io.ReadSeeker, offset, length int) ([]byte, error) {
if length < 1 {
return nil, errors.New("requested non positive length")
}
if offset > 0 {
if _, err := wholeMail.Seek(int64(offset), io.SeekStart); err != nil {
return nil, err
}
}
out := make([]byte, length)
_, err := wholeMail.Read(out)
return out, err
}
// GetSectionHeader returns the mime header of specified section.
func (bs *BodyStructure) GetSectionHeader(sectionPath []int) (header textproto.MIMEHeader, err error) {
info, err := bs.getInfo(sectionPath)
if err != nil {

18
pkg/message/section_test.go
View File

@ -108,6 +108,24 @@ func TestGetSection(t *testing.T) {
}
}
func TestGetMainHeaderBytes(t *testing.T) {
wantHeader := []byte(`Subject: Sample mail
From: John Doe <jdoe@machine.example>
To: Mary Smith <mary@example.net>
Date: Fri, 21 Nov 1997 09:55:06 -0600
Content-Type: multipart/mixed; boundary="0000MAIN"
`)
structReader := strings.NewReader(sampleMail)
bs, err := NewBodyStructure(structReader)
require.NoError(t, err)
haveHeader, err := bs.GetMailHeaderBytes(strings.NewReader(sampleMail))
require.NoError(t, err)
require.Equal(t, wantHeader, haveHeader)
}
/* Structure example:
HEADER ([RFC-2822] header of the message)
TEXT ([RFC-2822] text body of the message) MULTIPART/MIXED