yesBad/proton-bridge
1
0
Fork 0
forked from Silverfish/proton-bridge
Code Actions Activity

Other refactor: clean old builder

Browse Source
This commit is contained in:
Jakub Cuth
2021-04-27 08:12:50 +00:00
parent 286f51a4e7
commit 94b5799ba7
14 changed files with 790 additions and 935 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
pkg/message/boundary_reader.go Normal file
View File

@ -0,0 +1,130 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package message
import (
"bufio"
"bytes"
"io"
)
type boundaryReader struct {
reader *bufio.Reader
closed, first bool
skipped int
nl []byte // "\r\n" or "\n" (set after seeing first boundary line)
nlDashBoundary []byte // nl + "--boundary"
dashBoundaryDash []byte // "--boundary--"
dashBoundary []byte // "--boundary"
}
func newBoundaryReader(r *bufio.Reader, boundary string) (br *boundaryReader, err error) {
b := []byte("\r\n--" + boundary + "--")
br = &boundaryReader{
reader: r,
closed: false,
first: true,
nl: b[:2],
nlDashBoundary: b[:len(b)-2],
dashBoundaryDash: b[2:],
dashBoundary: b[2 : len(b)-2],
}
err = br.writeNextPartTo(nil)
return
}
// writeNextPartTo will copy the the bytes of next part and write them to
// writer. Will return EOF if the underlying reader is empty.
func (br *boundaryReader) writeNextPartTo(part io.Writer) (err error) {
if br.closed {
return io.EOF
}
var line, slice []byte
br.skipped = 0
for {
slice, err = br.reader.ReadSlice('\n')
line = append(line, slice...)
if err == bufio.ErrBufferFull {
continue
}
br.skipped += len(line)
if err == io.EOF && br.isFinalBoundary(line) {
err = nil
br.closed = true
return
}
if err != nil {
return
}
if br.isBoundaryDelimiterLine(line) {
br.first = false
return
}
if br.isFinalBoundary(line) {
br.closed = true
return
}
if part != nil {
if _, err = part.Write(line); err != nil {
return
}
}
line = []byte{}
}
}
func (br *boundaryReader) isFinalBoundary(line []byte) bool {
if !bytes.HasPrefix(line, br.dashBoundaryDash) {
return false
}
rest := line[len(br.dashBoundaryDash):]
rest = skipLWSPChar(rest)
return len(rest) == 0 || bytes.Equal(rest, br.nl)
}
func (br *boundaryReader) isBoundaryDelimiterLine(line []byte) (ret bool) {
if !bytes.HasPrefix(line, br.dashBoundary) {
return false
}
rest := line[len(br.dashBoundary):]
rest = skipLWSPChar(rest)
if br.first && len(rest) == 1 && rest[0] == '\n' {
br.nl = br.nl[1:]
br.nlDashBoundary = br.nlDashBoundary[1:]
}
return bytes.Equal(rest, br.nl)
}
func skipLWSPChar(b []byte) []byte {
for len(b) > 0 && (b[0] == ' ' || b[0] == '\t') {
b = b[1:]
}
return b
}

63
pkg/message/build_encrypted.go
View File

@ -22,30 +22,36 @@ import (
"encoding/base64"
"io"
"io/ioutil"
"mime"
"mime/multipart"
"net/http"
"net/textproto"
"strings"
"github.com/ProtonMail/gopenpgp/v2/crypto"
pmmime "github.com/ProtonMail/proton-bridge/pkg/mime"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/emersion/go-message"
"github.com/emersion/go-textwrapper"
)
// BuildEncrypted is used for importing encrypted message.
func BuildEncrypted(m *pmapi.Message, readers []io.Reader, kr *crypto.KeyRing) ([]byte, error) { //nolint[funlen]
b := &bytes.Buffer{}
boundary := newBoundary(m.ID).gen()
// Overwrite content for main header for import.
// Even if message has just simple body we should upload as multipart/mixed.
// Each part has encrypted body and header reflects the original header.
mainHeader := GetHeader(m)
mainHeader.Set("Content-Type", "multipart/mixed; boundary="+GetBoundary(m))
mainHeader := convertGoMessageToTextprotoHeader(getMessageHeader(m, JobOptions{}))
mainHeader.Set("Content-Type", "multipart/mixed; boundary="+boundary)
mainHeader.Del("Content-Disposition")
mainHeader.Del("Content-Transfer-Encoding")
if err := WriteHeader(b, mainHeader); err != nil {
return nil, err
}
mw := multipart.NewWriter(b)
if err := mw.SetBoundary(GetBoundary(m)); err != nil {
if err := mw.SetBoundary(boundary); err != nil {
return nil, err
}
@ -71,7 +77,7 @@ func BuildEncrypted(m *pmapi.Message, readers []io.Reader, kr *crypto.KeyRing) (
for i := 0; i < len(m.Attachments); i++ {
att := m.Attachments[i]
r := readers[i]
h := GetAttachmentHeader(att, false)
h := getAttachmentHeader(att, false)
p, err := mw.CreatePart(h)
if err != nil {
return nil, err
@ -105,6 +111,55 @@ func BuildEncrypted(m *pmapi.Message, readers []io.Reader, kr *crypto.KeyRing) (
return b.Bytes(), nil
}
func convertGoMessageToTextprotoHeader(h message.Header) textproto.MIMEHeader {
out := make(textproto.MIMEHeader)
hf := h.Fields()
for hf.Next() {
// go-message fields are in the reverse order.
// textproto.MIMEHeader is not ordered except for the values of
// the same key which are ordered
key := textproto.CanonicalMIMEHeaderKey(hf.Key())
out[key] = append([]string{hf.Value()}, out[key]...)
}
return out
}
func getAttachmentHeader(att *pmapi.Attachment, buildForIMAP bool) textproto.MIMEHeader {
mediaType := att.MIMEType
if mediaType == "application/pgp-encrypted" {
mediaType = "application/octet-stream"
}
transferEncoding := "base64"
if mediaType == rfc822Message && buildForIMAP {
transferEncoding = "8bit"
}
encodedName := pmmime.EncodeHeader(att.Name)
disposition := "attachment" //nolint[goconst]
if strings.Contains(att.Header.Get("Content-Disposition"), pmapi.DispositionInline) {
disposition = pmapi.DispositionInline
}
h := make(textproto.MIMEHeader)
h.Set("Content-Type", mime.FormatMediaType(mediaType, map[string]string{"name": encodedName}))
if transferEncoding != "" {
h.Set("Content-Transfer-Encoding", transferEncoding)
}
h.Set("Content-Disposition", mime.FormatMediaType(disposition, map[string]string{"filename": encodedName}))
// Forward some original header lines.
forward := []string{"Content-Id", "Content-Description", "Content-Location"}
for _, k := range forward {
v := att.Header.Get(k)
if v != "" {
h.Set(k, v)
}
}
return h
}
func WriteHeader(w io.Writer, h textproto.MIMEHeader) (err error) {
if err = http.Header(h).Write(w); err != nil {
return

6
pkg/message/build_rfc822.go
View File

@ -329,7 +329,7 @@ func getMessageHeader(msg *pmapi.Message, opts JobOptions) message.Header { // n
// Sanitize the date; it needs to have a valid unix timestamp.
if opts.SanitizeDate {
if date, err := rfc5322.ParseDateTime(hdr.Get("Date")); err != nil || date.Before(time.Unix(0, 0)) {
msgDate := sanitizeMessageDate(msg.Time)
msgDate := SanitizeMessageDate(msg.Time)
hdr.Set("Date", msgDate.In(time.UTC).Format(time.RFC1123Z))
// We clobbered the date so we save it under X-Original-Date.
hdr.Set("X-Original-Date", date.In(time.UTC).Format(time.RFC1123Z))
@ -364,10 +364,10 @@ func getMessageHeader(msg *pmapi.Message, opts JobOptions) message.Header { // n
return hdr
}
// sanitizeMessageDate will return time from msgTime timestamp. If timestamp is
// SanitizeMessageDate will return time from msgTime timestamp. If timestamp is
// not after epoch the RFC822 publish day will be used. No message should
// realistically be older than RFC822 itself.
func sanitizeMessageDate(msgTime int64) time.Time {
func SanitizeMessageDate(msgTime int64) time.Time {
if msgTime := time.Unix(msgTime, 0); msgTime.After(time.Unix(0, 0)) {
return msgTime
}

2
pkg/message/envelope.go
View File

@ -32,7 +32,7 @@ func GetEnvelope(msg *pmapi.Message, header textproto.MIMEHeader) *imap.Envelope
setMessageIDIfNeeded(msg, &hdr)
return &imap.Envelope{
Date: sanitizeMessageDate(msg.Time),
Date: SanitizeMessageDate(msg.Time),
Subject: msg.Subject,
From: getAddresses([]*mail.Address{msg.Sender}),
Sender: getAddresses([]*mail.Address{msg.Sender}),

12
pkg/message/flags.go
View File

@ -22,13 +22,14 @@ import (
"github.com/emersion/go-imap"
)
//nolint[gochecknoglobals]
var (
AppleMailJunkFlag = imap.CanonicalFlag("$Junk")
ThunderbirdJunkFlag = imap.CanonicalFlag("Junk")
ThunderbirdNonJunkFlag = imap.CanonicalFlag("NonJunk")
// Various client specific flags.
const (
AppleMailJunkFlag = "$Junk"
ThunderbirdJunkFlag = "Junk"
ThunderbirdNonJunkFlag = "NonJunk"
)
// GetFlags returns imap flags from pmapi message attributes.
func GetFlags(m *pmapi.Message) (flags []string) {
if m.Unread == 0 {
flags = append(flags, imap.SeenFlag)
@ -59,6 +60,7 @@ func GetFlags(m *pmapi.Message) (flags []string) {
return
}
// ParseFlags sets attributes to pmapi messages based on imap flags.
func ParseFlags(m *pmapi.Message, flags []string) {
if m.Header.Get("received") == "" {
m.Flags = pmapi.FlagSent

138
pkg/message/header.go
View File

@ -1,138 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package message
import (
"mime"
"net/textproto"
"strings"
"time"
pmmime "github.com/ProtonMail/proton-bridge/pkg/mime"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
)
// GetHeader builds the header for the message.
func GetHeader(msg *pmapi.Message) textproto.MIMEHeader { //nolint[funlen]
h := make(textproto.MIMEHeader)
// Copy the custom header fields if there are some.
if msg.Header != nil {
h = textproto.MIMEHeader(msg.Header)
}
// Add or rewrite fields.
h.Set("Subject", pmmime.EncodeHeader(msg.Subject))
if msg.Sender != nil {
h.Set("From", pmmime.EncodeHeader(msg.Sender.String()))
}
if len(msg.ReplyTos) > 0 {
h.Set("Reply-To", pmmime.EncodeHeader(toAddressList(msg.ReplyTos)))
}
if len(msg.ToList) > 0 {
h.Set("To", pmmime.EncodeHeader(toAddressList(msg.ToList)))
}
if len(msg.CCList) > 0 {
h.Set("Cc", pmmime.EncodeHeader(toAddressList(msg.CCList)))
}
if len(msg.BCCList) > 0 {
h.Set("Bcc", pmmime.EncodeHeader(toAddressList(msg.BCCList)))
}
// Add or rewrite date related fields.
if msg.Time > 0 {
h.Set("X-Pm-Date", time.Unix(msg.Time, 0).Format(time.RFC1123Z))
if d, err := msg.Header.Date(); err != nil || d.IsZero() { // Fix date if needed.
h.Set("Date", time.Unix(msg.Time, 0).Format(time.RFC1123Z))
}
}
// Use External-Id if available to ensure email clients:
// * build the conversations threads correctly (Thunderbird, Mac Outlook, Apple Mail)
// * do not think the message is lost (Apple Mail)
if msg.ExternalID != "" {
h.Set("X-Pm-External-Id", "<"+msg.ExternalID+">")
if h.Get("Message-Id") == "" {
h.Set("Message-Id", "<"+msg.ExternalID+">")
}
}
if msg.ID != "" {
if h.Get("Message-Id") == "" {
h.Set("Message-Id", "<"+msg.ID+"@"+pmapi.InternalIDDomain+">")
}
h.Set("X-Pm-Internal-Id", msg.ID)
// Forward References, and include the message ID here (to improve outlook support).
if references := h.Get("References"); !strings.Contains(references, msg.ID) {
references += " <" + msg.ID + "@" + pmapi.InternalIDDomain + ">"
h.Set("References", references)
}
}
if msg.ConversationID != "" {
h.Set("X-Pm-ConversationID-Id", msg.ConversationID)
}
return h
}
func SetBodyContentFields(h *textproto.MIMEHeader, m *pmapi.Message) {
h.Set("Content-Type", m.MIMEType+"; charset=utf-8")
h.Set("Content-Disposition", pmapi.DispositionInline)
h.Set("Content-Transfer-Encoding", "quoted-printable")
}
func GetBodyHeader(m *pmapi.Message) textproto.MIMEHeader {
h := make(textproto.MIMEHeader)
SetBodyContentFields(&h, m)
return h
}
func GetAttachmentHeader(att *pmapi.Attachment, buildForIMAP bool) textproto.MIMEHeader {
mediaType := att.MIMEType
if mediaType == "application/pgp-encrypted" {
mediaType = "application/octet-stream"
}
transferEncoding := "base64"
if mediaType == rfc822Message && buildForIMAP {
transferEncoding = "8bit"
}
encodedName := pmmime.EncodeHeader(att.Name)
disposition := "attachment" //nolint[goconst]
if strings.Contains(att.Header.Get("Content-Disposition"), pmapi.DispositionInline) {
disposition = pmapi.DispositionInline
}
h := make(textproto.MIMEHeader)
h.Set("Content-Type", mime.FormatMediaType(mediaType, map[string]string{"name": encodedName}))
if transferEncoding != "" {
h.Set("Content-Transfer-Encoding", transferEncoding)
}
h.Set("Content-Disposition", mime.FormatMediaType(disposition, map[string]string{"filename": encodedName}))
// Forward some original header lines.
forward := []string{"Content-Id", "Content-Description", "Content-Location"}
for _, k := range forward {
v := att.Header.Get(k)
if v != "" {
h.Set(k, v)
}
}
return h
}

22
pkg/message/message.go
View File

@ -15,12 +15,11 @@
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
// Package message contains set of tools to convert message between Proton API
// and IMAP format.
package message
import (
"strings"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/sirupsen/logrus"
)
@ -29,20 +28,3 @@ const (
)
var log = logrus.WithField("pkg", "pkg/message") //nolint[gochecknoglobals]
func GetBoundary(m *pmapi.Message) string {
// The boundary needs to be deterministic because messages are not supposed to
// change.
return newBoundary(m.ID).gen()
}
func SeparateInlineAttachments(m *pmapi.Message) (atts, inlines []*pmapi.Attachment) {
for _, att := range m.Attachments {
if strings.Contains(att.Header.Get("Content-Disposition"), pmapi.DispositionInline) {
inlines = append(inlines, att)
} else {
atts = append(atts, att)
}
}
return
}

129
pkg/message/section.go
View File

@ -32,13 +32,18 @@ import (
"github.com/vmihailenco/msgpack/v5"
)
// BodyStructure is used to parse an email into MIME sections and then generate
// body structure for IMAP server.
type BodyStructure map[string]*SectionInfo
// SectionInfo is used to hold data about parts of each section.
type SectionInfo struct {
Header textproto.MIMEHeader
Start, BSize, Size, Lines int
reader io.Reader
}
// Read and count.
// Read will also count the final size of section.
func (si *SectionInfo) Read(p []byte) (n int, err error) {
n, err = si.reader.Read(p)
si.Size += n
@ -46,118 +51,13 @@ func (si *SectionInfo) Read(p []byte) (n int, err error) {
return
}
type boundaryReader struct {
reader *bufio.Reader
closed, first bool
skipped int
nl []byte // "\r\n" or "\n" (set after seeing first boundary line)
nlDashBoundary []byte // nl + "--boundary"
dashBoundaryDash []byte // "--boundary--"
dashBoundary []byte // "--boundary"
}
func newBoundaryReader(r *bufio.Reader, boundary string) (br *boundaryReader, err error) {
b := []byte("\r\n--" + boundary + "--")
br = &boundaryReader{
reader: r,
closed: false,
first: true,
nl: b[:2],
nlDashBoundary: b[:len(b)-2],
dashBoundaryDash: b[2:],
dashBoundary: b[2 : len(b)-2],
}
err = br.WriteNextPartTo(nil)
return
}
func skipLWSPChar(b []byte) []byte {
for len(b) > 0 && (b[0] == ' ' || b[0] == '\t') {
b = b[1:]
}
return b
}
func (br *boundaryReader) isFinalBoundary(line []byte) bool {
if !bytes.HasPrefix(line, br.dashBoundaryDash) {
return false
}
rest := line[len(br.dashBoundaryDash):]
rest = skipLWSPChar(rest)
return len(rest) == 0 || bytes.Equal(rest, br.nl)
}
func (br *boundaryReader) isBoundaryDelimiterLine(line []byte) (ret bool) {
if !bytes.HasPrefix(line, br.dashBoundary) {
return false
}
rest := line[len(br.dashBoundary):]
rest = skipLWSPChar(rest)
if br.first && len(rest) == 1 && rest[0] == '\n' {
br.nl = br.nl[1:]
br.nlDashBoundary = br.nlDashBoundary[1:]
}
return bytes.Equal(rest, br.nl)
}
func (br *boundaryReader) WriteNextPartTo(part io.Writer) (err error) {
if br.closed {
return io.EOF
}
var line, slice []byte
br.skipped = 0
for {
slice, err = br.reader.ReadSlice('\n')
line = append(line, slice...)
if err == bufio.ErrBufferFull {
continue
}
br.skipped += len(line)
if err == io.EOF && br.isFinalBoundary(line) {
err = nil
br.closed = true
return
}
if err != nil {
return
}
if br.isBoundaryDelimiterLine(line) {
br.first = false
return
}
if br.isFinalBoundary(line) {
br.closed = true
return
}
if part != nil {
if _, err = part.Write(line); err != nil {
return
}
}
line = []byte{}
}
}
type BodyStructure map[string]*SectionInfo
func NewBodyStructure(reader io.Reader) (structure *BodyStructure, err error) {
structure = &BodyStructure{}
err = structure.Parse(reader)
return
}
// DeserializeBodyStructure will create new structure from msgpack bytes.
func DeserializeBodyStructure(raw []byte) (*BodyStructure, error) {
bs := &BodyStructure{}
err := msgpack.Unmarshal(raw, bs)
@ -167,6 +67,7 @@ func DeserializeBodyStructure(raw []byte) (*BodyStructure, error) {
return bs, err
}
// Serialize will write msgpack bytes.
func (bs *BodyStructure) Serialize() ([]byte, error) {
data, err := msgpack.Marshal(bs)
if err != nil {
@ -175,6 +76,7 @@ func (bs *BodyStructure) Serialize() ([]byte, error) {
return data, nil
}
// Parse will read the mail and create all body structures.
func (bs *BodyStructure) Parse(r io.Reader) error {
return bs.parseAllChildSections(r, []int{}, 0)
}
@ -215,7 +117,7 @@ func (bs *BodyStructure) parseAllChildSections(r io.Reader, currentPath []int, s
for err == nil {
start += br.skipped
part := &bytes.Buffer{}
err = br.WriteNextPartTo(part)
err = br.writeNextPartTo(part)
if err != nil {
break
}
@ -319,19 +221,16 @@ func (bs *BodyStructure) getInfo(sectionPath []int) (sectionInfo *SectionInfo, e
return
}
// GetSection returns bytes of section including MIME header.
func (bs *BodyStructure) GetSection(wholeMail io.ReadSeeker, sectionPath []int) (section []byte, err error) {
info, err := bs.getInfo(sectionPath)
if err != nil {
return
}
if _, err = wholeMail.Seek(int64(info.Start), io.SeekStart); err != nil {
return
}
section = make([]byte, info.Size)
_, err = wholeMail.Read(section)
return
return goToOffsetAndReadNBytes(wholeMail, info.Start, info.Size)
}
// GetSectionContent returns bytes of section content (excluding MIME header).
func (bs *BodyStructure) GetSectionContent(wholeMail io.ReadSeeker, sectionPath []int) (section []byte, err error) {
info, err := bs.getInfo(sectionPath)
if err != nil {
@ -380,6 +279,8 @@ func (bs *BodyStructure) GetSectionHeader(sectionPath []int) (header textproto.M
return
}
// IMAPBodyStructure will prepare imap bodystructure recurently for given part.
// Use empty path to create whole email structure.
func (bs *BodyStructure) IMAPBodyStructure(currentPart []int) (imapBS *imap.BodyStructure, err error) {
var info *SectionInfo
if info, err = bs.getInfo(currentPart); err != nil {