GODT-1044: lite parser

This commit is contained in:
James Houlahan
2021-05-17 15:56:22 +02:00
committed by Jakub Cuth
parent 509ba52ba2
commit e01dc77a61
12 changed files with 966 additions and 174 deletions

244
pkg/message/encrypt.go Normal file
View File

@ -0,0 +1,244 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package message
import (
"bytes"
"encoding/base64"
"io"
"io/ioutil"
"mime"
"mime/quotedprintable"
"strings"
"github.com/ProtonMail/gopenpgp/v2/crypto"
pmmime "github.com/ProtonMail/proton-bridge/pkg/mime"
"github.com/emersion/go-message/textproto"
)
func EncryptRFC822(kr *crypto.KeyRing, r io.Reader) ([]byte, error) {
b, err := ioutil.ReadAll(r)
if err != nil {
return nil, err
}
header, body, err := readHeaderBody(b)
if err != nil {
return nil, err
}
buf := new(bytes.Buffer)
result, err := writeEncryptedPart(kr, header, bytes.NewReader(body))
if err != nil {
return nil, err
}
if err := textproto.WriteHeader(buf, *header); err != nil {
return nil, err
}
if _, err := result.WriteTo(buf); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func writeEncryptedPart(kr *crypto.KeyRing, header *textproto.Header, r io.Reader) (io.WriterTo, error) {
decoder := getTransferDecoder(r, header.Get("Content-Transfer-Encoding"))
encoded := new(bytes.Buffer)
contentType, contentParams, err := parseContentType(header.Get("Content-Type"))
if err != nil {
return nil, err
}
switch {
case contentType == "", strings.HasPrefix(contentType, "text/"), strings.HasPrefix(contentType, "message/"):
header.Del("Content-Transfer-Encoding")
if charset, ok := contentParams["charset"]; ok {
if reader, err := pmmime.CharsetReader(charset, decoder); err == nil {
decoder = reader
// We can decode the charset to utf-8 so let's set that as the content type charset parameter.
contentParams["charset"] = "utf-8"
header.Set("Content-Type", mime.FormatMediaType(contentType, contentParams))
}
}
if err := encode(&writeCloser{encoded}, func(w io.Writer) error {
return writeEncryptedTextPart(w, decoder, kr)
}); err != nil {
return nil, err
}
case contentType == "multipart/encrypted":
if _, err := encoded.ReadFrom(decoder); err != nil {
return nil, err
}
case strings.HasPrefix(contentType, "multipart/"):
if err := encode(&writeCloser{encoded}, func(w io.Writer) error {
return writeEncryptedMultiPart(kr, w, header, decoder)
}); err != nil {
return nil, err
}
default:
header.Set("Content-Transfer-Encoding", "base64")
if err := encode(base64.NewEncoder(base64.StdEncoding, encoded), func(w io.Writer) error {
return writeEncryptedAttachmentPart(w, decoder, kr)
}); err != nil {
return nil, err
}
}
return encoded, nil
}
func writeEncryptedTextPart(w io.Writer, r io.Reader, kr *crypto.KeyRing) error {
dec, err := ioutil.ReadAll(r)
if err != nil {
return err
}
var arm string
if msg, err := crypto.NewPGPMessageFromArmored(string(dec)); err != nil {
enc, err := kr.Encrypt(crypto.NewPlainMessage(dec), kr)
if err != nil {
return err
}
if arm, err = enc.GetArmored(); err != nil {
return err
}
} else if arm, err = msg.GetArmored(); err != nil {
return err
}
if _, err := io.WriteString(w, arm); err != nil {
return err
}
return nil
}
func writeEncryptedAttachmentPart(w io.Writer, r io.Reader, kr *crypto.KeyRing) error {
dec, err := ioutil.ReadAll(r)
if err != nil {
return err
}
enc, err := kr.Encrypt(crypto.NewPlainMessage(dec), kr)
if err != nil {
return err
}
if _, err := w.Write(enc.GetBinary()); err != nil {
return err
}
return nil
}
func writeEncryptedMultiPart(kr *crypto.KeyRing, w io.Writer, header *textproto.Header, r io.Reader) error {
_, contentParams, err := parseContentType(header.Get("Content-Type"))
if err != nil {
return err
}
scanner, err := newPartScanner(r, contentParams["boundary"])
if err != nil {
return err
}
parts, err := scanner.scanAll()
if err != nil {
return err
}
writer := newPartWriter(w, contentParams["boundary"])
for _, part := range parts {
header, body, err := readHeaderBody(part.b)
if err != nil {
return err
}
result, err := writeEncryptedPart(kr, header, bytes.NewReader(body))
if err != nil {
return err
}
if err := writer.createPart(func(w io.Writer) error {
if err := textproto.WriteHeader(w, *header); err != nil {
return err
}
if _, err := result.WriteTo(w); err != nil {
return err
}
return nil
}); err != nil {
return err
}
}
return writer.done()
}
func getTransferDecoder(r io.Reader, encoding string) io.Reader {
switch strings.ToLower(encoding) {
case "base64":
return base64.NewDecoder(base64.StdEncoding, r)
case "quoted-printable":
return quotedprintable.NewReader(r)
default:
return r
}
}
func encode(wc io.WriteCloser, fn func(io.Writer) error) error {
if err := fn(wc); err != nil {
return err
}
return wc.Close()
}
type writeCloser struct {
io.Writer
}
func (writeCloser) Close() error { return nil }
func parseContentType(val string) (string, map[string]string, error) {
if val == "" {
val = "text/plain"
}
return pmmime.ParseMediaType(val)
}

101
pkg/message/encrypt_test.go Normal file
View File

@ -0,0 +1,101 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package message
import (
"bytes"
"io/ioutil"
"testing"
"github.com/ProtonMail/gopenpgp/v2/crypto"
"github.com/stretchr/testify/require"
)
func TestEncryptRFC822(t *testing.T) {
literal, err := ioutil.ReadFile("testdata/text_plain_latin1.eml")
require.NoError(t, err)
key, err := crypto.GenerateKey("name", "email", "rsa", 2048)
require.NoError(t, err)
kr, err := crypto.NewKeyRing(key)
require.NoError(t, err)
enc, err := EncryptRFC822(kr, bytes.NewReader(literal))
require.NoError(t, err)
section(t, enc).
expectContentType(is(`text/plain`)).
expectContentTypeParam(`charset`, is(`utf-8`)).
expectBody(decryptsTo(kr, `ééééééé`))
}
func TestEncryptRFC822Multipart(t *testing.T) {
literal, err := ioutil.ReadFile("testdata/multipart_alternative_nested.eml")
require.NoError(t, err)
key, err := crypto.GenerateKey("name", "email", "rsa", 2048)
require.NoError(t, err)
kr, err := crypto.NewKeyRing(key)
require.NoError(t, err)
enc, err := EncryptRFC822(kr, bytes.NewReader(literal))
require.NoError(t, err)
section(t, enc).
expectContentType(is(`multipart/alternative`))
section(t, enc, 1).
expectContentType(is(`multipart/alternative`))
section(t, enc, 1, 1).
expectContentType(is(`text/plain`)).
expectBody(decryptsTo(kr, "*multipart 1.1*\n\n"))
section(t, enc, 1, 2).
expectContentType(is(`text/html`)).
expectBody(decryptsTo(kr, `<html>
<head>
<meta http-equiv="content-type" content="text/html; charset=UTF-8">
</head>
<body>
<b>multipart 1.2</b>
</body>
</html>
`))
section(t, enc, 2).
expectContentType(is(`multipart/alternative`))
section(t, enc, 2, 1).
expectContentType(is(`text/plain`)).
expectBody(decryptsTo(kr, "*multipart 2.1*\n\n"))
section(t, enc, 2, 2).
expectContentType(is(`text/html`)).
expectBody(decryptsTo(kr, `<html>
<head>
<meta http-equiv="content-type" content="text/html; charset=UTF-8">
</head>
<body>
<b>multipart 2.2</b>
</body>
</html>
`))
}

View File

@ -59,30 +59,3 @@ func GetFlags(m *pmapi.Message) (flags []string) {
return
}
// ParseFlags sets attributes to pmapi messages based on imap flags.
func ParseFlags(m *pmapi.Message, flags []string) {
if m.Header.Get("received") == "" {
m.Flags = pmapi.FlagSent
} else {
m.Flags = pmapi.FlagReceived
}
m.Unread = true
for _, f := range flags {
switch f {
case imap.SeenFlag:
m.Unread = false
case imap.DraftFlag:
m.Flags &= ^pmapi.FlagSent
m.Flags &= ^pmapi.FlagReceived
m.LabelIDs = append(m.LabelIDs, pmapi.DraftLabel)
case imap.FlaggedFlag:
m.LabelIDs = append(m.LabelIDs, pmapi.StarredLabel)
case imap.AnsweredFlag:
m.Flags |= pmapi.FlagReplied
case AppleMailJunkFlag, ThunderbirdJunkFlag:
m.LabelIDs = append(m.LabelIDs, pmapi.SpamLabel)
}
}
}

123
pkg/message/header.go Normal file
View File

@ -0,0 +1,123 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package message
import (
"bufio"
"bytes"
"io"
"io/ioutil"
"github.com/emersion/go-message/textproto"
"github.com/pkg/errors"
)
// HeaderLines returns each line in the given header.
func HeaderLines(header []byte) [][]byte {
var (
lines [][]byte
quote int
)
forEachLine(bufio.NewReader(bytes.NewReader(header)), func(line []byte) {
switch {
case len(bytes.TrimSpace(line)) == 0:
lines = append(lines, line)
case quote%2 != 0, len(bytes.SplitN(line, []byte(`: `), 2)) != 2:
if len(lines) > 0 {
lines[len(lines)-1] = append(lines[len(lines)-1], line...)
} else {
lines = append(lines, line)
}
default:
lines = append(lines, line)
}
quote += bytes.Count(line, []byte(`"`))
})
return lines
}
func forEachLine(br *bufio.Reader, fn func([]byte)) {
for {
b, err := br.ReadBytes('\n')
if err != nil {
if !errors.Is(err, io.EOF) {
panic(err)
}
if len(b) > 0 {
fn(b)
}
return
}
fn(b)
}
}
func readHeaderBody(b []byte) (*textproto.Header, []byte, error) {
rawHeader, body, err := splitHeaderBody(b)
if err != nil {
return nil, nil, err
}
var header textproto.Header
for _, line := range HeaderLines(rawHeader) {
if len(bytes.TrimSpace(line)) > 0 {
header.AddRaw(line)
}
}
return &header, body, nil
}
func splitHeaderBody(b []byte) ([]byte, []byte, error) {
br := bufio.NewReader(bytes.NewReader(b))
var header []byte
for {
b, err := br.ReadBytes('\n')
if err != nil {
if !errors.Is(err, io.EOF) {
panic(err)
}
break
}
header = append(header, b...)
if len(bytes.TrimSpace(b)) == 0 {
break
}
}
body, err := ioutil.ReadAll(br)
if err != nil && !errors.Is(err, io.EOF) {
return nil, nil, err
}
return header, body, nil
}

View File

@ -0,0 +1,76 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package message
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestHeaderLines(t *testing.T) {
const header = "To: somebody\r\nFrom: somebody else\r\nSubject: this is\r\n\ta multiline field\r\n\r\n"
assert.Equal(t, [][]byte{
[]byte("To: somebody\r\n"),
[]byte("From: somebody else\r\n"),
[]byte("Subject: this is\r\n\ta multiline field\r\n"),
[]byte("\r\n"),
}, HeaderLines([]byte(header)))
}
func TestHeaderLinesMultilineFilename(t *testing.T) {
const header = "Content-Type: application/msword; name=\"this is a very long\nfilename.doc\""
assert.Equal(t, [][]byte{
[]byte("Content-Type: application/msword; name=\"this is a very long\nfilename.doc\""),
}, HeaderLines([]byte(header)))
}
func TestHeaderLinesMultilineFilenameWithColon(t *testing.T) {
const header = "Content-Type: application/msword; name=\"this is a very long\nfilename: too long.doc\""
assert.Equal(t, [][]byte{
[]byte("Content-Type: application/msword; name=\"this is a very long\nfilename: too long.doc\""),
}, HeaderLines([]byte(header)))
}
func TestHeaderLinesMultilineFilenameWithColonAndNewline(t *testing.T) {
const header = "Content-Type: application/msword; name=\"this is a very long\nfilename: too long.doc\"\n"
assert.Equal(t, [][]byte{
[]byte("Content-Type: application/msword; name=\"this is a very long\nfilename: too long.doc\"\n"),
}, HeaderLines([]byte(header)))
}
func TestHeaderLinesMultipleMultilineFilenames(t *testing.T) {
const header = `Content-Type: application/msword; name="=E5=B8=B6=E6=9C=89=E5=A4=96=E5=9C=8B=E5=AD=97=E7=AC=A6=E7=9A=84=E9=99=84=E4=
=BB=B6.DOC"
Content-Transfer-Encoding: base64
Content-Disposition: attachment; filename="=E5=B8=B6=E6=9C=89=E5=A4=96=E5=9C=8B=E5=AD=97=E7=AC=A6=E7=9A=84=E9=99=84=E4=
=BB=B6.DOC"
Content-ID: <>
`
assert.Equal(t, [][]byte{
[]byte("Content-Type: application/msword; name=\"=E5=B8=B6=E6=9C=89=E5=A4=96=E5=9C=8B=E5=AD=97=E7=AC=A6=E7=9A=84=E9=99=84=E4=\n=BB=B6.DOC\"\n"),
[]byte("Content-Transfer-Encoding: base64\n"),
[]byte("Content-Disposition: attachment; filename=\"=E5=B8=B6=E6=9C=89=E5=A4=96=E5=9C=8B=E5=AD=97=E7=AC=A6=E7=9A=84=E9=99=84=E4=\n=BB=B6.DOC\"\n"),
[]byte("Content-ID: <>\n"),
}, HeaderLines([]byte(header)))
}

96
pkg/message/scanner.go Normal file
View File

@ -0,0 +1,96 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package message
import (
"bufio"
"bytes"
"errors"
"io"
)
type partScanner struct {
r *bufio.Reader
boundary string
progress int
}
type part struct {
b []byte
offset int
}
func newPartScanner(r io.Reader, boundary string) (*partScanner, error) {
scanner := &partScanner{r: bufio.NewReader(r), boundary: boundary}
if _, _, err := scanner.readToBoundary(); err != nil {
return nil, err
}
return scanner, nil
}
func (s *partScanner) scanAll() ([]part, error) {
var parts []part
for {
offset := s.progress
b, more, err := s.readToBoundary()
if err != nil {
return nil, err
}
if !more {
return parts, nil
}
parts = append(parts, part{b: b, offset: offset})
}
}
func (s *partScanner) readToBoundary() ([]byte, bool, error) {
var res []byte
for {
line, err := s.r.ReadBytes('\n')
if err != nil {
if !errors.Is(err, io.EOF) {
return nil, false, err
}
if len(line) == 0 {
return nil, false, nil
}
}
s.progress += len(line)
switch {
case bytes.HasPrefix(bytes.TrimSpace(line), []byte("--"+s.boundary)):
return bytes.TrimSuffix(bytes.TrimSuffix(res, []byte("\n")), []byte("\r")), true, nil
case bytes.HasSuffix(bytes.TrimSpace(line), []byte(s.boundary+"--")):
return bytes.TrimSuffix(bytes.TrimSuffix(res, []byte("\n")), []byte("\r")), false, nil
default:
res = append(res, line...)
}
}
}

136
pkg/message/scanner_test.go Normal file
View File

@ -0,0 +1,136 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package message
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestScanner(t *testing.T) {
const literal = `this part of the text should be ignored
--longrandomstring
body1
--longrandomstring
body2
--longrandomstring--
`
scanner, err := newPartScanner(strings.NewReader(literal), "longrandomstring")
require.NoError(t, err)
parts, err := scanner.scanAll()
require.NoError(t, err)
assert.Equal(t, "\nbody1\n", string(parts[0].b))
assert.Equal(t, "\nbody2\n", string(parts[1].b))
assert.Equal(t, "\nbody1\n", literal[parts[0].offset:parts[0].offset+len(parts[0].b)])
assert.Equal(t, "\nbody2\n", literal[parts[1].offset:parts[1].offset+len(parts[1].b)])
}
func TestScannerNested(t *testing.T) {
const literal = `This is the preamble. It is to be ignored, though it
is a handy place for mail composers to include an
explanatory note to non-MIME compliant readers.
--simple boundary
Content-type: multipart/mixed; boundary="nested boundary"
This is the preamble. It is to be ignored, though it
is a handy place for mail composers to include an
explanatory note to non-MIME compliant readers.
--nested boundary
Content-type: text/plain; charset=us-ascii
This part does not end with a linebreak.
--nested boundary
Content-type: text/plain; charset=us-ascii
This part does end with a linebreak.
--nested boundary--
--simple boundary
Content-type: text/plain; charset=us-ascii
This part does end with a linebreak.
--simple boundary--
This is the epilogue. It is also to be ignored.
`
scanner, err := newPartScanner(strings.NewReader(literal), "simple boundary")
require.NoError(t, err)
parts, err := scanner.scanAll()
require.NoError(t, err)
assert.Equal(t, `Content-type: multipart/mixed; boundary="nested boundary"
This is the preamble. It is to be ignored, though it
is a handy place for mail composers to include an
explanatory note to non-MIME compliant readers.
--nested boundary
Content-type: text/plain; charset=us-ascii
This part does not end with a linebreak.
--nested boundary
Content-type: text/plain; charset=us-ascii
This part does end with a linebreak.
--nested boundary--`, string(parts[0].b))
assert.Equal(t, `Content-type: text/plain; charset=us-ascii
This part does end with a linebreak.
`, string(parts[1].b))
}
func TestScannerNoFinalLinebreak(t *testing.T) {
const literal = `--nested boundary
Content-type: text/plain; charset=us-ascii
This part does not end with a linebreak.
--nested boundary
Content-type: text/plain; charset=us-ascii
This part does end with a linebreak.
--nested boundary--`
scanner, err := newPartScanner(strings.NewReader(literal), "nested boundary")
require.NoError(t, err)
parts, err := scanner.scanAll()
require.NoError(t, err)
assert.Equal(t, `Content-type: text/plain; charset=us-ascii
This part does not end with a linebreak.`, string(parts[0].b))
assert.Equal(t, `Content-type: text/plain; charset=us-ascii
This part does end with a linebreak.
`, string(parts[1].b))
}

48
pkg/message/writer.go Normal file
View File

@ -0,0 +1,48 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package message
import (
"fmt"
"io"
)
type partWriter struct {
w io.Writer
boundary string
}
func newPartWriter(w io.Writer, boundary string) *partWriter {
return &partWriter{w: w, boundary: boundary}
}
func (w *partWriter) createPart(fn func(io.Writer) error) error {
if _, err := fmt.Fprintf(w.w, "\r\n--%v\r\n", w.boundary); err != nil {
return err
}
return fn(w.w)
}
func (w *partWriter) done() error {
if _, err := fmt.Fprintf(w.w, "\r\n--%v--\r\n", w.boundary); err != nil {
return err
}
return nil
}