GODT-1193: don't use message.Read; permit non-UTF-8 charsets

go.mod

@@ -71,5 +71,6 @@ require (
 replace (
 	github.com/docker/docker-credential-helpers => github.com/ProtonMail/docker-credential-helpers v1.1.0
 	github.com/emersion/go-imap => github.com/ProtonMail/go-imap v0.0.0-20201228133358-4db68cea0cac
+	github.com/emersion/go-message => github.com/ProtonMail/go-message v0.0.0-20210611055058-fabeff2ec753
 	github.com/jameskeane/bcrypt => github.com/ProtonMail/bcrypt v0.0.0-20210511135022-227b4adcab57
 )

go.sum

@@ -20,6 +20,8 @@ github.com/ProtonMail/go-imap v0.0.0-20201228133358-4db68cea0cac h1:2xU3QncAiS/W
 github.com/ProtonMail/go-imap v0.0.0-20201228133358-4db68cea0cac/go.mod h1:yKASt+C3ZiDAiCSssxg9caIckWF/JG7ZQTO7GAmvicU=
 github.com/ProtonMail/go-imap-id v0.0.0-20190926060100-f94a56b9ecde h1:5koQozTDELymYOyFbQ/VSubexAEXzDR8qGM5mO8GRdw=
 github.com/ProtonMail/go-imap-id v0.0.0-20190926060100-f94a56b9ecde/go.mod h1:795VPXcRUIQ9JyMNHP4el582VokQfippgjkQP3Gk0r0=
+github.com/ProtonMail/go-message v0.0.0-20210611055058-fabeff2ec753 h1:I8IsYA297x0QLU80G5I6aLYUu3JYNSpo8j5fkXtFDW0=
+github.com/ProtonMail/go-message v0.0.0-20210611055058-fabeff2ec753/go.mod h1:NBAn21zgCJ/52WLDyed18YvYFm5tEoeDauubFqLokM4=
 github.com/ProtonMail/go-mime v0.0.0-20190923161245-9b5a4261663a h1:W6RrgN/sTxg1msqzFFb+G80MFmpjMw61IU+slm+wln4=
 github.com/ProtonMail/go-mime v0.0.0-20190923161245-9b5a4261663a/go.mod h1:NYt+V3/4rEeDuaev/zw1zCq8uqVEuPHzDPo3OZrlGJ4=
 github.com/ProtonMail/go-rfc5322 v0.8.0 h1:7emrf75n3CDIduQflx7aT1nJa5h/kGsiFKUYX/+IAkU=
@@ -81,15 +83,11 @@ github.com/emersion/go-imap-unselect v0.0.0-20171113212723-b985794e5f26 h1:FiSb8
 github.com/emersion/go-imap-unselect v0.0.0-20171113212723-b985794e5f26/go.mod h1:+gnnZx3Mg3MnCzZrv0eZdp5puxXQUgGT/6N6L7ShKfM=
 github.com/emersion/go-mbox v1.0.2 h1:tE/rT+lEugK9y0myEymCCHnwlZN04hlXPrbKkxRBA5I=
 github.com/emersion/go-mbox v1.0.2/go.mod h1:Yp9IVuuOYLEuMv4yjgDHvhb5mHOcYH6x92Oas3QqEZI=
-github.com/emersion/go-message v0.11.1/go.mod h1:C4jnca5HOTo4bGN9YdqNQM9sITuT3Y0K6bSUw9RklvY=
-github.com/emersion/go-message v0.12.1-0.20201221184100-40c3f864532b h1:xYuhW6egTaCP+zjbUcfoy/Dr3ASdVPR9W7fmkHvZHPE=
-github.com/emersion/go-message v0.12.1-0.20201221184100-40c3f864532b/go.mod h1:N1JWdZQ2WRUalmdHAX308CWBq747VJ8oUorFI3VCBwU=
 github.com/emersion/go-sasl v0.0.0-20191210011802-430746ea8b9b/go.mod h1:G/dpzLu16WtQpBfQ/z3LYiYJn3ZhKSGWn83fyoyQe/k=
 github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 h1:OJyUGMJTzHTd1XQp98QTaHernxMYzRaOasRir9hUlFQ=
 github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ=
 github.com/emersion/go-smtp v0.14.0 h1:RYW203p+EcPjL8Z/ZpT9lZ6iOc8MG1MQzEx1UKEkXlA=
 github.com/emersion/go-smtp v0.14.0/go.mod h1:qm27SGYgoIPRot6ubfQ/GpiPy/g3PaZAVRxiO/sDUgQ=
-github.com/emersion/go-textwrapper v0.0.0-20160606182133-d0e65e56babe/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U=
 github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 h1:IbFBtwoTQyw0fIM5xv1HF+Y+3ZijDR839WMulgxCcUY=
 github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U=
 github.com/emersion/go-vcard v0.0.0-20190105225839-8856043f13c5 h1:n9qx98xiS5V4x2WIpPC2rr9mUM5ri9r/YhCEKbhCHro=
@@ -176,9 +174,6 @@ github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL
 github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
 github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/martinlindhe/base36 v1.0.0/go.mod h1:+AtEs8xrBpCeYgSLoY/aJ6Wf37jtBuR0s35750M27+8=
-github.com/martinlindhe/base36 v1.1.0 h1:cIwvvwYse/0+1CkUPYH5ZvVIYG3JrILmQEIbLuar02Y=
-github.com/martinlindhe/base36 v1.1.0/go.mod h1:+AtEs8xrBpCeYgSLoY/aJ6Wf37jtBuR0s35750M27+8=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=

pkg/message/build_framework_test.go

@@ -33,6 +33,7 @@ import (
 	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/text/encoding/htmlindex"
 )
 
 func newTestFetcher(
@@ -298,6 +299,25 @@ func decryptsTo(kr *crypto.KeyRing, want string) decryptsToMatcher {
 	return decryptsToMatcher{kr: kr, want: want}
 }
 
+type decodesToMatcher struct {
+	charset string
+	want    string
+}
+
+func (matcher decodesToMatcher) match(t *testing.T, have string) {
+	enc, err := htmlindex.Get(matcher.charset)
+	require.NoError(t, err)
+
+	dec, err := enc.NewDecoder().String(have)
+	require.NoError(t, err)
+
+	assert.Equal(t, matcher.want, dec)
+}
+
+func decodesTo(charset string, want string) decodesToMatcher {
+	return decodesToMatcher{charset: charset, want: want}
+}
+
 type verifiesAgainstMatcher struct {
 	kr  *crypto.KeyRing
 	sig *crypto.PGPSignature

pkg/message/build_rfc822.go

@@ -20,7 +20,6 @@ package message
 import (
 	"bytes"
 	"encoding/base64"
-	"io/ioutil"
 	"mime"
 	"net/mail"
 	"strings"
@@ -290,17 +289,12 @@ func writeMultipartSignedRFC822(header message.Header, body []byte, sig pmapi.Si
 		return nil, err
 	}
 
-	ent, err := message.Read(bytes.NewReader(body))
+	bodyHeader, bodyData, err := readHeaderBody(body)
 	if err != nil {
 		return nil, err
 	}
 
-	bodyPart, err := w.CreatePart(ent.Header)
+	bodyPart, err := w.CreatePart(message.Header{Header: *bodyHeader})
-	if err != nil {
-		return nil, err
-	}
-
-	bodyData, err := ioutil.ReadAll(ent.Body)
 	if err != nil {
 		return nil, err
 	}
@@ -347,12 +341,12 @@ func writeMultipartSignedRFC822(header message.Header, body []byte, sig pmapi.Si
 func writeMultipartEncryptedRFC822(header message.Header, body []byte) ([]byte, error) {
 	buf := new(bytes.Buffer)
 
-	ent, err := message.Read(bytes.NewReader(body))
+	bodyHeader, bodyData, err := readHeaderBody(body)
 	if err != nil {
 		return nil, err
 	}
 
-	entFields := ent.Header.Fields()
+	entFields := bodyHeader.Fields()
 
 	for entFields.Next() {
 		header.Set(entFields.Key(), entFields.Value())
@@ -363,11 +357,6 @@ func writeMultipartEncryptedRFC822(header message.Header, body []byte) ([]byte,
 		return nil, err
 	}
 
-	bodyData, err := ioutil.ReadAll(ent.Body)
-	if err != nil {
-		return nil, err
-	}
-
 	if _, err := w.Write(bodyData); err != nil {
 		return nil, err
 	}

pkg/message/build_test.go

@@ -98,6 +98,31 @@ func TestBuildPlainEncryptedMessage(t *testing.T) {
 		expectBody(contains(`Where do fruits go on vacation? Pear-is!`))
 }
 
+func TestBuildPlainEncryptedLatin2Message(t *testing.T) {
+	m := gomock.NewController(t)
+	defer m.Finish()
+
+	b := NewBuilder(1, 1, 1)
+	defer b.Done()
+
+	body := readerToString(getFileReader("pgp-mime-body-plaintext-latin2.eml"))
+
+	kr := tests.MakeKeyRing(t)
+	msg := newTestMessage(t, kr, "messageID", "addressID", "multipart/mixed", body, time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC))
+
+	res, err := b.NewJob(context.Background(), newTestFetcher(m, kr, msg), msg.ID).GetResult()
+	require.NoError(t, err)
+
+	section(t, res).
+		expectContentType(is(`text/plain`)).
+		expectContentTypeParam("charset", is(`iso-8859-2`)).
+		expectDate(is(`Wed, 01 Jan 2020 00:00:00 +0000`)).
+		expectHeader(`Subject`, is(`plain no pubkey no sign`)).
+		expectHeader(`From`, is(`"pm.bridge.qa" <pm.bridge.qa@gmail.com>`)).
+		expectHeader(`To`, is(`schizofrenic@pm.me`)).
+		expectBody(decodesTo("iso-8859-2", "řšřšřš\r\n"))
+}
+
 func TestBuildHTMLEncryptedMessage(t *testing.T) {
 	m := gomock.NewController(t)
 	defer m.Finish()

pkg/message/testdata/pgp-mime-body-plaintext-latin2.eml

@@ -0,0 +1,8 @@
+Subject: plain no pubkey no sign
+From: "pm.bridge.qa" <pm.bridge.qa@gmail.com>
+To: schizofrenic@pm.me
+Message-ID: <564b9c7c-91eb-6508-107a-35108f383a44@gmail.com>
+Content-Type: text/plain; charset=iso-8859-2; format=flowed
+Content-Language: en-US
+
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>