GODT-1193: don't use message.Read; permit non-UTF-8 charsets

pkg/message/build_framework_test.go

@@ -33,6 +33,7 @@ import (
 	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/text/encoding/htmlindex"
 )
 
 func newTestFetcher(
@@ -298,6 +299,25 @@ func decryptsTo(kr *crypto.KeyRing, want string) decryptsToMatcher {
 	return decryptsToMatcher{kr: kr, want: want}
 }
 
+type decodesToMatcher struct {
+	charset string
+	want    string
+}
+
+func (matcher decodesToMatcher) match(t *testing.T, have string) {
+	enc, err := htmlindex.Get(matcher.charset)
+	require.NoError(t, err)
+
+	dec, err := enc.NewDecoder().String(have)
+	require.NoError(t, err)
+
+	assert.Equal(t, matcher.want, dec)
+}
+
+func decodesTo(charset string, want string) decodesToMatcher {
+	return decodesToMatcher{charset: charset, want: want}
+}
+
 type verifiesAgainstMatcher struct {
 	kr  *crypto.KeyRing
 	sig *crypto.PGPSignature

pkg/message/build_rfc822.go

@@ -20,7 +20,6 @@ package message
 import (
 	"bytes"
 	"encoding/base64"
-	"io/ioutil"
 	"mime"
 	"net/mail"
 	"strings"
@@ -290,17 +289,12 @@ func writeMultipartSignedRFC822(header message.Header, body []byte, sig pmapi.Si
 		return nil, err
 	}
 
-	ent, err := message.Read(bytes.NewReader(body))
+	bodyHeader, bodyData, err := readHeaderBody(body)
 	if err != nil {
 		return nil, err
 	}
 
-	bodyPart, err := w.CreatePart(ent.Header)
-	if err != nil {
-		return nil, err
-	}
-
-	bodyData, err := ioutil.ReadAll(ent.Body)
+	bodyPart, err := w.CreatePart(message.Header{Header: *bodyHeader})
 	if err != nil {
 		return nil, err
 	}
@@ -347,12 +341,12 @@ func writeMultipartSignedRFC822(header message.Header, body []byte, sig pmapi.Si
 func writeMultipartEncryptedRFC822(header message.Header, body []byte) ([]byte, error) {
 	buf := new(bytes.Buffer)
 
-	ent, err := message.Read(bytes.NewReader(body))
+	bodyHeader, bodyData, err := readHeaderBody(body)
 	if err != nil {
 		return nil, err
 	}
 
-	entFields := ent.Header.Fields()
+	entFields := bodyHeader.Fields()
 
 	for entFields.Next() {
 		header.Set(entFields.Key(), entFields.Value())
@@ -363,11 +357,6 @@ func writeMultipartEncryptedRFC822(header message.Header, body []byte) ([]byte,
 		return nil, err
 	}
 
-	bodyData, err := ioutil.ReadAll(ent.Body)
-	if err != nil {
-		return nil, err
-	}
-
 	if _, err := w.Write(bodyData); err != nil {
 		return nil, err
 	}

pkg/message/build_test.go

@@ -98,6 +98,31 @@ func TestBuildPlainEncryptedMessage(t *testing.T) {
 		expectBody(contains(`Where do fruits go on vacation? Pear-is!`))
 }
 
+func TestBuildPlainEncryptedLatin2Message(t *testing.T) {
+	m := gomock.NewController(t)
+	defer m.Finish()
+
+	b := NewBuilder(1, 1, 1)
+	defer b.Done()
+
+	body := readerToString(getFileReader("pgp-mime-body-plaintext-latin2.eml"))
+
+	kr := tests.MakeKeyRing(t)
+	msg := newTestMessage(t, kr, "messageID", "addressID", "multipart/mixed", body, time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC))
+
+	res, err := b.NewJob(context.Background(), newTestFetcher(m, kr, msg), msg.ID).GetResult()
+	require.NoError(t, err)
+
+	section(t, res).
+		expectContentType(is(`text/plain`)).
+		expectContentTypeParam("charset", is(`iso-8859-2`)).
+		expectDate(is(`Wed, 01 Jan 2020 00:00:00 +0000`)).
+		expectHeader(`Subject`, is(`plain no pubkey no sign`)).
+		expectHeader(`From`, is(`"pm.bridge.qa" <pm.bridge.qa@gmail.com>`)).
+		expectHeader(`To`, is(`schizofrenic@pm.me`)).
+		expectBody(decodesTo("iso-8859-2", "řšřšřš\r\n"))
+}
+
 func TestBuildHTMLEncryptedMessage(t *testing.T) {
 	m := gomock.NewController(t)
 	defer m.Finish()

pkg/message/testdata/pgp-mime-body-plaintext-latin2.eml

@@ -0,0 +1,8 @@
+Subject: plain no pubkey no sign
+From: "pm.bridge.qa" <pm.bridge.qa@gmail.com>
+To: schizofrenic@pm.me
+Message-ID: <564b9c7c-91eb-6508-107a-35108f383a44@gmail.com>
+Content-Type: text/plain; charset=iso-8859-2; format=flowed
+Content-Language: en-US
+
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>