From 5bf359d34f0784a29eb190688afd6e194e100f26 Mon Sep 17 00:00:00 2001 From: James Houlahan Date: Wed, 9 Jun 2021 09:27:34 +0200 Subject: [PATCH] GODT-1193: don't use message.Read; permit non-UTF-8 charsets --- go.mod | 1 + go.sum | 9 ++----- pkg/message/build_framework_test.go | 20 +++++++++++++++ pkg/message/build_rfc822.go | 19 +++----------- pkg/message/build_test.go | 25 +++++++++++++++++++ .../pgp-mime-body-plaintext-latin2.eml | 8 ++++++ 6 files changed, 60 insertions(+), 22 deletions(-) create mode 100644 pkg/message/testdata/pgp-mime-body-plaintext-latin2.eml diff --git a/go.mod b/go.mod index fd0fb525..ae541d3b 100644 --- a/go.mod +++ b/go.mod @@ -71,5 +71,6 @@ require ( replace ( github.com/docker/docker-credential-helpers => github.com/ProtonMail/docker-credential-helpers v1.1.0 github.com/emersion/go-imap => github.com/ProtonMail/go-imap v0.0.0-20201228133358-4db68cea0cac + github.com/emersion/go-message => github.com/ProtonMail/go-message v0.0.0-20210611055058-fabeff2ec753 github.com/jameskeane/bcrypt => github.com/ProtonMail/bcrypt v0.0.0-20210511135022-227b4adcab57 ) diff --git a/go.sum b/go.sum index 02ee7bff..2f79a1c2 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ github.com/ProtonMail/go-imap v0.0.0-20201228133358-4db68cea0cac h1:2xU3QncAiS/W github.com/ProtonMail/go-imap v0.0.0-20201228133358-4db68cea0cac/go.mod h1:yKASt+C3ZiDAiCSssxg9caIckWF/JG7ZQTO7GAmvicU= github.com/ProtonMail/go-imap-id v0.0.0-20190926060100-f94a56b9ecde h1:5koQozTDELymYOyFbQ/VSubexAEXzDR8qGM5mO8GRdw= github.com/ProtonMail/go-imap-id v0.0.0-20190926060100-f94a56b9ecde/go.mod h1:795VPXcRUIQ9JyMNHP4el582VokQfippgjkQP3Gk0r0= +github.com/ProtonMail/go-message v0.0.0-20210611055058-fabeff2ec753 h1:I8IsYA297x0QLU80G5I6aLYUu3JYNSpo8j5fkXtFDW0= +github.com/ProtonMail/go-message v0.0.0-20210611055058-fabeff2ec753/go.mod h1:NBAn21zgCJ/52WLDyed18YvYFm5tEoeDauubFqLokM4= github.com/ProtonMail/go-mime v0.0.0-20190923161245-9b5a4261663a h1:W6RrgN/sTxg1msqzFFb+G80MFmpjMw61IU+slm+wln4= github.com/ProtonMail/go-mime v0.0.0-20190923161245-9b5a4261663a/go.mod h1:NYt+V3/4rEeDuaev/zw1zCq8uqVEuPHzDPo3OZrlGJ4= github.com/ProtonMail/go-rfc5322 v0.8.0 h1:7emrf75n3CDIduQflx7aT1nJa5h/kGsiFKUYX/+IAkU= @@ -81,15 +83,11 @@ github.com/emersion/go-imap-unselect v0.0.0-20171113212723-b985794e5f26 h1:FiSb8 github.com/emersion/go-imap-unselect v0.0.0-20171113212723-b985794e5f26/go.mod h1:+gnnZx3Mg3MnCzZrv0eZdp5puxXQUgGT/6N6L7ShKfM= github.com/emersion/go-mbox v1.0.2 h1:tE/rT+lEugK9y0myEymCCHnwlZN04hlXPrbKkxRBA5I= github.com/emersion/go-mbox v1.0.2/go.mod h1:Yp9IVuuOYLEuMv4yjgDHvhb5mHOcYH6x92Oas3QqEZI= -github.com/emersion/go-message v0.11.1/go.mod h1:C4jnca5HOTo4bGN9YdqNQM9sITuT3Y0K6bSUw9RklvY= -github.com/emersion/go-message v0.12.1-0.20201221184100-40c3f864532b h1:xYuhW6egTaCP+zjbUcfoy/Dr3ASdVPR9W7fmkHvZHPE= -github.com/emersion/go-message v0.12.1-0.20201221184100-40c3f864532b/go.mod h1:N1JWdZQ2WRUalmdHAX308CWBq747VJ8oUorFI3VCBwU= github.com/emersion/go-sasl v0.0.0-20191210011802-430746ea8b9b/go.mod h1:G/dpzLu16WtQpBfQ/z3LYiYJn3ZhKSGWn83fyoyQe/k= github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 h1:OJyUGMJTzHTd1XQp98QTaHernxMYzRaOasRir9hUlFQ= github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ= github.com/emersion/go-smtp v0.14.0 h1:RYW203p+EcPjL8Z/ZpT9lZ6iOc8MG1MQzEx1UKEkXlA= github.com/emersion/go-smtp v0.14.0/go.mod h1:qm27SGYgoIPRot6ubfQ/GpiPy/g3PaZAVRxiO/sDUgQ= -github.com/emersion/go-textwrapper v0.0.0-20160606182133-d0e65e56babe/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U= github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 h1:IbFBtwoTQyw0fIM5xv1HF+Y+3ZijDR839WMulgxCcUY= github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U= github.com/emersion/go-vcard v0.0.0-20190105225839-8856043f13c5 h1:n9qx98xiS5V4x2WIpPC2rr9mUM5ri9r/YhCEKbhCHro= @@ -176,9 +174,6 @@ github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/martinlindhe/base36 v1.0.0/go.mod h1:+AtEs8xrBpCeYgSLoY/aJ6Wf37jtBuR0s35750M27+8= -github.com/martinlindhe/base36 v1.1.0 h1:cIwvvwYse/0+1CkUPYH5ZvVIYG3JrILmQEIbLuar02Y= -github.com/martinlindhe/base36 v1.1.0/go.mod h1:+AtEs8xrBpCeYgSLoY/aJ6Wf37jtBuR0s35750M27+8= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= diff --git a/pkg/message/build_framework_test.go b/pkg/message/build_framework_test.go index 4e00ce00..16ddfed6 100644 --- a/pkg/message/build_framework_test.go +++ b/pkg/message/build_framework_test.go @@ -33,6 +33,7 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "golang.org/x/text/encoding/htmlindex" ) func newTestFetcher( @@ -298,6 +299,25 @@ func decryptsTo(kr *crypto.KeyRing, want string) decryptsToMatcher { return decryptsToMatcher{kr: kr, want: want} } +type decodesToMatcher struct { + charset string + want string +} + +func (matcher decodesToMatcher) match(t *testing.T, have string) { + enc, err := htmlindex.Get(matcher.charset) + require.NoError(t, err) + + dec, err := enc.NewDecoder().String(have) + require.NoError(t, err) + + assert.Equal(t, matcher.want, dec) +} + +func decodesTo(charset string, want string) decodesToMatcher { + return decodesToMatcher{charset: charset, want: want} +} + type verifiesAgainstMatcher struct { kr *crypto.KeyRing sig *crypto.PGPSignature diff --git a/pkg/message/build_rfc822.go b/pkg/message/build_rfc822.go index 36366ca9..50471a3b 100644 --- a/pkg/message/build_rfc822.go +++ b/pkg/message/build_rfc822.go @@ -20,7 +20,6 @@ package message import ( "bytes" "encoding/base64" - "io/ioutil" "mime" "net/mail" "strings" @@ -290,17 +289,12 @@ func writeMultipartSignedRFC822(header message.Header, body []byte, sig pmapi.Si return nil, err } - ent, err := message.Read(bytes.NewReader(body)) + bodyHeader, bodyData, err := readHeaderBody(body) if err != nil { return nil, err } - bodyPart, err := w.CreatePart(ent.Header) - if err != nil { - return nil, err - } - - bodyData, err := ioutil.ReadAll(ent.Body) + bodyPart, err := w.CreatePart(message.Header{Header: *bodyHeader}) if err != nil { return nil, err } @@ -347,12 +341,12 @@ func writeMultipartSignedRFC822(header message.Header, body []byte, sig pmapi.Si func writeMultipartEncryptedRFC822(header message.Header, body []byte) ([]byte, error) { buf := new(bytes.Buffer) - ent, err := message.Read(bytes.NewReader(body)) + bodyHeader, bodyData, err := readHeaderBody(body) if err != nil { return nil, err } - entFields := ent.Header.Fields() + entFields := bodyHeader.Fields() for entFields.Next() { header.Set(entFields.Key(), entFields.Value()) @@ -363,11 +357,6 @@ func writeMultipartEncryptedRFC822(header message.Header, body []byte) ([]byte, return nil, err } - bodyData, err := ioutil.ReadAll(ent.Body) - if err != nil { - return nil, err - } - if _, err := w.Write(bodyData); err != nil { return nil, err } diff --git a/pkg/message/build_test.go b/pkg/message/build_test.go index c9f7fe70..f692a33c 100644 --- a/pkg/message/build_test.go +++ b/pkg/message/build_test.go @@ -98,6 +98,31 @@ func TestBuildPlainEncryptedMessage(t *testing.T) { expectBody(contains(`Where do fruits go on vacation? Pear-is!`)) } +func TestBuildPlainEncryptedLatin2Message(t *testing.T) { + m := gomock.NewController(t) + defer m.Finish() + + b := NewBuilder(1, 1, 1) + defer b.Done() + + body := readerToString(getFileReader("pgp-mime-body-plaintext-latin2.eml")) + + kr := tests.MakeKeyRing(t) + msg := newTestMessage(t, kr, "messageID", "addressID", "multipart/mixed", body, time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC)) + + res, err := b.NewJob(context.Background(), newTestFetcher(m, kr, msg), msg.ID).GetResult() + require.NoError(t, err) + + section(t, res). + expectContentType(is(`text/plain`)). + expectContentTypeParam("charset", is(`iso-8859-2`)). + expectDate(is(`Wed, 01 Jan 2020 00:00:00 +0000`)). + expectHeader(`Subject`, is(`plain no pubkey no sign`)). + expectHeader(`From`, is(`"pm.bridge.qa" `)). + expectHeader(`To`, is(`schizofrenic@pm.me`)). + expectBody(decodesTo("iso-8859-2", "řšřšřš\r\n")) +} + func TestBuildHTMLEncryptedMessage(t *testing.T) { m := gomock.NewController(t) defer m.Finish() diff --git a/pkg/message/testdata/pgp-mime-body-plaintext-latin2.eml b/pkg/message/testdata/pgp-mime-body-plaintext-latin2.eml new file mode 100644 index 00000000..d971a446 --- /dev/null +++ b/pkg/message/testdata/pgp-mime-body-plaintext-latin2.eml @@ -0,0 +1,8 @@ +Subject: plain no pubkey no sign +From: "pm.bridge.qa" +To: schizofrenic@pm.me +Message-ID: <564b9c7c-91eb-6508-107a-35108f383a44@gmail.com> +Content-Type: text/plain; charset=iso-8859-2; format=flowed +Content-Language: en-US + +ø¹ø¹ø¹