GODT-948 Embedded messages

Makefile

@@ -294,6 +294,7 @@ LOG?=debug
 LOG_IMAP?=client # client/server/all, or empty to turn it off
 LOG_SMTP?=--log-smtp # empty to turn it off
 RUN_FLAGS?=-m -l=${LOG} --log-imap=${LOG_IMAP} ${LOG_SMTP}
+RUN_FLAGS_IE?=-m -l=${LOG}
 
 run: run-nogui-cli
 
@@ -316,11 +317,11 @@ run-ie-qml-preview:
 	$(MAKE) -C internal/frontend/qt-ie -f Makefile.local qmlpreview
 
 run-ie:
-	TARGET_CMD=Import-Export $(MAKE) run
+	TARGET_CMD=Import-Export RUN_FLAGS="${RUN_FLAGS_IE}" $(MAKE) run
 run-ie-qt:
-	TARGET_CMD=Import-Export $(MAKE) run-qt
+	TARGET_CMD=Import-Export RUN_FLAGS="${RUN_FLAGS_IE}" $(MAKE) run-qt
 run-ie-nogui:
-	TARGET_CMD=Import-Export $(MAKE) run-nogui
+	TARGET_CMD=Import-Export RUN_FLAGS="${RUN_FLAGS_IE}" $(MAKE) run-nogui
 
 clean-frontend-qt:
 	$(MAKE) -C internal/frontend/qt -f Makefile.local clean

internal/imap/mailbox_message.go

@@ -564,7 +564,7 @@ func (im *imapMailbox) writeRelatedPart(p io.Writer, m *pmapi.Message, inlines [
 			return
 		}
 
-		h := message.GetAttachmentHeader(inline)
+		h := message.GetAttachmentHeader(inline, true)
 		if p, err = related.CreatePart(h); err != nil {
 			return
 		}
@@ -738,7 +738,7 @@ func (im *imapMailbox) buildMessageInner(m *pmapi.Message, kr *crypto.KeyRing) (
 			defer buf.Reset()
 			att := atts[idx]
 
-			attachmentHeader := message.GetAttachmentHeader(att)
+			attachmentHeader := message.GetAttachmentHeader(att, true)
 			if partWriter, err = mw.CreatePart(attachmentHeader); err != nil {
 				return err
 			}

internal/store/event_loop.go

@@ -268,8 +268,6 @@ func (loop *eventLoop) processNextEvent() (more bool, err error) { // nolint[fun
 		return false, errors.New("received empty event")
 	}
 
-	l = l.WithField("newEventID", event.EventID)
-
 	if err = loop.processEvent(event); err != nil {
 		return false, errors.Wrap(err, "failed to process event")
 	}

internal/store/mailbox_message.go

@@ -67,10 +67,14 @@ func (storeMailbox *Mailbox) ImportMessage(msg *pmapi.Message, body []byte, labe
 	}
 
 	res, err := storeMailbox.client().Import([]*pmapi.ImportMsgReq{importReqs})
-	if err == nil && len(res) > 0 {
+	if err != nil {
-		msg.ID = res[0].MessageID
+		return err
 	}
-	return err
+	if len(res) == 0 {
+		return errors.New("no import response")
+	}
+	msg.ID = res[0].MessageID
+	return res[0].Error
 }
 
 // LabelMessages adds the label by calling an API.

pkg/message/body.go

@@ -57,19 +57,23 @@ func WriteAttachmentBody(w io.Writer, kr *crypto.KeyRing, m *pmapi.Message, att
 		att.Name += ".gpg"
 		att.MIMEType = "application/pgp-encrypted" //nolint
 	} else if err != nil && err != openpgperrors.ErrSignatureExpired {
-		err = fmt.Errorf("cannot decrypt attachment: %v", err)
+		return fmt.Errorf("cannot decrypt attachment: %v", err)
-		return
+	}
+
+	// Don't encode message/rfc822 attachments; they should be embedded and preserved.
+	if att.MIMEType == rfc822Message {
+		if n, err := io.Copy(w, dr); err != nil {
+			return fmt.Errorf("cannot write attached message: %v (wrote %v bytes)", err, n)
+		}
+		return nil
 	}
 
 	// Encode it.
 	ww := textwrapper.NewRFC822(w)
 	bw := base64.NewEncoder(base64.StdEncoding, ww)
 
-	var n int64
+	if n, err := io.Copy(bw, dr); err != nil {
-	if n, err = io.Copy(bw, dr); err != nil {
+		return fmt.Errorf("cannot write attachment: %v (wrote %v bytes)", err, n)
-		err = fmt.Errorf("cannot write attachment: %v (wrote %v bytes)", err, n)
 	}
-
+	return bw.Close()
-	_ = bw.Close()
-	return
 }

pkg/message/build.go

@@ -124,7 +124,7 @@ func (bld *Builder) writeRelatedPart(p io.Writer, inlines []*pmapi.Attachment) e
 			return err
 		}
 
-		h := GetAttachmentHeader(inline)
+		h := GetAttachmentHeader(inline, false)
 		if p, err = related.CreatePart(h); err != nil {
 			return err
 		}
@@ -194,7 +194,7 @@ func (bld *Builder) BuildMessage() (structure *BodyStructure, message []byte, er
 				return nil, nil, err
 			}
 
-			attachmentHeader := GetAttachmentHeader(att)
+			attachmentHeader := GetAttachmentHeader(att, false)
 			if partWriter, err = mw.CreatePart(attachmentHeader); err != nil {
 				return nil, nil, err
 			}
@@ -311,16 +311,11 @@ func BuildEncrypted(m *pmapi.Message, readers []io.Reader, kr *crypto.KeyRing) (
 	for i := 0; i < len(m.Attachments); i++ {
 		att := m.Attachments[i]
 		r := readers[i]
-		h := GetAttachmentHeader(att)
+		h := GetAttachmentHeader(att, false)
 		p, err := mw.CreatePart(h)
 		if err != nil {
 			return nil, err
 		}
-		// Create line wrapper writer.
-		ww := textwrapper.NewRFC822(p)
-
-		// Create base64 writer.
-		bw := base64.NewEncoder(base64.StdEncoding, ww)
 
 		data, err := ioutil.ReadAll(r)
 		if err != nil {
@@ -332,6 +327,9 @@ func BuildEncrypted(m *pmapi.Message, readers []io.Reader, kr *crypto.KeyRing) (
 		if err != nil {
 			return nil, err
 		}
+
+		ww := textwrapper.NewRFC822(p)
+		bw := base64.NewEncoder(base64.StdEncoding, ww)
 		if _, err := bw.Write(pgpMessage.GetBinary()); err != nil {
 			return nil, err
 		}

pkg/message/header.go

@@ -107,12 +107,17 @@ func GetRelatedHeader(m *pmapi.Message) textproto.MIMEHeader {
 	return h
 }
 
-func GetAttachmentHeader(att *pmapi.Attachment) textproto.MIMEHeader {
+func GetAttachmentHeader(att *pmapi.Attachment, buildForIMAP bool) textproto.MIMEHeader {
 	mediaType := att.MIMEType
 	if mediaType == "application/pgp-encrypted" {
 		mediaType = "application/octet-stream"
 	}
 
+	transferEncoding := "base64"
+	if mediaType == rfc822Message && buildForIMAP {
+		transferEncoding = "8bit"
+	}
+
 	encodedName := pmmime.EncodeHeader(att.Name)
 	disposition := "attachment" //nolint[goconst]
 	if strings.Contains(att.Header.Get("Content-Disposition"), "inline") {
@@ -121,7 +126,9 @@ func GetAttachmentHeader(att *pmapi.Attachment) textproto.MIMEHeader {
 
 	h := make(textproto.MIMEHeader)
 	h.Set("Content-Type", mime.FormatMediaType(mediaType, map[string]string{"name": encodedName}))
-	h.Set("Content-Transfer-Encoding", "base64")
+	if transferEncoding != "" {
+		h.Set("Content-Transfer-Encoding", transferEncoding)
+	}
 	h.Set("Content-Disposition", mime.FormatMediaType(disposition, map[string]string{"filename": encodedName}))
 
 	// Forward some original header lines.

pkg/message/message.go

@@ -26,6 +26,10 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+const (
+	rfc822Message = "message/rfc822"
+)
+
 var log = logrus.WithField("pkg", "pkg/message") //nolint[gochecknoglobals]
 
 func GetBoundary(m *pmapi.Message) string {

pkg/message/section.go

@@ -201,7 +201,7 @@ func (bs *BodyStructure) parseAllChildSections(r io.Reader, currentPath []int, s
 	mediaType, params, _ := pmmime.ParseMediaType(info.Header.Get("Content-Type"))
 
 	// If multipart, call getAllParts, else read to count lines.
-	if (strings.HasPrefix(mediaType, "multipart/") || mediaType == "message/rfc822") && params["boundary"] != "" {
+	if (strings.HasPrefix(mediaType, "multipart/") || mediaType == rfc822Message) && params["boundary"] != "" {
 		newPath := append(currentPath, 1)
 
 		var br *boundaryReader

test/features/bridge/imap/message/import.feature

@@ -117,3 +117,36 @@ Feature: IMAP import messages
     Then IMAP response is "OK"
     And API mailbox "INBOX" for "user" has 0 message
     And API mailbox "Sent" for "user" has 1 message
+
+  Scenario: Import embedded message
+    When IMAP client imports message to "INBOX"
+      """
+      From: Foo <foo@example.com>
+      To: Bridge Test <bridgetest@pm.test>
+      Subject: Embedded message
+      Content-Type: multipart/mixed; boundary="boundary"
+      Received: by 2002:0:0:0:0:0:0:0 with SMTP id 0123456789abcdef; Wed, 30 Dec 2020 01:23:45 0000
+
+      This is a multi-part message in MIME format.
+      --boundary
+      Content-Type: text/plain; charset=utf-8
+      Content-Transfer-Encoding: 7bit
+
+
+      --boundary
+      Content-Type: message/rfc822; name="embedded.eml"
+      Content-Transfer-Encoding: 7bit
+      Content-Disposition: attachment; filename="embedded.eml"
+
+      From: Bar <bar@example.com>
+      To: Bridge Test <bridgetest@pm.test>
+      Subject: (No Subject)
+      Content-Type: text/plain; charset=utf-8
+      Content-Transfer-Encoding: quoted-printable
+
+      hello
+
+      --boundary--
+
+      """
+    Then IMAP response is "OK"

test/features/bridge/smtp/send/embedded_message.feature

@@ -0,0 +1,38 @@
+Feature: SMTP sending embedded message
+  Scenario: Send it
+    Given there is connected user "user"
+    And there is SMTP client logged in as "user"
+    When SMTP client sends message
+      """
+      From: Bridge Test <[userAddress]>
+      To: Internal Bridge <bridgetest@protonmail.com>
+      Subject: Embedded message
+      Content-Type: multipart/mixed; boundary="boundary"
+
+      This is a multi-part message in MIME format.
+      --boundary
+      Content-Type: text/plain; charset=utf-8
+      Content-Transfer-Encoding: 7bit
+
+
+      --boundary
+      Content-Type: message/rfc822; name="embedded.eml"
+      Content-Transfer-Encoding: 7bit
+      Content-Disposition: attachment; filename="embedded.eml"
+
+      From: Bar <bar@example.com>
+      To: Bridge Test <bridgetest@pm.test>
+      Subject: (No Subject)
+      Content-Type: text/plain; charset=utf-8
+      Content-Transfer-Encoding: quoted-printable
+
+      hello
+
+      --boundary--
+
+
+      """
+    Then SMTP response is "OK"
+    And mailbox "Sent" for "user" has messages
+      | from          | to                        | subject          |
+      | [userAddress] | bridgetest@protonmail.com | Embedded message |

test/features/ie/transfer/import_embedded.feature

@@ -0,0 +1,43 @@
+Feature: Import embedded message
+  Background:
+    Given there is connected user "user"
+    And there is EML file "Inbox/hello.eml"
+      """
+      From: Foo <foo@example.com>
+      To: Bridge Test <bridgetest@pm.test>
+      Subject: Embedded message
+      Content-Type: multipart/mixed; boundary="boundary"
+      Received: by 2002:0:0:0:0:0:0:0 with SMTP id 0123456789abcdef; Wed, 30 Dec 2020 01:23:45 0000
+
+      This is a multi-part message in MIME format.
+      --boundary
+      Content-Type: text/plain; charset=utf-8
+      Content-Transfer-Encoding: 7bit
+
+
+      --boundary
+      Content-Type: message/rfc822; name="embedded.eml"
+      Content-Transfer-Encoding: 7bit
+      Content-Disposition: attachment; filename="embedded.eml"
+
+      From: Bar <bar@example.com>
+      To: Bridge Test <bridgetest@pm.test>
+      Subject: (No Subject)
+      Content-Type: text/plain; charset=utf-8
+      Content-Transfer-Encoding: quoted-printable
+
+      hello
+
+      --boundary--
+
+      """
+
+  Scenario: Import it
+    When user "user" imports local files
+    Then progress result is "OK"
+    And transfer exported 1 messages
+    And transfer imported 1 messages
+    And transfer failed for 0 messages
+    And API mailbox "INBOX" for "user" has messages
+      | from            | to                 | subject          |
+      | foo@example.com | bridgetest@pm.test | Embedded message |