GODT-829 fix apple mail subfolders

Changelog.md

@@ -4,12 +4,33 @@ Changelog [format](http://keepachangelog.com/en/1.0.0/)
 
 ## Unreleased
 
+### Fixed
+* GODT-829 Remove `NoInferior` to display sub-folders in apple mail.
+
+## [Bridge 1.4.4] Forth
+
+### Fixed
+* GODT-798 Replace, don't add, transfer encoding when making body 7-bit clean.
+* Move/Copy duplicate for emails with References in Outlook
+* CSB-247 Cannot update from 1.4.0
+
+
+## [Bridge 1.4.3] Forth
+
+### Changed
+* Reverted sending IMAP updates to be not blocking again.
+
+### Fixed
+* GODT-783 Settings flags by FLAGS (not using +/-FLAGS) do not change spam state.
+
+
 ## [Bridge 1.4.2] Forth
 
 ### Changed
 * GODT-761 Use label.Path instead of Name to partially support subfolders for webapp beta release.
 * GODT-765 Improve speed of checking whether message is deleted.
 
+
 ## [IE 1.1.1] Danube (beta 2020-09-xx) [Bridge 1.4.1] Forth (beta 2020-09-xx)
 
 ### Fixed
@@ -17,12 +38,14 @@ Changelog [format](http://keepachangelog.com/en/1.0.0/)
 * GODT-752 Parsing non-utf8 multipart/alternative message.
 * GODT-752 Parsing message with duplicate charset parameter.
 
+
 ## [IE 1.1.0] Danube
 
 ### Fixed
 * GODT-703 Import-Export showed always at least one total message.
 * GODT-738 Fix for mbox files with long lines.
 
+
 ## [Bridge 1.4.0] Forth
 
 ### Added

internal/imap/mailbox.go

@@ -80,7 +80,10 @@ func (im *imapMailbox) Info() (*imap.MailboxInfo, error) {
 }
 
 func (im *imapMailbox) getFlags() []string {
-	flags := []string{imap.NoInferiorsAttr} // Subfolders are not yet supported by API.
+	flags := []string{}
+	if !im.storeMailbox.IsFolder() || im.storeMailbox.IsSystem() {
+		flags = append(flags, imap.NoInferiorsAttr) // Subfolders are not supported for System or Label
+	}
 	switch im.storeMailbox.LabelID() {
 	case pmapi.SentLabel:
 		flags = append(flags, specialuse.Sent)

internal/imap/mailbox_message.go

@@ -24,7 +24,6 @@ import (
 	"mime/multipart"
 	"net/mail"
 	"net/textproto"
-	"regexp"
 	"sort"
 	"strings"
 	"time"
@@ -141,18 +140,19 @@ func (im *imapMailbox) CreateMessage(flags []string, date time.Time, body imap.L
 	references := m.Header.Get("References")
 	referenceList := strings.Fields(references)
 
-	if len(referenceList) > 0 {
+	// In case there is a mail client which corrupts headers, try
+	// "References" too.
+	if internalID == "" && len(referenceList) > 0 {
 		lastReference := referenceList[len(referenceList)-1]
-		// In case we are using a mail client which corrupts headers, try "References" too.
-		re := regexp.MustCompile(pmapi.InternalReferenceFormat)
-		match := re.FindStringSubmatch(lastReference)
-		if len(match) > 0 {
-			internalID = match[0]
+		match := pmapi.RxInternalReferenceFormat.FindStringSubmatch(lastReference)
+		if len(match) == 2 {
+			internalID = match[1]
 		}
 	}
 
-	// Avoid appending a message which is already on the server. Apply the new
-	// label instead. This sometimes happens which Outlook (it uses APPEND instead of COPY).
+	// Avoid appending a message which is already on the server. Apply the
+	// new label instead. This always happens with Outlook (it uses APPEND
+	// instead of COPY).
 	if internalID != "" {
 		// Check to see if this belongs to a different address in split mode or another ProtonMail account.
 		msg, err := im.storeMailbox.GetMessage(internalID)

internal/imap/mailbox_messages.go

@@ -57,6 +57,10 @@ func (im *imapMailbox) UpdateMessagesFlags(uid bool, seqSet *imap.SeqSet, operat
 	return im.addOrRemoveFlags(operation, messageIDs, flags)
 }
 
+// setFlags is used for FLAGS command (not +FLAGS or -FLAGS), which means
+// to set flags passed as an argument and unset the rest. For example,
+// if message is not read, is flagged and is not deleted, call FLAGS \Seen
+// should flag message as read, unflagged and keep undeleted.
 func (im *imapMailbox) setFlags(messageIDs, flags []string) error { //nolint
 	seen := false
 	flagged := false
@@ -106,18 +110,19 @@ func (im *imapMailbox) setFlags(messageIDs, flags []string) error { //nolint
 		}
 	}
 
+	// Spam should not be taken into action here as Outlook is using FLAGS
+	// without preserving junk flag. Probably it's because junk is not standard
+	// in the rfc3501 and thus Outlook expects calling FLAGS \Seen will not
+	// change the state of junk or other non-standard flags.
+	// Still, its safe to label as spam once any client sends the request.
+	if spam {
 		spamMailbox, err := im.storeAddress.GetMailbox("Spam")
 		if err != nil {
 			return err
 		}
-	if spam {
 		if err := spamMailbox.LabelMessages(messageIDs); err != nil {
 			return err
 		}
-	} else {
-		if err := spamMailbox.UnlabelMessages(messageIDs); err != nil {
-			return err
-		}
 	}
 
 	return nil

internal/smtp/user.go

@@ -25,7 +25,6 @@ import (
 	"io"
 	"mime"
 	"net/mail"
-	"regexp"
 	"strings"
 	"time"
 
@@ -408,9 +407,9 @@ func (su *smtpUser) handleReferencesHeader(m *pmapi.Message) (draftID, parentID
 		if !strings.Contains(reference, "@"+pmapi.InternalIDDomain) {
 			newReferences = append(newReferences, reference)
 		} else { // internalid is the parentID.
-			idMatch := regexp.MustCompile(pmapi.InternalReferenceFormat).FindStringSubmatch(reference)
-			if len(idMatch) > 0 {
-				lastID := strings.TrimSuffix(strings.Trim(idMatch[0], "<>"), "@protonmail.internalid")
+			idMatch := pmapi.RxInternalReferenceFormat.FindStringSubmatch(reference)
+			if len(idMatch) == 2 {
+				lastID := idMatch[1]
 				filter := &pmapi.MessagesFilter{ID: []string{lastID}}
 				if su.addressID != "" {
 					filter.AddressID = su.addressID

internal/store/change.go

@@ -122,22 +122,10 @@ func (store *Store) imapSendUpdate(update imapBackend.Update) {
 		return
 	}
 
-	done := update.Done()
-	go func() {
-		// This timeout is to not keep running many blocked goroutines.
-		// In case nothing listens to this channel, this thread should stop.
 	select {
-		case store.imapUpdates <- update:
 	case <-time.After(1 * time.Second):
-			store.log.Warn("IMAP update could not be sent (timeout).")
-		}
-	}()
-
-	// This timeout is to not block IMAP backend by wait for IMAP client.
-	select {
-	case <-done:
-	case <-time.After(1 * time.Second):
-		store.log.Warn("IMAP update could not be delivered (timeout).")
+		store.log.Warn("IMAP update could not be sent (timeout)")
 		return
+	case store.imapUpdates <- update:
 	}
 }

internal/updates/updates.go

@@ -310,7 +310,9 @@ func (u *Updates) StartUpgrade(currentStatus chan<- Progress) { // nolint[funlen
 	status.UpdateDescription(InfoUpgrading)
 	switch runtime.GOOS {
 	case "windows": //nolint[goconst]
-		installerFile := strings.Split(u.winInstallerFile, "/")[1]
+		// Cannot use filepath.Base on windows it has different delimiter
+		split := strings.Split(u.winInstallerFile, "/")
+		installerFile := split[len(split)-1]
 		cmd := exec.Command("./" + installerFile) // nolint[gosec]
 		cmd.Dir = u.updateTempDir
 		status.Err = cmd.Start()

pkg/message/parser/writer.go

@@ -35,7 +35,7 @@ func newWriter(root *Part) *Writer {
 
 func (w *Writer) Write(ww io.Writer) error {
 	if !w.root.is7BitClean() {
-		w.root.Header.Add("Content-Transfer-Encoding", "base64")
+		w.root.Header.Set("Content-Transfer-Encoding", "base64")
 	}
 
 	msgWriter, err := message.CreateWriter(ww, w.root.Header)
@@ -68,7 +68,7 @@ func (w *Writer) write(writer *message.Writer, p *Part) error {
 
 func (w *Writer) writeAsChild(writer *message.Writer, p *Part) error {
 	if !p.is7BitClean() {
-		p.Header.Add("Content-Transfer-Encoding", "base64")
+		p.Header.Set("Content-Transfer-Encoding", "base64")
 	}
 
 	childWriter, err := writer.CreatePart(p.Header)

pkg/pmapi/messages.go

@@ -29,6 +29,7 @@ import (
 	"net/http"
 	"net/mail"
 	"net/url"
+	"regexp"
 	"strconv"
 	"strings"
 
@@ -149,8 +150,9 @@ const ConversationIDDomain = `protonmail.conversationid`
 // InternalIDDomain is used as a placeholder for reference/message ID headers to improve compatibility with various clients.
 const InternalIDDomain = `protonmail.internalid`
 
-// InternalReferenceFormat describes format of the message ID (as regex) used for parsing reference headers.
-const InternalReferenceFormat = `(?U)<.*@` + InternalIDDomain + `>`
+// RxInternalReferenceFormat is compiled regexp which describes the match for
+// a message ID used in reference headers.
+var RxInternalReferenceFormat = regexp.MustCompile(`(?U)<(.+)@` + regexp.QuoteMeta(InternalIDDomain) + `>`) //nolint[gochecknoglobals]
 
 // Message structure.
 type Message struct {

test/features/bridge/imap/message/update_spam.feature

@@ -0,0 +1,21 @@
+Feature: IMAP update messages in Spam folder
+  Background:
+    Given there is connected user "user"
+    # Messages are inserted in opposite way to keep increasing ID.
+    # Sequence numbers are then opposite than listed above.
+    And there are messages in mailbox "Spam" for "user"
+      | from              | to         | subject | body  | read  | starred | deleted |
+      | john.doe@mail.com | user@pm.me | foo     | hello | false | false   | false   |
+      | jane.doe@mail.com | name@pm.me | bar     | world | true  | true    | false   |
+    And there is IMAP client logged in as "user"
+    And there is IMAP client selected in "Spam"
+
+  Scenario: Mark message as read only
+    When IMAP client marks message "2" with "\Seen"
+    Then IMAP response is "OK"
+    And message "1" in "Spam" for "user" is marked as read
+    And message "1" in "Spam" for "user" is marked as unstarred
+    And API mailbox "Spam" for "user" has messages
+      | from              | to         | subject |
+      | john.doe@mail.com | user@pm.me | foo     |
+      | jane.doe@mail.com | name@pm.me | bar     |