diff --git a/internal/imap/mailbox_append.go b/internal/imap/mailbox_append.go new file mode 100644 index 00000000..a4f9f6d8 --- /dev/null +++ b/internal/imap/mailbox_append.go @@ -0,0 +1,198 @@ +// Copyright (c) 2021 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package imap + +import ( + "io" + "net/mail" + "strings" + "time" + + "github.com/ProtonMail/gopenpgp/v2/crypto" + "github.com/ProtonMail/proton-bridge/internal/imap/uidplus" + "github.com/ProtonMail/proton-bridge/pkg/message" + "github.com/ProtonMail/proton-bridge/pkg/pmapi" + "github.com/emersion/go-imap" + "github.com/pkg/errors" +) + +// CreateMessage appends a new message to this mailbox. The \Recent flag will +// be added regardless of whether flags is empty or not. If date is nil, the +// current time will be used. +// +// If the Backend implements Updater, it must notify the client immediately +// via a mailbox update. +func (im *imapMailbox) CreateMessage(flags []string, date time.Time, body imap.Literal) error { + return im.logCommand(func() error { + return im.createMessage(flags, date, body) + }, "APPEND", flags, date) +} + +func (im *imapMailbox) createMessage(flags []string, date time.Time, body imap.Literal) error { //nolint[funlen] + // Called from go-imap in goroutines - we need to handle panics for each function. + defer im.panicHandler.HandlePanic() + + m, _, _, readers, err := message.Parse(body) + if err != nil { + return err + } + + addr := im.storeAddress.APIAddress() + if addr == nil { + return errors.New("no available address for encryption") + } + m.AddressID = addr.ID + + kr, err := im.user.client().KeyRingForAddressID(addr.ID) + if err != nil { + return err + } + + // Handle imported messages which have no "Sender" address. + // This sometimes occurs with outlook which reports errors as imported emails or for drafts. + if m.Sender == nil { + im.log.Warning("Append: Missing email sender. Will use main address") + m.Sender = &mail.Address{ + Name: "", + Address: addr.Email, + } + } + + // "Drafts" needs to call special API routes. + // Clients always append the whole message again and remove the old one. + if im.storeMailbox.LabelID() == pmapi.DraftLabel { + // Sender address needs to be sanitised (drafts need to match cases exactly). + m.Sender.Address = pmapi.ConstructAddress(m.Sender.Address, addr.Email) + + draft, _, err := im.user.storeUser.CreateDraft(kr, m, readers, "", "", "") + if err != nil { + return errors.Wrap(err, "failed to create draft") + } + + targetSeq := im.storeMailbox.GetUIDList([]string{draft.ID}) + return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), targetSeq) + } + + // We need to make sure this is an import, and not a sent message from this account + // (sent messages from the account will be added by the event loop). + if im.storeMailbox.LabelID() == pmapi.SentLabel { + sanitizedSender := pmapi.SanitizeEmail(m.Sender.Address) + + // Check whether this message was sent by a bridge user. + user, err := im.user.backend.bridge.GetUser(sanitizedSender) + if err == nil && user.ID() == im.storeUser.UserID() { + logEntry := im.log.WithField("addr", sanitizedSender).WithField("extID", m.Header.Get("Message-Id")) + + // If we find the message in the store already, we can skip importing it. + if foundUID := im.storeMailbox.GetUIDByHeader(&m.Header); foundUID != uint32(0) { + logEntry.Info("Ignoring APPEND of duplicate to Sent folder") + return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), &uidplus.OrderedSeq{foundUID}) + } + + // We didn't find the message in the store, so we are currently sending it. + logEntry.WithField("time", date).Info("No matching UID, continuing APPEND to Sent") + } + } + + message.ParseFlags(m, flags) + if !date.IsZero() { + m.Time = date.Unix() + } + + internalID := m.Header.Get("X-Pm-Internal-Id") + references := m.Header.Get("References") + referenceList := strings.Fields(references) + + // In case there is a mail client which corrupts headers, try + // "References" too. + if internalID == "" && len(referenceList) > 0 { + lastReference := referenceList[len(referenceList)-1] + match := pmapi.RxInternalReferenceFormat.FindStringSubmatch(lastReference) + if len(match) == 2 { + internalID = match[1] + } + } + + im.user.appendExpungeLock.Lock() + defer im.user.appendExpungeLock.Unlock() + + // Avoid appending a message which is already on the server. Apply the + // new label instead. This always happens with Outlook (it uses APPEND + // instead of COPY). + if internalID != "" { + // Check to see if this belongs to a different address in split mode or another ProtonMail account. + msg, err := im.storeMailbox.GetMessage(internalID) + if err == nil && (im.user.user.IsCombinedAddressMode() || (im.storeAddress.AddressID() == msg.Message().AddressID)) { + IDs := []string{internalID} + + // See the comment bellow. + if msg.IsMarkedDeleted() { + if err := im.storeMailbox.MarkMessagesUndeleted(IDs); err != nil { + log.WithError(err).Error("Failed to undelete re-imported internal message") + } + } + + err = im.storeMailbox.LabelMessages(IDs) + if err != nil { + return err + } + + targetSeq := im.storeMailbox.GetUIDList(IDs) + return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), targetSeq) + } + } + + im.log.Info("Importing external message") + if err := im.importMessage(m, readers, kr); err != nil { + im.log.Error("Import failed: ", err) + return err + } + + // IMAP clients can move message to local folder (setting \Deleted flag) + // and then move it back (IMAP client does not remember the message, + // so instead removing the flag it imports duplicate message). + // Regular IMAP server would keep the message twice and later EXPUNGE would + // not delete the message (EXPUNGE would delete the original message and + // the new duplicate one would stay). API detects duplicates; therefore + // we need to remove \Deleted flag if IMAP client re-imports. + msg, err := im.storeMailbox.GetMessage(m.ID) + if err == nil && msg.IsMarkedDeleted() { + if err := im.storeMailbox.MarkMessagesUndeleted([]string{m.ID}); err != nil { + log.WithError(err).Error("Failed to undelete re-imported message") + } + } + + targetSeq := im.storeMailbox.GetUIDList([]string{m.ID}) + return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), targetSeq) +} + +func (im *imapMailbox) importMessage(m *pmapi.Message, readers []io.Reader, kr *crypto.KeyRing) (err error) { + body, err := message.BuildEncrypted(m, readers, kr) + if err != nil { + return err + } + + labels := []string{} + for _, l := range m.LabelIDs { + if l == pmapi.StarredLabel { + labels = append(labels, pmapi.StarredLabel) + } + } + + return im.storeMailbox.ImportMessage(m, body, labels) +} diff --git a/internal/imap/mailbox_fetch.go b/internal/imap/mailbox_fetch.go new file mode 100644 index 00000000..31916198 --- /dev/null +++ b/internal/imap/mailbox_fetch.go @@ -0,0 +1,368 @@ +// Copyright (c) 2021 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package imap + +import ( + "bytes" + "context" + "fmt" + "net/textproto" + "sort" + + "github.com/ProtonMail/proton-bridge/internal/imap/cache" + "github.com/ProtonMail/proton-bridge/pkg/message" + "github.com/ProtonMail/proton-bridge/pkg/pmapi" + "github.com/emersion/go-imap" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +func (im *imapMailbox) getMessage( + storeMessage storeMessageProvider, + items []imap.FetchItem, + msgBuildCountHistogram *msgBuildCountHistogram, +) (msg *imap.Message, err error) { + msglog := im.log.WithField("msgID", storeMessage.ID()) + msglog.Trace("Getting message") + + seqNum, err := storeMessage.SequenceNumber() + if err != nil { + return + } + + m := storeMessage.Message() + + msg = imap.NewMessage(seqNum, items) + for _, item := range items { + switch item { + case imap.FetchEnvelope: + // No need to check IsFullHeaderCached here. API header + // contain enough information to build the envelope. + msg.Envelope = message.GetEnvelope(m, storeMessage.GetHeader()) + case imap.FetchBody, imap.FetchBodyStructure: + structure, err := im.getBodyStructure(storeMessage) + if err != nil { + return nil, err + } + if msg.BodyStructure, err = structure.IMAPBodyStructure([]int{}); err != nil { + return nil, err + } + case imap.FetchFlags: + msg.Flags = message.GetFlags(m) + if storeMessage.IsMarkedDeleted() { + msg.Flags = append(msg.Flags, imap.DeletedFlag) + } + case imap.FetchInternalDate: + // Apple Mail crashes fetching messages with date older than 1970. + // There is no point having message older than RFC itself, it's not possible. + msg.InternalDate = message.SanitizeMessageDate(m.Time) + case imap.FetchRFC822Size: + if msg.Size, err = im.getSize(storeMessage); err != nil { + return nil, err + } + case imap.FetchUid: + if msg.Uid, err = storeMessage.UID(); err != nil { + return nil, err + } + case imap.FetchAll, imap.FetchFast, imap.FetchFull, imap.FetchRFC822, imap.FetchRFC822Header, imap.FetchRFC822Text: + fallthrough // this is list of defined items by go-imap, but items can be also sections generated from requests + default: + if err = im.getLiteralForSection(item, msg, storeMessage, msgBuildCountHistogram); err != nil { + return + } + } + } + + return msg, err +} + +// getSize returns cached size or it will build the message, save the size in +// DB and then returns the size after build. +// +// We are storing size in DB as part of pmapi messages metada. The size +// attribute on the server represents size of encrypted body. The value is +// cleared in Bridge and the final decrypted size (including header, attachment +// and MIME structure) is computed after building the message. +func (im *imapMailbox) getSize(storeMessage storeMessageProvider) (uint32, error) { + m := storeMessage.Message() + if m.Size <= 0 { + im.log.WithField("msgID", m.ID).Debug("Size unknown - downloading body") + // We are sure the size is not a problem right now. Clients + // might not first check sizes of all messages so we couldn't + // be sure if seeing 1st or 2nd sync is all right or not. + // Therefore, it's better to exclude getting size from the + // counting and see build count as real message build. + if _, _, err := im.getBodyAndStructure(storeMessage, nil); err != nil { + return 0, err + } + } + return uint32(m.Size), nil +} + +func (im *imapMailbox) getLiteralForSection( + itemSection imap.FetchItem, + msg *imap.Message, + storeMessage storeMessageProvider, + msgBuildCountHistogram *msgBuildCountHistogram, +) error { + section, err := imap.ParseBodySectionName(itemSection) + if err != nil { + log.WithError(err).Warn("Failed to parse body section name; part will be skipped") + return nil //nolint[nilerr] ignore error + } + + var literal imap.Literal + if literal, err = im.getMessageBodySection(storeMessage, section, msgBuildCountHistogram); err != nil { + return err + } + + msg.Body[section] = literal + return nil +} + +// getBodyStructure returns the cached body structure or it will build the message, +// save the structure in DB and then returns the structure after build. +// +// Apple Mail requests body structure for all messages irregularly. We cache +// bodystructure in local database in order to not re-download all messages +// from server. +func (im *imapMailbox) getBodyStructure(storeMessage storeMessageProvider) (bs *message.BodyStructure, err error) { + bs, err = storeMessage.GetBodyStructure() + if err != nil { + im.log.WithError(err).Debug("Fail to retrieve bodystructure from database") + } + if bs == nil { + // We are sure the body structure is not a problem right now. + // Clients might do first fetch body structure so we couldn't + // be sure if seeing 1st or 2nd sync is all right or not. + // Therefore, it's better to exclude first body structure fetch + // from the counting and see build count as real message build. + if bs, _, err = im.getBodyAndStructure(storeMessage, nil); err != nil { + return + } + } + return +} + +func (im *imapMailbox) getBodyAndStructure( + storeMessage storeMessageProvider, msgBuildCountHistogram *msgBuildCountHistogram, +) ( + structure *message.BodyStructure, bodyReader *bytes.Reader, err error, +) { + m := storeMessage.Message() + id := im.storeUser.UserID() + m.ID + cache.BuildLock(id) + defer cache.BuildUnlock(id) + bodyReader, structure = cache.LoadMail(id) + + // return the message which was found in cache + if bodyReader.Len() != 0 && structure != nil { + return structure, bodyReader, nil + } + + structure, body, err := im.buildMessage(m) + bodyReader = bytes.NewReader(body) + size := int64(len(body)) + l := im.log.WithField("newSize", size).WithField("msgID", m.ID) + + if err != nil || structure == nil || size == 0 { + l.WithField("hasStructure", structure != nil).Warn("Failed to build message") + return structure, bodyReader, err + } + + // Save the size, body structure and header even for messages which + // were unable to decrypt. Hence they doesn't have to be computed every + // time. + m.Size = size + cacheMessageInStore(storeMessage, structure, body, l) + + if msgBuildCountHistogram != nil { + times, errCount := storeMessage.IncreaseBuildCount() + if errCount != nil { + l.WithError(errCount).Warn("Cannot increase build count") + } + msgBuildCountHistogram.add(times) + } + + // Drafts can change therefore we don't want to cache them. + if !isMessageInDraftFolder(m) { + cache.SaveMail(id, body, structure) + } + + return structure, bodyReader, err +} + +func cacheMessageInStore(storeMessage storeMessageProvider, structure *message.BodyStructure, body []byte, l *logrus.Entry) { + m := storeMessage.Message() + if errSize := storeMessage.SetSize(m.Size); errSize != nil { + l.WithError(errSize).Warn("Cannot update size while building") + } + if structure != nil && !isMessageInDraftFolder(m) { + if errStruct := storeMessage.SetBodyStructure(structure); errStruct != nil { + l.WithError(errStruct).Warn("Cannot update bodystructure while building") + } + } + header, errHead := structure.GetMailHeaderBytes(bytes.NewReader(body)) + if errHead == nil && len(header) != 0 { + if errStore := storeMessage.SetHeader(header); errStore != nil { + l.WithError(errStore).Warn("Cannot update header in store") + } + } else { + l.WithError(errHead).Warn("Cannot get header bytes from structure") + } +} + +func isMessageInDraftFolder(m *pmapi.Message) bool { + for _, labelID := range m.LabelIDs { + if labelID == pmapi.DraftLabel { + return true + } + } + return false +} + +// This will download message (or read from cache) and pick up the section, +// extract data (header,body, both) and trim the output if needed. +func (im *imapMailbox) getMessageBodySection( //nolint[funlen] + storeMessage storeMessageProvider, + section *imap.BodySectionName, + msgBuildCountHistogram *msgBuildCountHistogram, +) (imap.Literal, error) { + var header textproto.MIMEHeader + var extraNewlineAfterHeader bool + var response []byte + + im.log.WithField("msgID", storeMessage.ID()).Trace("Getting message body") + + isMainHeaderRequested := len(section.Path) == 0 && section.Specifier == imap.HeaderSpecifier + if isMainHeaderRequested && storeMessage.IsFullHeaderCached() { + // In order to speed up (avoid download and decryptions) we + // cache the header. If a mail header was requested and DB + // contains full header (it means it was already built once) + // the DB header can be used without downloading and decrypting. + // Otherwise header is incomplete and clients would have issues + // e.g. AppleMail expects `text/plain` in HTML mails. + header = storeMessage.GetHeader() + } else { + // For all other cases it is necessary to download and decrypt the message + // and drop the header which was obtained from cache. The header will + // will be stored in DB once successfully built. Check `getBodyAndStructure`. + structure, bodyReader, err := im.getBodyAndStructure(storeMessage, msgBuildCountHistogram) + if err != nil { + return nil, err + } + + switch { + case section.Specifier == imap.EntireSpecifier && len(section.Path) == 0: + // An empty section specification refers to the entire message, including the header. + response, err = structure.GetSection(bodyReader, section.Path) + case section.Specifier == imap.TextSpecifier || (section.Specifier == imap.EntireSpecifier && len(section.Path) != 0): + // The TEXT specifier refers to the content of the message (or section), omitting the [RFC-2822] header. + // Non-empty section with no specifier (imap.EntireSpecifier) refers to section content without header. + response, err = structure.GetSectionContent(bodyReader, section.Path) + case section.Specifier == imap.MIMESpecifier: // The MIME part specifier refers to the [MIME-IMB] header for this part. + fallthrough + case section.Specifier == imap.HeaderSpecifier: + if content, err := structure.GetSectionContent(bodyReader, section.Path); err == nil && content != nil { + extraNewlineAfterHeader = true + } + header, err = structure.GetSectionHeader(section.Path) + default: + err = errors.New("Unknown specifier " + string(section.Specifier)) + } + + if err != nil { + return nil, err + } + } + + if header != nil { + response = filteredHeaderAsBytes(header, section) + // The blank line is included in all header fetches, + // except in the case of a message which has no body. + if extraNewlineAfterHeader { + response = append(response, []byte("\r\n")...) + } + } + + // Trim any output if requested. + return bytes.NewBuffer(section.ExtractPartial(response)), nil +} + +// filteredHeaderAsBytes filters the header fields by section fields and it +// returns the filtered fields as bytes. +// Options are: all fields, only selected fields, all fields except selected. +func filteredHeaderAsBytes(header textproto.MIMEHeader, section *imap.BodySectionName) []byte { + // remove fields + if len(section.Fields) != 0 && section.NotFields { + for _, field := range section.Fields { + header.Del(field) + } + } + + fields := make([]string, 0, len(header)) + if len(section.Fields) == 0 || section.NotFields { // add all and sort + for f := range header { + fields = append(fields, f) + } + sort.Strings(fields) + } else { // add only requested (in requested order) + for _, f := range section.Fields { + fields = append(fields, textproto.CanonicalMIMEHeaderKey(f)) + } + } + + headerBuf := &bytes.Buffer{} + for _, canonical := range fields { + if values, ok := header[canonical]; !ok { + continue + } else { + for _, val := range values { + fmt.Fprintf(headerBuf, "%s: %s\r\n", canonical, val) + } + } + } + return headerBuf.Bytes() +} + +// buildMessage from PM to IMAP. +func (im *imapMailbox) buildMessage(m *pmapi.Message) (*message.BodyStructure, []byte, error) { + body, err := im.builder.NewJobWithOptions( + context.Background(), + im.user.client(), + m.ID, + message.JobOptions{ + IgnoreDecryptionErrors: true, // Whether to ignore decryption errors and create a "custom message" instead. + SanitizeDate: true, // Whether to replace all dates before 1970 with RFC822's birthdate. + AddInternalID: true, // Whether to include MessageID as X-Pm-Internal-Id. + AddExternalID: true, // Whether to include ExternalID as X-Pm-External-Id. + AddMessageDate: true, // Whether to include message time as X-Pm-Date. + AddMessageIDReference: true, // Whether to include the MessageID in References. + }, + ).GetResult() + if err != nil { + return nil, nil, err + } + + structure, err := message.NewBodyStructure(bytes.NewReader(body)) + if err != nil { + return nil, nil, err + } + + return structure, body, nil +} diff --git a/internal/imap/mailbox_message.go b/internal/imap/mailbox_message.go deleted file mode 100644 index 00b48e47..00000000 --- a/internal/imap/mailbox_message.go +++ /dev/null @@ -1,540 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package imap - -import ( - "bytes" - "context" - "fmt" - "io" - "net/mail" - "net/textproto" - "sort" - "strings" - "time" - - "github.com/ProtonMail/gopenpgp/v2/crypto" - "github.com/ProtonMail/proton-bridge/internal/imap/cache" - "github.com/ProtonMail/proton-bridge/internal/imap/uidplus" - "github.com/ProtonMail/proton-bridge/pkg/message" - "github.com/ProtonMail/proton-bridge/pkg/pmapi" - "github.com/emersion/go-imap" - "github.com/hashicorp/go-multierror" - "github.com/pkg/errors" -) - -var ( - rfc822Birthday = time.Date(1982, 8, 13, 0, 0, 0, 0, time.UTC) //nolint[gochecknoglobals] -) - -type doNotCacheError struct{ e error } - -func (dnc *doNotCacheError) Error() string { return dnc.e.Error() } -func (dnc *doNotCacheError) add(err error) { dnc.e = multierror.Append(dnc.e, err) } -func (dnc *doNotCacheError) errorOrNil() error { - if dnc == nil { - return nil - } - - if dnc.e != nil { - return dnc - } - - return nil -} - -// CreateMessage appends a new message to this mailbox. The \Recent flag will -// be added regardless of whether flags is empty or not. If date is nil, the -// current time will be used. -// -// If the Backend implements Updater, it must notify the client immediately -// via a mailbox update. -func (im *imapMailbox) CreateMessage(flags []string, date time.Time, body imap.Literal) error { - return im.logCommand(func() error { - return im.createMessage(flags, date, body) - }, "APPEND", flags, date) -} - -func (im *imapMailbox) createMessage(flags []string, date time.Time, body imap.Literal) error { // nolint[funlen] - // Called from go-imap in goroutines - we need to handle panics for each function. - defer im.panicHandler.HandlePanic() - - m, _, _, readers, err := message.Parse(body) - if err != nil { - return err - } - - addr := im.storeAddress.APIAddress() - if addr == nil { - return errors.New("no available address for encryption") - } - m.AddressID = addr.ID - - kr, err := im.user.client().KeyRingForAddressID(addr.ID) - if err != nil { - return err - } - - // Handle imported messages which have no "Sender" address. - // This sometimes occurs with outlook which reports errors as imported emails or for drafts. - if m.Sender == nil { - im.log.Warning("Append: Missing email sender. Will use main address") - m.Sender = &mail.Address{ - Name: "", - Address: addr.Email, - } - } - - // "Drafts" needs to call special API routes. - // Clients always append the whole message again and remove the old one. - if im.storeMailbox.LabelID() == pmapi.DraftLabel { - // Sender address needs to be sanitised (drafts need to match cases exactly). - m.Sender.Address = pmapi.ConstructAddress(m.Sender.Address, addr.Email) - - draft, _, err := im.user.storeUser.CreateDraft(kr, m, readers, "", "", "") - if err != nil { - return errors.Wrap(err, "failed to create draft") - } - - targetSeq := im.storeMailbox.GetUIDList([]string{draft.ID}) - return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), targetSeq) - } - - // We need to make sure this is an import, and not a sent message from this account - // (sent messages from the account will be added by the event loop). - if im.storeMailbox.LabelID() == pmapi.SentLabel { - sanitizedSender := pmapi.SanitizeEmail(m.Sender.Address) - - // Check whether this message was sent by a bridge user. - user, err := im.user.backend.bridge.GetUser(sanitizedSender) - if err == nil && user.ID() == im.storeUser.UserID() { - logEntry := im.log.WithField("addr", sanitizedSender).WithField("extID", m.Header.Get("Message-Id")) - - // If we find the message in the store already, we can skip importing it. - if foundUID := im.storeMailbox.GetUIDByHeader(&m.Header); foundUID != uint32(0) { - logEntry.Info("Ignoring APPEND of duplicate to Sent folder") - return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), &uidplus.OrderedSeq{foundUID}) - } - - // We didn't find the message in the store, so we are currently sending it. - logEntry.WithField("time", date).Info("No matching UID, continuing APPEND to Sent") - } - } - - message.ParseFlags(m, flags) - if !date.IsZero() { - m.Time = date.Unix() - } - - internalID := m.Header.Get("X-Pm-Internal-Id") - references := m.Header.Get("References") - referenceList := strings.Fields(references) - - // In case there is a mail client which corrupts headers, try - // "References" too. - if internalID == "" && len(referenceList) > 0 { - lastReference := referenceList[len(referenceList)-1] - match := pmapi.RxInternalReferenceFormat.FindStringSubmatch(lastReference) - if len(match) == 2 { - internalID = match[1] - } - } - - im.user.appendExpungeLock.Lock() - defer im.user.appendExpungeLock.Unlock() - - // Avoid appending a message which is already on the server. Apply the - // new label instead. This always happens with Outlook (it uses APPEND - // instead of COPY). - if internalID != "" { - // Check to see if this belongs to a different address in split mode or another ProtonMail account. - msg, err := im.storeMailbox.GetMessage(internalID) - if err == nil && (im.user.user.IsCombinedAddressMode() || (im.storeAddress.AddressID() == msg.Message().AddressID)) { - IDs := []string{internalID} - - // See the comment bellow. - if msg.IsMarkedDeleted() { - if err := im.storeMailbox.MarkMessagesUndeleted(IDs); err != nil { - log.WithError(err).Error("Failed to undelete re-imported internal message") - } - } - - err = im.storeMailbox.LabelMessages(IDs) - if err != nil { - return err - } - - targetSeq := im.storeMailbox.GetUIDList(IDs) - return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), targetSeq) - } - } - - im.log.Info("Importing external message") - if err := im.importMessage(m, readers, kr); err != nil { - im.log.Error("Import failed: ", err) - return err - } - - // IMAP clients can move message to local folder (setting \Deleted flag) - // and then move it back (IMAP client does not remember the message, - // so instead removing the flag it imports duplicate message). - // Regular IMAP server would keep the message twice and later EXPUNGE would - // not delete the message (EXPUNGE would delete the original message and - // the new duplicate one would stay). API detects duplicates; therefore - // we need to remove \Deleted flag if IMAP client re-imports. - msg, err := im.storeMailbox.GetMessage(m.ID) - if err == nil && msg.IsMarkedDeleted() { - if err := im.storeMailbox.MarkMessagesUndeleted([]string{m.ID}); err != nil { - log.WithError(err).Error("Failed to undelete re-imported message") - } - } - - targetSeq := im.storeMailbox.GetUIDList([]string{m.ID}) - return uidplus.AppendResponse(im.storeMailbox.UIDValidity(), targetSeq) -} - -func (im *imapMailbox) importMessage(m *pmapi.Message, readers []io.Reader, kr *crypto.KeyRing) (err error) { // nolint[funlen] - body, err := message.BuildEncrypted(m, readers, kr) - if err != nil { - return err - } - - labels := []string{} - for _, l := range m.LabelIDs { - if l == pmapi.StarredLabel { - labels = append(labels, pmapi.StarredLabel) - } - } - - return im.storeMailbox.ImportMessage(m, body, labels) -} - -func (im *imapMailbox) getMessage(storeMessage storeMessageProvider, items []imap.FetchItem, msgBuildCountHistogram *msgBuildCountHistogram) (msg *imap.Message, err error) { //nolint[funlen] - msglog := im.log.WithField("msgID", storeMessage.ID()) - msglog.Trace("Getting message") - - seqNum, err := storeMessage.SequenceNumber() - if err != nil { - return - } - - m := storeMessage.Message() - - msg = imap.NewMessage(seqNum, items) - for _, item := range items { - switch item { - case imap.FetchEnvelope: - // No need to check IsFullHeaderCached here. API header - // contain enough information to build the envelope. - msg.Envelope = message.GetEnvelope(m, storeMessage.GetHeader()) - case imap.FetchBody, imap.FetchBodyStructure: - var structure *message.BodyStructure - structure, err = im.getBodyStructure(storeMessage) - if err != nil { - return - } - if msg.BodyStructure, err = structure.IMAPBodyStructure([]int{}); err != nil { - return - } - case imap.FetchFlags: - msg.Flags = message.GetFlags(m) - if storeMessage.IsMarkedDeleted() { - msg.Flags = append(msg.Flags, imap.DeletedFlag) - } - case imap.FetchInternalDate: - msg.InternalDate = time.Unix(m.Time, 0) - - // Apple Mail crashes fetching messages with date older than 1970. - // There is no point having message older than RFC itself, it's not possible. - if msg.InternalDate.Before(rfc822Birthday) { - msg.InternalDate = rfc822Birthday - } - case imap.FetchRFC822Size: - // Size attribute on the server counts encrypted data. The value is cleared - // on our part and we need to compute "real" size of decrypted data. - if m.Size <= 0 { - msglog.Debug("Size unknown - downloading body") - // We are sure the size is not a problem right now. Clients - // might not first check sizes of all messages so we couldn't - // be sure if seeing 1st or 2nd sync is all right or not. - // Therefore, it's better to exclude getting size from the - // counting and see build count as real message build. - if _, _, err = im.getBodyAndStructure(storeMessage, nil); err != nil { - return - } - } - msg.Size = uint32(m.Size) - case imap.FetchUid: - msg.Uid, err = storeMessage.UID() - if err != nil { - return nil, err - } - case imap.FetchAll, imap.FetchFast, imap.FetchFull, imap.FetchRFC822, imap.FetchRFC822Header, imap.FetchRFC822Text: - fallthrough // this is list of defined items by go-imap, but items can be also sections generated from requests - default: - if err = im.getLiteralForSection(item, msg, storeMessage, msgBuildCountHistogram); err != nil { - return - } - } - } - - return msg, err -} - -func (im *imapMailbox) getLiteralForSection(itemSection imap.FetchItem, msg *imap.Message, storeMessage storeMessageProvider, msgBuildCountHistogram *msgBuildCountHistogram) error { - section, err := imap.ParseBodySectionName(itemSection) - if err != nil { - log.WithError(err).Warn("Failed to parse body section name; part will be skipped") - return nil //nolint[nilerr] ignore error - } - - var literal imap.Literal - if literal, err = im.getMessageBodySection(storeMessage, section, msgBuildCountHistogram); err != nil { - return err - } - - msg.Body[section] = literal - return nil -} - -func (im *imapMailbox) getBodyStructure(storeMessage storeMessageProvider) (bs *message.BodyStructure, err error) { - // Apple Mail requests body structure for all - // messages irregularly. We cache bodystructure in - // local database in order to not re-download all - // messages from server. - bs, err = storeMessage.GetBodyStructure() - if err != nil { - im.log.WithError(err).Debug("Fail to retrieve bodystructure from database") - } - if bs == nil { - // We are sure the body structure is not a problem right now. - // Clients might do first fetch body structure so we couldn't - // be sure if seeing 1st or 2nd sync is all right or not. - // Therefore, it's better to exclude first body structure fetch - // from the counting and see build count as real message build. - if bs, _, err = im.getBodyAndStructure(storeMessage, nil); err != nil { - return - } - } - return -} - -//nolint[funlen] Jakub will fix in refactor -func (im *imapMailbox) getBodyAndStructure(storeMessage storeMessageProvider, msgBuildCountHistogram *msgBuildCountHistogram) ( - structure *message.BodyStructure, - bodyReader *bytes.Reader, err error, -) { - m := storeMessage.Message() - id := im.storeUser.UserID() + m.ID - cache.BuildLock(id) - if bodyReader, structure = cache.LoadMail(id); bodyReader.Len() == 0 || structure == nil { - var body []byte - structure, body, err = im.buildMessage(m) - m.Size = int64(len(body)) - // Save size and body structure even for messages unable to decrypt - // so the size or body structure doesn't have to be computed every time. - if err := storeMessage.SetSize(m.Size); err != nil { - im.log.WithError(err). - WithField("newSize", m.Size). - WithField("msgID", m.ID). - Warn("Cannot update size while building") - } - if structure != nil && !isMessageInDraftFolder(m) { - if err := storeMessage.SetBodyStructure(structure); err != nil { - im.log.WithError(err). - WithField("msgID", m.ID). - Warn("Cannot update bodystructure while building") - } - } - if err == nil && structure != nil && len(body) > 0 { - header, errHead := structure.GetMailHeaderBytes(bytes.NewReader(body)) - if errHead == nil { - if errHead := storeMessage.SetHeader(header); errHead != nil { - im.log.WithError(errHead). - WithField("msgID", m.ID). - Warn("Cannot update header after building") - } - } else { - im.log.WithError(errHead). - WithField("msgID", m.ID). - Warn("Cannot get header bytes after building") - } - if msgBuildCountHistogram != nil { - times, err := storeMessage.IncreaseBuildCount() - if err != nil { - im.log.WithError(err). - WithField("msgID", m.ID). - Warn("Cannot increase build count") - } - msgBuildCountHistogram.add(times) - } - // Drafts can change and we don't want to cache them. - if !isMessageInDraftFolder(m) { - cache.SaveMail(id, body, structure) - } - bodyReader = bytes.NewReader(body) - } - if _, ok := err.(*doNotCacheError); ok { - im.log.WithField("msgID", m.ID).Errorf("do not cache message: %v", err) - err = nil - bodyReader = bytes.NewReader(body) - } - } - cache.BuildUnlock(id) - return structure, bodyReader, err -} - -func isMessageInDraftFolder(m *pmapi.Message) bool { - for _, labelID := range m.LabelIDs { - if labelID == pmapi.DraftLabel { - return true - } - } - return false -} - -// This will download message (or read from cache) and pick up the section, -// extract data (header,body, both) and trim the output if needed. -func (im *imapMailbox) getMessageBodySection( //nolint[funlen] - storeMessage storeMessageProvider, - section *imap.BodySectionName, - msgBuildCountHistogram *msgBuildCountHistogram, -) (imap.Literal, error) { - var header textproto.MIMEHeader - var extraNewlineAfterHeader bool - var response []byte - - im.log.WithField("msgID", storeMessage.ID()).Trace("Getting message body") - - isMainHeaderRequested := len(section.Path) == 0 && section.Specifier == imap.HeaderSpecifier - if isMainHeaderRequested && storeMessage.IsFullHeaderCached() { - // In order to speed up (avoid download and decryptions) we - // cache the header. If a mail header was requested and DB - // contains full header (it means it was already built once) - // the DB header can be used without downloading and decrypting. - // Otherwise header is incomplete and clients would have issues - // e.g. AppleMail expects `text/plain` in HTML mails. - header = storeMessage.GetHeader() - } else { - // For all other cases it is necessary to download and decrypt the message - // and drop the header which was obtained from cache. The header will - // will be stored in DB once successfully built. Check `getBodyAndStructure`. - structure, bodyReader, err := im.getBodyAndStructure(storeMessage, msgBuildCountHistogram) - if err != nil { - return nil, err - } - - switch { - case section.Specifier == imap.EntireSpecifier && len(section.Path) == 0: - // An empty section specification refers to the entire message, including the header. - response, err = structure.GetSection(bodyReader, section.Path) - case section.Specifier == imap.TextSpecifier || (section.Specifier == imap.EntireSpecifier && len(section.Path) != 0): - // The TEXT specifier refers to the content of the message (or section), omitting the [RFC-2822] header. - // Non-empty section with no specifier (imap.EntireSpecifier) refers to section content without header. - response, err = structure.GetSectionContent(bodyReader, section.Path) - case section.Specifier == imap.MIMESpecifier: // The MIME part specifier refers to the [MIME-IMB] header for this part. - fallthrough - case section.Specifier == imap.HeaderSpecifier: - if content, err := structure.GetSectionContent(bodyReader, section.Path); err == nil && content != nil { - extraNewlineAfterHeader = true - } - header, err = structure.GetSectionHeader(section.Path) - default: - err = errors.New("Unknown specifier " + string(section.Specifier)) - } - - if err != nil { - return nil, err - } - } - - if header != nil { - response = filteredHeaderAsBytes(header, section) - // The blank line is included in all header fetches, - // except in the case of a message which has no body. - if extraNewlineAfterHeader { - response = append(response, []byte("\r\n")...) - } - } - - // Trim any output if requested. - return bytes.NewBuffer(section.ExtractPartial(response)), nil -} - -// filteredHeaderAsBytes filters the header fields by section fields and it -// returns the filtered fields as bytes. -// Options are: all fields, only selected fields, all fields except selected. -func filteredHeaderAsBytes(header textproto.MIMEHeader, section *imap.BodySectionName) []byte { - // remove fields - if len(section.Fields) != 0 && section.NotFields { - for _, field := range section.Fields { - header.Del(field) - } - } - - fields := make([]string, 0, len(header)) - if len(section.Fields) == 0 || section.NotFields { // add all and sort - for f := range header { - fields = append(fields, f) - } - sort.Strings(fields) - } else { // add only requested (in requested order) - for _, f := range section.Fields { - fields = append(fields, textproto.CanonicalMIMEHeaderKey(f)) - } - } - - headerBuf := &bytes.Buffer{} - for _, canonical := range fields { - if values, ok := header[canonical]; !ok { - continue - } else { - for _, val := range values { - fmt.Fprintf(headerBuf, "%s: %s\r\n", canonical, val) - } - } - } - return headerBuf.Bytes() -} - -// buildMessage from PM to IMAP. -func (im *imapMailbox) buildMessage(m *pmapi.Message) (*message.BodyStructure, []byte, error) { - body, err := im.builder.NewJobWithOptions( - context.Background(), - im.user.client(), - m.ID, - message.JobOptions{ - IgnoreDecryptionErrors: true, // Whether to ignore decryption errors and create a "custom message" instead. - SanitizeDate: true, // Whether to replace all dates before 1970 with RFC822's birthdate. - AddInternalID: true, // Whether to include MessageID as X-Pm-Internal-Id. - AddExternalID: true, // Whether to include ExternalID as X-Pm-External-Id. - AddMessageDate: true, // Whether to include message time as X-Pm-Date. - AddMessageIDReference: true, // Whether to include the MessageID in References. - }, - ).GetResult() - if err != nil { - return nil, nil, err - } - - structure, err := message.NewBodyStructure(bytes.NewReader(body)) - if err != nil { - return nil, nil, err - } - - return structure, body, nil -} diff --git a/internal/imap/mailbox_message_test.go b/internal/imap/mailbox_message_test.go deleted file mode 100644 index a0689d73..00000000 --- a/internal/imap/mailbox_message_test.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package imap - -import ( - "errors" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestDoNotCache(t *testing.T) { - var dnc doNotCacheError - require.NoError(t, dnc.errorOrNil()) - _, ok := dnc.errorOrNil().(*doNotCacheError) - require.True(t, !ok, "should not be type doNotCacheError") - - dnc.add(errors.New("first")) - require.True(t, dnc.errorOrNil() != nil, "should be error") - _, ok = dnc.errorOrNil().(*doNotCacheError) - require.True(t, ok, "should be type doNotCacheError") - - dnc.add(errors.New("second")) - dnc.add(errors.New("third")) - t.Log(dnc.errorOrNil()) -} diff --git a/pkg/message/boundary_reader.go b/pkg/message/boundary_reader.go new file mode 100644 index 00000000..e4079d41 --- /dev/null +++ b/pkg/message/boundary_reader.go @@ -0,0 +1,130 @@ +// Copyright (c) 2021 Proton Technologies AG +// +// This file is part of ProtonMail Bridge. +// +// ProtonMail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ProtonMail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with ProtonMail Bridge. If not, see . + +package message + +import ( + "bufio" + "bytes" + "io" +) + +type boundaryReader struct { + reader *bufio.Reader + + closed, first bool + skipped int + + nl []byte // "\r\n" or "\n" (set after seeing first boundary line) + nlDashBoundary []byte // nl + "--boundary" + dashBoundaryDash []byte // "--boundary--" + dashBoundary []byte // "--boundary" +} + +func newBoundaryReader(r *bufio.Reader, boundary string) (br *boundaryReader, err error) { + b := []byte("\r\n--" + boundary + "--") + br = &boundaryReader{ + reader: r, + closed: false, + first: true, + nl: b[:2], + nlDashBoundary: b[:len(b)-2], + dashBoundaryDash: b[2:], + dashBoundary: b[2 : len(b)-2], + } + err = br.writeNextPartTo(nil) + return +} + +// writeNextPartTo will copy the the bytes of next part and write them to +// writer. Will return EOF if the underlying reader is empty. +func (br *boundaryReader) writeNextPartTo(part io.Writer) (err error) { + if br.closed { + return io.EOF + } + + var line, slice []byte + br.skipped = 0 + + for { + slice, err = br.reader.ReadSlice('\n') + line = append(line, slice...) + if err == bufio.ErrBufferFull { + continue + } + + br.skipped += len(line) + + if err == io.EOF && br.isFinalBoundary(line) { + err = nil + br.closed = true + return + } + + if err != nil { + return + } + + if br.isBoundaryDelimiterLine(line) { + br.first = false + return + } + + if br.isFinalBoundary(line) { + br.closed = true + return + } + + if part != nil { + if _, err = part.Write(line); err != nil { + return + } + } + + line = []byte{} + } +} + +func (br *boundaryReader) isFinalBoundary(line []byte) bool { + if !bytes.HasPrefix(line, br.dashBoundaryDash) { + return false + } + rest := line[len(br.dashBoundaryDash):] + rest = skipLWSPChar(rest) + return len(rest) == 0 || bytes.Equal(rest, br.nl) +} + +func (br *boundaryReader) isBoundaryDelimiterLine(line []byte) (ret bool) { + if !bytes.HasPrefix(line, br.dashBoundary) { + return false + } + rest := line[len(br.dashBoundary):] + rest = skipLWSPChar(rest) + + if br.first && len(rest) == 1 && rest[0] == '\n' { + br.nl = br.nl[1:] + br.nlDashBoundary = br.nlDashBoundary[1:] + } + return bytes.Equal(rest, br.nl) +} + +func skipLWSPChar(b []byte) []byte { + for len(b) > 0 && (b[0] == ' ' || b[0] == '\t') { + b = b[1:] + } + return b +} diff --git a/pkg/message/build_encrypted.go b/pkg/message/build_encrypted.go index 8adfedee..e67e73a6 100644 --- a/pkg/message/build_encrypted.go +++ b/pkg/message/build_encrypted.go @@ -22,30 +22,36 @@ import ( "encoding/base64" "io" "io/ioutil" + "mime" "mime/multipart" "net/http" "net/textproto" + "strings" "github.com/ProtonMail/gopenpgp/v2/crypto" + pmmime "github.com/ProtonMail/proton-bridge/pkg/mime" "github.com/ProtonMail/proton-bridge/pkg/pmapi" + "github.com/emersion/go-message" "github.com/emersion/go-textwrapper" ) +// BuildEncrypted is used for importing encrypted message. func BuildEncrypted(m *pmapi.Message, readers []io.Reader, kr *crypto.KeyRing) ([]byte, error) { //nolint[funlen] b := &bytes.Buffer{} + boundary := newBoundary(m.ID).gen() // Overwrite content for main header for import. // Even if message has just simple body we should upload as multipart/mixed. // Each part has encrypted body and header reflects the original header. - mainHeader := GetHeader(m) - mainHeader.Set("Content-Type", "multipart/mixed; boundary="+GetBoundary(m)) + mainHeader := convertGoMessageToTextprotoHeader(getMessageHeader(m, JobOptions{})) + mainHeader.Set("Content-Type", "multipart/mixed; boundary="+boundary) mainHeader.Del("Content-Disposition") mainHeader.Del("Content-Transfer-Encoding") if err := WriteHeader(b, mainHeader); err != nil { return nil, err } mw := multipart.NewWriter(b) - if err := mw.SetBoundary(GetBoundary(m)); err != nil { + if err := mw.SetBoundary(boundary); err != nil { return nil, err } @@ -71,7 +77,7 @@ func BuildEncrypted(m *pmapi.Message, readers []io.Reader, kr *crypto.KeyRing) ( for i := 0; i < len(m.Attachments); i++ { att := m.Attachments[i] r := readers[i] - h := GetAttachmentHeader(att, false) + h := getAttachmentHeader(att, false) p, err := mw.CreatePart(h) if err != nil { return nil, err @@ -105,6 +111,55 @@ func BuildEncrypted(m *pmapi.Message, readers []io.Reader, kr *crypto.KeyRing) ( return b.Bytes(), nil } +func convertGoMessageToTextprotoHeader(h message.Header) textproto.MIMEHeader { + out := make(textproto.MIMEHeader) + hf := h.Fields() + for hf.Next() { + // go-message fields are in the reverse order. + // textproto.MIMEHeader is not ordered except for the values of + // the same key which are ordered + key := textproto.CanonicalMIMEHeaderKey(hf.Key()) + out[key] = append([]string{hf.Value()}, out[key]...) + } + return out +} + +func getAttachmentHeader(att *pmapi.Attachment, buildForIMAP bool) textproto.MIMEHeader { + mediaType := att.MIMEType + if mediaType == "application/pgp-encrypted" { + mediaType = "application/octet-stream" + } + + transferEncoding := "base64" + if mediaType == rfc822Message && buildForIMAP { + transferEncoding = "8bit" + } + + encodedName := pmmime.EncodeHeader(att.Name) + disposition := "attachment" //nolint[goconst] + if strings.Contains(att.Header.Get("Content-Disposition"), pmapi.DispositionInline) { + disposition = pmapi.DispositionInline + } + + h := make(textproto.MIMEHeader) + h.Set("Content-Type", mime.FormatMediaType(mediaType, map[string]string{"name": encodedName})) + if transferEncoding != "" { + h.Set("Content-Transfer-Encoding", transferEncoding) + } + h.Set("Content-Disposition", mime.FormatMediaType(disposition, map[string]string{"filename": encodedName})) + + // Forward some original header lines. + forward := []string{"Content-Id", "Content-Description", "Content-Location"} + for _, k := range forward { + v := att.Header.Get(k) + if v != "" { + h.Set(k, v) + } + } + + return h +} + func WriteHeader(w io.Writer, h textproto.MIMEHeader) (err error) { if err = http.Header(h).Write(w); err != nil { return diff --git a/pkg/message/build_rfc822.go b/pkg/message/build_rfc822.go index 804b3d12..8d89b257 100644 --- a/pkg/message/build_rfc822.go +++ b/pkg/message/build_rfc822.go @@ -329,7 +329,7 @@ func getMessageHeader(msg *pmapi.Message, opts JobOptions) message.Header { // n // Sanitize the date; it needs to have a valid unix timestamp. if opts.SanitizeDate { if date, err := rfc5322.ParseDateTime(hdr.Get("Date")); err != nil || date.Before(time.Unix(0, 0)) { - msgDate := sanitizeMessageDate(msg.Time) + msgDate := SanitizeMessageDate(msg.Time) hdr.Set("Date", msgDate.In(time.UTC).Format(time.RFC1123Z)) // We clobbered the date so we save it under X-Original-Date. hdr.Set("X-Original-Date", date.In(time.UTC).Format(time.RFC1123Z)) @@ -364,10 +364,10 @@ func getMessageHeader(msg *pmapi.Message, opts JobOptions) message.Header { // n return hdr } -// sanitizeMessageDate will return time from msgTime timestamp. If timestamp is +// SanitizeMessageDate will return time from msgTime timestamp. If timestamp is // not after epoch the RFC822 publish day will be used. No message should // realistically be older than RFC822 itself. -func sanitizeMessageDate(msgTime int64) time.Time { +func SanitizeMessageDate(msgTime int64) time.Time { if msgTime := time.Unix(msgTime, 0); msgTime.After(time.Unix(0, 0)) { return msgTime } diff --git a/pkg/message/envelope.go b/pkg/message/envelope.go index 6f20de04..ff919fde 100644 --- a/pkg/message/envelope.go +++ b/pkg/message/envelope.go @@ -32,7 +32,7 @@ func GetEnvelope(msg *pmapi.Message, header textproto.MIMEHeader) *imap.Envelope setMessageIDIfNeeded(msg, &hdr) return &imap.Envelope{ - Date: sanitizeMessageDate(msg.Time), + Date: SanitizeMessageDate(msg.Time), Subject: msg.Subject, From: getAddresses([]*mail.Address{msg.Sender}), Sender: getAddresses([]*mail.Address{msg.Sender}), diff --git a/pkg/message/flags.go b/pkg/message/flags.go index 258a5d2e..b895c61a 100644 --- a/pkg/message/flags.go +++ b/pkg/message/flags.go @@ -22,13 +22,14 @@ import ( "github.com/emersion/go-imap" ) -//nolint[gochecknoglobals] -var ( - AppleMailJunkFlag = imap.CanonicalFlag("$Junk") - ThunderbirdJunkFlag = imap.CanonicalFlag("Junk") - ThunderbirdNonJunkFlag = imap.CanonicalFlag("NonJunk") +// Various client specific flags. +const ( + AppleMailJunkFlag = "$Junk" + ThunderbirdJunkFlag = "Junk" + ThunderbirdNonJunkFlag = "NonJunk" ) +// GetFlags returns imap flags from pmapi message attributes. func GetFlags(m *pmapi.Message) (flags []string) { if m.Unread == 0 { flags = append(flags, imap.SeenFlag) @@ -59,6 +60,7 @@ func GetFlags(m *pmapi.Message) (flags []string) { return } +// ParseFlags sets attributes to pmapi messages based on imap flags. func ParseFlags(m *pmapi.Message, flags []string) { if m.Header.Get("received") == "" { m.Flags = pmapi.FlagSent diff --git a/pkg/message/header.go b/pkg/message/header.go deleted file mode 100644 index 81569a9f..00000000 --- a/pkg/message/header.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright (c) 2021 Proton Technologies AG -// -// This file is part of ProtonMail Bridge. -// -// ProtonMail Bridge is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// ProtonMail Bridge is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with ProtonMail Bridge. If not, see . - -package message - -import ( - "mime" - "net/textproto" - "strings" - "time" - - pmmime "github.com/ProtonMail/proton-bridge/pkg/mime" - "github.com/ProtonMail/proton-bridge/pkg/pmapi" -) - -// GetHeader builds the header for the message. -func GetHeader(msg *pmapi.Message) textproto.MIMEHeader { //nolint[funlen] - h := make(textproto.MIMEHeader) - - // Copy the custom header fields if there are some. - if msg.Header != nil { - h = textproto.MIMEHeader(msg.Header) - } - - // Add or rewrite fields. - h.Set("Subject", pmmime.EncodeHeader(msg.Subject)) - if msg.Sender != nil { - h.Set("From", pmmime.EncodeHeader(msg.Sender.String())) - } - if len(msg.ReplyTos) > 0 { - h.Set("Reply-To", pmmime.EncodeHeader(toAddressList(msg.ReplyTos))) - } - if len(msg.ToList) > 0 { - h.Set("To", pmmime.EncodeHeader(toAddressList(msg.ToList))) - } - if len(msg.CCList) > 0 { - h.Set("Cc", pmmime.EncodeHeader(toAddressList(msg.CCList))) - } - if len(msg.BCCList) > 0 { - h.Set("Bcc", pmmime.EncodeHeader(toAddressList(msg.BCCList))) - } - - // Add or rewrite date related fields. - if msg.Time > 0 { - h.Set("X-Pm-Date", time.Unix(msg.Time, 0).Format(time.RFC1123Z)) - if d, err := msg.Header.Date(); err != nil || d.IsZero() { // Fix date if needed. - h.Set("Date", time.Unix(msg.Time, 0).Format(time.RFC1123Z)) - } - } - - // Use External-Id if available to ensure email clients: - // * build the conversations threads correctly (Thunderbird, Mac Outlook, Apple Mail) - // * do not think the message is lost (Apple Mail) - if msg.ExternalID != "" { - h.Set("X-Pm-External-Id", "<"+msg.ExternalID+">") - if h.Get("Message-Id") == "" { - h.Set("Message-Id", "<"+msg.ExternalID+">") - } - } - if msg.ID != "" { - if h.Get("Message-Id") == "" { - h.Set("Message-Id", "<"+msg.ID+"@"+pmapi.InternalIDDomain+">") - } - h.Set("X-Pm-Internal-Id", msg.ID) - // Forward References, and include the message ID here (to improve outlook support). - if references := h.Get("References"); !strings.Contains(references, msg.ID) { - references += " <" + msg.ID + "@" + pmapi.InternalIDDomain + ">" - h.Set("References", references) - } - } - if msg.ConversationID != "" { - h.Set("X-Pm-ConversationID-Id", msg.ConversationID) - } - - return h -} - -func SetBodyContentFields(h *textproto.MIMEHeader, m *pmapi.Message) { - h.Set("Content-Type", m.MIMEType+"; charset=utf-8") - h.Set("Content-Disposition", pmapi.DispositionInline) - h.Set("Content-Transfer-Encoding", "quoted-printable") -} - -func GetBodyHeader(m *pmapi.Message) textproto.MIMEHeader { - h := make(textproto.MIMEHeader) - SetBodyContentFields(&h, m) - return h -} - -func GetAttachmentHeader(att *pmapi.Attachment, buildForIMAP bool) textproto.MIMEHeader { - mediaType := att.MIMEType - if mediaType == "application/pgp-encrypted" { - mediaType = "application/octet-stream" - } - - transferEncoding := "base64" - if mediaType == rfc822Message && buildForIMAP { - transferEncoding = "8bit" - } - - encodedName := pmmime.EncodeHeader(att.Name) - disposition := "attachment" //nolint[goconst] - if strings.Contains(att.Header.Get("Content-Disposition"), pmapi.DispositionInline) { - disposition = pmapi.DispositionInline - } - - h := make(textproto.MIMEHeader) - h.Set("Content-Type", mime.FormatMediaType(mediaType, map[string]string{"name": encodedName})) - if transferEncoding != "" { - h.Set("Content-Transfer-Encoding", transferEncoding) - } - h.Set("Content-Disposition", mime.FormatMediaType(disposition, map[string]string{"filename": encodedName})) - - // Forward some original header lines. - forward := []string{"Content-Id", "Content-Description", "Content-Location"} - for _, k := range forward { - v := att.Header.Get(k) - if v != "" { - h.Set(k, v) - } - } - - return h -} diff --git a/pkg/message/message.go b/pkg/message/message.go index 453d14a2..1969a215 100644 --- a/pkg/message/message.go +++ b/pkg/message/message.go @@ -15,12 +15,11 @@ // You should have received a copy of the GNU General Public License // along with ProtonMail Bridge. If not, see . +// Package message contains set of tools to convert message between Proton API +// and IMAP format. package message import ( - "strings" - - "github.com/ProtonMail/proton-bridge/pkg/pmapi" "github.com/sirupsen/logrus" ) @@ -29,20 +28,3 @@ const ( ) var log = logrus.WithField("pkg", "pkg/message") //nolint[gochecknoglobals] - -func GetBoundary(m *pmapi.Message) string { - // The boundary needs to be deterministic because messages are not supposed to - // change. - return newBoundary(m.ID).gen() -} - -func SeparateInlineAttachments(m *pmapi.Message) (atts, inlines []*pmapi.Attachment) { - for _, att := range m.Attachments { - if strings.Contains(att.Header.Get("Content-Disposition"), pmapi.DispositionInline) { - inlines = append(inlines, att) - } else { - atts = append(atts, att) - } - } - return -} diff --git a/pkg/message/section.go b/pkg/message/section.go index 738bfae4..05049c2c 100644 --- a/pkg/message/section.go +++ b/pkg/message/section.go @@ -32,13 +32,18 @@ import ( "github.com/vmihailenco/msgpack/v5" ) +// BodyStructure is used to parse an email into MIME sections and then generate +// body structure for IMAP server. +type BodyStructure map[string]*SectionInfo + +// SectionInfo is used to hold data about parts of each section. type SectionInfo struct { Header textproto.MIMEHeader Start, BSize, Size, Lines int reader io.Reader } -// Read and count. +// Read will also count the final size of section. func (si *SectionInfo) Read(p []byte) (n int, err error) { n, err = si.reader.Read(p) si.Size += n @@ -46,118 +51,13 @@ func (si *SectionInfo) Read(p []byte) (n int, err error) { return } -type boundaryReader struct { - reader *bufio.Reader - - closed, first bool - skipped int - - nl []byte // "\r\n" or "\n" (set after seeing first boundary line) - nlDashBoundary []byte // nl + "--boundary" - dashBoundaryDash []byte // "--boundary--" - dashBoundary []byte // "--boundary" -} - -func newBoundaryReader(r *bufio.Reader, boundary string) (br *boundaryReader, err error) { - b := []byte("\r\n--" + boundary + "--") - br = &boundaryReader{ - reader: r, - closed: false, - first: true, - nl: b[:2], - nlDashBoundary: b[:len(b)-2], - dashBoundaryDash: b[2:], - dashBoundary: b[2 : len(b)-2], - } - err = br.WriteNextPartTo(nil) - return -} - -func skipLWSPChar(b []byte) []byte { - for len(b) > 0 && (b[0] == ' ' || b[0] == '\t') { - b = b[1:] - } - return b -} - -func (br *boundaryReader) isFinalBoundary(line []byte) bool { - if !bytes.HasPrefix(line, br.dashBoundaryDash) { - return false - } - rest := line[len(br.dashBoundaryDash):] - rest = skipLWSPChar(rest) - return len(rest) == 0 || bytes.Equal(rest, br.nl) -} - -func (br *boundaryReader) isBoundaryDelimiterLine(line []byte) (ret bool) { - if !bytes.HasPrefix(line, br.dashBoundary) { - return false - } - rest := line[len(br.dashBoundary):] - rest = skipLWSPChar(rest) - - if br.first && len(rest) == 1 && rest[0] == '\n' { - br.nl = br.nl[1:] - br.nlDashBoundary = br.nlDashBoundary[1:] - } - return bytes.Equal(rest, br.nl) -} - -func (br *boundaryReader) WriteNextPartTo(part io.Writer) (err error) { - if br.closed { - return io.EOF - } - - var line, slice []byte - br.skipped = 0 - - for { - slice, err = br.reader.ReadSlice('\n') - line = append(line, slice...) - if err == bufio.ErrBufferFull { - continue - } - - br.skipped += len(line) - - if err == io.EOF && br.isFinalBoundary(line) { - err = nil - br.closed = true - return - } - - if err != nil { - return - } - - if br.isBoundaryDelimiterLine(line) { - br.first = false - return - } - - if br.isFinalBoundary(line) { - br.closed = true - return - } - - if part != nil { - if _, err = part.Write(line); err != nil { - return - } - } - - line = []byte{} - } -} - -type BodyStructure map[string]*SectionInfo - func NewBodyStructure(reader io.Reader) (structure *BodyStructure, err error) { structure = &BodyStructure{} err = structure.Parse(reader) return } +// DeserializeBodyStructure will create new structure from msgpack bytes. func DeserializeBodyStructure(raw []byte) (*BodyStructure, error) { bs := &BodyStructure{} err := msgpack.Unmarshal(raw, bs) @@ -167,6 +67,7 @@ func DeserializeBodyStructure(raw []byte) (*BodyStructure, error) { return bs, err } +// Serialize will write msgpack bytes. func (bs *BodyStructure) Serialize() ([]byte, error) { data, err := msgpack.Marshal(bs) if err != nil { @@ -175,6 +76,7 @@ func (bs *BodyStructure) Serialize() ([]byte, error) { return data, nil } +// Parse will read the mail and create all body structures. func (bs *BodyStructure) Parse(r io.Reader) error { return bs.parseAllChildSections(r, []int{}, 0) } @@ -215,7 +117,7 @@ func (bs *BodyStructure) parseAllChildSections(r io.Reader, currentPath []int, s for err == nil { start += br.skipped part := &bytes.Buffer{} - err = br.WriteNextPartTo(part) + err = br.writeNextPartTo(part) if err != nil { break } @@ -319,19 +221,16 @@ func (bs *BodyStructure) getInfo(sectionPath []int) (sectionInfo *SectionInfo, e return } +// GetSection returns bytes of section including MIME header. func (bs *BodyStructure) GetSection(wholeMail io.ReadSeeker, sectionPath []int) (section []byte, err error) { info, err := bs.getInfo(sectionPath) if err != nil { return } - if _, err = wholeMail.Seek(int64(info.Start), io.SeekStart); err != nil { - return - } - section = make([]byte, info.Size) - _, err = wholeMail.Read(section) - return + return goToOffsetAndReadNBytes(wholeMail, info.Start, info.Size) } +// GetSectionContent returns bytes of section content (excluding MIME header). func (bs *BodyStructure) GetSectionContent(wholeMail io.ReadSeeker, sectionPath []int) (section []byte, err error) { info, err := bs.getInfo(sectionPath) if err != nil { @@ -380,6 +279,8 @@ func (bs *BodyStructure) GetSectionHeader(sectionPath []int) (header textproto.M return } +// IMAPBodyStructure will prepare imap bodystructure recurently for given part. +// Use empty path to create whole email structure. func (bs *BodyStructure) IMAPBodyStructure(currentPart []int) (imapBS *imap.BodyStructure, err error) { var info *SectionInfo if info, err = bs.getInfo(currentPart); err != nil { diff --git a/test/fakeapi/controller_control.go b/test/fakeapi/controller_control.go index 250309f7..977d4ea6 100644 --- a/test/fakeapi/controller_control.go +++ b/test/fakeapi/controller_control.go @@ -20,10 +20,8 @@ package fakeapi import ( "errors" "fmt" - "net/mail" "strings" - messageUtils "github.com/ProtonMail/proton-bridge/pkg/message" "github.com/ProtonMail/proton-bridge/pkg/pmapi" ) @@ -141,7 +139,6 @@ func (ctl *Controller) AddUserMessage(username string, message *pmapi.Message) ( } message.ID = ctl.messageIDGenerator.next("") message.LabelIDs = append(message.LabelIDs, pmapi.AllMailLabel) - message.Header = mail.Header(messageUtils.GetHeader(message)) for iAtt := 0; iAtt < message.NumAttachments; iAtt++ { message.Attachments = append(message.Attachments, newTestAttachment(iAtt, message.ID)) diff --git a/test/liveapi/messages.go b/test/liveapi/messages.go index 41ebf6e1..f49fe26b 100644 --- a/test/liveapi/messages.go +++ b/test/liveapi/messages.go @@ -18,11 +18,7 @@ package liveapi import ( - "bytes" "fmt" - "io" - "mime/multipart" - "net/http" messageUtils "github.com/ProtonMail/proton-bridge/pkg/message" "github.com/ProtonMail/proton-bridge/pkg/pmapi" @@ -43,14 +39,19 @@ func (ctl *Controller) AddUserMessage(username string, message *pmapi.Message) ( message.Flags = pmapi.ComputeMessageFlagsByLabels(message.LabelIDs) } - body, err := buildMessage(client, message) + kr, err := client.KeyRingForAddressID(message.AddressID) + if err != nil { + return "", errors.Wrap(err, "failed to get keyring while adding user message") + } + + body, err := messageUtils.BuildEncrypted(message, nil, kr) if err != nil { return "", errors.Wrap(err, "failed to build message") } req := &pmapi.ImportMsgReq{ AddressID: message.AddressID, - Body: body.Bytes(), + Body: body, Unread: message.Unread, Time: message.Time, Flags: message.Flags, @@ -70,66 +71,6 @@ func (ctl *Controller) AddUserMessage(username string, message *pmapi.Message) ( return result.MessageID, nil } -func buildMessage(client pmapi.Client, message *pmapi.Message) (*bytes.Buffer, error) { - if err := encryptMessage(client, message); err != nil { - return nil, errors.Wrap(err, "failed to encrypt message") - } - - body := &bytes.Buffer{} - if err := buildMessageHeader(message, body); err != nil { - return nil, errors.Wrap(err, "failed to build message header") - } - if err := buildMessageBody(message, body); err != nil { - return nil, errors.Wrap(err, "failed to build message body") - } - return body, nil -} - -func encryptMessage(client pmapi.Client, message *pmapi.Message) error { - kr, err := client.KeyRingForAddressID(message.AddressID) - if err != nil { - return errors.Wrap(err, "failed to get keyring for address") - } - - if err = message.Encrypt(kr, nil); err != nil { - return errors.Wrap(err, "failed to encrypt message body") - } - return nil -} - -func buildMessageHeader(message *pmapi.Message, body *bytes.Buffer) error { - header := messageUtils.GetHeader(message) - header.Set("Content-Type", "multipart/mixed; boundary="+messageUtils.GetBoundary(message)) - header.Del("Content-Disposition") - header.Del("Content-Transfer-Encoding") - - if err := http.Header(header).Write(body); err != nil { - return errors.Wrap(err, "failed to write header") - } - _, _ = body.WriteString("\r\n") - return nil -} - -func buildMessageBody(message *pmapi.Message, body *bytes.Buffer) error { - mw := multipart.NewWriter(body) - if err := mw.SetBoundary(messageUtils.GetBoundary(message)); err != nil { - return errors.Wrap(err, "failed to set boundary") - } - - bodyHeader := messageUtils.GetBodyHeader(message) - bodyHeader.Set("Content-Transfer-Encoding", "7bit") - - part, err := mw.CreatePart(bodyHeader) - if err != nil { - return errors.Wrap(err, "failed to create message body part") - } - if _, err := io.WriteString(part, message.Body); err != nil { - return errors.Wrap(err, "failed to write message body") - } - _ = mw.Close() - return nil -} - func (ctl *Controller) GetMessages(username, labelID string) ([]*pmapi.Message, error) { client, ok := ctl.pmapiByUsername[username] if !ok {