Silverfish/proton-bridge
1
0
Fork 1
mirror of https://github.com/ProtonMail/proton-bridge.git synced 2025-12-11 13:16:53 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

GODT-1177: remove Import-Export from repo

Browse Source
This commit is contained in:
James Houlahan
2021-05-28 09:56:52 +02:00
committed by Jakub
parent 649195cc2b
commit ffb18adfd0
141 changed files with 140 additions and 8781 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
internal/bridge/bridge.go
View File

@ -68,8 +68,15 @@ func New(
clientManager.AllowProxy()
}
storeFactory := newStoreFactory(cache, sentryReporter, panicHandler, eventListener)
u := users.New(locations, panicHandler, eventListener, clientManager, credStorer, storeFactory, true)
u := users.New(
locations,
panicHandler,
eventListener,
clientManager,
credStorer,
newStoreFactory(cache, sentryReporter, panicHandler, eventListener),
)
b := &Bridge{
Users: u,

36
internal/frontend/share/info.rc Normal file
View File

@ -0,0 +1,36 @@
#define STRINGIZE_(x) #x
#define STRINGIZE(x) STRINGIZE_(x)
IDI_ICON1 ICON DISCARDABLE STRINGIZE(ICO_FILE)
#define FILE_COMMENTS "The Bridge is an application that runs on your computer in the background and seamlessly encrypts and decrypts your mail as it enters and leaves your computer."
#define FILE_DESCRIPTION "ProtonMail Bridge"
#define INTERNAL_NAME STRINGIZE(EXE_NAME)
#define PRODUCT_NAME "ProtonMail Bridge for Windows"
#define LEGAL_COPYRIGHT "(C) " STRINGIZE(YEAR) " Proton Technologies AG"
1 VERSIONINFO
FILEVERSION FILE_VERSION_COMMA,0
PRODUCTVERSION FILE_VERSION_COMMA,0
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0"
BEGIN
VALUE "Comments", FILE_COMMENTS
VALUE "CompanyName", "Proton Technologies AG"
VALUE "FileDescription", FILE_DESCRIPTION
VALUE "FileVersion", STRINGIZE(FILE_VERSION)
VALUE "InternalName", INTERNAL_NAME
VALUE "LegalCopyright", LEGAL_COPYRIGHT
VALUE "OriginalFilename", STRINGIZE(ORIGINAL_FILE_NAME)
VALUE "ProductName", PRODUCT_NAME
VALUE "ProductVersion", STRINGIZE(PRODUCT_VERSION)
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x0409, 0x04B0
END
END

187
internal/importexport/importexport.go
View File

@ -1,187 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
// Package importexport provides core functionality of Import-Export app.
package importexport
import (
"bytes"
"context"
"github.com/ProtonMail/proton-bridge/internal/events"
"github.com/ProtonMail/proton-bridge/internal/transfer"
"github.com/ProtonMail/proton-bridge/internal/users"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/ProtonMail/proton-bridge/pkg/listener"
logrus "github.com/sirupsen/logrus"
)
var (
log = logrus.WithField("pkg", "importexport") //nolint[gochecknoglobals]
)
type ImportExport struct {
*users.Users
locations Locator
cache Cacher
panicHandler users.PanicHandler
eventListener listener.Listener
clientManager pmapi.Manager
}
func New(
locations Locator,
cache Cacher,
panicHandler users.PanicHandler,
eventListener listener.Listener,
clientManager pmapi.Manager,
credStorer users.CredentialsStorer,
) *ImportExport {
u := users.New(locations, panicHandler, eventListener, clientManager, credStorer, &storeFactory{}, false)
return &ImportExport{
Users: u,
locations: locations,
cache: cache,
panicHandler: panicHandler,
eventListener: eventListener,
clientManager: clientManager,
}
}
// ReportBug reports a new bug from the user.
func (ie *ImportExport) ReportBug(osType, osVersion, description, accountName, address, emailClient string) error {
return ie.clientManager.ReportBug(context.Background(), pmapi.ReportBugReq{
OS: osType,
OSVersion: osVersion,
Browser: emailClient,
Title: "[Import-Export] Bug",
Description: description,
Username: accountName,
Email: address,
})
}
// ReportFile submits import report file.
func (ie *ImportExport) ReportFile(osType, osVersion, accountName, address string, logdata []byte) error {
report := pmapi.ReportBugReq{
OS: osType,
OSVersion: osVersion,
Description: "An Import-Export report from the user swam down the river.",
Title: "[Import-Export] report file",
Username: accountName,
Email: address,
}
report.AddAttachment("log", "report.log", bytes.NewReader(logdata))
return ie.clientManager.ReportBug(context.Background(), report)
}
// GetLocalImporter returns transferrer from local EML or MBOX structure to ProtonMail account.
func (ie *ImportExport) GetLocalImporter(username, address, path string) (*transfer.Transfer, error) {
source := transfer.NewLocalProvider(path)
target, err := ie.getPMAPIProvider(username, address)
if err != nil {
return nil, err
}
logsPath, err := ie.locations.ProvideLogsPath()
if err != nil {
return nil, err
}
return transfer.New(ie.panicHandler, newImportMetricsManager(ie), logsPath, ie.cache.GetTransferDir(), source, target)
}
// GetRemoteImporter returns transferrer from remote IMAP to ProtonMail account.
func (ie *ImportExport) GetRemoteImporter(username, address, remoteUsername, remotePassword, host, port string) (*transfer.Transfer, error) {
source, err := transfer.NewIMAPProvider(remoteUsername, remotePassword, host, port)
if err != nil {
return nil, err
}
target, err := ie.getPMAPIProvider(username, address)
if err != nil {
return nil, err
}
logsPath, err := ie.locations.ProvideLogsPath()
if err != nil {
return nil, err
}
return transfer.New(ie.panicHandler, newImportMetricsManager(ie), logsPath, ie.cache.GetTransferDir(), source, target)
}
// GetEMLExporter returns transferrer from ProtonMail account to local EML structure.
func (ie *ImportExport) GetEMLExporter(username, address, path string) (*transfer.Transfer, error) {
source, err := ie.getPMAPIProvider(username, address)
if err != nil {
return nil, err
}
target := transfer.NewEMLProvider(path)
logsPath, err := ie.locations.ProvideLogsPath()
if err != nil {
return nil, err
}
return transfer.New(ie.panicHandler, newExportMetricsManager(ie), logsPath, ie.cache.GetTransferDir(), source, target)
}
// GetMBOXExporter returns transferrer from ProtonMail account to local MBOX structure.
func (ie *ImportExport) GetMBOXExporter(username, address, path string) (*transfer.Transfer, error) {
source, err := ie.getPMAPIProvider(username, address)
if err != nil {
return nil, err
}
target := transfer.NewMBOXProvider(path)
logsPath, err := ie.locations.ProvideLogsPath()
if err != nil {
return nil, err
}
return transfer.New(ie.panicHandler, newExportMetricsManager(ie), logsPath, ie.cache.GetTransferDir(), source, target)
}
func (ie *ImportExport) getPMAPIProvider(username, address string) (*transfer.PMAPIProvider, error) {
user, err := ie.Users.GetUser(username)
if err != nil {
return nil, err
}
addressID, err := user.GetAddressID(address)
if err != nil {
log.WithError(err).Info("Address does not exist, using all addresses")
}
provider, err := transfer.NewPMAPIProvider(user.GetClient(), user.ID(), addressID)
if err != nil {
return nil, err
}
go func() {
internetOffCh := ie.eventListener.ProvideChannel(events.InternetOffEvent)
internetOnCh := ie.eventListener.ProvideChannel(events.InternetOnEvent)
for {
select {
case <-internetOffCh:
provider.SetConnectionDown()
case <-internetOnCh:
provider.SetConnectionUp()
}
}
}()
return provider, nil
}

75
internal/importexport/metrics.go
View File

@ -1,75 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package importexport
import (
"strconv"
"github.com/ProtonMail/proton-bridge/internal/metrics"
"github.com/sirupsen/logrus"
)
type metricsManager struct {
ie *ImportExport
category metrics.Category
}
func newImportMetricsManager(ie *ImportExport) *metricsManager {
return &metricsManager{
ie: ie,
category: metrics.Import,
}
}
func newExportMetricsManager(ie *ImportExport) *metricsManager {
return &metricsManager{
ie: ie,
category: metrics.Export,
}
}
func (m *metricsManager) Load(numberOfMailboxes int) {
label := strconv.Itoa(numberOfMailboxes)
if err := m.ie.SendMetric(metrics.New(m.category, metrics.TransferLoad, metrics.Label(label))); err != nil {
logrus.WithError(err).Error("Failed to send metric")
}
}
func (m *metricsManager) Start() {
if err := m.ie.SendMetric(metrics.New(m.category, metrics.TransferStart, metrics.NoLabel)); err != nil {
logrus.WithError(err).Error("Failed to send metric")
}
}
func (m *metricsManager) Complete() {
if err := m.ie.SendMetric(metrics.New(m.category, metrics.TransferComplete, metrics.NoLabel)); err != nil {
logrus.WithError(err).Error("Failed to send metric")
}
}
func (m *metricsManager) Cancel() {
if err := m.ie.SendMetric(metrics.New(m.category, metrics.TransferCancel, metrics.NoLabel)); err != nil {
logrus.WithError(err).Error("Failed to send metric")
}
}
func (m *metricsManager) Fail() {
if err := m.ie.SendMetric(metrics.New(m.category, metrics.TransferFail, metrics.NoLabel)); err != nil {
logrus.WithError(err).Error("Failed to send metric")
}
}

35
internal/importexport/store_factory.go
View File

@ -1,35 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package importexport
import (
"github.com/ProtonMail/proton-bridge/internal/store"
)
// storeFactory implements dummy factory creating no store (not needed by Import-Export).
type storeFactory struct{}
// New does nothing.
func (f *storeFactory) New(user store.BridgeUser) (*store.Store, error) {
return nil, nil
}
// Remove does nothing.
func (f *storeFactory) Remove(userID string) error {
return nil
}

27
internal/importexport/types.go
View File

@ -1,27 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package importexport
type Locator interface {
ProvideLogsPath() (string, error)
Clear() error
}
type Cacher interface {
GetTransferDir() string
}

110
internal/transfer/mailbox.go
View File

@ -1,110 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"crypto/sha256"
"fmt"
"strings"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
)
var systemFolderMapping = map[string]string{ //nolint[gochecknoglobals]
"bin": "Trash",
"junk": "Spam",
"all": "All Mail",
"sent mail": "Sent",
"draft": "Drafts",
"important": "Starred",
// Add more translations.
}
// LeastUsedColor is intended to return color for creating a new inbox or label.
func LeastUsedColor(mailboxes []Mailbox) string {
usedColors := []string{}
for _, m := range mailboxes {
usedColors = append(usedColors, m.Color)
}
return pmapi.LeastUsedColor(usedColors)
}
// Mailbox is universal data holder of mailbox details for every provider.
type Mailbox struct {
ID string
Name string
Color string
IsExclusive bool
}
// IsSystemFolder returns true when ID corresponds to PM system folder.
func (m Mailbox) IsSystemFolder() bool {
return pmapi.IsSystemLabel(m.ID)
}
// Hash returns unique identifier to be used for matching.
func (m Mailbox) Hash() string {
return fmt.Sprintf("%x", sha256.Sum256([]byte(m.Name)))
}
// findMatchingMailboxes returns all matching mailboxes from `mailboxes`.
// Only one exclusive mailbox is included.
func (m Mailbox) findMatchingMailboxes(mailboxes []Mailbox) []Mailbox {
nameVariants := m.nameVariants()
isExclusiveIncluded := false
matches := []Mailbox{}
for i := range nameVariants {
nameVariant := nameVariants[len(nameVariants)-1-i]
for _, mailbox := range mailboxes {
if mailbox.IsExclusive && isExclusiveIncluded {
continue
}
if strings.ToLower(mailbox.Name) == nameVariant {
matches = append(matches, mailbox)
if mailbox.IsExclusive {
isExclusiveIncluded = true
}
}
}
}
return matches
}
// nameVariants returns all possible variants of the mailbox name.
// The best match (original name) is at the end of the slice.
// Variants are all in lower case. Examples:
// * Foo/bar -> [foo, bar, foo/bar]
// * x/Bin -> [x, trash, bin, x/bin]
// * a|b/c -> [a, b, c, a|b/c]
func (m Mailbox) nameVariants() (nameVariants []string) {
name := strings.ToLower(m.Name)
if strings.Contains(name, "/") || strings.Contains(name, "|") {
for _, slashPart := range strings.Split(name, "/") {
for _, part := range strings.Split(slashPart, "|") {
if mappedPart, ok := systemFolderMapping[part]; ok {
nameVariants = append(nameVariants, strings.ToLower(mappedPart))
}
nameVariants = append(nameVariants, part)
}
}
}
if mappedName, ok := systemFolderMapping[name]; ok {
nameVariants = append(nameVariants, strings.ToLower(mappedName))
}
return append(nameVariants, name)
}

111
internal/transfer/mailbox_test.go
View File

@ -1,111 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"testing"
r "github.com/stretchr/testify/require"
)
func TestLeastUsedColor(t *testing.T) {
var mailboxes []Mailbox
// Unset mailboxes, should use first available color
mailboxes = nil
r.Equal(t, "#7272a7", LeastUsedColor(mailboxes))
// No mailboxes at all, should use first available color
mailboxes = []Mailbox{}
r.Equal(t, "#7272a7", LeastUsedColor(mailboxes))
// All colors have same frequency, should use first available color
mailboxes = []Mailbox{
{Name: "Mbox1", Color: "#7272a7"},
{Name: "Mbox2", Color: "#cf5858"},
{Name: "Mbox3", Color: "#c26cc7"},
{Name: "Mbox4", Color: "#7569d1"},
{Name: "Mbox5", Color: "#69a9d1"},
{Name: "Mbox6", Color: "#5ec7b7"},
{Name: "Mbox7", Color: "#72bb75"},
{Name: "Mbox8", Color: "#c3d261"},
{Name: "Mbox9", Color: "#e6c04c"},
{Name: "Mbox10", Color: "#e6984c"},
{Name: "Mbox11", Color: "#8989ac"},
{Name: "Mbox12", Color: "#cf7e7e"},
{Name: "Mbox13", Color: "#c793ca"},
{Name: "Mbox14", Color: "#9b94d1"},
{Name: "Mbox15", Color: "#a8c4d5"},
{Name: "Mbox16", Color: "#97c9c1"},
{Name: "Mbox17", Color: "#9db99f"},
{Name: "Mbox18", Color: "#c6cd97"},
{Name: "Mbox19", Color: "#e7d292"},
{Name: "Mbox20", Color: "#dfb286"},
}
r.Equal(t, "#7272a7", LeastUsedColor(mailboxes))
// First three colors already used, but others wasn't. Should use first non-used one.
mailboxes = []Mailbox{
{Name: "Mbox1", Color: "#7272a7"},
{Name: "Mbox2", Color: "#cf5858"},
{Name: "Mbox3", Color: "#c26cc7"},
}
r.Equal(t, "#7569d1", LeastUsedColor(mailboxes))
}
func TestFindMatchingMailboxes(t *testing.T) {
mailboxes := []Mailbox{
{Name: "Inbox", IsExclusive: true},
{Name: "Sent", IsExclusive: true},
{Name: "Archive", IsExclusive: true},
{Name: "Foo", IsExclusive: false},
{Name: "hello/world", IsExclusive: true},
{Name: "Hello", IsExclusive: false},
{Name: "WORLD", IsExclusive: true},
{Name: "Trash", IsExclusive: true},
{Name: "Drafts", IsExclusive: true},
}
tests := []struct {
name string
wantNames []string
}{
{"inbox", []string{"Inbox"}},
{"foo", []string{"Foo"}},
{"hello", []string{"Hello"}},
{"world", []string{"WORLD"}},
{"hello/world", []string{"hello/world", "Hello"}},
{"hello|world", []string{"WORLD", "Hello"}},
{"nomailbox", []string{}},
{"bin", []string{"Trash"}},
{"root/bin", []string{"Trash"}},
{"draft", []string{"Drafts"}},
{"root/draft", []string{"Drafts"}},
}
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
mailbox := Mailbox{Name: tc.name}
got := mailbox.findMatchingMailboxes(mailboxes)
gotNames := []string{}
for _, m := range got {
gotNames = append(gotNames, m.Name)
}
r.Equal(t, tc.wantNames, gotNames)
})
}
}

125
internal/transfer/message.go
View File

@ -1,125 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"fmt"
"mime"
"net/mail"
"time"
)
// Message is data holder passed between import and export.
type Message struct {
ID string
Unread bool
Body []byte
Sources []Mailbox
Targets []Mailbox
}
// sourceNames returns array of source mailbox names.
func (msg Message) sourceNames() (names []string) {
for _, mailbox := range msg.Sources {
names = append(names, mailbox.Name)
}
return
}
// targetNames returns array of target mailbox names.
func (msg Message) targetNames() (names []string) {
for _, mailbox := range msg.Targets {
names = append(names, mailbox.Name)
}
return
}
// MessageStatus holds status for message used by progress manager.
type MessageStatus struct {
eventTime time.Time // Time of adding message to the process.
sourceNames []string // Source mailbox names message is in.
SourceID string // Message ID at the source.
targetNames []string // Target mailbox names message is in.
targetID string // Message ID at the target (if any).
bodyHash string // Hash of the message body.
skipped bool
exported bool
imported bool
exportErr error
importErr error
// Info about message displayed to user.
// This is needed only for failed messages, but we cannot know in advance
// which message will fail. We could clear it once the message passed
// without any error. However, if we say one message takes about 100 bytes
// in average, it's about 100 MB per million of messages, which is fine.
Subject string
From string
Time time.Time
}
func (status *MessageStatus) String() string {
return fmt.Sprintf("%s (%s, %s, %s): %s", status.SourceID, status.Subject, status.From, status.Time, status.GetErrorMessage())
}
func (status *MessageStatus) setDetailsFromHeader(header mail.Header) {
dec := &mime.WordDecoder{}
status.Subject = header.Get("subject")
if subject, err := dec.Decode(status.Subject); err == nil {
status.Subject = subject
}
status.From = header.Get("from")
if from, err := dec.Decode(status.From); err == nil {
status.From = from
}
if msgTime, err := header.Date(); err == nil {
status.Time = msgTime
}
}
func (status *MessageStatus) hasError(includeMissing bool) bool {
return status.exportErr != nil || status.importErr != nil || (includeMissing && !status.skipped && !status.imported)
}
// GetErrorMessage returns error message.
func (status *MessageStatus) GetErrorMessage() string {
return status.getErrorMessage(true)
}
func (status *MessageStatus) getErrorMessage(includeMissing bool) string {
if status.skipped {
return ""
}
if status.exportErr != nil {
return fmt.Sprintf("failed to export: %s", status.exportErr)
}
if status.importErr != nil {
return fmt.Sprintf("failed to import: %s", status.importErr)
}
if includeMissing && !status.imported {
if !status.exported {
return "failed to import: lost before read"
}
return "failed to import: lost in the process"
}
return ""
}

397
internal/transfer/progress.go
View File

@ -1,397 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"crypto/sha256"
"fmt"
"sync"
"time"
"github.com/sirupsen/logrus"
)
// Progress maintains progress between import, export and user interface.
// Import and export update progress about processing messages and progress
// informs user interface, vice versa action (such as pause or resume) from
// user interface is passed down to import and export.
type Progress struct { //nolint[maligned]
log *logrus.Entry
lock sync.Locker
updateCh chan struct{}
messageCounted bool
messageCounts map[string]uint
messageStatuses map[string]*MessageStatus
pauseReason string
isStopped bool
fatalError error
fileReport *fileReport
}
func newProgress(log *logrus.Entry, fileReport *fileReport) Progress {
return Progress{
log: log,
lock: &sync.Mutex{},
updateCh: make(chan struct{}),
messageCounts: map[string]uint{},
messageStatuses: map[string]*MessageStatus{},
fileReport: fileReport,
}
}
// update is helper to notify listener for updates.
func (p *Progress) update() {
if p.updateCh == nil {
return
}
// In case no one listens for an update, do not block the whole progress.
go func() {
defer func() {
// updateCh can be closed at the end of progress which is fine.
if r := recover(); r != nil {
log.WithField("r", r).Warn("Failed to send update")
}
}()
select {
case p.updateCh <- struct{}{}:
case <-time.After(5 * time.Millisecond):
}
}()
}
// finish should be called as the last call once everything is done.
func (p *Progress) finish() {
p.lock.Lock()
defer p.lock.Unlock()
log.Debug("Progress finished")
p.cleanUpdateCh()
}
// fatal should be called once there is error with no possible continuation.
func (p *Progress) fatal(err error) {
p.lock.Lock()
defer p.lock.Unlock()
log.WithError(err).Error("Progress finished")
p.setStop()
p.fatalError = err
p.cleanUpdateCh()
}
func (p *Progress) cleanUpdateCh() {
if p.updateCh == nil {
return
}
close(p.updateCh)
p.updateCh = nil
}
func (p *Progress) countsFinal() {
p.lock.Lock()
defer p.lock.Unlock()
defer p.update()
log.Info("Estimating count finished")
p.messageCounted = true
}
func (p *Progress) updateCount(mailbox string, count uint) {
p.lock.Lock()
defer p.lock.Unlock()
defer p.update()
log.WithField("mailbox", mailbox).WithField("count", count).Debug("Mailbox count updated")
p.messageCounts[mailbox] = count
}
// addMessage should be called as soon as there is ID of the message.
func (p *Progress) addMessage(messageID string, sourceNames, targetNames []string) {
p.lock.Lock()
defer p.lock.Unlock()
defer p.update()
p.log.WithField("id", messageID).Trace("Message added")
p.messageStatuses[messageID] = &MessageStatus{
eventTime: time.Now(),
sourceNames: sourceNames,
SourceID: messageID,
targetNames: targetNames,
}
}
// messageSkipped should be called once the message is skipped due to some
// filter such as time or folder and so on.
func (p *Progress) messageSkipped(messageID string) {
p.lock.Lock()
defer p.lock.Unlock()
defer p.update()
p.log.WithField("id", messageID).Debug("Message skipped")
p.messageStatuses[messageID].skipped = true
p.logMessage(messageID)
}
// messageExported should be called right before message is exported.
func (p *Progress) messageExported(messageID string, body []byte, err error) {
p.lock.Lock()
defer p.lock.Unlock()
defer p.update()
log := p.log.WithField("id", messageID)
if err != nil {
log = log.WithError(err)
}
log.Debug("Message exported")
status := p.messageStatuses[messageID]
status.exportErr = err
if err == nil {
status.exported = true
}
if len(body) > 0 {
status.bodyHash = fmt.Sprintf("%x", sha256.Sum256(body))
if header, err := getMessageHeader(body); err != nil {
log.WithError(err).Warning("Failed to parse headers for reporting")
} else {
status.setDetailsFromHeader(header)
}
}
// If export failed, no other step will be done with message and we can log it to the report file.
if err != nil {
p.logMessage(messageID)
}
}
// messageImported should be called right after message is imported.
func (p *Progress) messageImported(messageID, importID string, err error) {
p.lock.Lock()
defer p.lock.Unlock()
defer p.update()
log := p.log.WithField("id", messageID)
if err != nil {
log = log.WithError(err)
}
log.Debug("Message imported")
p.messageStatuses[messageID].targetID = importID
p.messageStatuses[messageID].importErr = err
if err == nil {
p.messageStatuses[messageID].imported = true
}
// Import is the last step, now we can log the result to the report file.
p.logMessage(messageID)
}
// logMessage writes message status to log file.
func (p *Progress) logMessage(messageID string) {
if p.fileReport == nil {
return
}
p.fileReport.writeMessageStatus(p.messageStatuses[messageID])
}
// callWrap calls the callback and in case of problem it pause the process.
// Then it waits for user action to fix it and click on continue or abort.
// Every function doing I/O should be wrapped by this function to provide
// stopping and pausing functionality.
func (p *Progress) callWrap(callback func() error) {
for {
if p.shouldStop() {
break
}
err := callback()
if err == nil {
break
}
p.Pause("paused due to " + err.Error())
}
}
// shouldStop is utility for providers to automatically wait during pause
// and returned value determines whether the process shouls be fully stopped.
func (p *Progress) shouldStop() bool {
for p.IsPaused() {
time.Sleep(time.Second)
}
return p.IsStopped()
}
// GetUpdateChannel returns channel notifying any update from import or export.
func (p *Progress) GetUpdateChannel() chan struct{} {
p.lock.Lock()
defer p.lock.Unlock()
return p.updateCh
}
// Pause pauses the progress.
func (p *Progress) Pause(reason string) {
p.lock.Lock()
defer p.lock.Unlock()
defer p.update()
p.log.Info("Progress paused")
p.pauseReason = reason
}
// Resume resumes the progress.
func (p *Progress) Resume() {
p.lock.Lock()
defer p.lock.Unlock()
defer p.update()
p.log.Info("Progress resumed")
p.pauseReason = ""
}
// IsPaused returns whether progress is paused.
func (p *Progress) IsPaused() bool {
p.lock.Lock()
defer p.lock.Unlock()
return p.pauseReason != ""
}
// PauseReason returns pause reason.
func (p *Progress) PauseReason() string {
p.lock.Lock()
defer p.lock.Unlock()
return p.pauseReason
}
// Stop stops the process.
func (p *Progress) Stop() {
p.lock.Lock()
defer p.lock.Unlock()
defer p.update()
p.log.Info("Progress stopped")
p.setStop()
// Once progress is stopped, some calls might be in progress. Results from
// those calls are irrelevant so we can close update channel sooner to not
// propagate any progress to user interface anymore.
p.cleanUpdateCh()
}
func (p *Progress) setStop() {
p.isStopped = true
p.pauseReason = "" // Clear pause to run paused code and stop it.
}
// IsStopped returns whether progress is stopped.
func (p *Progress) IsStopped() bool {
p.lock.Lock()
defer p.lock.Unlock()
return p.isStopped
}
// GetFatalError returns fatal error (progress failed and did not finish).
func (p *Progress) GetFatalError() error {
p.lock.Lock()
defer p.lock.Unlock()
return p.fatalError
}
// GetFailedMessages returns statuses of failed messages.
func (p *Progress) GetFailedMessages() []*MessageStatus {
p.lock.Lock()
defer p.lock.Unlock()
// Include lost messages in the process only when transfer is done.
includeMissing := p.updateCh == nil
statuses := []*MessageStatus{}
for _, status := range p.messageStatuses {
if status.hasError(includeMissing) {
statuses = append(statuses, status)
}
}
return statuses
}
// GetCounts returns counts of exported and imported messages.
func (p *Progress) GetCounts() ProgressCounts {
p.lock.Lock()
defer p.lock.Unlock()
counts := ProgressCounts{}
// Return counts only once total is estimated or the process already
// ended (for a case when it ended quickly to report it correctly).
if p.updateCh != nil && !p.messageCounted {
return counts
}
// Include lost messages in the process only when transfer is done.
includeMissing := p.updateCh == nil
for _, mailboxCount := range p.messageCounts {
counts.Total += mailboxCount
}
for _, status := range p.messageStatuses {
counts.Added++
if status.skipped {
counts.Skipped++
}
if status.exported {
counts.Exported++
}
if status.imported {
counts.Imported++
}
if status.hasError(includeMissing) {
counts.Failed++
}
}
return counts
}
// GenerateBugReport generates similar file to import log except private information.
func (p *Progress) GenerateBugReport() []byte {
bugReport := bugReport{}
for _, status := range p.messageStatuses {
bugReport.writeMessageStatus(status)
}
return bugReport.getData()
}
// FileReport returns path to generated defailed file report.
func (p *Progress) FileReport() string {
if p.fileReport == nil {
return ""
}
return p.fileReport.path
}

35
internal/transfer/progress_counts.go
View File

@ -1,35 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
// ProgressCounts holds counts counted by Progress.
type ProgressCounts struct {
Failed,
Skipped,
Imported,
Exported,
Added,
Total uint
}
// Progress returns ratio between processed messages (fully imported, skipped
// and failed ones) and total number of messages as percentage (0 - 1).
func (c *ProgressCounts) Progress() float32 {
progressed := c.Imported + c.Skipped + c.Failed
return float32(progressed) / float32(c.Total)
}

142
internal/transfer/progress_test.go
View File

@ -1,142 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"testing"
"time"
"github.com/pkg/errors"
a "github.com/stretchr/testify/assert"
r "github.com/stretchr/testify/require"
)
func TestProgressUpdateCount(t *testing.T) {
progress := newProgress(log, nil)
drainProgressUpdateChannel(&progress)
progress.updateCount("inbox", 10)
progress.updateCount("archive", 20)
progress.updateCount("inbox", 12)
progress.updateCount("sent", 5)
progress.updateCount("foo", 4)
progress.updateCount("foo", 5)
progress.finish()
counts := progress.GetCounts()
r.Equal(t, uint(42), counts.Total)
}
func TestProgressAddingMessages(t *testing.T) {
progress := newProgress(log, nil)
drainProgressUpdateChannel(&progress)
// msg1 has no problem.
progress.addMessage("msg1", []string{}, []string{})
progress.messageExported("msg1", []byte(""), nil)
progress.messageImported("msg1", "", nil)
// msg2 has an import problem.
progress.addMessage("msg2", []string{}, []string{})
progress.messageExported("msg2", []byte(""), nil)
progress.messageImported("msg2", "", errors.New("failed import"))
// msg3 has an export problem.
progress.addMessage("msg3", []string{}, []string{})
progress.messageExported("msg3", []byte(""), errors.New("failed export"))
// msg4 has an export problem and import is also called.
progress.addMessage("msg4", []string{}, []string{})
progress.messageExported("msg4", []byte(""), errors.New("failed export"))
progress.messageImported("msg4", "", nil)
// msg5 is skipped.
progress.addMessage("msg5", []string{}, []string{})
progress.messageSkipped("msg5")
progress.finish()
counts := progress.GetCounts()
a.Equal(t, uint(5), counts.Added)
a.Equal(t, uint(2), counts.Exported)
a.Equal(t, uint(2), counts.Imported)
a.Equal(t, uint(1), counts.Skipped)
a.Equal(t, uint(3), counts.Failed)
errorsMap := map[string]string{}
for _, status := range progress.GetFailedMessages() {
errorsMap[status.SourceID] = status.GetErrorMessage()
}
a.Equal(t, map[string]string{
"msg2": "failed to import: failed import",
"msg3": "failed to export: failed export",
"msg4": "failed to export: failed export",
}, errorsMap)
}
func TestProgressFinish(t *testing.T) {
progress := newProgress(log, nil)
drainProgressUpdateChannel(&progress)
progress.finish()
r.Nil(t, progress.updateCh)
r.NotPanics(t, func() { progress.addMessage("msg", []string{}, []string{}) })
}
func TestProgressFatalError(t *testing.T) {
progress := newProgress(log, nil)
drainProgressUpdateChannel(&progress)
progress.fatal(errors.New("fatal error"))
r.Nil(t, progress.updateCh)
r.NotPanics(t, func() { progress.addMessage("msg", []string{}, []string{}) })
}
func TestFailUnpauseAndStops(t *testing.T) {
progress := newProgress(log, nil)
drainProgressUpdateChannel(&progress)
progress.Pause("pausing")
progress.fatal(errors.New("fatal error"))
r.Nil(t, progress.updateCh)
r.True(t, progress.isStopped)
r.False(t, progress.IsPaused())
r.Eventually(t, progress.shouldStop, time.Second, 10*time.Millisecond)
}
func TestStopClosesUpdates(t *testing.T) {
progress := newProgress(log, nil)
ch := progress.updateCh
progress.Stop()
r.Nil(t, progress.updateCh)
r.PanicsWithError(t, "send on closed channel", func() { ch <- struct{}{} })
}
func drainProgressUpdateChannel(progress *Progress) {
// updateCh is not needed to drain under tests - timeout is implemented.
// But timeout takes time which would slow down tests.
go func() {
for range progress.updateCh {
}
}()
}

51
internal/transfer/provider.go
View File

@ -1,51 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
// Provider provides interface for common operation with provider.
type Provider interface {
// ID is used for generating transfer ID by combining source and target ID.
ID() string
// Mailboxes returns all available mailboxes.
// Provider used as source returns only non-empty maibloxes.
// Provider used as target does not return all mail maiblox.
Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error)
}
// SourceProvider provides interface of provider with support of export.
type SourceProvider interface {
Provider
// TransferTo exports messages based on rules to channel.
TransferTo(transferRules, *Progress, chan<- Message)
}
// TargetProvider provides interface of provider with support of import.
type TargetProvider interface {
Provider
// DefaultMailboxes returns the default mailboxes for default rules if no other is found.
DefaultMailboxes(sourceMailbox Mailbox) (targetMailboxes []Mailbox)
// CreateMailbox creates new mailbox to be used as target in transfer rules.
CreateMailbox(Mailbox) (Mailbox, error)
// TransferFrom imports messages from channel.
TransferFrom(transferRules, *Progress, <-chan Message)
}

68
internal/transfer/provider_eml.go
View File

@ -1,68 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
// EMLProvider implements import and export to/from EML file structure.
type EMLProvider struct {
root string
}
// NewEMLProvider creates EMLProvider.
func NewEMLProvider(root string) *EMLProvider {
return &EMLProvider{
root: root,
}
}
// ID is used for generating transfer ID by combining source and target ID.
// We want to keep the same rules for import from or export to local files
// no matter exact path, therefore it returns constant. The same as EML.
func (p *EMLProvider) ID() string {
return "local" //nolint[goconst]
}
// Mailboxes returns all available folder names from root of EML files.
// In case the same folder name is used more than once (for example root/a/foo
// and root/b/foo), it's treated as the same folder.
func (p *EMLProvider) Mailboxes(includeEmpty, includeAllMail bool) (mailboxes []Mailbox, err error) {
// Special case for exporting--we don't know the path before setup if finished.
if p.root == "" {
return
}
var folderNames []string
if includeEmpty {
folderNames, err = getFolderNames(p.root)
} else {
folderNames, err = getFolderNamesWithFileSuffix(p.root, ".eml")
}
if err != nil {
return nil, err
}
for _, folderName := range folderNames {
mailboxes = append(mailboxes, Mailbox{
ID: "",
Name: folderName,
Color: "",
IsExclusive: false,
})
}
return mailboxes, nil
}

134
internal/transfer/provider_eml_source.go
View File

@ -1,134 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"io/ioutil"
"os"
"path/filepath"
"github.com/pkg/errors"
)
// TransferTo exports messages based on rules to channel.
func (p *EMLProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) {
log.Info("Started transfer from EML to channel")
defer log.Info("Finished transfer from EML to channel")
filePathsPerFolder, err := p.getFilePathsPerFolder(rules)
if err != nil {
progress.fatal(err)
return
}
if len(filePathsPerFolder) == 0 {
return
}
// This list is not filtered by time but instead going throgh each file
// twice or keeping all in memory we will tell rough estimation which
// will be updated during processing each file.
for folderName, filePaths := range filePathsPerFolder {
if progress.shouldStop() {
break
}
progress.updateCount(folderName, uint(len(filePaths)))
}
progress.countsFinal()
for folderName, filePaths := range filePathsPerFolder {
// No error guaranteed by getFilePathsPerFolder.
rule, _ := rules.getRuleBySourceMailboxName(folderName)
log.WithField("rule", rule).Debug("Processing rule")
p.exportMessages(rule, filePaths, progress, ch)
}
}
func (p *EMLProvider) getFilePathsPerFolder(rules transferRules) (map[string][]string, error) {
filePaths, err := getFilePathsWithSuffix(p.root, ".eml")
if err != nil {
return nil, err
}
filePathsMap := map[string][]string{}
for _, filePath := range filePaths {
folder := filepath.Base(filepath.Dir(filepath.Join(p.root, filePath)))
_, err := rules.getRuleBySourceMailboxName(folder)
if err != nil {
log.WithField("msg", filePath).Trace("Message skipped due to folder name")
continue
}
filePathsMap[folder] = append(filePathsMap[folder], filePath)
}
return filePathsMap, nil
}
func (p *EMLProvider) exportMessages(rule *Rule, filePaths []string, progress *Progress, ch chan<- Message) {
for _, filePath := range filePaths {
if progress.shouldStop() {
break
}
msg, err := p.exportMessage(rule, filePath)
progress.addMessage(filePath, msg.sourceNames(), msg.targetNames())
// Read and check time in body only if the rule specifies it
// to not waste energy.
if err == nil && rule.HasTimeLimit() {
msgTime, msgTimeErr := getMessageTime(msg.Body)
if msgTimeErr != nil {
err = msgTimeErr
} else if !rule.isTimeInRange(msgTime) {
log.WithField("msg", filePath).Debug("Message skipped due to time")
progress.messageSkipped(filePath)
continue
}
}
progress.messageExported(filePath, msg.Body, err)
if err == nil {
ch <- msg
}
}
}
func (p *EMLProvider) exportMessage(rule *Rule, filePath string) (Message, error) {
fullFilePath := filepath.Clean(filepath.Join(p.root, filePath))
file, err := os.Open(fullFilePath) //nolint[gosec]
if err != nil {
return Message{}, errors.Wrap(err, "failed to open message")
}
defer file.Close() //nolint[errcheck]
body, err := ioutil.ReadAll(file)
if err != nil {
return Message{}, errors.Wrap(err, "failed to read message")
}
return Message{
ID: filePath,
Unread: false,
Body: body,
Sources: []Mailbox{rule.SourceMailbox},
Targets: rule.TargetMailboxes,
}, nil
}

88
internal/transfer/provider_eml_target.go
View File

@ -1,88 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"io/ioutil"
"os"
"path/filepath"
"github.com/hashicorp/go-multierror"
)
// DefaultMailboxes returns the default mailboxes for default rules if no other is found.
func (p *EMLProvider) DefaultMailboxes(sourceMailbox Mailbox) []Mailbox {
return []Mailbox{{
Name: sourceMailbox.Name,
}}
}
// CreateMailbox does nothing. Folders are created dynamically during the import.
func (p *EMLProvider) CreateMailbox(mailbox Mailbox) (Mailbox, error) {
return mailbox, nil
}
// TransferFrom imports messages from channel.
func (p *EMLProvider) TransferFrom(rules transferRules, progress *Progress, ch <-chan Message) {
log.Info("Started transfer from channel to EML")
defer log.Info("Finished transfer from channel to EML")
err := p.createFolders(rules)
if err != nil {
progress.fatal(err)
return
}
for msg := range ch {
for progress.shouldStop() {
break
}
err := p.writeFile(msg)
progress.messageImported(msg.ID, "", err)
}
}
func (p *EMLProvider) createFolders(rules transferRules) error {
for rule := range rules.iterateActiveRules() {
for _, mailbox := range rule.TargetMailboxes {
path := filepath.Join(p.root, sanitizeFileName(mailbox.Name))
if err := os.MkdirAll(path, os.ModePerm); err != nil {
return err
}
}
}
return nil
}
func (p *EMLProvider) writeFile(msg Message) error {
fileName := sanitizeFileName(filepath.Base(msg.ID))
if filepath.Ext(fileName) != ".eml" {
fileName += ".eml"
}
var err error
for _, mailbox := range msg.Targets {
path := filepath.Join(p.root, mailbox.Name, fileName)
if localErr := ioutil.WriteFile(path, msg.Body, 0600); localErr != nil {
err = multierror.Append(err, localErr)
}
}
return err
}

126
internal/transfer/provider_eml_test.go
View File

@ -1,126 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"fmt"
"io/ioutil"
"os"
"testing"
"time"
r "github.com/stretchr/testify/require"
)
func newTestEMLProvider(path string) *EMLProvider {
if path == "" {
path = "testdata/eml"
}
return NewEMLProvider(path)
}
func TestEMLProviderMailboxes(t *testing.T) {
provider := newTestEMLProvider("")
tests := []struct {
includeEmpty bool
wantMailboxes []Mailbox
}{
{true, []Mailbox{
{Name: "Foo"},
{Name: "Inbox"},
{Name: "eml"},
}},
{false, []Mailbox{
{Name: "Foo"},
{Name: "Inbox"},
}},
}
for _, tc := range tests {
tc := tc
t.Run(fmt.Sprintf("%v", tc.includeEmpty), func(t *testing.T) {
mailboxes, err := provider.Mailboxes(tc.includeEmpty, false)
r.NoError(t, err)
r.Equal(t, tc.wantMailboxes, mailboxes)
})
}
}
func TestEMLProviderTransferTo(t *testing.T) {
provider := newTestEMLProvider("")
rules, rulesClose := newTestRules(t)
defer rulesClose()
setupEMLRules(rules)
testTransferTo(t, rules, provider, []string{
"Foo/msg.eml",
"Inbox/msg.eml",
})
}
func TestEMLProviderTransferFrom(t *testing.T) {
dir, err := ioutil.TempDir("", "eml")
r.NoError(t, err)
defer os.RemoveAll(dir) //nolint[errcheck]
provider := newTestEMLProvider(dir)
rules, rulesClose := newTestRules(t)
defer rulesClose()
setupEMLRules(rules)
testTransferFrom(t, rules, provider, []Message{
{ID: "Foo/msg.eml", Body: getTestMsgBody("msg"), Targets: []Mailbox{{Name: "Foo"}}},
})
checkEMLFileStructure(t, dir, []string{
"Foo/msg.eml",
})
}
func TestEMLProviderTransferFromTo(t *testing.T) {
dir, err := ioutil.TempDir("", "eml")
r.NoError(t, err)
defer os.RemoveAll(dir) //nolint[errcheck]
source := newTestEMLProvider("")
target := newTestEMLProvider(dir)
rules, rulesClose := newTestRules(t)
defer rulesClose()
setupEMLRules(rules)
testTransferFromTo(t, rules, source, target, 5*time.Second)
checkEMLFileStructure(t, dir, []string{
"Foo/msg.eml",
"Inbox/msg.eml",
})
}
func setupEMLRules(rules transferRules) {
_ = rules.setRule(Mailbox{Name: "Inbox"}, []Mailbox{{Name: "Inbox"}}, 0, 0)
_ = rules.setRule(Mailbox{Name: "Foo"}, []Mailbox{{Name: "Foo"}}, 0, 0)
}
func checkEMLFileStructure(t *testing.T, root string, expectedFiles []string) {
files, err := getFilePathsWithSuffix(root, ".eml")
r.NoError(t, err)
r.Equal(t, expectedFiles, files)
}

123
internal/transfer/provider_imap.go
View File

@ -1,123 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"net"
"strings"
"github.com/emersion/go-imap"
"github.com/emersion/go-sasl"
)
type IMAPClientProvider interface {
Capability() (map[string]bool, error)
Support(capability string) (bool, error)
State() imap.ConnState
SupportAuth(mech string) (bool, error)
Authenticate(auth sasl.Client) error
Login(username, password string) error
List(ref, name string, ch chan *imap.MailboxInfo) error
Select(name string, readOnly bool) (*imap.MailboxStatus, error)
Fetch(seqset *imap.SeqSet, items []imap.FetchItem, ch chan *imap.Message) error
UidFetch(seqset *imap.SeqSet, items []imap.FetchItem, ch chan *imap.Message) error
}
// IMAPProvider implements export from IMAP server.
type IMAPProvider struct {
username string
password string
addr string
clientDialer func(addr string) (IMAPClientProvider, error)
client IMAPClientProvider
timeIt *timeIt
}
// NewIMAPProvider returns new IMAPProvider.
func NewIMAPProvider(username, password, host, port string) (*IMAPProvider, error) {
return newIMAPProvider(imapClientDial, username, password, host, port)
}
func newIMAPProvider(clientDialer func(string) (IMAPClientProvider, error), username, password, host, port string) (*IMAPProvider, error) {
p := &IMAPProvider{
username: username,
password: password,
addr: net.JoinHostPort(host, port),
timeIt: newTimeIt("imap"),
clientDialer: clientDialer,
}
if err := p.auth(); err != nil {
return nil, err
}
return p, nil
}
// ID is used for generating transfer ID by combining source and target ID.
// We want to keep the same rules for import from any IMAP server, therefore
// it returns constant.
func (p *IMAPProvider) ID() string {
return "imap"
}
// Mailboxes returns all available folder names from root of EML files.
// In case the same folder name is used more than once (for example root/a/foo
// and root/b/foo), it's treated as the same folder.
func (p *IMAPProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) {
mailboxesInfo, err := p.list()
if err != nil {
return nil, err
}
mailboxes := []Mailbox{}
for _, mailbox := range mailboxesInfo {
hasNoSelect := false
for _, attrib := range mailbox.Attributes {
if strings.ToLower(attrib) == "\\noselect" {
hasNoSelect = true
break
}
}
if hasNoSelect {
continue
}
if !includeEmpty || true {
mailboxStatus, err := p.selectIn(mailbox.Name)
if err != nil {
return nil, err
}
if mailboxStatus.Messages == 0 {
continue
}
}
mailboxes = append(mailboxes, Mailbox{
ID: "",
Name: mailbox.Name,
Color: "",
IsExclusive: false,
})
}
return mailboxes, nil
}

66
internal/transfer/provider_imap_errors.go
View File

@ -1,66 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
// imapError is base for all IMAP errors.
type imapError struct {
Message string
Err error
}
func (e imapError) Error() string {
return e.Message + ": " + e.Err.Error()
}
func (e imapError) Unwrap() error {
return e.Err
}
func (e imapError) Cause() error {
return e.Err
}
// ErrIMAPConnection is error representing connection issues.
type ErrIMAPConnection struct {
imapError
}
func (e ErrIMAPConnection) Is(target error) bool {
_, ok := target.(*ErrIMAPConnection)
return ok
}
// ErrIMAPAuth is error representing authentication issues.
type ErrIMAPAuth struct {
imapError
}
func (e ErrIMAPAuth) Is(target error) bool {
_, ok := target.(*ErrIMAPAuth)
return ok
}
// ErrIMAPAuthMethod is error representing wrong auth method.
type ErrIMAPAuthMethod struct {
imapError
}
func (e ErrIMAPAuthMethod) Is(target error) bool {
_, ok := target.(*ErrIMAPAuthMethod)
return ok
}

250
internal/transfer/provider_imap_source.go
View File

@ -1,250 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"fmt"
"io/ioutil"
"github.com/emersion/go-imap"
)
type imapMessageInfo struct {
id string
uid uint32
size uint32
}
const (
imapPageSize = uint32(2000) // Optimized on Gmail.
imapMaxFetchSize = uint32(50 * 1000 * 1000) // Size in octets. If 0, it will use one fetch per message.
)
// TransferTo exports messages based on rules to channel.
func (p *IMAPProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) {
log.Info("Started transfer from IMAP to channel")
defer log.Info("Finished transfer from IMAP to channel")
p.timeIt.clear()
defer p.timeIt.logResults()
imapMessageInfoMap := p.loadMessageInfoMap(rules, progress)
for rule := range rules.iterateActiveRules() {
messagesInfo := imapMessageInfoMap[rule.SourceMailbox.Name]
if messagesInfo == nil {
log.WithField("rule", rule).Warn("Rule has no message info")
continue
}
log.WithField("rule", rule).Debug("Processing rule")
p.transferTo(rule, messagesInfo, progress, ch)
}
}
func (p *IMAPProvider) loadMessageInfoMap(rules transferRules, progress *Progress) map[string]map[string]imapMessageInfo {
res := map[string]map[string]imapMessageInfo{}
for rule := range rules.iterateActiveRules() {
if progress.shouldStop() {
break
}
mailboxName := rule.SourceMailbox.Name
var mailbox *imap.MailboxStatus
progress.callWrap(func() error {
var err error
mailbox, err = p.selectIn(mailboxName)
return err
})
if mailbox == nil {
log.WithField("rule", rule.SourceMailbox.Name).Warn("Failed to select into mailbox")
continue
}
if mailbox.Messages == 0 {
continue
}
messagesInfo := p.loadMessagesInfo(rule, progress, mailbox.UidValidity, mailbox.Messages)
res[rule.SourceMailbox.Name] = messagesInfo
progress.updateCount(rule.SourceMailbox.Name, uint(len(messagesInfo)))
}
progress.countsFinal()
return res
}
func (p *IMAPProvider) loadMessagesInfo(rule *Rule, progress *Progress, uidValidity, count uint32) map[string]imapMessageInfo {
p.timeIt.start("load", rule.SourceMailbox.Name)
defer p.timeIt.stop("load", rule.SourceMailbox.Name)
log := log.WithField("mailbox", rule.SourceMailbox.Name)
messagesInfo := map[string]imapMessageInfo{}
fetchItems := []imap.FetchItem{imap.FetchUid, imap.FetchRFC822Size}
if rule.HasTimeLimit() {
fetchItems = append(fetchItems, imap.FetchEnvelope)
}
processMessageCallback := func(imapMessage *imap.Message) {
if rule.HasTimeLimit() {
t := imapMessage.Envelope.Date.Unix()
if t != 0 && !rule.isTimeInRange(t) {
log.WithField("uid", imapMessage.Uid).Debug("Message skipped due to time")
return
}
}
id := getUniqueMessageID(rule.SourceMailbox.Name, uidValidity, imapMessage.Uid)
// We use ID as key to ensure we have every unique message only once.
// Some IMAP servers responded twice the same message...
messagesInfo[id] = imapMessageInfo{
id: id,
uid: imapMessage.Uid,
size: imapMessage.Size,
}
progress.addMessage(id, []string{rule.SourceMailbox.Name}, rule.TargetMailboxNames())
}
pageStart := uint32(1)
pageEnd := imapPageSize
for {
if progress.shouldStop() || pageStart > count {
break
}
// Some servers do not accept message sequence number higher than the total count.
if pageEnd > count {
pageEnd = count
}
seqSet := &imap.SeqSet{}
seqSet.AddRange(pageStart, pageEnd)
err := p.fetch(rule.SourceMailbox.Name, seqSet, fetchItems, processMessageCallback)
if err != nil {
log.WithError(err).WithField("idx", seqSet).Warning("Load batch fetch failed, trying one by one")
for ; pageStart <= pageEnd; pageStart++ {
seqSet := &imap.SeqSet{}
seqSet.AddNum(pageStart)
if err := p.fetch(rule.SourceMailbox.Name, seqSet, fetchItems, processMessageCallback); err != nil {
log.WithError(err).WithField("idx", seqSet).Warning("Load fetch failed, skipping the message")
}
}
}
pageStart = pageEnd + 1
pageEnd += imapPageSize
}
return messagesInfo
}
func (p *IMAPProvider) transferTo(rule *Rule, messagesInfo map[string]imapMessageInfo, progress *Progress, ch chan<- Message) {
progress.callWrap(func() error {
_, err := p.selectIn(rule.SourceMailbox.Name)
return err
})
seqSet := &imap.SeqSet{}
seqSetSize := uint32(0)
uidToID := map[uint32]string{}
for _, messageInfo := range messagesInfo {
if progress.shouldStop() {
break
}
if seqSetSize != 0 && (seqSetSize+messageInfo.size) > imapMaxFetchSize {
log.WithField("mailbox", rule.SourceMailbox.Name).WithField("seq", seqSet).WithField("size", seqSetSize).Debug("Fetching messages")
p.exportMessages(rule, progress, ch, seqSet, uidToID)
seqSet = &imap.SeqSet{}
seqSetSize = 0
uidToID = map[uint32]string{}
}
seqSet.AddNum(messageInfo.uid)
seqSetSize += messageInfo.size
uidToID[messageInfo.uid] = messageInfo.id
}
if len(uidToID) != 0 {
log.WithField("mailbox", rule.SourceMailbox.Name).WithField("seq", seqSet).WithField("size", seqSetSize).Debug("Fetching messages")
p.exportMessages(rule, progress, ch, seqSet, uidToID)
}
}
func (p *IMAPProvider) exportMessages(rule *Rule, progress *Progress, ch chan<- Message, seqSet *imap.SeqSet, uidToID map[uint32]string) {
section := &imap.BodySectionName{}
items := []imap.FetchItem{imap.FetchUid, imap.FetchFlags, section.FetchItem()}
processMessageCallback := func(imapMessage *imap.Message) {
if progress.shouldStop() {
return
}
id, ok := uidToID[imapMessage.Uid]
// Sometimes, server sends not requested messages.
if !ok {
log.WithField("uid", imapMessage.Uid).Warning("Message skipped: not requested")
return
}
// Sometimes, server sends message twice, once with body and once without it.
bodyReader := imapMessage.GetBody(section)
if bodyReader == nil {
log.WithField("uid", imapMessage.Uid).Warning("Message skipped: no body")
return
}
body, err := ioutil.ReadAll(bodyReader)
progress.messageExported(id, body, err)
if err == nil {
msg := p.exportMessage(rule, id, imapMessage, body)
p.timeIt.stop("fetch", rule.SourceMailbox.Name)
ch <- msg
p.timeIt.start("fetch", rule.SourceMailbox.Name)
}
}
p.timeIt.start("fetch", rule.SourceMailbox.Name)
progress.callWrap(func() error {
return p.uidFetch(rule.SourceMailbox.Name, seqSet, items, processMessageCallback)
})
p.timeIt.stop("fetch", rule.SourceMailbox.Name)
}
func (p *IMAPProvider) exportMessage(rule *Rule, id string, imapMessage *imap.Message, body []byte) Message {
unread := true
for _, flag := range imapMessage.Flags {
if flag == imap.SeenFlag {
unread = false
}
}
return Message{
ID: id,
Unread: unread,
Body: body,
Sources: []Mailbox{rule.SourceMailbox},
Targets: rule.TargetMailboxes,
}
}
func getUniqueMessageID(mailboxName string, uidValidity, uid uint32) string {
return fmt.Sprintf("%s_%d:%d", mailboxName, uidValidity, uid)
}

100
internal/transfer/provider_imap_test.go
View File

@ -1,100 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"fmt"
"testing"
"github.com/emersion/go-imap"
gomock "github.com/golang/mock/gomock"
"github.com/pkg/errors"
r "github.com/stretchr/testify/require"
)
func newTestIMAPProvider(t *testing.T, m mocks) *IMAPProvider {
m.imapClientProvider.EXPECT().State().Return(imap.ConnectedState).AnyTimes()
m.imapClientProvider.EXPECT().Capability().Return(map[string]bool{
"AUTH": true,
}, nil).AnyTimes()
dialer := func(string) (IMAPClientProvider, error) {
return m.imapClientProvider, nil
}
provider, err := newIMAPProvider(dialer, "user", "pass", "host", "42")
r.NoError(t, err)
return provider
}
func TestProviderIMAPLoadMessagesInfo(t *testing.T) {
m := initMocks(t)
defer m.ctrl.Finish()
provider := newTestIMAPProvider(t, m)
progress := newProgress(log, nil)
drainProgressUpdateChannel(&progress)
rule := &Rule{SourceMailbox: Mailbox{Name: "Mailbox"}}
uidValidity := 1
count := 2200
failingIndex := 2100
m.imapClientProvider.EXPECT().Select(rule.SourceMailbox.Name, gomock.Any()).Return(&imap.MailboxStatus{}, nil).AnyTimes()
m.imapClientProvider.EXPECT().
Fetch(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(seqSet *imap.SeqSet, items []imap.FetchItem, ch chan *imap.Message) error {
defer close(ch)
for _, seq := range seqSet.Set {
for i := seq.Start; i <= seq.Stop; i++ {
if int(i) == failingIndex {
return errors.New("internal server error")
}
ch <- &imap.Message{
SeqNum: i,
Uid: i * 10,
Size: i * 100,
}
}
}
return nil
}).
// 2200 messages is split into two batches (2000 and 200),
// the second one fails and makes 200 calls (one-by-one).
// Plus two failed requests are repeated `imapRetries` times.
Times(2 + 200 + (2 * (imapRetries - 1)))
messageInfo := provider.loadMessagesInfo(rule, &progress, uint32(uidValidity), uint32(count))
r.Equal(t, count-1, len(messageInfo)) // One message produces internal server error.
for index := 1; index <= count; index++ {
uid := index * 10
key := fmt.Sprintf("%s_%d:%d", rule.SourceMailbox.Name, uidValidity, uid)
if index == failingIndex {
r.Empty(t, messageInfo[key])
continue
}
r.Equal(t, imapMessageInfo{
id: key,
uid: uint32(uid),
size: uint32(index * 100),
}, messageInfo[key])
}
}

311
internal/transfer/provider_imap_utils.go
View File

@ -1,311 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"crypto/tls"
"fmt"
"net"
"net/http"
"strings"
"time"
imapID "github.com/ProtonMail/go-imap-id"
"github.com/ProtonMail/proton-bridge/internal/constants"
"github.com/emersion/go-imap"
imapClient "github.com/emersion/go-imap/client"
"github.com/emersion/go-sasl"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
const (
imapDialTimeout = 5 * time.Second
imapRetries = 10
imapReconnectTimeout = 30 * time.Minute
imapReconnectSleep = time.Minute
protonStatusURL = "http://protonstatus.com/vpn_status"
)
type imapErrorLogger struct {
log *logrus.Entry
}
func (l *imapErrorLogger) Printf(f string, v ...interface{}) {
l.log.Errorf(f, v...)
}
func (l *imapErrorLogger) Println(v ...interface{}) {
l.log.Errorln(v...)
}
func imapClientDial(addr string) (IMAPClientProvider, error) {
if _, err := net.DialTimeout("tcp", addr, imapDialTimeout); err != nil {
return nil, errors.Wrap(err, "failed to dial server")
}
client, err := imapClientDialHelper(addr)
if err == nil {
client.ErrorLog = &imapErrorLogger{logrus.WithField("pkg", "imap-client")}
// Logrus `WriterLevel` fails for big messages because of bufio.MaxScanTokenSize limit.
// Also, this spams a lot, uncomment once needed during development.
// client.SetDebug(imap.NewDebugWriter(
// logrus.WithField("pkg", "imap/client").WriterLevel(logrus.TraceLevel),
// logrus.WithField("pkg", "imap/server").WriterLevel(logrus.TraceLevel),
// ))
}
return client, err
}
func imapClientDialHelper(addr string) (*imapClient.Client, error) {
host, _, _ := net.SplitHostPort(addr)
if host == "127.0.0.1" {
return imapClient.Dial(addr)
}
// IMAP mail.yahoo.com has problem with golang TLS 1.3 implementation
// with weird behaviour, i.e., Yahoo does not return error during dial
// or handshake but server does logs out right after successful login
// leaving no time to perform any action.
// Limiting TLS to version 1.2 is working just fine.
var tlsConf *tls.Config
if strings.Contains(strings.ToLower(host), "yahoo") {
log.Warning("Yahoo server detected: limiting maximal TLS version to 1.2.")
tlsConf = &tls.Config{MaxVersion: tls.VersionTLS12} //nolint[gosec] G402
}
return imapClient.DialTLS(addr, tlsConf)
}
func (p *IMAPProvider) ensureConnection(callback func() error) error {
return p.ensureConnectionAndSelection(callback, "")
}
func (p *IMAPProvider) ensureConnectionAndSelection(callback func() error, ensureSelectedIn string) error {
var callErr error
for i := 1; i <= imapRetries; i++ {
callErr = callback()
if callErr == nil {
return nil
}
log.WithField("attempt", i).WithError(callErr).Warning("IMAP call failed, trying reconnect")
err := p.tryReconnect(ensureSelectedIn)
if err != nil {
return err
}
}
return errors.Wrap(callErr, "too many retries")
}
func (p *IMAPProvider) tryReconnect(ensureSelectedIn string) error {
start := time.Now()
var previousErr error
for {
if time.Since(start) > imapReconnectTimeout {
return previousErr
}
err := checkConnection()
log.WithError(err).Debug("Connection check")
if err != nil {
time.Sleep(imapReconnectSleep)
previousErr = err
continue
}
err = p.reauth()
log.WithError(err).Debug("Reauth")
if err != nil {
time.Sleep(imapReconnectSleep)
previousErr = err
continue
}
if ensureSelectedIn != "" {
_, err = p.client.Select(ensureSelectedIn, true)
log.WithError(err).Debug("Reselect")
if err != nil {
previousErr = err
continue
}
}
break
}
return nil
}
func (p *IMAPProvider) reauth() error {
var state imap.ConnState
// In some cases once go-imap fails, we cannot issue another command
// because it would dead-lock. Let's simply ignore it, we want to open
// new connection anyway.
ch := make(chan struct{})
go func() {
defer close(ch)
if _, err := p.client.Capability(); err != nil {
state = p.client.State()
}
}()
select {
case <-ch:
case <-time.After(30 * time.Second):
}
log.WithField("addr", p.addr).WithField("state", state).Debug("Reconnecting")
p.client = nil
return p.auth()
}
func (p *IMAPProvider) auth() error { //nolint[funlen]
log := log.WithField("addr", p.addr)
log.Info("Connecting to server")
client, err := p.clientDialer(p.addr)
if err != nil {
return ErrIMAPConnection{imapError{Err: err, Message: "failed to connect to server"}}
}
p.client = client
log.Info("Connected")
if (p.client.State() & imap.AuthenticatedState) == imap.AuthenticatedState {
return nil
}
capability, err := p.client.Capability()
log.WithField("capability", capability).WithError(err).Debug("Server capability")
if err != nil {
return ErrIMAPConnection{imapError{Err: err, Message: "failed to get capabilities"}}
}
// SASL AUTH PLAIN
if ok, _ := p.client.SupportAuth("PLAIN"); p.client.State() == imap.NotAuthenticatedState && ok {
log.Debug("Trying plain auth")
authPlain := sasl.NewPlainClient("", p.username, p.password)
if err = p.client.Authenticate(authPlain); err != nil {
return ErrIMAPAuth{imapError{Err: err, Message: "plain auth failed"}}
}
}
// LOGIN: if the server reports the IMAP4rev1 capability then it is standards conformant and must support login.
if ok, _ := p.client.Support("IMAP4rev1"); p.client.State() == imap.NotAuthenticatedState && ok {
log.Debug("Trying login")
if err = p.client.Login(p.username, p.password); err != nil {
return ErrIMAPAuth{imapError{Err: err, Message: "login failed"}}
}
}
if p.client.State() == imap.NotAuthenticatedState {
return ErrIMAPAuthMethod{imapError{Err: err, Message: "unknown auth method"}}
}
log.Info("Logged in")
if c, ok := p.client.(*imapClient.Client); ok {
idClient := imapID.NewClient(c)
if ok, err := idClient.SupportID(); err == nil && ok {
serverID, err := idClient.ID(imapID.ID{
imapID.FieldName: "ImportExport",
imapID.FieldVersion: constants.Version,
})
log.WithField("ID", serverID).WithError(err).Debug("Server info")
}
}
return err
}
func (p *IMAPProvider) list() (mailboxes []*imap.MailboxInfo, err error) {
err = p.ensureConnection(func() error {
mailboxesCh := make(chan *imap.MailboxInfo)
doneCh := make(chan error)
go func() {
doneCh <- p.client.List("", "*", mailboxesCh)
}()
for mailbox := range mailboxesCh {
mailboxes = append(mailboxes, mailbox)
}
return <-doneCh
})
return
}
func (p *IMAPProvider) selectIn(mailboxName string) (mailbox *imap.MailboxStatus, err error) {
err = p.ensureConnection(func() error {
mailbox, err = p.client.Select(mailboxName, true)
return err
})
return
}
func (p *IMAPProvider) fetch(ensureSelectedIn string, seqSet *imap.SeqSet, items []imap.FetchItem, processMessageCallback func(m *imap.Message)) error {
return p.fetchHelper(false, ensureSelectedIn, seqSet, items, processMessageCallback)
}
func (p *IMAPProvider) uidFetch(ensureSelectedIn string, seqSet *imap.SeqSet, items []imap.FetchItem, processMessageCallback func(m *imap.Message)) error {
return p.fetchHelper(true, ensureSelectedIn, seqSet, items, processMessageCallback)
}
func (p *IMAPProvider) fetchHelper(uid bool, ensureSelectedIn string, seqSet *imap.SeqSet, items []imap.FetchItem, processMessageCallback func(m *imap.Message)) error {
return p.ensureConnectionAndSelection(func() error {
messagesCh := make(chan *imap.Message)
doneCh := make(chan error)
go func() {
if uid {
doneCh <- p.client.UidFetch(seqSet, items, messagesCh)
} else {
doneCh <- p.client.Fetch(seqSet, items, messagesCh)
}
}()
for message := range messagesCh {
processMessageCallback(message)
}
err := <-doneCh
return err
}, ensureSelectedIn)
}
// checkConnection returns an error if there is no internet connection.
// Note we don't want to use client manager because it only reports connection
// issues with API; we are only interested here whether we can reach
// third-party IMAP servers.
func checkConnection() error {
client := &http.Client{Timeout: time.Second * 10}
resp, err := client.Get(protonStatusURL)
if err != nil {
return err
}
_ = resp.Body.Close()
if resp.StatusCode != 200 {
return fmt.Errorf("HTTP status code %d", resp.StatusCode)
}
return nil
}

68
internal/transfer/provider_local.go
View File

@ -1,68 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
// LocalProvider implements import from local EML and MBOX file structure.
type LocalProvider struct {
root string
emlProvider *EMLProvider
mboxProvider *MBOXProvider
}
func NewLocalProvider(root string) *LocalProvider {
return &LocalProvider{
root: root,
emlProvider: NewEMLProvider(root),
mboxProvider: NewMBOXProvider(root),
}
}
// ID is used for generating transfer ID by combining source and target ID.
// We want to keep the same rules for import from or export to local files
// no matter exact path, therefore it returns constant.
// The same as EML and MBOX.
func (p *LocalProvider) ID() string {
return "local" //nolint[goconst]
}
// Mailboxes returns all available folder names from root of EML and MBOX files.
func (p *LocalProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) {
mailboxes, err := p.emlProvider.Mailboxes(includeEmpty, includeAllMail)
if err != nil {
return nil, err
}
mboxMailboxes, err := p.mboxProvider.Mailboxes(includeEmpty, includeAllMail)
if err != nil {
return nil, err
}
for _, mboxMailbox := range mboxMailboxes {
found := false
for _, mailboxes := range mailboxes {
if mboxMailbox.Name == mailboxes.Name {
found = true
break
}
}
if !found {
mailboxes = append(mailboxes, mboxMailbox)
}
}
return mailboxes, nil
}

42
internal/transfer/provider_local_source.go
View File

@ -1,42 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"sync"
)
// TransferTo exports messages based on rules to channel.
func (p *LocalProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) {
log.Info("Started transfer from EML and MBOX to channel")
defer log.Info("Finished transfer from EML and MBOX to channel")
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
p.emlProvider.TransferTo(rules, progress, ch)
}()
go func() {
defer wg.Done()
p.mboxProvider.TransferTo(rules, progress, ch)
}()
wg.Wait()
}

77
internal/transfer/provider_local_test.go
View File

@ -1,77 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"fmt"
"testing"
r "github.com/stretchr/testify/require"
)
func newTestLocalProvider(path string) *LocalProvider {
if path == "" {
path = "testdata/emlmbox"
}
return NewLocalProvider(path)
}
func TestLocalProviderMailboxes(t *testing.T) {
provider := newTestLocalProvider("")
tests := []struct {
includeEmpty bool
wantMailboxes []Mailbox
}{
{true, []Mailbox{
{Name: "Foo"},
{Name: "emlmbox"},
{Name: "Inbox"},
}},
{false, []Mailbox{
{Name: "Foo"},
{Name: "Inbox"},
}},
}
for _, tc := range tests {
tc := tc
t.Run(fmt.Sprintf("%v", tc.includeEmpty), func(t *testing.T) {
mailboxes, err := provider.Mailboxes(tc.includeEmpty, false)
r.NoError(t, err)
r.Equal(t, tc.wantMailboxes, mailboxes)
})
}
}
func TestLocalProviderTransferTo(t *testing.T) {
provider := newTestLocalProvider("")
rules, rulesClose := newTestRules(t)
defer rulesClose()
setupEMLMBOXRules(rules)
testTransferTo(t, rules, provider, []string{
"Foo/msg.eml",
"Inbox.mbox:1",
})
}
func setupEMLMBOXRules(rules transferRules) {
_ = rules.setRule(Mailbox{Name: "Inbox"}, []Mailbox{{Name: "Inbox"}}, 0, 0)
_ = rules.setRule(Mailbox{Name: "Foo"}, []Mailbox{{Name: "Foo"}}, 0, 0)
}

102
internal/transfer/provider_mbox.go
View File

@ -1,102 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"os"
"path/filepath"
"strings"
"github.com/pkg/errors"
)
// MBOXProvider implements import and export to/from MBOX structure.
type MBOXProvider struct {
root string
}
func NewMBOXProvider(root string) *MBOXProvider {
return &MBOXProvider{
root: root,
}
}
// ID is used for generating transfer ID by combining source and target ID.
// We want to keep the same rules for import from or export to local files
// no matter exact path, therefore it returns constant. The same as EML.
func (p *MBOXProvider) ID() string {
return "local" //nolint[goconst]
}
// Mailboxes returns all available folder names from root of EML files.
// In case the same folder name is used more than once (for example root/a/foo
// and root/b/foo), it's treated as the same folder.
func (p *MBOXProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) {
filePaths, err := getAllPathsWithSuffix(p.root, ".mbox")
if err != nil {
return nil, err
}
mailboxNames := map[string]bool{}
for _, filePath := range filePaths {
fileName := filepath.Base(filePath)
filePath, err := p.handleAppleMailMBOXStructure(filePath)
if err != nil {
log.WithError(err).Warn("Failed to handle MBOX structure")
continue
}
mailboxName := strings.TrimSuffix(fileName, ".mbox")
mailboxNames[mailboxName] = true
labels, err := getGmailLabelsFromMboxFile(filepath.Join(p.root, filePath))
if err != nil {
log.WithError(err).Error("Failed to get gmail labels from mbox file")
continue
}
for label := range labels {
mailboxNames[label] = true
}
}
mailboxes := []Mailbox{}
for mailboxName := range mailboxNames {
mailboxes = append(mailboxes, Mailbox{
ID: "",
Name: mailboxName,
Color: "",
IsExclusive: false,
})
}
return mailboxes, nil
}
// handleAppleMailMBOXStructure changes the path of mailbox directory to
// the path of mbox file. Apple Mail MBOX exports has this structure:
// `Folder.mbox` directory with `mbox` file inside.
// Example: `Folder.mbox/mbox` (and this function converts `Folder.mbox`
// to `Folder.mbox/mbox`).
func (p *MBOXProvider) handleAppleMailMBOXStructure(filePath string) (string, error) {
if info, err := os.Stat(filepath.Join(p.root, filePath)); err == nil && info.IsDir() {
if _, err := os.Stat(filepath.Join(p.root, filePath, "mbox")); err != nil {
return "", errors.Wrap(err, "wrong mbox structure")
}
return filepath.Join(filePath, "mbox"), nil
}
return filePath, nil
}

118
internal/transfer/provider_mbox_gmail_labels.go
View File

@ -1,118 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"bufio"
"bytes"
"io"
"mime"
"os"
"strings"
)
type stringSet map[string]bool
const xGmailLabelsHeader = "X-Gmail-Labels"
// filteredOutGmailLabels is set of labels which we don't want to show to users
// as they might be auto-generated by Gmail and unwanted.
var filteredOutGmailLabels = []string{ //nolint[gochecknoglobals]
"Unread",
"Opened",
"IMAP_Junk",
"IMAP_NonJunk",
"IMAP_NotJunk",
"IMAP_$NotJunk",
}
func getGmailLabelsFromMboxFile(filePath string) (stringSet, error) {
f, err := os.Open(filePath) //nolint[gosec]
if err != nil {
return nil, err
}
return getGmailLabelsFromMboxReader(f)
}
func getGmailLabelsFromMboxReader(f io.Reader) (stringSet, error) {
allLabels := stringSet{}
// Scanner is not used as it does not support long lines and some mbox
// files contain very long lines even though that should not be happening.
r := bufio.NewReader(f)
for {
b, isPrefix, err := r.ReadLine()
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
for isPrefix {
_, isPrefix, err = r.ReadLine()
if err != nil {
break
}
}
if bytes.HasPrefix(b, []byte(xGmailLabelsHeader)) {
for label := range getGmailLabelsFromValue(string(b)) {
allLabels[label] = true
}
}
}
return allLabels, nil
}
func getGmailLabelsFromMessage(body []byte) (stringSet, error) {
header, err := getMessageHeader(body)
if err != nil {
return nil, err
}
labels := header.Get(xGmailLabelsHeader)
return getGmailLabelsFromValue(labels), nil
}
func getGmailLabelsFromValue(value string) stringSet {
value = strings.TrimPrefix(value, xGmailLabelsHeader+":")
if decoded, err := new(mime.WordDecoder).DecodeHeader(value); err != nil {
log.WithError(err).Error("Failed to decode header")
} else {
value = decoded
}
labels := stringSet{}
for _, label := range strings.Split(value, ",") {
label = strings.TrimSpace(label)
if label == "" {
continue
}
skip := false
for _, filteredOutLabel := range filteredOutGmailLabels {
if label == filteredOutLabel {
skip = true
break
}
}
if skip {
continue
}
labels[label] = true
}
return labels
}

135
internal/transfer/provider_mbox_gmail_labels_test.go
View File

@ -1,135 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"fmt"
"strings"
"testing"
r "github.com/stretchr/testify/require"
)
func TestGetGmailLabelsFromMboxReader(t *testing.T) {
mboxFile := `From - Mon May 4 16:40:31 2020
Subject: Test 1
X-Gmail-Labels: Foo,Bar
hello
From - Mon May 4 16:40:31 2020
Subject: Test 2
X-Gmail-Labels: Foo , Baz
hello
From - Mon May 4 16:40:31 2020
Subject: Test 3
X-Gmail-Labels: ,
hello
From - Mon May 4 16:40:31 2020
Subject: Test 4
X-Gmail-Labels:
hello
From - Mon May 4 16:40:31 2020
Subject: Test 5
hello
`
mboxReader := strings.NewReader(mboxFile)
labels, err := getGmailLabelsFromMboxReader(mboxReader)
r.NoError(t, err)
r.Equal(t, toSet("Foo", "Bar", "Baz"), labels)
}
func TestGetGmailLabelsFromMessage(t *testing.T) {
tests := []struct {
body string
wantLabels stringSet
}{
{`Subject: One
X-Gmail-Labels: Foo,Bar
Hello
`, toSet("Foo", "Bar")},
{`Subject: Two
X-Gmail-Labels: Foo , Bar ,
Hello
`, toSet("Foo", "Bar")},
{`Subject: Three
X-Gmail-Labels: ,
Hello
`, toSet()},
{`Subject: Four
X-Gmail-Labels:
Hello
`, toSet()},
{`Subject: Five
Hello
`, toSet()},
}
for _, tc := range tests {
tc := tc
t.Run(fmt.Sprintf("%v", tc.body), func(t *testing.T) {
labels, err := getGmailLabelsFromMessage([]byte(tc.body))
r.NoError(t, err)
r.Equal(t, tc.wantLabels, labels)
})
}
}
func TestGetGmailLabelsFromValue(t *testing.T) {
tests := []struct {
value string
wantLabels stringSet
}{
{"Foo,Bar", toSet("Foo", "Bar")},
{" Foo , Bar ", toSet("Foo", "Bar")},
{" Foo , Bar , ", toSet("Foo", "Bar")},
{" Foo Bar ", toSet("Foo Bar")},
{" , ", toSet()},
{" ", toSet()},
{"", toSet()},
{"=?UTF-8?Q?Archived,Category_personal,test_=F0=9F=98=80=F0=9F=99=83?=", toSet("Archived", "Category personal", "test 😀🙃")},
{"IMAP_NotJunk,Foo,Opened,bar,Unread", toSet("Foo", "bar")},
}
for _, tc := range tests {
tc := tc
t.Run(fmt.Sprintf("%v", tc.value), func(t *testing.T) {
labels := getGmailLabelsFromValue(tc.value)
r.Equal(t, tc.wantLabels, labels)
})
}
}
func toSet(items ...string) stringSet {
set := map[string]bool{}
for _, item := range items {
set[item] = true
}
return set
}

255
internal/transfer/provider_mbox_source.go
View File

@ -1,255 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
"github.com/emersion/go-mbox"
"github.com/pkg/errors"
)
// TransferTo exports messages based on rules to channel.
func (p *MBOXProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) {
log.Info("Started transfer from MBOX to channel")
defer log.Info("Finished transfer from MBOX to channel")
filePathsPerFolder, err := p.getFilePathsPerFolder()
if err != nil {
progress.fatal(err)
return
}
if len(filePathsPerFolder) == 0 {
return
}
for folderName, filePaths := range filePathsPerFolder {
log.WithField("folder", folderName).Debug("Estimating folder counts")
for _, filePath := range filePaths {
if progress.shouldStop() {
break
}
p.updateCount(progress, filePath)
}
}
progress.countsFinal()
for folderName, filePaths := range filePathsPerFolder {
log.WithField("folder", folderName).Debug("Processing folder")
for _, filePath := range filePaths {
if progress.shouldStop() {
break
}
p.transferTo(rules, progress, ch, folderName, filePath)
}
}
}
func (p *MBOXProvider) getFilePathsPerFolder() (map[string][]string, error) {
filePaths, err := getAllPathsWithSuffix(p.root, ".mbox")
if err != nil {
return nil, err
}
filePathsMap := map[string][]string{}
for _, filePath := range filePaths {
fileName := filepath.Base(filePath)
filePath, err := p.handleAppleMailMBOXStructure(filePath)
// Skip unsupported MBOX structures. It was already filtered out in configuration step.
if err != nil {
continue
}
folder := strings.TrimSuffix(fileName, ".mbox")
filePathsMap[folder] = append(filePathsMap[folder], filePath)
}
return filePathsMap, nil
}
func (p *MBOXProvider) updateCount(progress *Progress, filePath string) {
mboxReader := p.openMbox(progress, filePath)
if mboxReader == nil {
return
}
count := 0
for {
_, err := mboxReader.NextMessage()
if err == io.EOF {
break
} else if err != nil {
progress.fatal(err)
break
}
count++
}
progress.updateCount(filePath, uint(count))
}
func (p *MBOXProvider) transferTo(rules transferRules, progress *Progress, ch chan<- Message, folderName, filePath string) {
mboxReader := p.openMbox(progress, filePath)
if mboxReader == nil {
return
}
index := 0
for {
if progress.shouldStop() {
break
}
index++
id := fmt.Sprintf("%s:%d", filePath, index)
msgReader, err := mboxReader.NextMessage()
if err == io.EOF {
break
} else if err != nil {
progress.fatal(err)
break
}
msg, err := p.exportMessage(rules, folderName, id, msgReader)
progress.addMessage(id, msg.sourceNames(), msg.targetNames())
if err == nil && len(msg.Targets) == 0 {
progress.messageSkipped(id)
continue
}
progress.messageExported(id, msg.Body, err)
if err == nil {
ch <- msg
}
}
}
func (p *MBOXProvider) exportMessage(rules transferRules, folderName, id string, msgReader io.Reader) (Message, error) {
body, err := ioutil.ReadAll(msgReader)
if err != nil {
return Message{}, errors.Wrap(err, "failed to read message")
}
msgRules := p.getMessageRules(rules, folderName, id, body)
sources := p.getMessageSources(msgRules)
targets := p.getMessageTargets(msgRules, id, body)
return Message{
ID: id,
Unread: false,
Body: body,
Sources: sources,
Targets: targets,
}, nil
}
func (p *MBOXProvider) getMessageRules(rules transferRules, folderName, id string, body []byte) []*Rule {
msgRules := []*Rule{}
folderRule, err := rules.getRuleBySourceMailboxName(folderName)
if err != nil {
log.WithField("msg", id).WithField("source", folderName).Debug("Message source doesn't have a rule")
} else if folderRule.Active {
msgRules = append(msgRules, folderRule)
}
gmailLabels, err := getGmailLabelsFromMessage(body)
if err != nil {
log.WithError(err).Error("Failed to get gmail labels, ")
} else {
for label := range gmailLabels {
rule, err := rules.getRuleBySourceMailboxName(label)
if err != nil {
log.WithField("msg", id).WithField("source", label).Debug("Message source doesn't have a rule")
continue
}
if rule.Active {
msgRules = append(msgRules, rule)
}
}
}
return msgRules
}
func (p *MBOXProvider) getMessageSources(msgRules []*Rule) []Mailbox {
sources := []Mailbox{}
for _, rule := range msgRules {
sources = append(sources, rule.SourceMailbox)
}
return sources
}
func (p *MBOXProvider) getMessageTargets(msgRules []*Rule, id string, body []byte) []Mailbox {
targets := []Mailbox{}
haveExclusiveMailbox := false
for _, rule := range msgRules {
// Read and check time in body only if the rule specifies it
// to not waste energy.
if rule.HasTimeLimit() {
msgTime, err := getMessageTime(body)
if err != nil {
log.WithError(err).Error("Failed to parse time, time check skipped")
} else if !rule.isTimeInRange(msgTime) {
log.WithField("msg", id).WithField("source", rule.SourceMailbox.Name).Debug("Message skipped due to time")
continue
}
}
for _, newTarget := range rule.TargetMailboxes {
// msgRules is sorted. The first rule is based on the folder name,
// followed by the order from X-Gmail-Labels. The rule based on
// the folder name should have priority for exclusive target.
if newTarget.IsExclusive && haveExclusiveMailbox {
continue
}
found := false
for _, target := range targets {
if target.Hash() == newTarget.Hash() {
found = true
break
}
}
if found {
continue
}
if newTarget.IsExclusive {
haveExclusiveMailbox = true
}
targets = append(targets, newTarget)
}
}
return targets
}
func (p *MBOXProvider) openMbox(progress *Progress, mboxPath string) *mbox.Reader {
mboxPath = filepath.Join(p.root, mboxPath)
mboxFile, err := os.Open(mboxPath) //nolint[gosec]
if os.IsNotExist(err) {
return nil
} else if err != nil {
progress.fatal(err)
return nil
}
return mbox.NewReader(mboxFile)
}

97
internal/transfer/provider_mbox_target.go
View File

@ -1,97 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"os"
"path/filepath"
"strings"
"time"
"github.com/emersion/go-mbox"
"github.com/hashicorp/go-multierror"
)
// DefaultMailboxes returns the default mailboxes for default rules if no other is found.
func (p *MBOXProvider) DefaultMailboxes(sourceMailbox Mailbox) []Mailbox {
return []Mailbox{{
Name: sourceMailbox.Name,
}}
}
// CreateMailbox does nothing. Files are created dynamically during the import.
func (p *MBOXProvider) CreateMailbox(mailbox Mailbox) (Mailbox, error) {
return mailbox, nil
}
// TransferFrom imports messages from channel.
func (p *MBOXProvider) TransferFrom(rules transferRules, progress *Progress, ch <-chan Message) {
log.Info("Started transfer from channel to MBOX")
defer log.Info("Finished transfer from channel to MBOX")
for msg := range ch {
if progress.shouldStop() {
break
}
err := p.writeMessage(msg)
progress.messageImported(msg.ID, "", err)
}
}
func (p *MBOXProvider) writeMessage(msg Message) error {
var multiErr error
for _, mailbox := range msg.Targets {
mboxName := sanitizeFileName(mailbox.Name)
if !strings.HasSuffix(mboxName, ".mbox") {
mboxName += ".mbox"
}
mboxPath := filepath.Join(p.root, mboxName)
mboxFile, err := os.OpenFile(filepath.Clean(mboxPath), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
if err != nil {
multiErr = multierror.Append(multiErr, err)
continue
}
msgFrom := ""
msgTime := time.Now()
if header, err := getMessageHeader(msg.Body); err == nil {
if date, err := header.Date(); err == nil {
msgTime = date
}
if addresses, err := header.AddressList("from"); err == nil && len(addresses) > 0 {
msgFrom = addresses[0].Address
}
}
mboxWriter := mbox.NewWriter(mboxFile)
messageWriter, err := mboxWriter.CreateMessage(msgFrom, msgTime)
if err != nil {
multiErr = multierror.Append(multiErr, err)
continue
}
_, err = messageWriter.Write(msg.Body)
if err != nil {
multiErr = multierror.Append(multiErr, err)
continue
}
}
return multiErr
}

223
internal/transfer/provider_mbox_test.go
View File

@ -1,223 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"fmt"
"io/ioutil"
"os"
"testing"
"time"
r "github.com/stretchr/testify/require"
)
func newTestMBOXProvider(path string) *MBOXProvider {
if path == "" {
path = "testdata/mbox"
}
return NewMBOXProvider(path)
}
func TestMBOXProviderMailboxes(t *testing.T) {
tests := []struct {
provider *MBOXProvider
includeEmpty bool
wantMailboxes []Mailbox
}{
{newTestMBOXProvider(""), true, []Mailbox{
{Name: "All Mail"},
{Name: "Foo"},
{Name: "Bar"},
{Name: "Inbox"},
}},
{newTestMBOXProvider(""), false, []Mailbox{
{Name: "All Mail"},
{Name: "Foo"},
{Name: "Bar"},
{Name: "Inbox"},
}},
{newTestMBOXProvider("testdata/mbox-applemail"), true, []Mailbox{
{Name: "All Mail"},
{Name: "Foo"},
{Name: "Bar"},
}},
}
for _, tc := range tests {
tc := tc
t.Run(fmt.Sprintf("%v", tc.includeEmpty), func(t *testing.T) {
mailboxes, err := tc.provider.Mailboxes(tc.includeEmpty, false)
r.NoError(t, err)
r.ElementsMatch(t, tc.wantMailboxes, mailboxes)
})
}
}
func TestMBOXProviderTransferTo(t *testing.T) {
provider := newTestMBOXProvider("")
rules, rulesClose := newTestRules(t)
defer rulesClose()
setupMBOXRules(rules)
msgs := testTransferTo(t, rules, provider, []string{
"All Mail.mbox:1",
"All Mail.mbox:2",
"Foo.mbox:1",
"Inbox.mbox:1",
})
got := map[string][]string{}
for _, msg := range msgs {
got[msg.ID] = msg.targetNames()
}
r.Equal(t, map[string][]string{
"All Mail.mbox:1": {"Archive", "Foo"}, // Bar is not in rules.
"All Mail.mbox:2": {"Archive", "Foo"},
"Foo.mbox:1": {"Foo"},
"Inbox.mbox:1": {"Inbox"},
}, got)
}
func TestMBOXProviderTransferToAppleMail(t *testing.T) {
provider := newTestMBOXProvider("testdata/mbox-applemail")
rules, rulesClose := newTestRules(t)
defer rulesClose()
setupMBOXRules(rules)
msgs := testTransferTo(t, rules, provider, []string{
"All Mail.mbox/mbox:1",
"All Mail.mbox/mbox:2",
})
got := map[string][]string{}
for _, msg := range msgs {
got[msg.ID] = msg.targetNames()
}
r.Equal(t, map[string][]string{
"All Mail.mbox/mbox:1": {"Archive", "Foo"}, // Bar is not in rules.
"All Mail.mbox/mbox:2": {"Archive", "Foo"},
}, got)
}
func TestMBOXProviderTransferFrom(t *testing.T) {
dir, err := ioutil.TempDir("", "mbox")
r.NoError(t, err)
defer os.RemoveAll(dir) //nolint[errcheck]
provider := newTestMBOXProvider(dir)
rules, rulesClose := newTestRules(t)
defer rulesClose()
setupMBOXRules(rules)
testTransferFrom(t, rules, provider, []Message{
{ID: "Foo.mbox:1", Body: getTestMsgBody("msg"), Targets: []Mailbox{{Name: "Foo"}}},
})
checkMBOXFileStructure(t, dir, []string{
"Foo.mbox",
})
}
func TestMBOXProviderTransferFromTo(t *testing.T) {
dir, err := ioutil.TempDir("", "mbox")
r.NoError(t, err)
defer os.RemoveAll(dir) //nolint[errcheck]
source := newTestMBOXProvider("")
target := newTestMBOXProvider(dir)
rules, rulesClose := newTestRules(t)
defer rulesClose()
setupMBOXRules(rules)
testTransferFromTo(t, rules, source, target, 5*time.Second)
checkMBOXFileStructure(t, dir, []string{
"Archive.mbox",
"Foo.mbox",
"Inbox.mbox",
})
}
func TestMBOXProviderGetMessageRules(t *testing.T) {
provider := newTestMBOXProvider("")
body := []byte(`Subject: Test
X-Gmail-Labels: foo,bar
`)
rules := transferRules{
rules: map[string]*Rule{
"1": {Active: true, SourceMailbox: Mailbox{Name: "folder"}},
"2": {Active: false, SourceMailbox: Mailbox{Name: "foo"}},
"3": {Active: true, SourceMailbox: Mailbox{Name: "bar"}},
"4": {Active: false, SourceMailbox: Mailbox{Name: "baz"}},
"5": {Active: true, SourceMailbox: Mailbox{Name: "other"}},
},
}
gotRules := provider.getMessageRules(rules, "folder", "id", body)
r.Equal(t, 2, len(gotRules))
r.Equal(t, "folder", gotRules[0].SourceMailbox.Name)
r.Equal(t, "bar", gotRules[1].SourceMailbox.Name)
}
func TestMBOXProviderGetMessageTargetsReturnsOnlyOneFolder(t *testing.T) {
provider := newTestMBOXProvider("")
folderA := Mailbox{Name: "Folder A", IsExclusive: true}
folderB := Mailbox{Name: "Folder B", IsExclusive: true}
labelA := Mailbox{Name: "Label A", IsExclusive: false}
labelB := Mailbox{Name: "Label B", IsExclusive: false}
labelC := Mailbox{Name: "Label C", IsExclusive: false}
rule1 := &Rule{TargetMailboxes: []Mailbox{folderA, labelA, labelB}}
rule2 := &Rule{TargetMailboxes: []Mailbox{folderB, labelC}}
rule3 := &Rule{TargetMailboxes: []Mailbox{folderB}}
tests := []struct {
rules []*Rule
wantMailboxes []Mailbox
}{
{[]*Rule{}, []Mailbox{}},
{[]*Rule{rule1}, []Mailbox{folderA, labelA, labelB}},
{[]*Rule{rule1, rule2}, []Mailbox{folderA, labelA, labelB, labelC}},
{[]*Rule{rule1, rule3}, []Mailbox{folderA, labelA, labelB}},
{[]*Rule{rule3, rule1}, []Mailbox{folderB, labelA, labelB}},
}
for _, tc := range tests {
tc := tc
t.Run(fmt.Sprintf("%v", tc.rules), func(t *testing.T) {
mailboxes := provider.getMessageTargets(tc.rules, "", []byte(""))
r.Equal(t, tc.wantMailboxes, mailboxes)
})
}
}
func setupMBOXRules(rules transferRules) {
_ = rules.setRule(Mailbox{Name: "All Mail"}, []Mailbox{{Name: "Archive"}}, 0, 0)
_ = rules.setRule(Mailbox{Name: "Inbox"}, []Mailbox{{Name: "Inbox"}}, 0, 0)
_ = rules.setRule(Mailbox{Name: "Foo"}, []Mailbox{{Name: "Foo"}}, 0, 0)
}
func checkMBOXFileStructure(t *testing.T, root string, expectedFiles []string) {
files, err := getAllPathsWithSuffix(root, ".mbox")
r.NoError(t, err)
r.Equal(t, expectedFiles, files)
}

169
internal/transfer/provider_pmapi.go
View File

@ -1,169 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"context"
"sort"
"github.com/ProtonMail/gopenpgp/v2/crypto"
"github.com/ProtonMail/proton-bridge/pkg/message"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/pkg/errors"
)
const (
fetchWorkers = 20 // In how many workers to fetch message (group list on IMAP).
attachWorkers = 5 // In how many workers to fetch attachments (for one message).
buildWorkers = 20 // In how many workers to build messages.
)
// PMAPIProvider implements import and export to/from ProtonMail server.
type PMAPIProvider struct {
client pmapi.Client
userID string
addressID string
keyRing *crypto.KeyRing
builder *message.Builder
nextImportRequests map[string]*pmapi.ImportMsgReq // Key is msg transfer ID.
nextImportRequestsSize int
timeIt *timeIt
connection bool
}
// NewPMAPIProvider returns new PMAPIProvider.
func NewPMAPIProvider(client pmapi.Client, userID, addressID string) (*PMAPIProvider, error) {
provider := &PMAPIProvider{
client: client,
userID: userID,
addressID: addressID,
builder: message.NewBuilder(fetchWorkers, attachWorkers, buildWorkers),
nextImportRequests: map[string]*pmapi.ImportMsgReq{},
nextImportRequestsSize: 0,
timeIt: newTimeIt("pmapi"),
}
if addressID != "" {
keyRing, err := client.KeyRingForAddressID(addressID)
if err != nil {
return nil, errors.Wrap(err, "failed to get key ring")
}
provider.keyRing = keyRing
}
return provider, nil
}
// ID returns identifier of current setup of PMAPI provider.
// Identification is unique per user.
func (p *PMAPIProvider) ID() string {
return p.userID
}
// Mailboxes returns all available labels in ProtonMail account.
func (p *PMAPIProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) {
labels, err := p.client.ListLabels(context.Background())
if err != nil {
return nil, err
}
sortedLabels := byFoldersLabels(labels)
sort.Sort(sortedLabels)
emptyLabelsMap := map[string]bool{}
if !includeEmpty {
messagesCounts, err := p.client.CountMessages(context.Background(), p.addressID)
if err != nil {
return nil, err
}
for _, messagesCount := range messagesCounts {
if messagesCount.Total == 0 {
emptyLabelsMap[messagesCount.LabelID] = true
}
}
}
mailboxes := []Mailbox{}
for _, mailbox := range getSystemMailboxes(includeAllMail) {
if !includeEmpty && emptyLabelsMap[mailbox.ID] {
continue
}
mailboxes = append(mailboxes, mailbox)
}
for _, label := range sortedLabels {
if !includeEmpty && emptyLabelsMap[label.ID] {
continue
}
mailboxes = append(mailboxes, Mailbox{
ID: label.ID,
Name: label.Name,
Color: label.Color,
IsExclusive: bool(label.Exclusive),
})
}
return mailboxes, nil
}
func getSystemMailboxes(includeAllMail bool) []Mailbox {
mailboxes := []Mailbox{
{ID: pmapi.InboxLabel, Name: "Inbox", IsExclusive: true},
{ID: pmapi.DraftLabel, Name: "Drafts", IsExclusive: true},
{ID: pmapi.SentLabel, Name: "Sent", IsExclusive: true},
{ID: pmapi.StarredLabel, Name: "Starred", IsExclusive: true},
{ID: pmapi.ArchiveLabel, Name: "Archive", IsExclusive: true},
{ID: pmapi.SpamLabel, Name: "Spam", IsExclusive: true},
{ID: pmapi.TrashLabel, Name: "Trash", IsExclusive: true},
}
if includeAllMail {
mailboxes = append(mailboxes, Mailbox{
ID: pmapi.AllMailLabel,
Name: "All Mail",
IsExclusive: true,
})
}
return mailboxes
}
type byFoldersLabels []*pmapi.Label
func (l byFoldersLabels) Len() int {
return len(l)
}
func (l byFoldersLabels) Swap(i, j int) {
l[i], l[j] = l[j], l[i]
}
// Less sorts first folders, then labels, by user order.
func (l byFoldersLabels) Less(i, j int) bool {
if l[i].Exclusive && !l[j].Exclusive {
return true
}
if !l[i].Exclusive && l[j].Exclusive {
return false
}
return l[i].Order < l[j].Order
}

179
internal/transfer/provider_pmapi_source.go
View File

@ -1,179 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"context"
"errors"
"fmt"
"sync"
"github.com/ProtonMail/proton-bridge/pkg/message"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/sirupsen/logrus"
)
const pmapiListPageSize = 150
// TransferTo exports messages based on rules to channel.
func (p *PMAPIProvider) TransferTo(rules transferRules, progress *Progress, ch chan<- Message) {
log.Info("Started transfer from PMAPI to channel")
defer log.Info("Finished transfer from PMAPI to channel")
p.timeIt.clear()
defer p.timeIt.logResults()
// TransferTo cannot end sooner than loadCounts goroutine because
// loadCounts writes to channel in progress which would be closed.
// That can happen for really small accounts.
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
p.loadCounts(rules, progress)
}()
for rule := range rules.iterateActiveRules() {
p.transferTo(rule, progress, ch, rules.skipEncryptedMessages)
}
wg.Wait()
}
func (p *PMAPIProvider) loadCounts(rules transferRules, progress *Progress) {
for rule := range rules.iterateActiveRules() {
if progress.shouldStop() {
break
}
rule := rule
progress.callWrap(func() error {
_, total, err := p.listMessages(&pmapi.MessagesFilter{
AddressID: p.addressID,
LabelID: rule.SourceMailbox.ID,
Begin: rule.FromTime,
End: rule.ToTime,
Limit: 0,
})
if err != nil {
log.WithError(err).Warning("Problem to load counts")
return err
}
progress.updateCount(rule.SourceMailbox.Name, uint(total))
return nil
})
}
progress.countsFinal()
}
func (p *PMAPIProvider) transferTo(rule *Rule, progress *Progress, ch chan<- Message, skipEncryptedMessages bool) {
page := 0
for {
if progress.shouldStop() {
break
}
isLastPage := true
progress.callWrap(func() error {
// Would be better to filter by Begin and BeginID to be sure
// in case user deletes messages during the process, no message
// is skipped (paging is off then), but API does not support
// filtering by both mentioned fields at the same time.
desc := false
pmapiMessages, total, err := p.listMessages(&pmapi.MessagesFilter{
AddressID: p.addressID,
LabelID: rule.SourceMailbox.ID,
Begin: rule.FromTime,
End: rule.ToTime,
PageSize: pmapiListPageSize,
Page: page,
Sort: "ID",
Desc: &desc,
})
if err != nil {
return err
}
log.WithFields(logrus.Fields{
"label": rule.SourceMailbox.ID,
"page": page,
"total": total,
"count": len(pmapiMessages),
}).Debug("Listing messages")
isLastPage = len(pmapiMessages) < pmapiListPageSize
for _, pmapiMessage := range pmapiMessages {
if progress.shouldStop() {
break
}
msgID := fmt.Sprintf("%s_%s", rule.SourceMailbox.ID, pmapiMessage.ID)
progress.addMessage(msgID, []string{rule.SourceMailbox.Name}, rule.TargetMailboxNames())
msg, err := p.exportMessage(rule, progress, pmapiMessage.ID, msgID, skipEncryptedMessages)
progress.messageExported(msgID, msg.Body, err)
if err == nil {
ch <- msg
}
}
page++
return nil
})
if isLastPage {
break
}
}
}
func (p *PMAPIProvider) exportMessage(rule *Rule, progress *Progress, pmapiMsgID, msgID string, skipEncryptedMessages bool) (Message, error) {
var msg *pmapi.Message
progress.callWrap(func() error {
var err error
msg, err = p.getMessage(pmapiMsgID)
return err
})
p.timeIt.start("build", msgID)
defer p.timeIt.stop("build", msgID)
body, err := p.builder.NewJobWithOptions(
context.Background(),
p.client,
msg.ID,
message.JobOptions{IgnoreDecryptionErrors: !skipEncryptedMessages},
).GetResult()
if err != nil {
if errors.Is(err, message.ErrDecryptionFailed) && skipEncryptedMessages {
err = errors.New("skipping encrypted message")
}
return Message{Body: []byte(msg.Body)}, err
}
return Message{
ID: msgID,
Unread: bool(msg.Unread),
Body: body,
Sources: []Mailbox{rule.SourceMailbox},
Targets: rule.TargetMailboxes,
}, nil
}

339
internal/transfer/provider_pmapi_target.go
View File

@ -1,339 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"bytes"
"context"
"fmt"
"io"
"io/ioutil"
"net/mail"
"sync"
pkgMsg "github.com/ProtonMail/proton-bridge/pkg/message"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/pkg/errors"
)
const (
pmapiImportBatchMaxItems = 10
pmapiImportBatchMaxSize = 25 * 1000 * 1000 // 25 MB
pmapiImportWorkers = 4 // To keep memory under 1 GB.
)
// DefaultMailboxes returns the default mailboxes for default rules if no other is found.
func (p *PMAPIProvider) DefaultMailboxes(_ Mailbox) []Mailbox {
return []Mailbox{{
ID: pmapi.ArchiveLabel,
Name: "Archive",
IsExclusive: true,
}}
}
// CreateMailbox creates label in ProtonMail account.
func (p *PMAPIProvider) CreateMailbox(mailbox Mailbox) (Mailbox, error) {
if mailbox.ID != "" {
return Mailbox{}, errors.New("mailbox is already created")
}
label, err := p.client.CreateLabel(context.Background(), &pmapi.Label{
Name: mailbox.Name,
Color: mailbox.Color,
Exclusive: pmapi.Boolean(mailbox.IsExclusive),
Type: pmapi.LabelTypeMailbox,
})
if err != nil {
return Mailbox{}, errors.Wrap(err, fmt.Sprintf("failed to create mailbox %s", mailbox.Name))
}
mailbox.ID = label.ID
return mailbox, nil
}
// TransferFrom imports messages from channel.
func (p *PMAPIProvider) TransferFrom(rules transferRules, progress *Progress, ch <-chan Message) {
log.Info("Started transfer from channel to PMAPI")
defer log.Info("Finished transfer from channel to PMAPI")
p.timeIt.clear()
defer p.timeIt.logResults()
// Cache has to be cleared before each transfer to not contain
// old stuff from previous cancelled run.
p.nextImportRequests = map[string]*pmapi.ImportMsgReq{}
p.nextImportRequestsSize = 0
preparedImportRequestsCh := make(chan map[string]*pmapi.ImportMsgReq)
wg := p.startImportWorkers(progress, preparedImportRequestsCh)
for msg := range ch {
if progress.shouldStop() {
break
}
if p.isMessageDraft(msg) {
p.transferDraft(rules, progress, msg)
} else {
p.transferMessage(rules, progress, msg, preparedImportRequestsCh)
}
}
if len(p.nextImportRequests) > 0 {
preparedImportRequestsCh <- p.nextImportRequests
}
close(preparedImportRequestsCh)
wg.Wait()
}
func (p *PMAPIProvider) isMessageDraft(msg Message) bool {
for _, target := range msg.Targets {
if target.ID == pmapi.DraftLabel {
return true
}
}
return false
}
func (p *PMAPIProvider) transferDraft(rules transferRules, progress *Progress, msg Message) {
importedID, err := p.importDraft(msg, rules.globalMailbox)
progress.messageImported(msg.ID, importedID, err)
}
func (p *PMAPIProvider) importDraft(msg Message, globalMailbox *Mailbox) (string, error) { //nolint[funlen]
message, attachmentReaders, err := p.parseMessage(msg)
if err != nil {
return "", errors.Wrap(err, "failed to parse message")
}
if message.Sender == nil {
mainAddress := p.client.Addresses().Main()
message.Sender = &mail.Address{
Name: mainAddress.DisplayName,
Address: mainAddress.Email,
}
}
// Trying to encrypt an encrypted draft will return an error;
// users are forbidden to import messages encrypted with foreign keys to drafts.
if message.IsEncrypted() {
return "", errors.New("refusing to import draft encrypted by foreign key")
}
p.timeIt.start("encrypt", msg.ID)
err = message.Encrypt(p.keyRing, nil)
p.timeIt.stop("encrypt", msg.ID)
if err != nil {
return "", errors.Wrap(err, "failed to encrypt draft")
}
if globalMailbox != nil {
message.LabelIDs = append(message.LabelIDs, globalMailbox.ID)
}
attachments := message.Attachments
message.Attachments = nil
draft, err := p.createDraft(msg.ID, message, "", pmapi.DraftActionReply)
if err != nil {
return "", errors.Wrap(err, "failed to create draft")
}
for idx, attachment := range attachments {
attachment.MessageID = draft.ID
attachmentBody, _ := ioutil.ReadAll(attachmentReaders[idx])
r := bytes.NewReader(attachmentBody)
sigReader, err := attachment.DetachedSign(p.keyRing, r)
if err != nil {
return "", errors.Wrap(err, "failed to sign attachment")
}
p.timeIt.start("encrypt", msg.ID)
r = bytes.NewReader(attachmentBody)
encReader, err := attachment.Encrypt(p.keyRing, r)
p.timeIt.stop("encrypt", msg.ID)
if err != nil {
return "", errors.Wrap(err, "failed to encrypt attachment")
}
_, err = p.createAttachment(msg.ID, attachment, encReader, sigReader)
if err != nil {
return "", errors.Wrap(err, "failed to create attachment")
}
}
return draft.ID, nil
}
func (p *PMAPIProvider) transferMessage(rules transferRules, progress *Progress, msg Message, preparedImportRequestsCh chan map[string]*pmapi.ImportMsgReq) {
importMsgReq, err := p.generateImportMsgReq(rules, progress, msg)
if err != nil {
progress.messageImported(msg.ID, "", err)
return
}
if importMsgReq == nil || progress.shouldStop() {
return
}
importMsgReqSize := len(importMsgReq.Message)
if p.nextImportRequestsSize+importMsgReqSize > pmapiImportBatchMaxSize || len(p.nextImportRequests) == pmapiImportBatchMaxItems {
preparedImportRequestsCh <- p.nextImportRequests
p.nextImportRequests = map[string]*pmapi.ImportMsgReq{}
p.nextImportRequestsSize = 0
}
p.nextImportRequests[msg.ID] = importMsgReq
p.nextImportRequestsSize += importMsgReqSize
}
func (p *PMAPIProvider) generateImportMsgReq(rules transferRules, progress *Progress, msg Message) (*pmapi.ImportMsgReq, error) {
message, attachmentReaders, err := p.parseMessage(msg)
if err != nil {
return nil, errors.Wrap(err, "failed to parse message")
}
var body []byte
if message.IsEncrypted() {
if rules.skipEncryptedMessages {
progress.messageSkipped(msg.ID)
return nil, nil
}
body = msg.Body
} else {
p.timeIt.start("encrypt", msg.ID)
body, err = p.encryptMessage(message, attachmentReaders)
p.timeIt.stop("encrypt", msg.ID)
if err != nil {
return nil, errors.Wrap(err, "failed to encrypt message")
}
}
labelIDs := []string{}
for _, target := range msg.Targets {
// Frontend should not set All Mail to Rules, but to be sure...
if target.ID != pmapi.AllMailLabel {
labelIDs = append(labelIDs, target.ID)
}
}
if rules.globalMailbox != nil {
labelIDs = append(labelIDs, rules.globalMailbox.ID)
}
return &pmapi.ImportMsgReq{
Metadata: &pmapi.ImportMetadata{
AddressID: p.addressID,
Unread: pmapi.Boolean(msg.Unread),
Time: message.Time,
Flags: computeMessageFlags(message.Header),
LabelIDs: labelIDs,
},
Message: body,
}, nil
}
func (p *PMAPIProvider) parseMessage(msg Message) (m *pmapi.Message, r []io.Reader, err error) {
p.timeIt.start("parse", msg.ID)
defer p.timeIt.stop("parse", msg.ID)
message, _, _, attachmentReaders, err := pkgMsg.Parse(bytes.NewBuffer(msg.Body))
return message, attachmentReaders, err
}
func (p *PMAPIProvider) encryptMessage(msg *pmapi.Message, attachmentReaders []io.Reader) ([]byte, error) {
if msg.MIMEType == pmapi.ContentTypeMultipartEncrypted {
return []byte(msg.Body), nil
}
return pkgMsg.BuildEncrypted(msg, attachmentReaders, p.keyRing)
}
func computeMessageFlags(header mail.Header) (flag int64) {
if header.Get("received") == "" {
return pmapi.FlagSent
}
return pmapi.FlagReceived
}
func (p *PMAPIProvider) startImportWorkers(progress *Progress, preparedImportRequestsCh chan map[string]*pmapi.ImportMsgReq) *sync.WaitGroup {
var wg sync.WaitGroup
wg.Add(pmapiImportWorkers)
for i := 0; i < pmapiImportWorkers; i++ {
go func() {
for importRequests := range preparedImportRequestsCh {
p.importMessages(progress, importRequests)
}
wg.Done()
}()
}
return &wg
}
func (p *PMAPIProvider) importMessages(progress *Progress, importRequests map[string]*pmapi.ImportMsgReq) {
if progress.shouldStop() {
return
}
importMsgIDs := []string{}
importMsgRequests := pmapi.ImportMsgReqs{}
for msgID, req := range importRequests {
importMsgIDs = append(importMsgIDs, msgID)
importMsgRequests = append(importMsgRequests, req)
}
log.WithField("msgIDs", importMsgIDs).Trace("Importing messages")
results, err := p.importRequest(importMsgIDs[0], importMsgRequests)
// In case the whole request failed, try to import every message one by one.
if err != nil || len(results) == 0 {
log.WithError(err).Warning("Importing messages failed, trying one by one")
for msgID, req := range importRequests {
importedID, err := p.importMessage(msgID, progress, req)
progress.messageImported(msgID, importedID, err)
}
return
}
// In case request passed but some messages failed, try to import the failed ones alone.
for index, result := range results {
msgID := importMsgIDs[index]
if result.Error != nil {
log.WithError(result.Error).WithField("msg", msgID).Warning("Importing message failed, trying alone")
req := importMsgRequests[index]
importedID, err := p.importMessage(msgID, progress, req)
progress.messageImported(msgID, importedID, err)
} else {
progress.messageImported(msgID, result.MessageID, nil)
}
}
}
func (p *PMAPIProvider) importMessage(msgSourceID string, progress *Progress, req *pmapi.ImportMsgReq) (importedID string, importedErr error) {
progress.callWrap(func() error {
results, err := p.importRequest(msgSourceID, pmapi.ImportMsgReqs{req})
if err != nil {
return errors.Wrap(err, "failed to import messages")
}
if len(results) == 0 {
importedErr = errors.New("import ended with no result")
return nil // This should not happen, only when there is bug which means we should skip this one.
}
if results[0].Error != nil {
importedErr = errors.Wrap(results[0].Error, "failed to import message")
return nil //nolint[nilerr] Call passed but API refused this message, skip this one.
}
importedID = results[0].MessageID
return nil
})
return importedID, importedErr
}

201
internal/transfer/provider_pmapi_test.go
View File

@ -1,201 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"bytes"
"context"
"fmt"
"testing"
"time"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
gomock "github.com/golang/mock/gomock"
r "github.com/stretchr/testify/require"
)
func TestPMAPIProviderMailboxes(t *testing.T) {
m := initMocks(t)
defer m.ctrl.Finish()
setupPMAPIClientExpectationForExport(&m)
provider, err := NewPMAPIProvider(m.pmapiClient, "user", "addressID")
r.NoError(t, err)
tests := []struct {
includeEmpty bool
includeAllMail bool
wantMailboxes []Mailbox
}{
{true, false, []Mailbox{
{ID: "folder1", Name: "One", Color: "red", IsExclusive: true},
{ID: "folder2", Name: "Two", Color: "orange", IsExclusive: true},
{ID: "label2", Name: "Bar", Color: "green", IsExclusive: false},
{ID: "label1", Name: "Foo", Color: "blue", IsExclusive: false},
}},
{false, true, []Mailbox{
{ID: pmapi.AllMailLabel, Name: "All Mail", IsExclusive: true},
{ID: "folder1", Name: "One", Color: "red", IsExclusive: true},
{ID: "folder2", Name: "Two", Color: "orange", IsExclusive: true},
{ID: "label1", Name: "Foo", Color: "blue", IsExclusive: false},
}},
}
for _, tc := range tests {
tc := tc
t.Run(fmt.Sprintf("%v-%v", tc.includeEmpty, tc.includeAllMail), func(t *testing.T) {
mailboxes, err := provider.Mailboxes(tc.includeEmpty, tc.includeAllMail)
r.NoError(t, err)
r.Equal(t, []Mailbox{
{ID: pmapi.InboxLabel, Name: "Inbox", IsExclusive: true},
{ID: pmapi.DraftLabel, Name: "Drafts", IsExclusive: true},
{ID: pmapi.SentLabel, Name: "Sent", IsExclusive: true},
{ID: pmapi.StarredLabel, Name: "Starred", IsExclusive: true},
{ID: pmapi.ArchiveLabel, Name: "Archive", IsExclusive: true},
{ID: pmapi.SpamLabel, Name: "Spam", IsExclusive: true},
{ID: pmapi.TrashLabel, Name: "Trash", IsExclusive: true},
}, mailboxes[:7])
r.Equal(t, tc.wantMailboxes, mailboxes[7:])
})
}
}
func TestPMAPIProviderTransferTo(t *testing.T) {
m := initMocks(t)
defer m.ctrl.Finish()
setupPMAPIClientExpectationForExport(&m)
provider, err := NewPMAPIProvider(m.pmapiClient, "user", "addressID")
r.NoError(t, err)
rules, rulesClose := newTestRules(t)
defer rulesClose()
setupPMAPIRules(rules)
testTransferTo(t, rules, provider, []string{
"0_msg1",
"0_msg2",
})
}
func TestPMAPIProviderTransferFrom(t *testing.T) {
m := initMocks(t)
defer m.ctrl.Finish()
setupPMAPIClientExpectationForImport(&m)
provider, err := NewPMAPIProvider(m.pmapiClient, "user", "addressID")
r.NoError(t, err)
rules, rulesClose := newTestRules(t)
defer rulesClose()
setupPMAPIRules(rules)
testTransferFrom(t, rules, provider, []Message{
{ID: "msg1", Body: getTestMsgBody("msg1"), Targets: []Mailbox{{ID: pmapi.InboxLabel}}},
{ID: "msg2", Body: getTestMsgBody("msg2"), Targets: []Mailbox{{ID: pmapi.InboxLabel}}},
})
}
func TestPMAPIProviderTransferFromDraft(t *testing.T) {
m := initMocks(t)
defer m.ctrl.Finish()
setupPMAPIClientExpectationForImportDraft(&m)
provider, err := NewPMAPIProvider(m.pmapiClient, "user", "addressID")
r.NoError(t, err)
rules, rulesClose := newTestRules(t)
defer rulesClose()
setupPMAPIRules(rules)
testTransferFrom(t, rules, provider, []Message{
{ID: "draft1", Body: getTestMsgBody("draft1"), Targets: []Mailbox{{ID: pmapi.DraftLabel}}},
})
}
func TestPMAPIProviderTransferFromTo(t *testing.T) {
m := initMocks(t)
defer m.ctrl.Finish()
setupPMAPIClientExpectationForExport(&m)
setupPMAPIClientExpectationForImport(&m)
source, err := NewPMAPIProvider(m.pmapiClient, "user", "addressID")
r.NoError(t, err)
target, err := NewPMAPIProvider(m.pmapiClient, "user", "addressID")
r.NoError(t, err)
rules, rulesClose := newTestRules(t)
defer rulesClose()
setupPMAPIRules(rules)
testTransferFromTo(t, rules, source, target, 5*time.Second)
}
func setupPMAPIRules(rules transferRules) {
_ = rules.setRule(Mailbox{ID: pmapi.InboxLabel}, []Mailbox{{ID: pmapi.InboxLabel}}, 0, 0)
}
func setupPMAPIClientExpectationForExport(m *mocks) {
m.pmapiClient.EXPECT().KeyRingForAddressID(gomock.Any()).Return(m.keyring, nil).AnyTimes()
m.pmapiClient.EXPECT().ListLabels(gomock.Any()).Return([]*pmapi.Label{
{ID: "label1", Name: "Foo", Color: "blue", Exclusive: false, Order: 2},
{ID: "label2", Name: "Bar", Color: "green", Exclusive: false, Order: 1},
{ID: "folder1", Name: "One", Color: "red", Exclusive: true, Order: 1},
{ID: "folder2", Name: "Two", Color: "orange", Exclusive: true, Order: 2},
}, nil).AnyTimes()
m.pmapiClient.EXPECT().CountMessages(gomock.Any(), gomock.Any()).Return([]*pmapi.MessagesCount{
{LabelID: "label1", Total: 10},
{LabelID: "label2", Total: 0},
{LabelID: "folder1", Total: 20},
}, nil).AnyTimes()
m.pmapiClient.EXPECT().ListMessages(gomock.Any(), gomock.Any()).Return([]*pmapi.Message{
{ID: "msg1"},
{ID: "msg2"},
}, 2, nil).AnyTimes()
m.pmapiClient.EXPECT().GetMessage(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, msgID string) (*pmapi.Message, error) {
return &pmapi.Message{
ID: msgID,
Body: string(getTestMsgBody(msgID)),
MIMEType: pmapi.ContentTypeMultipartMixed,
}, nil
}).AnyTimes()
}
func setupPMAPIClientExpectationForImport(m *mocks) {
m.pmapiClient.EXPECT().KeyRingForAddressID(gomock.Any()).Return(m.keyring, nil).AnyTimes()
m.pmapiClient.EXPECT().Import(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, requests pmapi.ImportMsgReqs) ([]*pmapi.ImportMsgRes, error) {
results := []*pmapi.ImportMsgRes{}
for _, request := range requests {
for _, msgID := range []string{"msg1", "msg2"} {
if bytes.Contains(request.Message, []byte(msgID)) {
results = append(results, &pmapi.ImportMsgRes{MessageID: msgID, Error: nil})
}
}
}
return results, nil
}).AnyTimes()
}
func setupPMAPIClientExpectationForImportDraft(m *mocks) {
m.pmapiClient.EXPECT().KeyRingForAddressID(gomock.Any()).Return(m.keyring, nil).AnyTimes()
m.pmapiClient.EXPECT().CreateDraft(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, msg *pmapi.Message, parentID string, action int) (*pmapi.Message, error) {
r.Equal(m.t, msg.Subject, "draft1")
msg.ID = "draft1"
return msg, nil
})
}

136
internal/transfer/provider_pmapi_utils.go
View File

@ -1,136 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"context"
"fmt"
"io"
"time"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/pkg/errors"
)
const (
pmapiRetries = 10
pmapiReconnectTimeout = 30 * time.Minute
pmapiReconnectSleep = 10 * time.Second
)
func (p *PMAPIProvider) SetConnectionUp() {
p.connection = true
}
func (p *PMAPIProvider) SetConnectionDown() {
p.connection = false
}
func (p *PMAPIProvider) ensureConnection(callback func() error) error {
var callErr error
for i := 1; i <= pmapiRetries; i++ {
callErr = callback()
if callErr == nil {
return nil
}
log.WithField("attempt", i).WithError(callErr).Warning("API call failed, trying reconnect")
err := p.tryReconnect()
if err != nil {
return err
}
}
return errors.Wrap(callErr, "too many retries")
}
func (p *PMAPIProvider) tryReconnect() error {
start := time.Now()
var previousErr error
for {
if time.Since(start) > pmapiReconnectTimeout {
return previousErr
}
if !p.connection {
time.Sleep(pmapiReconnectSleep)
continue
}
break
}
return nil
}
func (p *PMAPIProvider) listMessages(filter *pmapi.MessagesFilter) (messages []*pmapi.Message, count int, err error) {
err = p.ensureConnection(func() error {
// Sort is used in the key so the filter is different for estimating and real fetching.
key := fmt.Sprintf("%s_%s_%d", filter.LabelID, filter.Sort, filter.Page)
p.timeIt.start("listing", key)
defer p.timeIt.stop("listing", key)
messages, count, err = p.client.ListMessages(context.Background(), filter)
return err
})
return
}
func (p *PMAPIProvider) getMessage(msgID string) (message *pmapi.Message, err error) {
err = p.ensureConnection(func() error {
p.timeIt.start("download", msgID)
defer p.timeIt.stop("download", msgID)
message, err = p.client.GetMessage(context.Background(), msgID)
return err
})
return
}
func (p *PMAPIProvider) importRequest(msgSourceID string, req pmapi.ImportMsgReqs) (res []*pmapi.ImportMsgRes, err error) {
err = p.ensureConnection(func() error {
p.timeIt.start("upload", msgSourceID)
defer p.timeIt.stop("upload", msgSourceID)
res, err = p.client.Import(context.Background(), req)
return err
})
return
}
func (p *PMAPIProvider) createDraft(msgSourceID string, message *pmapi.Message, parent string, action int) (draft *pmapi.Message, err error) {
err = p.ensureConnection(func() error {
p.timeIt.start("upload", msgSourceID)
defer p.timeIt.stop("upload", msgSourceID)
draft, err = p.client.CreateDraft(context.Background(), message, parent, action)
return err
})
return
}
func (p *PMAPIProvider) createAttachment(msgSourceID string, att *pmapi.Attachment, r io.Reader, sig io.Reader) (created *pmapi.Attachment, err error) {
err = p.ensureConnection(func() error {
// Use some attributes from attachment to have unique key for each call.
key := fmt.Sprintf("%s_%s_%d", msgSourceID, att.Name, att.Size)
p.timeIt.start("upload", key)
defer p.timeIt.stop("upload", key)
created, err = p.client.CreateAttachment(context.Background(), att, r, sig)
return err
})
return
}

115
internal/transfer/provider_test.go
View File

@ -1,115 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"fmt"
"testing"
"time"
a "github.com/stretchr/testify/assert"
r "github.com/stretchr/testify/require"
)
func getTestMsgBody(subject string) []byte {
return []byte(fmt.Sprintf(`Subject: %s
From: Bridge Test <bridgetest@pm.test>
To: Bridge Test <bridgetest@protonmail.com>
Content-Type: multipart/mixed; boundary=c672b8d1ef56ed28ab87c3622c5114069bdd3ad7b8f9737498d0c01ecef0967a
--c672b8d1ef56ed28ab87c3622c5114069bdd3ad7b8f9737498d0c01ecef0967a
Content-Disposition: inline
Content-Transfer-Encoding: 7bit
Content-Type: text/plain; charset=utf-8
hello
--c672b8d1ef56ed28ab87c3622c5114069bdd3ad7b8f9737498d0c01ecef0967a--
`, subject))
}
func testTransferTo(t *testing.T, rules transferRules, provider SourceProvider, expectedMessageIDs []string) []Message {
progress := newProgress(log, nil)
drainProgressUpdateChannel(&progress)
ch := make(chan Message)
go func() {
provider.TransferTo(rules, &progress, ch)
close(ch)
}()
msgs := []Message{}
gotMessageIDs := []string{}
for msg := range ch {
msgs = append(msgs, msg)
gotMessageIDs = append(gotMessageIDs, msg.ID)
}
r.ElementsMatch(t, expectedMessageIDs, gotMessageIDs)
r.Empty(t, progress.GetFailedMessages())
return msgs
}
func testTransferFrom(t *testing.T, rules transferRules, provider TargetProvider, messages []Message) {
progress := newProgress(log, nil)
drainProgressUpdateChannel(&progress)
ch := make(chan Message)
go func() {
for _, message := range messages {
progress.addMessage(message.ID, []string{}, []string{})
progress.messageExported(message.ID, []byte(""), nil)
ch <- message
}
close(ch)
}()
go func() {
provider.TransferFrom(rules, &progress, ch)
progress.finish()
}()
maxWait := time.Duration(len(messages)*2) * time.Second
a.Eventually(t, func() bool {
return progress.updateCh == nil
}, maxWait, 10*time.Millisecond, "Waiting for imported messages timed out")
r.Empty(t, progress.GetFailedMessages())
}
func testTransferFromTo(t *testing.T, rules transferRules, source SourceProvider, target TargetProvider, maxWait time.Duration) {
progress := newProgress(log, nil)
drainProgressUpdateChannel(&progress)
ch := make(chan Message)
go func() {
source.TransferTo(rules, &progress, ch)
close(ch)
}()
go func() {
target.TransferFrom(rules, &progress, ch)
progress.finish()
}()
a.Eventually(t, func() bool {
return progress.updateCh == nil
}, maxWait, 10*time.Millisecond, "Waiting for export and import timed out")
r.Empty(t, progress.GetFailedMessages())
}

145
internal/transfer/report.go
View File

@ -1,145 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"bytes"
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"time"
"github.com/pkg/errors"
)
// fileReport is struct which can write and read message details.
// File report includes private information.
type fileReport struct {
path string
}
func openLastFileReport(reportsPath, importID string) (*fileReport, error) { //nolint[deadcode]
allLogFileNames, err := getFilePathsWithSuffix(reportsPath, ".log")
if err != nil {
return nil, err
}
reportFileNames := []string{}
for _, fileName := range allLogFileNames {
if strings.HasPrefix(fileName, fmt.Sprintf("import_%s_", importID)) {
reportFileNames = append(reportFileNames, fileName)
}
}
if len(reportFileNames) == 0 {
return nil, errors.New("no report found")
}
sort.Strings(reportFileNames)
reportFileName := reportFileNames[len(reportFileNames)-1]
path := filepath.Join(reportsPath, reportFileName)
return &fileReport{
path: path,
}, nil
}
func newFileReport(reportsPath, importID string) *fileReport {
fileName := fmt.Sprintf("import_%s_%d.log", importID, time.Now().Unix())
path := filepath.Join(reportsPath, fileName)
return &fileReport{
path: path,
}
}
func (r *fileReport) writeMessageStatus(messageStatus *MessageStatus) {
f, err := os.OpenFile(r.path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
if err != nil {
log.WithError(err).Error("Failed to open report file")
}
defer f.Close() //nolint[errcheck]
messageReport := newMessageReportFromMessageStatus(messageStatus, true)
data, err := json.Marshal(messageReport)
if err != nil {
log.WithError(err).Error("Failed to marshall message details")
}
data = append(data, '\n')
if _, err = f.Write(data); err != nil {
log.WithError(err).Error("Failed to write to report file")
}
}
// bugReport is struct which can create report for bug reporting.
// Bug report does NOT include private information.
type bugReport struct {
data bytes.Buffer
}
func (r *bugReport) writeMessageStatus(messageStatus *MessageStatus) {
messageReport := newMessageReportFromMessageStatus(messageStatus, false)
data, err := json.Marshal(messageReport)
if err != nil {
log.WithError(err).Error("Failed to marshall message details")
}
_, _ = r.data.Write(data)
_, _ = r.data.Write([]byte("\n"))
}
func (r *bugReport) getData() []byte {
return r.data.Bytes()
}
// messageReport is struct which holds data used by `fileReport` and `bugReport`.
type messageReport struct {
EventTime int64
SourceID string
TargetID string
BodyHash string
SourceMailboxes []string
TargetMailboxes []string
Error string
// Private information for user.
Subject string
From string
Time string
}
func newMessageReportFromMessageStatus(messageStatus *MessageStatus, includePrivateInfo bool) messageReport {
md := messageReport{
EventTime: messageStatus.eventTime.Unix(),
SourceID: messageStatus.SourceID,
TargetID: messageStatus.targetID,
BodyHash: messageStatus.bodyHash,
SourceMailboxes: messageStatus.sourceNames,
TargetMailboxes: messageStatus.targetNames,
Error: messageStatus.GetErrorMessage(),
}
if includePrivateInfo {
md.Subject = messageStatus.Subject
md.From = messageStatus.From
md.Time = messageStatus.Time.Format(time.RFC1123Z)
}
return md
}

366
internal/transfer/rules.go
View File

@ -1,366 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"time"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/pkg/errors"
)
// transferRules maintains import rules, e.g. to which target mailbox should be
// source mailbox imported or what time spans.
type transferRules struct {
filePath string
// rules is map with key as hash of source mailbox to its rule.
// Every source mailbox should have rule, at least disabled one.
rules map[string]*Rule
// globalMailbox is applied to every message in the import phase.
// E.g., every message will be imported into this mailbox.
globalMailbox *Mailbox
// globalFromTime and globalToTime is applied to every rule right
// before the transfer (propagateGlobalTime has to be called).
globalFromTime int64
globalToTime int64
// skipEncryptedMessages determines whether message which cannot
// be decrypted should be imported/exported or skipped.
skipEncryptedMessages bool
}
// loadRules loads rules from `rulesPath` based on `ruleID`.
func loadRules(rulesPath, ruleID string) transferRules {
fileName := fmt.Sprintf("rules_%s.json", ruleID)
filePath := filepath.Join(rulesPath, fileName)
var rules map[string]*Rule
f, err := os.Open(filePath) //nolint[gosec]
if err != nil {
log.WithError(err).Debug("Problem to read rules")
} else {
defer f.Close() //nolint[errcheck]
if err := json.NewDecoder(f).Decode(&rules); err != nil {
log.WithError(err).Warn("Problem to umarshal rules")
}
}
if rules == nil {
rules = map[string]*Rule{}
}
return transferRules{
filePath: filePath,
rules: rules,
}
}
func (r *transferRules) setSkipEncryptedMessages(skip bool) {
r.skipEncryptedMessages = skip
}
func (r *transferRules) setGlobalMailbox(mailbox *Mailbox) {
r.globalMailbox = mailbox
}
func (r *transferRules) setGlobalTimeLimit(fromTime, toTime int64) {
r.globalFromTime = fromTime
r.globalToTime = toTime
}
func (r *transferRules) propagateGlobalTime() {
if r.globalFromTime == 0 && r.globalToTime == 0 {
return
}
for _, rule := range r.rules {
if !rule.HasTimeLimit() {
rule.FromTime = r.globalFromTime
rule.ToTime = r.globalToTime
}
}
}
func (r *transferRules) getRuleBySourceMailboxName(name string) (*Rule, error) {
for _, rule := range r.rules {
if rule.SourceMailbox.Name == name {
return rule, nil
}
}
return nil, fmt.Errorf("no rule for mailbox %s", name)
}
func (r *transferRules) iterateActiveRules() chan *Rule {
ch := make(chan *Rule)
go func() {
for _, rule := range r.rules {
if rule.Active {
ch <- rule
}
}
close(ch)
}()
return ch
}
// setDefaultRules iterates `sourceMailboxes` and sets missing rules with
// matching mailboxes from `targetMailboxes`. In case no matching mailbox
// is found, `defaultCallback` with a source mailbox as a parameter is used.
func (r *transferRules) setDefaultRules(sourceMailboxes []Mailbox, targetMailboxes []Mailbox, defaultCallback func(Mailbox) []Mailbox) {
for _, sourceMailbox := range sourceMailboxes {
h := sourceMailbox.Hash()
if _, ok := r.rules[h]; ok {
continue
}
targetMailboxes := sourceMailbox.findMatchingMailboxes(targetMailboxes)
if !containsExclusive(targetMailboxes) {
targetMailboxes = append(targetMailboxes, defaultCallback(sourceMailbox)...)
}
active := true
if len(targetMailboxes) == 0 {
active = false
}
// For both import to or export from ProtonMail, spam and draft
// mailboxes are by default deactivated.
for _, mailbox := range append([]Mailbox{sourceMailbox}, targetMailboxes...) {
if mailbox.ID == pmapi.SpamLabel || mailbox.ID == pmapi.DraftLabel || mailbox.ID == pmapi.TrashLabel {
active = false
break
}
}
r.rules[h] = &Rule{
Active: active,
SourceMailbox: sourceMailbox,
TargetMailboxes: targetMailboxes,
}
}
// There is no point showing rule which has no action (i.e., source mailbox
// is not available).
// A good reason to keep all rules and only deactivate them would be for
// multiple imports from different sources with the same or similar enough
// mailbox setup to reuse configuration. That is very minor feature which
// can be implemented in more reasonable way by allowing users to save and
// load configurations.
for key, rule := range r.rules {
found := false
for _, sourceMailbox := range sourceMailboxes {
if sourceMailbox.Name == rule.SourceMailbox.Name {
found = true
}
}
if !found {
delete(r.rules, key)
}
}
r.save()
}
// setRule sets messages from `sourceMailbox` between `fromData` and `toDate`
// (if used) to be imported to all `targetMailboxes`.
func (r *transferRules) setRule(sourceMailbox Mailbox, targetMailboxes []Mailbox, fromTime, toTime int64) error {
numberOfExclusiveMailboxes := 0
for _, mailbox := range targetMailboxes {
if mailbox.IsExclusive {
numberOfExclusiveMailboxes++
}
}
if numberOfExclusiveMailboxes > 1 {
return errors.New("rule can have only one exclusive target mailbox")
}
h := sourceMailbox.Hash()
r.rules[h] = &Rule{
Active: true,
SourceMailbox: sourceMailbox,
TargetMailboxes: targetMailboxes,
FromTime: fromTime,
ToTime: toTime,
}
r.save()
return nil
}
// unsetRule unsets messages from `sourceMailbox` to be exported.
func (r *transferRules) unsetRule(sourceMailbox Mailbox) {
h := sourceMailbox.Hash()
if rule, ok := r.rules[h]; ok {
rule.Active = false
} else {
r.rules[h] = &Rule{
Active: false,
SourceMailbox: sourceMailbox,
}
}
r.save()
}
// getRule returns rule for `sourceMailbox` or nil if it does not exist.
func (r *transferRules) getRule(sourceMailbox Mailbox) *Rule {
h := sourceMailbox.Hash()
return r.rules[h]
}
// getSortedRules returns all set rules in order by `byRuleOrder`.
func (r *transferRules) getSortedRules() []*Rule {
rules := []*Rule{}
for _, rule := range r.rules {
rules = append(rules, rule)
}
sort.Sort(byRuleOrder(rules))
return rules
}
// reset wipes our all rules.
func (r *transferRules) reset() {
r.rules = map[string]*Rule{}
r.save()
}
// save saves rules to file.
func (r *transferRules) save() {
f, err := os.Create(r.filePath)
if err != nil {
log.WithError(err).Warn("Problem to write rules")
return
}
defer f.Close() //nolint[errcheck]
if err := json.NewEncoder(f).Encode(r.rules); err != nil {
log.WithError(err).Warn("Problem to marshal rules")
}
}
// Rule is data holder of rule for one source mailbox used by `transferRules`.
type Rule struct {
Active bool `json:"active"`
SourceMailbox Mailbox `json:"source"`
TargetMailboxes []Mailbox `json:"targets"`
FromTime int64 `json:"from"`
ToTime int64 `json:"to"`
}
// String returns textual representation for log purposes.
func (r *Rule) String() string {
return fmt.Sprintf(
"%s -> %s (%d - %d)",
r.SourceMailbox.Name,
strings.Join(r.TargetMailboxNames(), ", "),
r.FromTime,
r.ToTime,
)
}
func (r *Rule) isTimeInRange(t int64) bool {
if !r.HasTimeLimit() {
return true
}
return r.FromTime <= t && t <= r.ToTime
}
// HasTimeLimit returns whether rule defines time limit.
func (r *Rule) HasTimeLimit() bool {
return r.FromTime != 0 || r.ToTime != 0
}
// FromDate returns time struct based on `FromTime`.
func (r *Rule) FromDate() time.Time {
return time.Unix(r.FromTime, 0)
}
// ToDate returns time struct based on `ToTime`.
func (r *Rule) ToDate() time.Time {
return time.Unix(r.ToTime, 0)
}
// TargetMailboxNames returns array of target mailbox names.
func (r *Rule) TargetMailboxNames() (names []string) {
for _, mailbox := range r.TargetMailboxes {
names = append(names, mailbox.Name)
}
return
}
// byRuleOrder implements sort.Interface. Sort order:
// * System folders first (as defined in getSystemMailboxes).
// * Custom folders by name.
// * Custom labels by name.
type byRuleOrder []*Rule
func (a byRuleOrder) Len() int {
return len(a)
}
func (a byRuleOrder) Swap(i, j int) {
a[i], a[j] = a[j], a[i]
}
func (a byRuleOrder) Less(i, j int) bool {
if a[i].SourceMailbox.IsExclusive && !a[j].SourceMailbox.IsExclusive {
return true
}
if !a[i].SourceMailbox.IsExclusive && a[j].SourceMailbox.IsExclusive {
return false
}
iSystemIndex := -1
jSystemIndex := -1
for index, systemFolders := range getSystemMailboxes(true) {
if a[i].SourceMailbox.Name == systemFolders.Name {
iSystemIndex = index
}
if a[j].SourceMailbox.Name == systemFolders.Name {
jSystemIndex = index
}
}
if iSystemIndex != -1 && jSystemIndex == -1 {
return true
}
if iSystemIndex == -1 && jSystemIndex != -1 {
return false
}
if iSystemIndex != -1 && jSystemIndex != -1 {
return iSystemIndex < jSystemIndex
}
return a[i].SourceMailbox.Name < a[j].SourceMailbox.Name
}
// containsExclusive returns true if there is at least one exclusive mailbox.
func containsExclusive(mailboxes []Mailbox) bool {
for _, m := range mailboxes {
if m.IsExclusive {
return true
}
}
return false
}

247
internal/transfer/rules_test.go
View File

@ -1,247 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"fmt"
"io/ioutil"
"os"
"testing"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
r "github.com/stretchr/testify/require"
)
func newTestRules(t *testing.T) (transferRules, func()) {
path, err := ioutil.TempDir("", "rules")
r.NoError(t, err)
ruleID := "rule"
rules := loadRules(path, ruleID)
return rules, func() {
_ = os.RemoveAll(path)
}
}
func TestLoadRules(t *testing.T) {
path, err := ioutil.TempDir("", "rules")
r.NoError(t, err)
defer os.RemoveAll(path) //nolint[errcheck]
ruleID := "rule"
rules := loadRules(path, ruleID)
mailboxA := Mailbox{ID: "1", Name: "One", Color: "orange", IsExclusive: true}
mailboxB := Mailbox{ID: "2", Name: "Two", Color: "", IsExclusive: true}
mailboxC := Mailbox{ID: "3", Name: "Three", Color: "", IsExclusive: false}
r.NoError(t, rules.setRule(mailboxA, []Mailbox{mailboxB, mailboxC}, 0, 0))
r.NoError(t, rules.setRule(mailboxB, []Mailbox{mailboxB}, 10, 20))
r.NoError(t, rules.setRule(mailboxC, []Mailbox{}, 0, 30))
rules2 := loadRules(path, ruleID)
r.Equal(t, map[string]*Rule{
mailboxA.Hash(): {Active: true, SourceMailbox: mailboxA, TargetMailboxes: []Mailbox{mailboxB, mailboxC}, FromTime: 0, ToTime: 0},
mailboxB.Hash(): {Active: true, SourceMailbox: mailboxB, TargetMailboxes: []Mailbox{mailboxB}, FromTime: 10, ToTime: 20},
mailboxC.Hash(): {Active: true, SourceMailbox: mailboxC, TargetMailboxes: []Mailbox{}, FromTime: 0, ToTime: 30},
}, rules2.rules)
rules2.unsetRule(mailboxA)
rules2.unsetRule(mailboxC)
rules3 := loadRules(path, ruleID)
r.Equal(t, map[string]*Rule{
mailboxA.Hash(): {Active: false, SourceMailbox: mailboxA, TargetMailboxes: []Mailbox{mailboxB, mailboxC}, FromTime: 0, ToTime: 0},
mailboxB.Hash(): {Active: true, SourceMailbox: mailboxB, TargetMailboxes: []Mailbox{mailboxB}, FromTime: 10, ToTime: 20},
mailboxC.Hash(): {Active: false, SourceMailbox: mailboxC, TargetMailboxes: []Mailbox{}, FromTime: 0, ToTime: 30},
}, rules3.rules)
}
func TestSetGlobalTimeLimit(t *testing.T) {
path, err := ioutil.TempDir("", "rules")
r.NoError(t, err)
defer os.RemoveAll(path) //nolint[errcheck]
rules := loadRules(path, "rule")
mailboxA := Mailbox{Name: "One"}
mailboxB := Mailbox{Name: "Two"}
r.NoError(t, rules.setRule(mailboxA, []Mailbox{}, 10, 20))
r.NoError(t, rules.setRule(mailboxB, []Mailbox{}, 0, 0))
rules.setGlobalTimeLimit(30, 40)
rules.propagateGlobalTime()
r.Equal(t, map[string]*Rule{
mailboxA.Hash(): {Active: true, SourceMailbox: mailboxA, TargetMailboxes: []Mailbox{}, FromTime: 10, ToTime: 20},
mailboxB.Hash(): {Active: true, SourceMailbox: mailboxB, TargetMailboxes: []Mailbox{}, FromTime: 30, ToTime: 40},
}, rules.rules)
}
func TestSetDefaultRules(t *testing.T) {
path, err := ioutil.TempDir("", "rules")
r.NoError(t, err)
defer os.RemoveAll(path) //nolint[errcheck]
rules := loadRules(path, "rule")
mailbox1 := Mailbox{Name: "One"} // Set manually, default will not override it.
mailbox2 := Mailbox{Name: "Two"} // Matched by `targetMailboxes`.
mailbox3 := Mailbox{Name: "Three"} // Matched by `defaultCallback`, not included in `targetMailboxes`.
mailbox4 := Mailbox{Name: "Four"} // Matched by nothing, will not be active.
mailbox5 := Mailbox{Name: "Spam", ID: pmapi.SpamLabel} // Spam is inactive by default (ID found in source).
mailbox6a := Mailbox{Name: "Draft"} // Draft is inactive by default (ID found in target, mailbox6b).
mailbox6b := Mailbox{Name: "Draft", ID: pmapi.DraftLabel}
sourceMailboxes := []Mailbox{mailbox1, mailbox2, mailbox3, mailbox4, mailbox5, mailbox6a}
targetMailboxes := []Mailbox{mailbox1, mailbox2, mailbox6b}
r.NoError(t, rules.setRule(mailbox1, []Mailbox{mailbox3}, 0, 0))
defaultCallback := func(mailbox Mailbox) []Mailbox {
if mailbox.Name == "Three" {
return []Mailbox{mailbox3}
}
return []Mailbox{}
}
rules.setDefaultRules(sourceMailboxes, targetMailboxes, defaultCallback)
r.Equal(t, map[string]*Rule{
mailbox1.Hash(): {Active: true, SourceMailbox: mailbox1, TargetMailboxes: []Mailbox{mailbox3}},
mailbox2.Hash(): {Active: true, SourceMailbox: mailbox2, TargetMailboxes: []Mailbox{mailbox2}},
mailbox3.Hash(): {Active: true, SourceMailbox: mailbox3, TargetMailboxes: []Mailbox{mailbox3}},
mailbox4.Hash(): {Active: false, SourceMailbox: mailbox4, TargetMailboxes: []Mailbox{}},
mailbox5.Hash(): {Active: false, SourceMailbox: mailbox5, TargetMailboxes: []Mailbox{}},
mailbox6a.Hash(): {Active: false, SourceMailbox: mailbox6a, TargetMailboxes: []Mailbox{mailbox6b}},
}, rules.rules)
}
func TestSetDefaultRulesDeactivateMissing(t *testing.T) {
path, err := ioutil.TempDir("", "rules")
r.NoError(t, err)
defer os.RemoveAll(path) //nolint[errcheck]
rules := loadRules(path, "rule")
mailboxA := Mailbox{ID: "1", Name: "One", Color: "", IsExclusive: true}
mailboxB := Mailbox{ID: "2", Name: "Two", Color: "", IsExclusive: true}
r.NoError(t, rules.setRule(mailboxA, []Mailbox{mailboxB}, 0, 0))
r.NoError(t, rules.setRule(mailboxB, []Mailbox{mailboxB}, 0, 0))
sourceMailboxes := []Mailbox{mailboxA}
targetMailboxes := []Mailbox{mailboxA, mailboxB}
defaultCallback := func(mailbox Mailbox) (mailboxes []Mailbox) {
return
}
rules.setDefaultRules(sourceMailboxes, targetMailboxes, defaultCallback)
r.Equal(t, map[string]*Rule{
mailboxA.Hash(): {Active: true, SourceMailbox: mailboxA, TargetMailboxes: []Mailbox{mailboxB}, FromTime: 0, ToTime: 0},
}, rules.rules)
}
func TestIsTimeInRange(t *testing.T) {
tests := []struct {
rule Rule
time int64
want bool
}{
{generateTimeRule(0, 0), 0, true},
{generateTimeRule(0, 0), 10, true},
{generateTimeRule(0, 15), 10, true},
{generateTimeRule(5, 15), 10, true},
{generateTimeRule(0, 5), 10, false},
{generateTimeRule(5, 7), 10, false},
{generateTimeRule(15, 30), 10, false},
{generateTimeRule(15, 0), 10, false},
}
for _, tc := range tests {
tc := tc
t.Run(fmt.Sprintf("%v / %d", tc.rule, tc.time), func(t *testing.T) {
got := tc.rule.isTimeInRange(tc.time)
r.Equal(t, tc.want, got)
})
}
}
func TestHasTimeLimit(t *testing.T) {
tests := []struct {
rule Rule
want bool
}{
{generateTimeRule(0, 0), false},
{generateTimeRule(0, 1), true},
{generateTimeRule(1, 2), true},
{generateTimeRule(1, 0), true},
}
for _, tc := range tests {
tc := tc
t.Run(fmt.Sprintf("%v", tc.rule), func(t *testing.T) {
r.Equal(t, tc.want, tc.rule.HasTimeLimit())
})
}
}
func generateTimeRule(from, to int64) Rule {
return Rule{
SourceMailbox: Mailbox{},
TargetMailboxes: []Mailbox{},
FromTime: from,
ToTime: to,
}
}
func TestOrderRules(t *testing.T) {
wantMailboxOrder := []Mailbox{
{Name: "Inbox", IsExclusive: true},
{Name: "Drafts", IsExclusive: true},
{Name: "Sent", IsExclusive: true},
{Name: "Starred", IsExclusive: true},
{Name: "Archive", IsExclusive: true},
{Name: "Spam", IsExclusive: true},
{Name: "All Mail", IsExclusive: true},
{Name: "Folder A", IsExclusive: true},
{Name: "Folder B", IsExclusive: true},
{Name: "Folder C", IsExclusive: true},
{Name: "Label A", IsExclusive: false},
{Name: "Label B", IsExclusive: false},
{Name: "Label C", IsExclusive: false},
}
wantMailboxNames := []string{}
rules := map[string]*Rule{}
for _, mailbox := range wantMailboxOrder {
wantMailboxNames = append(wantMailboxNames, mailbox.Name)
rules[mailbox.Hash()] = &Rule{
SourceMailbox: mailbox,
}
}
transferRules := transferRules{
rules: rules,
}
gotMailboxNames := []string{}
for _, rule := range transferRules.getSortedRules() {
gotMailboxNames = append(gotMailboxNames, rule.SourceMailbox.Name)
}
r.Equal(t, wantMailboxNames, gotMailboxNames)
}

4
internal/transfer/testdata/eml/Foo/msg.eml vendored
View File

@ -1,4 +0,0 @@
From: Bridge Test <bridgetest@pm.test>
To: Bridge Test <bridgetest@protonmail.com>
hello

4
internal/transfer/testdata/eml/Inbox/msg.eml vendored
View File

@ -1,4 +0,0 @@
From: Bridge Test <bridgetest@pm.test>
To: Bridge Test <bridgetest@protonmail.com>
hello

4
internal/transfer/testdata/emlmbox/Foo/msg.eml vendored
View File

@ -1,4 +0,0 @@
From: Bridge Test <bridgetest@pm.test>
To: Bridge Test <bridgetest@protonmail.com>
hello

5
internal/transfer/testdata/emlmbox/Inbox.mbox vendored
View File

@ -1,5 +0,0 @@
From - Mon May 4 16:40:31 2020
From: Bridge Test <bridgetest@pm.test>
To: Bridge Test <bridgetest@protonmail.com>
hello

62
internal/transfer/testdata/keyring_userKey vendored
View File

@ -1,62 +0,0 @@
-----BEGIN PGP PRIVATE KEY BLOCK-----
Version: OpenPGP.js v4.4.5
Comment: testpassphrase
xcLYBFzGzhEBCADBxfqTFMqfQzT77A5tuuhPFwPq8dfC2evs8u1OvTqFbztY
5FOuSxzduyeDqQ1Fx6dKEOKgcYE8t1Uh4VSS7z6bTdY8j9yrL81kCVB46sE1
OzStzyx/5l7OdH/pM4F+aKslnLvqlw0UeJr+UNizVtOCEUaNfVjPK3cc1ocx
v+36K4RnnyfEtjUW9gDZbhgaF02G5ILHmWmbgM7I+77gCd2wI0EdY9s/JZQ+
VmkMFqoMdY9PyBchoOIPUkkGQi1SaF4IEzMaAUSbnCYkHHY/SbfDTcR46VGq
cXlkB1rq5xskaUQ9r+giCC/K4pc7bBkI1lQ7ADVuWvdrWnWapK0FO6CfABEB
AAEAB/0YPhPJ0phA/EWviN+16bmGVOZNaVapjt2zMMybWmrtEQv3OeWgO3nP
4cohRi/zaCBCphcm+dxbLhftW7AFi/9PVcR09436MB+oTCQFugpUWw+4TmA5
BidxTpDxf4X2vH3rquQLBufWL6U7JlPeKAGL1xZ2aCq0DIeOk5D+xTjZizV2
GIyQRVCLWb+LfDmvvcp3Y94X60KXdBAMuS1ZMKcY3Sl8VAXNB4KQsC/kByzf
6FCB097XZRYV7lvJJQ7+6Wisb3yVi8sEQx2sFm5fAp+0qi3a6zRTEp49r6Hr
gyWViH5zOOpA7DcNwx1Bwhi7GG0tak6EUnnKUNLfOupglcphBADmpXCgT4nc
uSBYTiZSVcB/ICCkTxVsHL1WcXtPK2Ikzussx2n9kb0rapvuC0YLipX9lUkQ
fyeC3jQJeCyN79AkDGkOfWaESueT2hM0Po+RwDgMibKn6yJ1zebz4Lc2J3C9
oVFcAnql+9KyGsAPn03fyQzDnvhNnJvHJi4Hx8AWoQQA1xLoXeVBjRi0IjjU
E6Mqaq5RLEog4kXRp86VSSEGHBwyIYnDiM//gjseo/CXuVyHwL7UXitp8s1B
D1uE3APrhqUS66fD5pkF+z+RcSqiIv7I76NJ24Cdg38L6seGSjOHrq7/dEeG
K6WqfQUCEjta3yNSg7pXb2wn2WZqKIK+rz8EALZRuMXeql/FtO3Cjb0sv7oT
9dLP4cn1bskGRJ+Vok9lfCERbfXGccoAk3V+qSfpHgKxsebkRbUhf+trOGnw
tW+kBWo/5hYGQuN+A9JogSJViT+nuZyE+x1/rKswDFmlMSdf2GIDARWIV0gc
b1yOEwUmNBSthPcnFXvBr4BG3XTtNPTNLSJhcm9uMjEtM0BzYWRlbWJlLm9y
ZyIgPGFyb24yMS0zQHNhZGVtYmUub3JnPsLAdQQQAQgAHwUCXMbOEQYLCQcI
AwIEFQgKAgMWAgECGQECGwMCHgEACgkQZ/B4v2b2xB6XUgf/dHGRHimyMR78
QYbEm2cuaEvOtq4a+J6Zv3P4VOWAbvkGWS9LDKSvVi60vq4oYOmF54HgPzur
nA4OtZDf0HKwQK45VZ7CYD693o70jkKPrAAJG3yTsbesfiS7RbFyGKzKJ7EL
nsUIJkfgm/SlKmXU/u8MOBO5Wg7/TcsS33sRWHl90j+9jbhqdl92R+vY/CwC
ieFkQA7/TDv1u+NAalH+Lpkd8AIuEcki+TAogZ7oi/SnofwnoB7BxRm+mIkp
ZZhIDSCaPOzLG8CSZ81d3HVHhqbf8dh0DFKFoUYyKdbOqIkNWWASf+c/ZEme
IWcekY8hqwf/raZ56tGM/bRwYPcotMfC1wRcxs4RAQgAsMb5/ELWmrfPy3ba
5qif+RXhGSbjitATNgHpoPUHrfTC7cn4JWHqehoXLAQpFAoKd+O/ZNpZozK9
ilpqGUx05yMw06jNQEhYIbgIF4wzPpz02Lp6YeMwdF5LF+Rw83PHdHrA/wRV
/QjL04+kZnN+G5HmzMlhFY+oZSpL+Gp1bTXgtAVDkhCnMB5tP2VwULMGyJ+X
vRYxwTK2CrLjIVZv5n1VYY+caCowU6j/XFqvlCJj+G5oV+UhFOWffaMRXhOh
a64RrhqT1Np7wCLvLMP2wpys9xlMcLQJLqDNxqOTp504V7dm67ncC0fKUsT4
m4oTktnxKPd6MU+4VYveaLCquwARAQABAAf4u9s7gpGErs1USxmDO9TlyGZK
aBlri8nMf3s+hOJCOo3cRaRHJBfdY6pu/baG6H6JTsWzeY4MHwr6N+dhVIEh
FPMa9EZAjagyc4GugxWGiMVTfU+2AEfdrdynhQKMgXSctnnNCdkRuX0nwqb3
nlupm1hsz2ze4+Wg0BKSLS0FQdoUbITdJUR69OHr4dNJVHWYI0JSBx4SdhV3
y9163dDvmc+lW9AEaD53vyZWfzCHZxsR/gI32VmT0z5gn1t8w9AOdXo2lA1H
bf7wh4/qCyujGu64ToZtiEny/GCyM6PofLtiZuJNLw3s/y+B2tKv22aTJ760
+Gib1xB9WcWjKyrxBADoeCyq+nHGrl0CwOkmjanlFymgo7mnBOXuiFOvGrKk
M1meMU1TI4TEBWkVnDVMcSejgjAf/bX1dtouba1tMAMu7DlaV/0EwbSADRel
RSqEbIzIOys+y9TY/BMI/uCKNyEKHvu1KUXADb+CBpdBpCfMBWDANFlo9xLz
Ajcmu2dyawQAwquwC0VXQcvzfs+Hd5au5XvHdm1KidOiAdu6PH1SrOgenIN4
lkEjHrJD9jmloO2/GVcxDBB2pmf0B4HEg7DuY9LXBrksP5eSbbRc5+UH1HUv
u82AqQnfNKTd/jae+lLwaOS++ohtwMkkD6W0LdWnHPjyyXg4Oi9zPID3asRu
3PED/3CYyjl6S8GTMY4FNH7Yxu9+NV2xpKE92Hf8K/hnYlmSSVKDCEeOJtLt
BkkcSqY6liCNSMmJdVyAF2GrR+zmDac7UQRssf57oOWtSsGozt0aqJXuspMT
6aB+P1UhZ8Ly9rWZNiJ0jwyfnQNOLCYDaqjFmiSpqrNnJ2Q1Xge3+k80P9DC
wF8EGAEIAAkFAlzGzhECGwwACgkQZ/B4v2b2xB5wlwgAjZA1zdv5irFjyWVo
4/itONtyO1NbdpyYpcct7vD0oV+a4wahQP0J3Kk1GhZ5tvAoZF/jakQQOM5o
GjUYpXAGnr09Mv9EiQ2pDwXc2yq0WfXnGxNrpzOqdtV+IqY9NYkl55Tme7x+
WRvrkPSUeUsyEGvxwR1stdv8eg9jUmxdl8Io3PYoFJJlrM/6aXeC1r3KOj7q
XAnR0XHJ+QBSNKCWLlQv5hui9BKfcLiVKFK/dNhs82nRyhPr4sWFw6MTqdAK
4zkn7l0jmy6Evi1AiiGPiHPnxeNErnofOIEh4REQj00deZADHrixTLtx2FuR
uaSC3IcBmBsj1fNb4eYXElILjQ==
=fMOl
-----END PGP PRIVATE KEY BLOCK-----

16
internal/transfer/testdata/mbox-applemail/All Mail.mbox/mbox vendored
View File

@ -1,16 +0,0 @@
From - Mon May 4 16:40:31 2020
From: Bridge Test <bridgetest@pm.test>
To: Bridge Test <bridgetest@protonmail.com>
Subject: Test 1
X-Gmail-Labels: Foo,Bar
hello
From - Mon May 4 16:40:31 2020
From: Bridge Test <bridgetest@pm.test>
To: Bridge Test <bridgetest@protonmail.com>
Subject: Test 2
X-Gmail-Labels: Foo
hello

0
internal/transfer/testdata/mbox-applemail/Inbox.mbox/.keep vendored
View File

16
internal/transfer/testdata/mbox/All Mail.mbox vendored
View File

@ -1,16 +0,0 @@
From - Mon May 4 16:40:31 2020
From: Bridge Test <bridgetest@pm.test>
To: Bridge Test <bridgetest@protonmail.com>
Subject: Test 1
X-Gmail-Labels: Foo,Bar
hello
From - Mon May 4 16:40:31 2020
From: Bridge Test <bridgetest@pm.test>
To: Bridge Test <bridgetest@protonmail.com>
Subject: Test 2
X-Gmail-Labels: Foo
hello

5
internal/transfer/testdata/mbox/Foo.mbox vendored
View File

@ -1,5 +0,0 @@
From - Mon May 4 16:40:31 2020
From: Bridge Test <bridgetest@pm.test>
To: Bridge Test <bridgetest@protonmail.com>
hello

5
internal/transfer/testdata/mbox/Inbox.mbox vendored
View File

@ -1,5 +0,0 @@
From - Mon May 4 16:40:31 2020
From: Bridge Test <bridgetest@pm.test>
To: Bridge Test <bridgetest@protonmail.com>
hello

80
internal/transfer/timeit.go
View File

@ -1,80 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"sync"
"time"
)
type timeIt struct {
lock sync.Locker
name string
groups map[string]int64
ongoing map[string]time.Time
}
func newTimeIt(name string) *timeIt {
return &timeIt{
lock: &sync.Mutex{},
name: name,
groups: map[string]int64{},
ongoing: map[string]time.Time{},
}
}
func (t *timeIt) clear() {
t.lock.Lock()
defer t.lock.Unlock()
t.groups = map[string]int64{}
t.ongoing = map[string]time.Time{}
}
func (t *timeIt) start(group, id string) {
t.lock.Lock()
defer t.lock.Unlock()
t.ongoing[group+"/"+id] = time.Now()
}
func (t *timeIt) stop(group, id string) {
endTime := time.Now()
t.lock.Lock()
defer t.lock.Unlock()
startTime, ok := t.ongoing[group+"/"+id]
if !ok {
log.WithField("group", group).WithField("id", id).Error("Stop called before start")
return
}
delete(t.ongoing, group+"/"+id)
diff := endTime.Sub(startTime).Milliseconds()
t.groups[group] += diff
}
func (t *timeIt) logResults() {
t.lock.Lock()
defer t.lock.Unlock()
// Print also ongoing to be sure that nothing was left out.
// Basically ongoing should be empty.
log.WithField("name", t.name).WithField("result", t.groups).WithField("ongoing", t.ongoing).Debug("Time measurement")
}

214
internal/transfer/transfer.go
View File

@ -1,214 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
// Package transfer provides tools to export messages from one provider and
// import them to another provider. Provider can be EML, MBOX, IMAP or PMAPI.
package transfer
import (
"crypto/sha256"
"fmt"
"github.com/sirupsen/logrus"
)
var log = logrus.WithField("pkg", "transfer") //nolint[gochecknoglobals]
// Transfer is facade on top of import rules, progress manager and source
// and target providers. This is the main object which should be used.
type Transfer struct {
panicHandler PanicHandler
metrics MetricsManager
id string
logDir string
rules transferRules
source SourceProvider
target TargetProvider
rulesCache []*Rule
sourceMboxCache []Mailbox
targetMboxCache []Mailbox
}
// New creates Transfer for specific source and target. Usage:
//
// source := transfer.NewEMLProvider(...)
// target := transfer.NewPMAPIProvider(...)
// transfer.New(source, target, ...)
func New(panicHandler PanicHandler, metrics MetricsManager, logDir, rulesDir string, source SourceProvider, target TargetProvider) (*Transfer, error) {
transferID := fmt.Sprintf("%x", sha256.Sum256([]byte(source.ID()+"-"+target.ID())))
rules := loadRules(rulesDir, transferID)
transfer := &Transfer{
panicHandler: panicHandler,
metrics: metrics,
id: transferID,
logDir: logDir,
rules: rules,
source: source,
target: target,
}
if err := transfer.setDefaultRules(); err != nil {
return nil, err
}
metrics.Load(len(transfer.sourceMboxCache))
return transfer, nil
}
// SetDefaultRules sets missing rules for source mailboxes with matching
// target mailboxes. In case no matching mailbox is found, `defaultCallback`
// with a source mailbox as a parameter is used.
func (t *Transfer) setDefaultRules() error {
sourceMailboxes, err := t.SourceMailboxes()
if err != nil {
return err
}
targetMailboxes, err := t.TargetMailboxes()
if err != nil {
return err
}
defaultCallback := func(sourceMailbox Mailbox) []Mailbox {
return t.target.DefaultMailboxes(sourceMailbox)
}
t.rules.setDefaultRules(sourceMailboxes, targetMailboxes, defaultCallback)
return nil
}
// SetSkipEncryptedMessages sets whether message which cannot be decrypted
// should be imported/exported or skipped.
func (t *Transfer) SetSkipEncryptedMessages(skip bool) {
t.rules.setSkipEncryptedMessages(skip)
}
// SetGlobalMailbox sets mailbox that is applied to every message in
// the import phase.
func (t *Transfer) SetGlobalMailbox(mailbox *Mailbox) {
t.rules.setGlobalMailbox(mailbox)
}
// SetGlobalTimeLimit sets time limit that is applied to rules without any
// specified time limit.
func (t *Transfer) SetGlobalTimeLimit(fromTime, toTime int64) {
t.rules.setGlobalTimeLimit(fromTime, toTime)
}
// SetRule sets sourceMailbox for transfer.
func (t *Transfer) SetRule(sourceMailbox Mailbox, targetMailboxes []Mailbox, fromTime, toTime int64) error {
t.rulesCache = nil
return t.rules.setRule(sourceMailbox, targetMailboxes, fromTime, toTime)
}
// UnsetRule unsets sourceMailbox from transfer.
func (t *Transfer) UnsetRule(sourceMailbox Mailbox) {
t.rulesCache = nil
t.rules.unsetRule(sourceMailbox)
}
// ResetRules unsets all rules.
func (t *Transfer) ResetRules() {
t.rulesCache = nil
t.rules.reset()
}
// GetRule returns rule for given mailbox.
func (t *Transfer) GetRule(sourceMailbox Mailbox) *Rule {
return t.rules.getRule(sourceMailbox)
}
// GetRules returns all set transfer rules.
func (t *Transfer) GetRules() []*Rule {
if t.rulesCache == nil {
t.rulesCache = t.rules.getSortedRules()
}
return t.rulesCache
}
// SourceMailboxes returns mailboxes available at source side.
func (t *Transfer) SourceMailboxes() (m []Mailbox, err error) {
if t.sourceMboxCache == nil {
t.sourceMboxCache, err = t.source.Mailboxes(false, true)
}
return t.sourceMboxCache, err
}
// TargetMailboxes returns mailboxes available at target side.
func (t *Transfer) TargetMailboxes() (m []Mailbox, err error) {
if t.targetMboxCache == nil {
t.targetMboxCache, err = t.target.Mailboxes(true, false)
}
return t.targetMboxCache, err
}
// CreateTargetMailbox creates mailbox in target provider.
func (t *Transfer) CreateTargetMailbox(mailbox Mailbox) (Mailbox, error) {
t.targetMboxCache = nil
return t.target.CreateMailbox(mailbox)
}
// ChangeTarget changes the target. It is safe to change target for export,
// must not be changed for import. Do not set after you started transfer.
func (t *Transfer) ChangeTarget(target TargetProvider) {
t.targetMboxCache = nil
t.target = target
}
// Start starts the transfer from source to target.
func (t *Transfer) Start() *Progress {
log.Debug("Transfer started")
t.rules.save()
t.rules.propagateGlobalTime()
t.metrics.Start()
log := log.WithField("id", t.id)
reportFile := newFileReport(t.logDir, t.id)
progress := newProgress(log, reportFile)
// Small queue to prevent having idle source while target is blocked.
// E.g., when upload to PM is in progress, we can in meantime download
// the next batch from remote IMAP server.
ch := make(chan Message, 10)
go func() {
defer t.panicHandler.HandlePanic()
t.source.TransferTo(t.rules, &progress, ch)
close(ch)
}()
go func() {
defer t.panicHandler.HandlePanic()
t.target.TransferFrom(t.rules, &progress, ch)
progress.finish()
if progress.isStopped {
if progress.fatalError != nil {
t.metrics.Fail()
} else {
t.metrics.Cancel()
}
} else {
t.metrics.Complete()
}
}()
return &progress
}

71
internal/transfer/transfer_test.go
View File

@ -1,71 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"io/ioutil"
"testing"
"github.com/ProtonMail/gopenpgp/v2/crypto"
transfermocks "github.com/ProtonMail/proton-bridge/internal/transfer/mocks"
pmapimocks "github.com/ProtonMail/proton-bridge/pkg/pmapi/mocks"
gomock "github.com/golang/mock/gomock"
)
type mocks struct {
t *testing.T
ctrl *gomock.Controller
panicHandler *transfermocks.MockPanicHandler
imapClientProvider *transfermocks.MockIMAPClientProvider
pmapiClient *pmapimocks.MockClient
keyring *crypto.KeyRing
}
func initMocks(t *testing.T) mocks {
mockCtrl := gomock.NewController(t)
m := mocks{
t: t,
ctrl: mockCtrl,
panicHandler: transfermocks.NewMockPanicHandler(mockCtrl),
imapClientProvider: transfermocks.NewMockIMAPClientProvider(mockCtrl),
pmapiClient: pmapimocks.NewMockClient(mockCtrl),
keyring: newTestKeyring(),
}
return m
}
func newTestKeyring() *crypto.KeyRing {
data, err := ioutil.ReadFile("testdata/keyring_userKey")
if err != nil {
panic(err)
}
key, err := crypto.NewKeyFromArmored(string(data))
if err != nil {
panic(err)
}
userKey, err := crypto.NewKeyRing(key)
if err != nil {
panic(err)
}
return userKey
}

30
internal/transfer/types.go
View File

@ -1,30 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
type PanicHandler interface {
HandlePanic()
}
type MetricsManager interface {
Load(int)
Start()
Complete()
Cancel()
Fail()
}

186
internal/transfer/utils.go
View File

@ -1,186 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"bufio"
"bytes"
"io/ioutil"
"net/mail"
"net/textproto"
"path/filepath"
"runtime"
"sort"
"strings"
"github.com/ProtonMail/go-rfc5322"
"github.com/pkg/errors"
)
// getFolderNames collects all folder names under `root`.
// Folder names will be without a path.
func getFolderNames(root string) ([]string, error) {
return getFolderNamesWithFileSuffix(root, "")
}
// getFolderNamesWithFileSuffix collects all folder names under `root`, which
// contains some file with a give `fileSuffix`. Names will be without a path.
func getFolderNamesWithFileSuffix(root, fileSuffix string) ([]string, error) {
folders := []string{}
files, err := ioutil.ReadDir(root)
if err != nil {
return nil, err
}
hasFileWithSuffix := fileSuffix == ""
for _, file := range files {
if file.IsDir() {
subfolders, err := getFolderNamesWithFileSuffix(filepath.Join(root, file.Name()), fileSuffix)
if err != nil {
return nil, err
}
for _, subfolder := range subfolders {
match := false
for _, folder := range folders {
if folder == subfolder {
match = true
break
}
}
if !match {
folders = append(folders, subfolder)
}
}
} else if fileSuffix == "" || strings.HasSuffix(file.Name(), fileSuffix) {
hasFileWithSuffix = true
}
}
if hasFileWithSuffix {
folders = append(folders, filepath.Base(root))
}
sort.Strings(folders)
return folders, nil
}
// getFilePathsWithSuffix collects all file names with `suffix` under `root`.
// File names will be with relative path based to `root`.
func getFilePathsWithSuffix(root, suffix string) ([]string, error) {
fileNames, err := getFilePathsWithSuffixInner("", root, suffix, false)
if err != nil {
return nil, err
}
sort.Strings(fileNames)
return fileNames, err
}
// getAllPathsWithSuffix is the same as getFilePathsWithSuffix but includes
// also directories.
func getAllPathsWithSuffix(root, suffix string) ([]string, error) {
fileNames, err := getFilePathsWithSuffixInner("", root, suffix, true)
if err != nil {
return nil, err
}
sort.Strings(fileNames)
return fileNames, err
}
func getFilePathsWithSuffixInner(prefix, root, suffix string, includeDir bool) ([]string, error) {
fileNames := []string{}
files, err := ioutil.ReadDir(root)
if err != nil {
return nil, err
}
for _, file := range files {
if !file.IsDir() {
if strings.HasSuffix(file.Name(), suffix) {
fileNames = append(fileNames, filepath.Join(prefix, file.Name()))
}
} else {
if includeDir && strings.HasSuffix(file.Name(), suffix) {
fileNames = append(fileNames, filepath.Join(prefix, file.Name()))
}
subfolderFileNames, err := getFilePathsWithSuffixInner(
filepath.Join(prefix, file.Name()),
filepath.Join(root, file.Name()),
suffix,
includeDir,
)
if err != nil {
return nil, err
}
fileNames = append(fileNames, subfolderFileNames...)
}
}
return fileNames, nil
}
// getMessageTime returns time of the message specified in the message header.
func getMessageTime(body []byte) (int64, error) {
hdr, err := getMessageHeader(body)
if err != nil {
return 0, err
}
t, err := rfc5322.ParseDateTime(hdr.Get("Date"))
if err != nil {
return 0, err
}
if t.IsZero() {
return 0, nil
}
return t.Unix(), nil
}
// getMessageHeader returns headers of the message body.
func getMessageHeader(body []byte) (mail.Header, error) {
tpr := textproto.NewReader(bufio.NewReader(bytes.NewBuffer(body)))
header, err := tpr.ReadMIMEHeader()
if err != nil {
return nil, errors.Wrap(err, "failed to read headers")
}
return mail.Header(header), nil
}
// sanitizeFileName replaces problematic special characters with underscore.
func sanitizeFileName(fileName string) string {
if len(fileName) == 0 {
return fileName
}
if runtime.GOOS != "windows" && (fileName[0] == '-' || fileName[0] == '.') { //nolint[goconst]
fileName = "_" + fileName[1:]
}
return strings.Map(func(r rune) rune {
switch r {
case '\\', '/', ':', '*', '?', '"', '<', '>', '|':
return '_'
case '[', ']', '(', ')', '{', '}', '^', '#', '%', '&', '!', '@', '+', '=', '\'', '~':
if runtime.GOOS != "windows" {
return '_'
}
}
return r
}, fileName)
}

225
internal/transfer/utils_test.go
View File

@ -1,225 +0,0 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"io/ioutil"
"os"
"path/filepath"
"runtime"
"testing"
r "github.com/stretchr/testify/require"
)
func TestGetFolderNames(t *testing.T) {
root, clean := createTestingFolderStructure(t)
defer clean()
tests := []struct {
suffix string
wantNames []string
}{
{
"",
[]string{
"bar",
"bar.mbox",
"baz",
filepath.Base(root),
"foo",
"qwerty",
"test",
},
},
{
".eml",
[]string{
"bar",
"baz",
filepath.Base(root),
"foo",
},
},
{
".txt",
[]string{
filepath.Base(root),
},
},
}
for _, tc := range tests {
tc := tc
t.Run(tc.suffix, func(t *testing.T) {
names, err := getFolderNamesWithFileSuffix(root, tc.suffix)
r.NoError(t, err)
r.Equal(t, tc.wantNames, names)
})
}
}
func TestGetFilePathsWithSuffix(t *testing.T) {
root, clean := createTestingFolderStructure(t)
defer clean()
tests := []struct {
suffix string
wantPaths []string
}{
{
".eml",
[]string{
"foo/bar/baz/msg1.eml",
"foo/bar/baz/msg2.eml",
"foo/bar/baz/msg3.eml",
"foo/bar/msg4.eml",
"foo/bar/msg5.eml",
"foo/baz/msg6.eml",
"foo/msg7.eml",
"msg10.eml",
"test/foo/msg8.eml",
"test/foo/msg9.eml",
},
},
{
".mbox",
[]string{
"bar.mbox",
"foo.mbox",
},
},
{
".txt",
[]string{
"info.txt",
},
},
{
".hello",
[]string{},
},
}
for _, tc := range tests {
tc := tc
t.Run(tc.suffix, func(t *testing.T) {
paths, err := getAllPathsWithSuffix(root, tc.suffix)
r.NoError(t, err)
r.Equal(t, tc.wantPaths, paths)
})
}
}
func createTestingFolderStructure(t *testing.T) (string, func()) {
root, err := ioutil.TempDir("", "folderstructure")
r.NoError(t, err)
for _, path := range []string{
"foo/bar/baz",
"foo/baz",
"test/foo",
"qwerty",
"bar.mbox",
} {
err = os.MkdirAll(filepath.Join(root, path), os.ModePerm)
r.NoError(t, err)
}
for _, path := range []string{
"foo/bar/baz/msg1.eml",
"foo/bar/baz/msg2.eml",
"foo/bar/baz/msg3.eml",
"foo/bar/msg4.eml",
"foo/bar/msg5.eml",
"foo/baz/msg6.eml",
"foo/msg7.eml",
"test/foo/msg8.eml",
"test/foo/msg9.eml",
"msg10.eml",
"info.txt",
"foo.mbox",
"bar.mbox/mbox", // Apple Mail mbox export format.
} {
f, err := os.Create(filepath.Join(root, path))
r.NoError(t, err)
err = f.Close()
r.NoError(t, err)
}
return root, func() {
_ = os.RemoveAll(root)
}
}
func TestGetMessageTime(t *testing.T) {
tests := []struct {
body string
wantTime int64
wantErr string
}{
{"", 0, "failed to read headers: EOF"},
{"Subject: hello\n\n", 0, ""},
{"Date: Thu, 23 Apr 2020 04:52:44 +0000\n\n", 1587617564, ""},
}
for _, tc := range tests {
tc := tc
t.Run(tc.body, func(t *testing.T) {
time, err := getMessageTime([]byte(tc.body))
if tc.wantErr == "" {
r.NoError(t, err)
} else {
r.EqualError(t, err, tc.wantErr)
}
r.Equal(t, tc.wantTime, time)
})
}
}
func TestGetMessageHeader(t *testing.T) {
body := `Subject: Hello
From: user@example.com
Body
`
header, err := getMessageHeader([]byte(body))
r.NoError(t, err)
r.Equal(t, header.Get("subject"), "Hello")
r.Equal(t, header.Get("from"), "user@example.com")
}
func TestSanitizeFileName(t *testing.T) {
tests := map[string]string{
"hello": "hello",
"a\\b/c:*?d\"<>|e": "a_b_c___d____e",
}
if runtime.GOOS == "darwin" || runtime.GOOS == "linux" {
tests[".hello"] = "_hello"
tests["-hello"] = "_hello"
}
if runtime.GOOS == "windows" {
tests["[hello]&@=~~"] = "_hello______"
}
for path, wantPath := range tests {
path := path
wantPath := wantPath
t.Run(path, func(t *testing.T) {
gotPath := sanitizeFileName(path)
r.Equal(t, wantPath, gotPath)
})
}
}

18
internal/users/user.go
View File

@ -50,8 +50,6 @@ type User struct {
creds *credentials.Credentials
lock sync.RWMutex
useOnlyActiveAddresses bool
}
// newUser creates a new user.
@ -62,7 +60,6 @@ func newUser(
eventListener listener.Listener,
credStorer CredentialsStorer,
storeFactory StoreMaker,
useOnlyActiveAddresses bool,
) (*User, *credentials.Credentials, error) {
log := log.WithField("user", userID)
@ -74,14 +71,13 @@ func newUser(
}
return &User{
log: log,
panicHandler: panicHandler,
listener: eventListener,
credStorer: credStorer,
storeFactory: storeFactory,
userID: userID,
creds: creds,
useOnlyActiveAddresses: useOnlyActiveAddresses,
log: log,
panicHandler: panicHandler,
listener: eventListener,
credStorer: credStorer,
storeFactory: storeFactory,
userID: userID,
creds: creds,
}, creds, nil
}

2
internal/users/user_credentials_test.go
View File

@ -172,7 +172,7 @@ func TestCheckBridgeLoginLoggedOut(t *testing.T) {
m.eventListener.EXPECT().Emit(events.LogoutEvent, "user"),
)
user, _, err := newUser(m.PanicHandler, "user", m.eventListener, m.credentialsStore, m.storeMaker, false)
user, _, err := newUser(m.PanicHandler, "user", m.eventListener, m.credentialsStore, m.storeMaker)
r.NoError(t, err)
err = user.connect(m.pmapiClient, testCredentialsDisconnected)

4
internal/users/user_new_test.go
View File

@ -33,7 +33,7 @@ func TestNewUserNoCredentialsStore(t *testing.T) {
m.credentialsStore.EXPECT().Get("user").Return(nil, errors.New("fail"))
_, _, err := newUser(m.PanicHandler, "user", m.eventListener, m.credentialsStore, m.storeMaker, false)
_, _, err := newUser(m.PanicHandler, "user", m.eventListener, m.credentialsStore, m.storeMaker)
r.Error(t, err)
}
@ -71,7 +71,7 @@ func TestNewUser(t *testing.T) {
}
func checkNewUserHasCredentials(m mocks, wantErr string, wantCreds *credentials.Credentials) {
user, _, err := newUser(m.PanicHandler, "user", m.eventListener, m.credentialsStore, m.storeMaker, false)
user, _, err := newUser(m.PanicHandler, "user", m.eventListener, m.credentialsStore, m.storeMaker)
r.NoError(m.t, err)
defer cleanUpUserData(user)

2
internal/users/user_test.go
View File

@ -27,7 +27,7 @@ func testNewUser(m mocks) *User {
mockInitConnectedUser(m)
mockEventLoopNoAction(m)
user, creds, err := newUser(m.PanicHandler, "user", m.eventListener, m.credentialsStore, m.storeMaker, false)
user, creds, err := newUser(m.PanicHandler, "user", m.eventListener, m.credentialsStore, m.storeMaker)
r.NoError(m.t, err)
err = user.connect(m.pmapiClient, creds)

36
internal/users/users.go
View File

@ -58,12 +58,6 @@ type Users struct {
// People are used to that and so we preserve that ordering here.
users []*User
// useOnlyActiveAddresses determines whether credentials keeps only active
// addresses or all of them. Each usage has to be consisteng, e.g., once
// user is added, it saves address list to credentials and next time loads
// as is, without requesting server again.
useOnlyActiveAddresses bool
lock sync.RWMutex
}
@ -74,19 +68,17 @@ func New(
clientManager pmapi.Manager,
credStorer CredentialsStorer,
storeFactory StoreMaker,
useOnlyActiveAddresses bool,
) *Users {
log.Trace("Creating new users")
u := &Users{
locations: locations,
panicHandler: panicHandler,
events: eventListener,
clientManager: clientManager,
credStorer: credStorer,
storeFactory: storeFactory,
useOnlyActiveAddresses: useOnlyActiveAddresses,
lock: sync.RWMutex{},
locations: locations,
panicHandler: panicHandler,
events: eventListener,
clientManager: clientManager,
credStorer: credStorer,
storeFactory: storeFactory,
lock: sync.RWMutex{},
}
go func() {
@ -135,7 +127,7 @@ func (u *Users) loadUsersFromCredentialsStore() error {
for _, userID := range userIDs {
l := log.WithField("user", userID)
user, creds, err := newUser(u.panicHandler, userID, u.events, u.credStorer, u.storeFactory, u.useOnlyActiveAddresses)
user, creds, err := newUser(u.panicHandler, userID, u.events, u.credStorer, u.storeFactory)
if err != nil {
l.WithError(err).Warn("Could not create user, skipping")
continue
@ -256,19 +248,11 @@ func (u *Users) addNewUser(client pmapi.Client, apiUser *pmapi.User, auth *pmapi
u.lock.Lock()
defer u.lock.Unlock()
var emails []string
if u.useOnlyActiveAddresses {
emails = client.Addresses().ActiveEmails()
} else {
emails = client.Addresses().AllEmails()
}
if _, err := u.credStorer.Add(apiUser.ID, apiUser.Name, auth.UID, auth.RefreshToken, passphrase, emails); err != nil {
if _, err := u.credStorer.Add(apiUser.ID, apiUser.Name, auth.UID, auth.RefreshToken, passphrase, client.Addresses().ActiveEmails()); err != nil {
return errors.Wrap(err, "failed to add user credentials to credentials store")
}
user, creds, err := newUser(u.panicHandler, apiUser.ID, u.events, u.credStorer, u.storeFactory, u.useOnlyActiveAddresses)
user, creds, err := newUser(u.panicHandler, apiUser.ID, u.events, u.credStorer, u.storeFactory)
if err != nil {
return errors.Wrap(err, "failed to create new user")
}

2
internal/users/users_test.go
View File

@ -223,7 +223,7 @@ func testNewUsers(t *testing.T, m mocks) *Users { //nolint[unparam]
m.eventListener.EXPECT().ProvideChannel(events.UpgradeApplicationEvent)
m.eventListener.EXPECT().ProvideChannel(events.InternetOnEvent)
users := New(m.locator, m.PanicHandler, m.eventListener, m.clientManager, m.credentialsStore, m.storeMaker, true)
users := New(m.locator, m.PanicHandler, m.eventListener, m.clientManager, m.credentialsStore, m.storeMaker)
waitForEvents()