yesBad/proton-bridge
1
0
Fork 0
forked from Silverfish/proton-bridge
Code Actions Activity

Merge branch 'master' into release/elbe

Browse Source
This commit is contained in:
Jakub
2020-11-27 07:46:40 +01:00
parent 6230200218 072ce54fe1
commit a50266cdc0
46 changed files with 1870 additions and 646 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
internal/bridge/credits.go
View File

@ -15,8 +15,8 @@
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
// Code generated by ./credits.sh at Wed Nov 4 13:57:47 CET 2020. DO NOT EDIT.
// Code generated by ./credits.sh at Tue Nov 24 08:56:01 AM CET 2020. DO NOT EDIT.
package bridge
const Credits = "github.com/0xAX/notificator;github.com/Masterminds/semver/v3;github.com/ProtonMail/bcrypt;github.com/ProtonMail/crypto;github.com/ProtonMail/docker-credential-helpers;github.com/ProtonMail/go-appdir;github.com/ProtonMail/go-apple-mobileconfig;github.com/ProtonMail/go-autostart;github.com/ProtonMail/go-imap;github.com/ProtonMail/go-imap-id;github.com/ProtonMail/go-smtp;github.com/ProtonMail/go-vcard;github.com/ProtonMail/gopenpgp/v2;github.com/PuerkitoBio/goquery;github.com/abiosoft/ishell;github.com/abiosoft/readline;github.com/allan-simon/go-singleinstance;github.com/antlr/antlr4;github.com/certifi/gocertifi;github.com/chzyer/logex;github.com/chzyer/test;github.com/cucumber/godog;github.com/docker/docker-credential-helpers;github.com/emersion/go-imap;github.com/emersion/go-imap-appendlimit;github.com/emersion/go-imap-idle;github.com/emersion/go-imap-move;github.com/emersion/go-imap-quota;github.com/emersion/go-imap-specialuse;github.com/emersion/go-imap-unselect;github.com/emersion/go-mbox;github.com/emersion/go-message;github.com/emersion/go-sasl;github.com/emersion/go-smtp;github.com/emersion/go-textwrapper;github.com/emersion/go-vcard;github.com/fatih/color;github.com/flynn-archive/go-shlex;github.com/getsentry/raven-go;github.com/go-resty/resty/v2;github.com/golang/mock;github.com/google/go-cmp;github.com/google/uuid;github.com/gopherjs/gopherjs;github.com/hashicorp/go-multierror;github.com/jameskeane/bcrypt;github.com/jaytaylor/html2text;github.com/kardianos/osext;github.com/keybase/go-keychain;github.com/logrusorgru/aurora;github.com/mattn/go-runewidth;github.com/miekg/dns;github.com/myesui/uuid;github.com/nsf/jsondiff;github.com/olekukonko/tablewriter;github.com/pkg/errors;github.com/sirupsen/logrus;github.com/skratchdot/open-golang;github.com/ssor/bom;github.com/stretchr/testify;github.com/therecipe/qt;github.com/twinj/uuid;github.com/urfave/cli;go.etcd.io/bbolt;golang.org/x/crypto;golang.org/x/net;golang.org/x/text;gopkg.in/stretchr/testify.v1;;Font Awesome 4.7.0;;Qt 5.13 by Qt group;"
const Credits = "github.com/0xAX/notificator;github.com/abiosoft/ishell;github.com/abiosoft/readline;github.com/allan-simon/go-singleinstance;github.com/chzyer/logex;github.com/chzyer/test;github.com/cucumber/godog;github.com/docker/docker-credential-helpers;github.com/emersion/go-imap;github.com/emersion/go-imap-appendlimit;github.com/emersion/go-imap-idle;github.com/emersion/go-imap-move;github.com/emersion/go-imap-quota;github.com/emersion/go-imap-specialuse;github.com/emersion/go-imap-unselect;github.com/emersion/go-mbox;github.com/emersion/go-message;github.com/emersion/go-sasl;github.com/emersion/go-smtp;github.com/emersion/go-textwrapper;github.com/emersion/go-vcard;github.com/fatih/color;github.com/flynn-archive/go-shlex;github.com/getsentry/sentry-go;github.com/golang/mock;github.com/google/go-cmp;github.com/google/uuid;github.com/gopherjs/gopherjs;github.com/go-resty/resty/v2;github.com/hashicorp/go-multierror;github.com/jameskeane/bcrypt;github.com/jaytaylor/html2text;github.com/kardianos/osext;github.com/keybase/go-keychain;github.com/logrusorgru/aurora;github.com/Masterminds/semver/v3;github.com/mattn/go-runewidth;github.com/miekg/dns;github.com/myesui/uuid;github.com/nsf/jsondiff;github.com/olekukonko/tablewriter;github.com/pkg/errors;github.com/ProtonMail/bcrypt;github.com/ProtonMail/crypto;github.com/ProtonMail/docker-credential-helpers;github.com/ProtonMail/go-appdir;github.com/ProtonMail/go-apple-mobileconfig;github.com/ProtonMail/go-autostart;github.com/ProtonMail/go-imap;github.com/ProtonMail/go-imap-id;github.com/ProtonMail/gopenpgp/v2;github.com/ProtonMail/go-rfc5322;github.com/ProtonMail/go-smtp;github.com/ProtonMail/go-vcard;github.com/PuerkitoBio/goquery;github.com/sirupsen/logrus;github.com/skratchdot/open-golang;github.com/ssor/bom;github.com/stretchr/testify;github.com/therecipe/qt;github.com/twinj/uuid;github.com/urfave/cli;go.etcd.io/bbolt;golang.org/x/crypto;golang.org/x/net;golang.org/x/text;gopkg.in/stretchr/testify.v1;;Font Awesome 4.7.0;;Qt 5.13 by Qt group;"

17
internal/bridge/release_notes.go
View File

@ -15,17 +15,18 @@
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
// Code generated by ./release-notes.sh at 'Wed Nov 4 12:24:35 PM CET 2020'. DO NOT EDIT.
// Code generated by ./release-notes.sh at 'Mon Nov 23 07:38:53 AM CET 2020'. DO NOT EDIT.
package bridge
const ReleaseNotes = `• Ensured better message flow by refactoring both address and date parsing
• Improved secure connectivity checks
• Better deb packaging
• More robust error handling
const ReleaseNotes = `• Improved package creation logic
• Refactor of sending functions to simplify code maintenance
• Added tests for package creation
• For more detailed summary of the changes see https://github.com/ProtonMail/proton-bridge/blob/master/Changelog.md
`
const ReleaseFixedBugs = `• Ensured that conversations are properly threaded
• Fixed Linux font issues (Fedora)
• Better handling of Mime encrypted messages
const ReleaseFixedBugs = `• Bridge crashes related to labels handling
• GUI popup related to TLS connection error
• An issue where a random session key is included in the data payload
• Error handling (including improved detection)
`

14
internal/cmd/main.go
View File

@ -22,7 +22,7 @@ import (
"runtime"
"github.com/ProtonMail/proton-bridge/pkg/constants"
"github.com/getsentry/raven-go"
"github.com/getsentry/sentry-go"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
)
@ -51,10 +51,18 @@ var (
// Main sets up Sentry, filters out unwanted args, creates app and runs it.
func Main(appName, usage string, extraFlags []cli.Flag, run func(*cli.Context) error) {
if err := raven.SetDSN(constants.DSNSentry); err != nil {
err := sentry.Init(sentry.ClientOptions{
Dsn: constants.DSNSentry,
Release: constants.Revision,
})
sentry.ConfigureScope(func(scope *sentry.Scope) {
scope.SetFingerprint([]string{"{{ default }}"})
})
if err != nil {
log.WithError(err).Errorln("Can not setup sentry DSN")
}
raven.SetRelease(constants.Revision)
filterProcessSerialNumberFromArgs()
filterRestartNumberFromArgs()

2
internal/frontend/qml/ImportExportUI/InlineLabelSelect.qml
View File

@ -45,7 +45,7 @@ Row {
}
InfoToolTip {
info: qsTr( "When master import lablel is selected then all imported email will have this label.", "Tooltip text for master import label")
info: qsTr( "When master import label is selected then all imported emails will have this label.", "Tooltip text for master import label")
anchors.verticalCenter: parent.verticalCenter
}

4
internal/importexport/credits.go
View File

@ -15,8 +15,8 @@
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
// Code generated by ./credits.sh at Wed Nov 4 13:57:47 CET 2020. DO NOT EDIT.
// Code generated by ./credits.sh at Tue Nov 24 08:56:01 AM CET 2020. DO NOT EDIT.
package importexport
const Credits = "github.com/0xAX/notificator;github.com/Masterminds/semver/v3;github.com/ProtonMail/bcrypt;github.com/ProtonMail/crypto;github.com/ProtonMail/docker-credential-helpers;github.com/ProtonMail/go-appdir;github.com/ProtonMail/go-apple-mobileconfig;github.com/ProtonMail/go-autostart;github.com/ProtonMail/go-imap;github.com/ProtonMail/go-imap-id;github.com/ProtonMail/go-smtp;github.com/ProtonMail/go-vcard;github.com/ProtonMail/gopenpgp/v2;github.com/PuerkitoBio/goquery;github.com/abiosoft/ishell;github.com/abiosoft/readline;github.com/allan-simon/go-singleinstance;github.com/antlr/antlr4;github.com/certifi/gocertifi;github.com/chzyer/logex;github.com/chzyer/test;github.com/cucumber/godog;github.com/docker/docker-credential-helpers;github.com/emersion/go-imap;github.com/emersion/go-imap-appendlimit;github.com/emersion/go-imap-idle;github.com/emersion/go-imap-move;github.com/emersion/go-imap-quota;github.com/emersion/go-imap-specialuse;github.com/emersion/go-imap-unselect;github.com/emersion/go-mbox;github.com/emersion/go-message;github.com/emersion/go-sasl;github.com/emersion/go-smtp;github.com/emersion/go-textwrapper;github.com/emersion/go-vcard;github.com/fatih/color;github.com/flynn-archive/go-shlex;github.com/getsentry/raven-go;github.com/go-resty/resty/v2;github.com/golang/mock;github.com/google/go-cmp;github.com/google/uuid;github.com/gopherjs/gopherjs;github.com/hashicorp/go-multierror;github.com/jameskeane/bcrypt;github.com/jaytaylor/html2text;github.com/kardianos/osext;github.com/keybase/go-keychain;github.com/logrusorgru/aurora;github.com/mattn/go-runewidth;github.com/miekg/dns;github.com/myesui/uuid;github.com/nsf/jsondiff;github.com/olekukonko/tablewriter;github.com/pkg/errors;github.com/sirupsen/logrus;github.com/skratchdot/open-golang;github.com/ssor/bom;github.com/stretchr/testify;github.com/therecipe/qt;github.com/twinj/uuid;github.com/urfave/cli;go.etcd.io/bbolt;golang.org/x/crypto;golang.org/x/net;golang.org/x/text;gopkg.in/stretchr/testify.v1;;Font Awesome 4.7.0;;Qt 5.13 by Qt group;"
const Credits = "github.com/0xAX/notificator;github.com/abiosoft/ishell;github.com/abiosoft/readline;github.com/allan-simon/go-singleinstance;github.com/chzyer/logex;github.com/chzyer/test;github.com/cucumber/godog;github.com/docker/docker-credential-helpers;github.com/emersion/go-imap;github.com/emersion/go-imap-appendlimit;github.com/emersion/go-imap-idle;github.com/emersion/go-imap-move;github.com/emersion/go-imap-quota;github.com/emersion/go-imap-specialuse;github.com/emersion/go-imap-unselect;github.com/emersion/go-mbox;github.com/emersion/go-message;github.com/emersion/go-sasl;github.com/emersion/go-smtp;github.com/emersion/go-textwrapper;github.com/emersion/go-vcard;github.com/fatih/color;github.com/flynn-archive/go-shlex;github.com/getsentry/sentry-go;github.com/golang/mock;github.com/google/go-cmp;github.com/google/uuid;github.com/gopherjs/gopherjs;github.com/go-resty/resty/v2;github.com/hashicorp/go-multierror;github.com/jameskeane/bcrypt;github.com/jaytaylor/html2text;github.com/kardianos/osext;github.com/keybase/go-keychain;github.com/logrusorgru/aurora;github.com/Masterminds/semver/v3;github.com/mattn/go-runewidth;github.com/miekg/dns;github.com/myesui/uuid;github.com/nsf/jsondiff;github.com/olekukonko/tablewriter;github.com/pkg/errors;github.com/ProtonMail/bcrypt;github.com/ProtonMail/crypto;github.com/ProtonMail/docker-credential-helpers;github.com/ProtonMail/go-appdir;github.com/ProtonMail/go-apple-mobileconfig;github.com/ProtonMail/go-autostart;github.com/ProtonMail/go-imap;github.com/ProtonMail/go-imap-id;github.com/ProtonMail/gopenpgp/v2;github.com/ProtonMail/go-rfc5322;github.com/ProtonMail/go-smtp;github.com/ProtonMail/go-vcard;github.com/PuerkitoBio/goquery;github.com/sirupsen/logrus;github.com/skratchdot/open-golang;github.com/ssor/bom;github.com/stretchr/testify;github.com/therecipe/qt;github.com/twinj/uuid;github.com/urfave/cli;go.etcd.io/bbolt;golang.org/x/crypto;golang.org/x/net;golang.org/x/text;gopkg.in/stretchr/testify.v1;;Font Awesome 4.7.0;;Qt 5.13 by Qt group;"

10
internal/smtp/preferences.go
View File

@ -45,7 +45,7 @@ type SendPreferences struct {
// internal emails (including the so-called encrypted-to-outside emails,
// which even though meant for external users, they don't really get out of
// our platform). If the email is sent unencrypted, no PGP scheme is needed.
Scheme int
Scheme pmapi.PackageFlag
// MIMEType is the MIME type to use for formatting the body of the email
// (before encryption/after decryption). The standard possibilities are the
@ -191,8 +191,12 @@ func (b *sendPreferencesBuilder) build() (p SendPreferences) {
p.Scheme = pmapi.PGPMIMEPackage
}
case b.shouldSign() && !b.shouldEncrypt() && b.getScheme() == pgpMIME:
p.Scheme = pmapi.ClearMIMEPackage
case b.shouldSign() && !b.shouldEncrypt():
if b.getScheme() == pgpInline {
p.Scheme = pmapi.ClearPackage
} else {
p.Scheme = pmapi.ClearMIMEPackage
}
default:
p.Scheme = pmapi.ClearPackage

16
internal/smtp/preferences_test.go
View File

@ -41,7 +41,7 @@ func TestPreferencesBuilder(t *testing.T) {
wantEncrypt bool
wantSign bool
wantScheme int
wantScheme pmapi.PackageFlag
wantMIMEType string
wantPublicKey string
}{
@ -254,6 +254,20 @@ func TestPreferencesBuilder(t *testing.T) {
wantMIMEType: "multipart/mixed",
},
{
name: "external with contact sign enabled and plain text",
contactMeta: &ContactMetadata{MIMEType: "text/plain", Scheme: pgpInline, Sign: true, SignIsSet: true},
receivedKeys: []pmapi.PublicKey{},
isInternal: false,
mailSettings: pmapi.MailSettings{PGPScheme: pmapi.PGPMIMEPackage, DraftMIMEType: "text/html"},
wantEncrypt: false,
wantSign: true,
wantScheme: pmapi.ClearPackage,
wantMIMEType: "text/plain",
},
{
name: "external with sign enabled, sending plaintext, should still send as ClearMIME",

103
internal/smtp/user.go
View File

@ -187,7 +187,7 @@ func (su *smtpUser) Send(from string, to []string, messageReader io.Reader) (err
log.WithError(err).Error("Failed to parse message")
return
}
clearBody := message.Body
richBody := message.Body
externalID := message.Header.Get("Message-Id")
externalID = strings.Trim(externalID, "<>")
@ -256,7 +256,6 @@ func (su *smtpUser) Send(from string, to []string, messageReader io.Reader) (err
atts = append(atts, message.Attachments...)
// Decrypt attachment keys, because we will need to re-encrypt them with the recipients' public keys.
attkeys := make(map[string]*crypto.SessionKey)
attkeysEncoded := make(map[string]pmapi.AlgoKey)
for _, att := range atts {
var keyPackets []byte
@ -266,23 +265,9 @@ func (su *smtpUser) Send(from string, to []string, messageReader io.Reader) (err
if attkeys[att.ID], err = kr.DecryptSessionKey(keyPackets); err != nil {
return errors.Wrap(err, "decrypting attachment session key")
}
attkeysEncoded[att.ID] = pmapi.AlgoKey{
Key: attkeys[att.ID].GetBase64Key(),
Algorithm: attkeys[att.ID].Algo,
}
}
plainSharedScheme := 0
htmlSharedScheme := 0
mimeSharedType := 0
plainAddressMap := make(map[string]*pmapi.MessageAddress)
htmlAddressMap := make(map[string]*pmapi.MessageAddress)
mimeAddressMap := make(map[string]*pmapi.MessageAddress)
var plainKey, htmlKey, mimeKey *crypto.SessionKey
var plainData, htmlData, mimeData []byte
req := pmapi.NewSendMessageReq(kr, mimeBody, plainBody, richBody, attkeys)
containsUnencryptedRecipients := false
for _, email := range to {
@ -298,61 +283,15 @@ func (su *smtpUser) Send(from string, to []string, messageReader io.Reader) (err
return err
}
var signature int
var signature pmapi.SignatureFlag
if sendPreferences.Sign {
signature = pmapi.YesSignature
signature = pmapi.SignatureDetached
} else {
signature = pmapi.NoSignature
signature = pmapi.SignatureNone
}
if sendPreferences.Scheme == pmapi.PGPMIMEPackage || sendPreferences.Scheme == pmapi.ClearMIMEPackage {
if mimeKey == nil {
if mimeKey, mimeData, err = encryptSymmetric(kr, mimeBody, true); err != nil {
return err
}
}
if sendPreferences.Scheme == pmapi.PGPMIMEPackage {
mimeBodyPacket, _, err := createPackets(sendPreferences.PublicKey, mimeKey, map[string]*crypto.SessionKey{})
if err != nil {
return err
}
mimeAddressMap[email] = &pmapi.MessageAddress{Type: sendPreferences.Scheme, BodyKeyPacket: mimeBodyPacket, Signature: signature}
} else {
mimeAddressMap[email] = &pmapi.MessageAddress{Type: sendPreferences.Scheme, Signature: signature}
}
mimeSharedType |= sendPreferences.Scheme
} else {
switch sendPreferences.MIMEType {
case pmapi.ContentTypePlainText:
if plainKey == nil {
if plainKey, plainData, err = encryptSymmetric(kr, plainBody, true); err != nil {
return err
}
}
newAddress := &pmapi.MessageAddress{Type: sendPreferences.Scheme, Signature: signature}
if sendPreferences.Encrypt && sendPreferences.PublicKey != nil {
newAddress.BodyKeyPacket, newAddress.AttachmentKeyPackets, err = createPackets(sendPreferences.PublicKey, plainKey, attkeys)
if err != nil {
return err
}
}
plainAddressMap[email] = newAddress
plainSharedScheme |= sendPreferences.Scheme
case pmapi.ContentTypeHTML:
if htmlKey == nil {
if htmlKey, htmlData, err = encryptSymmetric(kr, clearBody, true); err != nil {
return err
}
}
newAddress := &pmapi.MessageAddress{Type: sendPreferences.Scheme, Signature: signature}
if sendPreferences.Encrypt && sendPreferences.PublicKey != nil {
newAddress.BodyKeyPacket, newAddress.AttachmentKeyPackets, err = createPackets(sendPreferences.PublicKey, htmlKey, attkeys)
if err != nil {
return err
}
}
htmlAddressMap[email] = newAddress
htmlSharedScheme |= sendPreferences.Scheme
}
if err := req.AddRecipient(email, sendPreferences.Scheme, sendPreferences.PublicKey, signature, sendPreferences.MIMEType, sendPreferences.Encrypt); err != nil {
return errors.Wrap(err, "failed to add recipient")
}
}
@ -370,31 +309,7 @@ func (su *smtpUser) Send(from string, to []string, messageReader io.Reader) (err
}
}
req := &pmapi.SendMessageReq{}
plainPkg := buildPackage(plainAddressMap, plainSharedScheme, pmapi.ContentTypePlainText, plainData, plainKey, attkeysEncoded)
if plainPkg != nil {
req.Packages = append(req.Packages, plainPkg)
}
htmlPkg := buildPackage(htmlAddressMap, htmlSharedScheme, pmapi.ContentTypeHTML, htmlData, htmlKey, attkeysEncoded)
if htmlPkg != nil {
req.Packages = append(req.Packages, htmlPkg)
}
if len(mimeAddressMap) > 0 {
pkg := &pmapi.MessagePackage{
Body: base64.StdEncoding.EncodeToString(mimeData),
Addresses: mimeAddressMap,
MIMEType: pmapi.ContentTypeMultipartMixed,
Type: mimeSharedType,
BodyKey: pmapi.AlgoKey{
Key: mimeKey.GetBase64Key(),
Algorithm: mimeKey.Algo,
},
}
req.Packages = append(req.Packages, pkg)
}
req.PreparePackages()
return su.storeUser.SendMessage(message.ID, req)
}

86
internal/smtp/utils.go
View File

@ -18,11 +18,7 @@
package smtp
import (
"encoding/base64"
"regexp"
"github.com/ProtonMail/gopenpgp/v2/crypto"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
)
//nolint:gochecknoglobals // Used like a constant
@ -35,85 +31,3 @@ var mailFormat = regexp.MustCompile(`.+@.+\..+`)
func looksLikeEmail(e string) bool {
return mailFormat.MatchString(e)
}
func createPackets(
pubkey *crypto.KeyRing,
bodyKey *crypto.SessionKey,
attkeys map[string]*crypto.SessionKey,
) (bodyPacket string, attachmentPackets map[string]string, err error) {
// Encrypt message body keys.
packetBytes, err := pubkey.EncryptSessionKey(bodyKey)
if err != nil {
return
}
bodyPacket = base64.StdEncoding.EncodeToString(packetBytes)
// Encrypt attachment keys.
attachmentPackets = make(map[string]string)
for id, attkey := range attkeys {
var packets []byte
if packets, err = pubkey.EncryptSessionKey(attkey); err != nil {
return
}
attachmentPackets[id] = base64.StdEncoding.EncodeToString(packets)
}
return
}
func encryptSymmetric(
kr *crypto.KeyRing,
textToEncrypt string,
canonicalizeText bool, // nolint[unparam]
) (key *crypto.SessionKey, symEncryptedData []byte, err error) {
// We use only primary key to encrypt the message. Our keyring contains all keys (primary, old and deacivated ones).
firstKey, err := kr.FirstKey()
if err != nil {
return
}
pgpMessage, err := firstKey.Encrypt(crypto.NewPlainMessageFromString(textToEncrypt), kr)
if err != nil {
return
}
pgpSplitMessage, err := pgpMessage.SeparateKeyAndData(len(textToEncrypt), 0)
if err != nil {
return
}
key, err = kr.DecryptSessionKey(pgpSplitMessage.GetBinaryKeyPacket())
if err != nil {
return
}
symEncryptedData = pgpSplitMessage.GetBinaryDataPacket()
return
}
func buildPackage(
addressMap map[string]*pmapi.MessageAddress,
sharedScheme int,
mimeType string,
bodyData []byte,
bodyKey *crypto.SessionKey,
attKeys map[string]pmapi.AlgoKey,
) (pkg *pmapi.MessagePackage) {
if len(addressMap) == 0 {
return nil
}
pkg = &pmapi.MessagePackage{
Body: base64.StdEncoding.EncodeToString(bodyData),
Addresses: addressMap,
MIMEType: mimeType,
Type: sharedScheme,
}
if sharedScheme|pmapi.ClearPackage > 0 {
pkg.BodyKey.Key = bodyKey.GetBase64Key()
pkg.BodyKey.Algorithm = bodyKey.Algo
pkg.AttachmentKeys = attKeys
}
return pkg
}

171
internal/transfer/mocks/mocks.go
View File

@ -1,5 +1,5 @@
// Code generated by MockGen. DO NOT EDIT.
// Source: github.com/ProtonMail/proton-bridge/internal/transfer (interfaces: PanicHandler,ClientManager)
// Source: github.com/ProtonMail/proton-bridge/internal/transfer (interfaces: PanicHandler,ClientManager,IMAPClientProvider)
// Package mocks is a generated GoMock package.
package mocks
@ -8,6 +8,8 @@ import (
reflect "reflect"
pmapi "github.com/ProtonMail/proton-bridge/pkg/pmapi"
imap "github.com/emersion/go-imap"
sasl "github.com/emersion/go-sasl"
gomock "github.com/golang/mock/gomock"
)
@ -96,3 +98,170 @@ func (mr *MockClientManagerMockRecorder) GetClient(arg0 interface{}) *gomock.Cal
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetClient", reflect.TypeOf((*MockClientManager)(nil).GetClient), arg0)
}
// MockIMAPClientProvider is a mock of IMAPClientProvider interface
type MockIMAPClientProvider struct {
ctrl *gomock.Controller
recorder *MockIMAPClientProviderMockRecorder
}
// MockIMAPClientProviderMockRecorder is the mock recorder for MockIMAPClientProvider
type MockIMAPClientProviderMockRecorder struct {
mock *MockIMAPClientProvider
}
// NewMockIMAPClientProvider creates a new mock instance
func NewMockIMAPClientProvider(ctrl *gomock.Controller) *MockIMAPClientProvider {
mock := &MockIMAPClientProvider{ctrl: ctrl}
mock.recorder = &MockIMAPClientProviderMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use
func (m *MockIMAPClientProvider) EXPECT() *MockIMAPClientProviderMockRecorder {
return m.recorder
}
// Authenticate mocks base method
func (m *MockIMAPClientProvider) Authenticate(arg0 sasl.Client) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Authenticate", arg0)
ret0, _ := ret[0].(error)
return ret0
}
// Authenticate indicates an expected call of Authenticate
func (mr *MockIMAPClientProviderMockRecorder) Authenticate(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Authenticate", reflect.TypeOf((*MockIMAPClientProvider)(nil).Authenticate), arg0)
}
// Capability mocks base method
func (m *MockIMAPClientProvider) Capability() (map[string]bool, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Capability")
ret0, _ := ret[0].(map[string]bool)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// Capability indicates an expected call of Capability
func (mr *MockIMAPClientProviderMockRecorder) Capability() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Capability", reflect.TypeOf((*MockIMAPClientProvider)(nil).Capability))
}
// Fetch mocks base method
func (m *MockIMAPClientProvider) Fetch(arg0 *imap.SeqSet, arg1 []imap.FetchItem, arg2 chan *imap.Message) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Fetch", arg0, arg1, arg2)
ret0, _ := ret[0].(error)
return ret0
}
// Fetch indicates an expected call of Fetch
func (mr *MockIMAPClientProviderMockRecorder) Fetch(arg0, arg1, arg2 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Fetch", reflect.TypeOf((*MockIMAPClientProvider)(nil).Fetch), arg0, arg1, arg2)
}
// List mocks base method
func (m *MockIMAPClientProvider) List(arg0, arg1 string, arg2 chan *imap.MailboxInfo) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "List", arg0, arg1, arg2)
ret0, _ := ret[0].(error)
return ret0
}
// List indicates an expected call of List
func (mr *MockIMAPClientProviderMockRecorder) List(arg0, arg1, arg2 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockIMAPClientProvider)(nil).List), arg0, arg1, arg2)
}
// Login mocks base method
func (m *MockIMAPClientProvider) Login(arg0, arg1 string) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Login", arg0, arg1)
ret0, _ := ret[0].(error)
return ret0
}
// Login indicates an expected call of Login
func (mr *MockIMAPClientProviderMockRecorder) Login(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Login", reflect.TypeOf((*MockIMAPClientProvider)(nil).Login), arg0, arg1)
}
// Select mocks base method
func (m *MockIMAPClientProvider) Select(arg0 string, arg1 bool) (*imap.MailboxStatus, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Select", arg0, arg1)
ret0, _ := ret[0].(*imap.MailboxStatus)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// Select indicates an expected call of Select
func (mr *MockIMAPClientProviderMockRecorder) Select(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Select", reflect.TypeOf((*MockIMAPClientProvider)(nil).Select), arg0, arg1)
}
// State mocks base method
func (m *MockIMAPClientProvider) State() imap.ConnState {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "State")
ret0, _ := ret[0].(imap.ConnState)
return ret0
}
// State indicates an expected call of State
func (mr *MockIMAPClientProviderMockRecorder) State() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "State", reflect.TypeOf((*MockIMAPClientProvider)(nil).State))
}
// Support mocks base method
func (m *MockIMAPClientProvider) Support(arg0 string) (bool, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Support", arg0)
ret0, _ := ret[0].(bool)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// Support indicates an expected call of Support
func (mr *MockIMAPClientProviderMockRecorder) Support(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Support", reflect.TypeOf((*MockIMAPClientProvider)(nil).Support), arg0)
}
// SupportAuth mocks base method
func (m *MockIMAPClientProvider) SupportAuth(arg0 string) (bool, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "SupportAuth", arg0)
ret0, _ := ret[0].(bool)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// SupportAuth indicates an expected call of SupportAuth
func (mr *MockIMAPClientProviderMockRecorder) SupportAuth(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SupportAuth", reflect.TypeOf((*MockIMAPClientProvider)(nil).SupportAuth), arg0)
}
// UidFetch mocks base method
func (m *MockIMAPClientProvider) UidFetch(arg0 *imap.SeqSet, arg1 []imap.FetchItem, arg2 chan *imap.Message) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "UidFetch", arg0, arg1, arg2)
ret0, _ := ret[0].(error)
return ret0
}
// UidFetch indicates an expected call of UidFetch
func (mr *MockIMAPClientProviderMockRecorder) UidFetch(arg0, arg1, arg2 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UidFetch", reflect.TypeOf((*MockIMAPClientProvider)(nil).UidFetch), arg0, arg1, arg2)
}

25
internal/transfer/provider_imap.go
View File

@ -21,28 +21,49 @@ import (
"net"
"strings"
imapClient "github.com/emersion/go-imap/client"
"github.com/emersion/go-imap"
"github.com/emersion/go-sasl"
)
type IMAPClientProvider interface {
Capability() (map[string]bool, error)
Support(cap string) (bool, error)
State() imap.ConnState
SupportAuth(mech string) (bool, error)
Authenticate(auth sasl.Client) error
Login(username, password string) error
List(ref, name string, ch chan *imap.MailboxInfo) error
Select(name string, readOnly bool) (*imap.MailboxStatus, error)
Fetch(seqset *imap.SeqSet, items []imap.FetchItem, ch chan *imap.Message) error
UidFetch(seqset *imap.SeqSet, items []imap.FetchItem, ch chan *imap.Message) error
}
// IMAPProvider implements export from IMAP server.
type IMAPProvider struct {
username string
password string
addr string
client *imapClient.Client
clientDialer func(addr string) (IMAPClientProvider, error)
client IMAPClientProvider
timeIt *timeIt
}
// NewIMAPProvider returns new IMAPProvider.
func NewIMAPProvider(username, password, host, port string) (*IMAPProvider, error) {
return newIMAPProvider(imapClientDial, username, password, host, port)
}
func newIMAPProvider(clientDialer func(string) (IMAPClientProvider, error), username, password, host, port string) (*IMAPProvider, error) {
p := &IMAPProvider{
username: username,
password: password,
addr: net.JoinHostPort(host, port),
timeIt: newTimeIt("imap"),
clientDialer: clientDialer,
}
if err := p.auth(); err != nil {

69
internal/transfer/provider_imap_source.go
View File

@ -84,12 +84,37 @@ func (p *IMAPProvider) loadMessagesInfo(rule *Rule, progress *Progress, uidValid
p.timeIt.start("load", rule.SourceMailbox.Name)
defer p.timeIt.stop("load", rule.SourceMailbox.Name)
log := log.WithField("mailbox", rule.SourceMailbox.Name)
messagesInfo := map[string]imapMessageInfo{}
fetchItems := []imap.FetchItem{imap.FetchUid, imap.FetchRFC822Size}
if rule.HasTimeLimit() {
fetchItems = append(fetchItems, imap.FetchEnvelope)
}
processMessageCallback := func(imapMessage *imap.Message) {
if rule.HasTimeLimit() {
t := imapMessage.Envelope.Date.Unix()
if t != 0 && !rule.isTimeInRange(t) {
log.WithField("uid", imapMessage.Uid).Debug("Message skipped due to time")
return
}
}
id := getUniqueMessageID(rule.SourceMailbox.Name, uidValidity, imapMessage.Uid)
// We use ID as key to ensure we have every unique message only once.
// Some IMAP servers responded twice the same message...
messagesInfo[id] = imapMessageInfo{
id: id,
uid: imapMessage.Uid,
size: imapMessage.Size,
}
progress.addMessage(id, []string{rule.SourceMailbox.Name}, rule.TargetMailboxNames())
}
pageStart := uint32(1)
pageEnd := imapPageSize
for {
if progress.shouldStop() {
if progress.shouldStop() || pageStart > count {
break
}
@ -100,45 +125,21 @@ func (p *IMAPProvider) loadMessagesInfo(rule *Rule, progress *Progress, uidValid
seqSet := &imap.SeqSet{}
seqSet.AddRange(pageStart, pageEnd)
items := []imap.FetchItem{imap.FetchUid, imap.FetchRFC822Size}
if rule.HasTimeLimit() {
items = append(items, imap.FetchEnvelope)
}
pageMsgCount := uint32(0)
processMessageCallback := func(imapMessage *imap.Message) {
pageMsgCount++
if rule.HasTimeLimit() {
t := imapMessage.Envelope.Date.Unix()
if t != 0 && !rule.isTimeInRange(t) {
log.WithField("uid", imapMessage.Uid).Debug("Message skipped due to time")
return
err := p.fetch(rule.SourceMailbox.Name, seqSet, fetchItems, processMessageCallback)
if err != nil {
log.WithError(err).WithField("idx", seqSet).Warning("Load batch fetch failed, trying one by one")
for ; pageStart <= pageEnd; pageStart++ {
seqSet := &imap.SeqSet{}
seqSet.AddNum(pageStart)
if err := p.fetch(rule.SourceMailbox.Name, seqSet, fetchItems, processMessageCallback); err != nil {
log.WithError(err).WithField("idx", seqSet).Warning("Load fetch failed, skipping the message")
}
}
id := getUniqueMessageID(rule.SourceMailbox.Name, uidValidity, imapMessage.Uid)
// We use ID as key to ensure we have every unique message only once.
// Some IMAP servers responded twice the same message...
messagesInfo[id] = imapMessageInfo{
id: id,
uid: imapMessage.Uid,
size: imapMessage.Size,
}
progress.addMessage(id, []string{rule.SourceMailbox.Name}, rule.TargetMailboxNames())
}
progress.callWrap(func() error {
return p.fetch(rule.SourceMailbox.Name, seqSet, items, processMessageCallback)
})
if pageMsgCount < imapPageSize {
break
}
pageStart = pageEnd
pageStart = pageEnd + 1
pageEnd += imapPageSize
}
return messagesInfo
}

100
internal/transfer/provider_imap_test.go Normal file
View File

@ -0,0 +1,100 @@
// Copyright (c) 2020 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package transfer
import (
"fmt"
"testing"
"github.com/emersion/go-imap"
gomock "github.com/golang/mock/gomock"
"github.com/pkg/errors"
r "github.com/stretchr/testify/require"
)
func newTestIMAPProvider(t *testing.T, m mocks) *IMAPProvider {
m.imapClientProvider.EXPECT().State().Return(imap.ConnectedState).AnyTimes()
m.imapClientProvider.EXPECT().Capability().Return(map[string]bool{
"AUTH": true,
}, nil).AnyTimes()
dialer := func(string) (IMAPClientProvider, error) {
return m.imapClientProvider, nil
}
provider, err := newIMAPProvider(dialer, "user", "pass", "host", "42")
r.NoError(t, err)
return provider
}
func TestProviderIMAPLoadMessagesInfo(t *testing.T) {
m := initMocks(t)
defer m.ctrl.Finish()
provider := newTestIMAPProvider(t, m)
progress := newProgress(log, nil)
drainProgressUpdateChannel(&progress)
rule := &Rule{SourceMailbox: Mailbox{Name: "Mailbox"}}
uidValidity := 1
count := 2200
failingIndex := 2100
m.imapClientProvider.EXPECT().Select(rule.SourceMailbox.Name, gomock.Any()).Return(&imap.MailboxStatus{}, nil).AnyTimes()
m.imapClientProvider.EXPECT().
Fetch(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(seqSet *imap.SeqSet, items []imap.FetchItem, ch chan *imap.Message) error {
defer close(ch)
for _, seq := range seqSet.Set {
for i := seq.Start; i <= seq.Stop; i++ {
if int(i) == failingIndex {
return errors.New("internal server error")
}
ch <- &imap.Message{
SeqNum: i,
Uid: i * 10,
Size: i * 100,
}
}
}
return nil
}).
// 2200 messages is split into two batches (2000 and 200),
// the second one fails and makes 200 calls (one-by-one).
// Plus two failed requests are repeated `imapRetries` times.
Times(2 + 200 + (2 * (imapRetries - 1)))
messageInfo := provider.loadMessagesInfo(rule, &progress, uint32(uidValidity), uint32(count))
r.Equal(t, count-1, len(messageInfo)) // One message produces internal server error.
for index := 1; index <= count; index++ {
uid := index * 10
key := fmt.Sprintf("%s_%d:%d", rule.SourceMailbox.Name, uidValidity, uid)
if index == failingIndex {
r.Empty(t, messageInfo[key])
continue
}
r.Equal(t, imapMessageInfo{
id: key,
uid: uint32(uid),
size: uint32(index * 100),
}, messageInfo[key])
}
}

89
internal/transfer/provider_imap_utils.go
View File

@ -24,10 +24,11 @@ import (
"time"
imapID "github.com/ProtonMail/go-imap-id"
"github.com/ProtonMail/proton-bridge/pkg/constants"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/emersion/go-imap"
imapClient "github.com/emersion/go-imap/client"
sasl "github.com/emersion/go-sasl"
"github.com/emersion/go-sasl"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
@ -51,6 +52,43 @@ func (l *imapErrorLogger) Println(v ...interface{}) {
l.log.Errorln(v...)
}
func imapClientDial(addr string) (IMAPClientProvider, error) {
if _, err := net.DialTimeout("tcp", addr, imapDialTimeout); err != nil {
return nil, errors.Wrap(err, "failed to dial server")
}
client, err := imapClientDialHelper(addr)
if err == nil {
client.ErrorLog = &imapErrorLogger{logrus.WithField("pkg", "imap-client")}
// Logrus `WriterLevel` fails for big messages because of bufio.MaxScanTokenSize limit.
// Also, this spams a lot, uncomment once needed during development.
//client.SetDebug(imap.NewDebugWriter(
// logrus.WithField("pkg", "imap/client").WriterLevel(logrus.TraceLevel),
// logrus.WithField("pkg", "imap/server").WriterLevel(logrus.TraceLevel),
//))
}
return client, err
}
func imapClientDialHelper(addr string) (*imapClient.Client, error) {
host, _, _ := net.SplitHostPort(addr)
if host == "127.0.0.1" {
return imapClient.Dial(addr)
}
// IMAP mail.yahoo.com has problem with golang TLS 1.3 implementation
// with weird behaviour, i.e., Yahoo does not return error during dial
// or handshake but server does logs out right after successful login
// leaving no time to perform any action.
// Limiting TLS to version 1.2 is working just fine.
var tlsConf *tls.Config
if strings.Contains(strings.ToLower(host), "yahoo") {
log.Warning("Yahoo server detected: limiting maximal TLS version to 1.2.")
tlsConf = &tls.Config{MaxVersion: tls.VersionTLS12}
}
return imapClient.DialTLS(addr, tlsConf)
}
func (p *IMAPProvider) ensureConnection(callback func() error) error {
return p.ensureConnectionAndSelection(callback, "")
}
@ -138,41 +176,10 @@ func (p *IMAPProvider) auth() error { //nolint[funlen]
log.Info("Connecting to server")
if _, err := net.DialTimeout("tcp", p.addr, imapDialTimeout); err != nil {
return ErrIMAPConnection{imapError{Err: err, Message: "failed to dial server"}}
}
var client *imapClient.Client
var err error
host, _, _ := net.SplitHostPort(p.addr)
if host == "127.0.0.1" {
client, err = imapClient.Dial(p.addr)
} else {
// IMAP.mail.yahoo.com have problem with golang TLS1.3
// implementation with weird behaviour i.e. Yahoo
// no error during dial or handshake but server logs out right
// after successful login leaving no time to perform any
// action. It was discovered that limiting to maximum TLS
// version 1.2 for yahoo servers is working solution.
var tlsConf *tls.Config
if strings.Contains(strings.ToLower(host), "yahoo") {
log.Warning("Yahoo server detected: limiting maximal TLS version to 1.2.")
tlsConf = &tls.Config{MaxVersion: tls.VersionTLS12}
}
client, err = imapClient.DialTLS(p.addr, tlsConf)
}
client, err := p.clientDialer(p.addr)
if err != nil {
return ErrIMAPConnection{imapError{Err: err, Message: "failed to connect to server"}}
}
client.ErrorLog = &imapErrorLogger{logrus.WithField("pkg", "imap-client")}
// Logrus `WriterLevel` fails for big messages because of bufio.MaxScanTokenSize limit.
// Also, this spams a lot, uncomment once needed during development.
//client.SetDebug(imap.NewDebugWriter(
// logrus.WithField("pkg", "imap/client").WriterLevel(logrus.TraceLevel),
// logrus.WithField("pkg", "imap/server").WriterLevel(logrus.TraceLevel),
//))
p.client = client
log.Info("Connected")
@ -210,13 +217,15 @@ func (p *IMAPProvider) auth() error { //nolint[funlen]
log.Info("Logged in")
idClient := imapID.NewClient(p.client)
if ok, err := idClient.SupportID(); err == nil && ok {
serverID, err := idClient.ID(imapID.ID{
imapID.FieldName: "ImportExport",
imapID.FieldVersion: "beta",
})
log.WithField("ID", serverID).WithError(err).Debug("Server info")
if c, ok := p.client.(*imapClient.Client); ok {
idClient := imapID.NewClient(c)
if ok, err := idClient.SupportID(); err == nil && ok {
serverID, err := idClient.ID(imapID.ID{
imapID.FieldName: "ImportExport",
imapID.FieldVersion: constants.Version,
})
log.WithField("ID", serverID).WithError(err).Debug("Server info")
}
}
return err

24
internal/transfer/transfer_test.go
View File

@ -31,11 +31,12 @@ import (
type mocks struct {
t *testing.T
ctrl *gomock.Controller
panicHandler *transfermocks.MockPanicHandler
clientManager *transfermocks.MockClientManager
pmapiClient *pmapimocks.MockClient
pmapiConfig *pmapi.ClientConfig
ctrl *gomock.Controller
panicHandler *transfermocks.MockPanicHandler
clientManager *transfermocks.MockClientManager
imapClientProvider *transfermocks.MockIMAPClientProvider
pmapiClient *pmapimocks.MockClient
pmapiConfig *pmapi.ClientConfig
keyring *crypto.KeyRing
}
@ -46,12 +47,13 @@ func initMocks(t *testing.T) mocks {
m := mocks{
t: t,
ctrl: mockCtrl,
panicHandler: transfermocks.NewMockPanicHandler(mockCtrl),
clientManager: transfermocks.NewMockClientManager(mockCtrl),
pmapiClient: pmapimocks.NewMockClient(mockCtrl),
pmapiConfig: &pmapi.ClientConfig{},
keyring: newTestKeyring(),
ctrl: mockCtrl,
panicHandler: transfermocks.NewMockPanicHandler(mockCtrl),
clientManager: transfermocks.NewMockClientManager(mockCtrl),
imapClientProvider: transfermocks.NewMockIMAPClientProvider(mockCtrl),
pmapiClient: pmapimocks.NewMockClient(mockCtrl),
pmapiConfig: &pmapi.ClientConfig{},
keyring: newTestKeyring(),
}
m.clientManager.EXPECT().GetClient("user").Return(m.pmapiClient).AnyTimes()