GODT-1036 Event loop Sentry reporting of failures and refresh

go.mod

@@ -54,6 +54,7 @@ require (
 	github.com/nsf/jsondiff v0.0.0-20200515183724-f29ed568f4ce
 	github.com/olekukonko/tablewriter v0.0.4 // indirect
 	github.com/pkg/errors v0.9.1
+	github.com/pkg/math v0.0.0-20141027224758-f2ed9e40e245
 	github.com/sirupsen/logrus v1.7.0
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
 	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect

go.sum

@@ -223,6 +223,8 @@ github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTw
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/math v0.0.0-20141027224758-f2ed9e40e245 h1:gk/AF9SGRj+RafNCoDcS3RRscb8S4BVbvqODOgWA7/8=
+github.com/pkg/math v0.0.0-20141027224758-f2ed9e40e245/go.mod h1:2dhPPj2Li3DXrSY2U2ADdZy2B7sjQsT57lqENx1+FSE=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=

internal/app/base/base.go

@@ -76,6 +76,7 @@ const (
 )
 
 type Base struct {
+	SentryReporter *sentry.Reporter
 	CrashHandler   *crash.Handler
 	Locations      *locations.Locations
 	Settings       *settings.Settings
@@ -234,6 +235,7 @@ func New( // nolint[funlen]
 	}
 
 	return &Base{
+		SentryReporter: sentryReporter,
 		CrashHandler:   crashHandler,
 		Locations:      locations,
 		Settings:       settingsObj,

internal/app/bridge/bridge.go

@@ -71,8 +71,7 @@ func run(b *base.Base, c *cli.Context) error { // nolint[funlen]
 	if err != nil {
 		logrus.WithError(err).Fatal("Failed to load TLS config")
 	}
-
+	bridge := bridge.New(b.Locations, b.Cache, b.Settings, b.SentryReporter, b.CrashHandler, b.Listener, b.CM, b.Creds, b.Updater, b.Versioner)
-	bridge := bridge.New(b.Locations, b.Cache, b.Settings, b.CrashHandler, b.Listener, b.CM, b.Creds, b.Updater, b.Versioner)
 	imapBackend := imap.NewIMAPBackend(b.CrashHandler, b.Listener, b.Cache, bridge)
 	smtpBackend := smtp.NewSMTPBackend(b.CrashHandler, b.Listener, b.Settings, bridge)
 

internal/bridge/bridge.go

@@ -29,6 +29,7 @@ import (
 	"github.com/ProtonMail/proton-bridge/internal/updater"
 	"github.com/ProtonMail/proton-bridge/internal/users"
 	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/ProtonMail/proton-bridge/pkg/sentry"
 
 	"github.com/ProtonMail/proton-bridge/pkg/listener"
 	logrus "github.com/sirupsen/logrus"
@@ -56,6 +57,7 @@ func New(
 	locations Locator,
 	cache Cacher,
 	s SettingsProvider,
+	sentryReporter *sentry.Reporter,
 	panicHandler users.PanicHandler,
 	eventListener listener.Listener,
 	clientManager users.ClientManager,
@@ -69,7 +71,7 @@ func New(
 		clientManager.AllowProxy()
 	}
 
-	storeFactory := newStoreFactory(cache, panicHandler, clientManager, eventListener)
+	storeFactory := newStoreFactory(cache, sentryReporter, panicHandler, clientManager, eventListener)
 	u := users.New(locations, panicHandler, eventListener, clientManager, credStorer, storeFactory, true)
 	b := &Bridge{
 		Users: u,

internal/bridge/store_factory.go

@@ -23,12 +23,13 @@ import (
 
 	"github.com/ProtonMail/proton-bridge/internal/store"
 	"github.com/ProtonMail/proton-bridge/internal/users"
-
 	"github.com/ProtonMail/proton-bridge/pkg/listener"
+	"github.com/ProtonMail/proton-bridge/pkg/sentry"
 )
 
 type storeFactory struct {
 	cache          Cacher
+	sentryReporter *sentry.Reporter
 	panicHandler   users.PanicHandler
 	clientManager  users.ClientManager
 	eventListener  listener.Listener
@@ -37,12 +38,14 @@ type storeFactory struct {
 
 func newStoreFactory(
 	cache Cacher,
+	sentryReporter *sentry.Reporter,
 	panicHandler users.PanicHandler,
 	clientManager users.ClientManager,
 	eventListener listener.Listener,
 ) *storeFactory {
 	return &storeFactory{
 		cache:          cache,
+		sentryReporter: sentryReporter,
 		panicHandler:   panicHandler,
 		clientManager:  clientManager,
 		eventListener:  eventListener,
@@ -53,7 +56,7 @@ func newStoreFactory(
 // New creates new store for given user.
 func (f *storeFactory) New(user store.BridgeUser) (*store.Store, error) {
 	storePath := getUserStorePath(f.cache.GetDBDir(), user.ID())
-	return store.New(f.panicHandler, user, f.clientManager, f.eventListener, storePath, f.storeCache)
+	return store.New(f.sentryReporter, f.panicHandler, user, f.clientManager, f.eventListener, storePath, f.storeCache)
 }
 
 // Remove removes all store files for given user.

internal/store/event_loop.go

@@ -28,8 +28,13 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-const pollInterval = 30 * time.Second
+const (
-const pollIntervalSpread = 5 * time.Second
+	pollInterval       = 30 * time.Second
+	pollIntervalSpread = 5 * time.Second
+
+	// errMaxSentry defines after how many errors in a row to report it to sentry.
+	errMaxSentry = 20
+)
 
 type eventLoop struct {
 	cache          *Cache
@@ -41,6 +46,7 @@ type eventLoop struct {
 	isRunning      bool // The whole event loop is running.
 
 	pollCounter int
+	errCounter  int
 
 	log *logrus.Entry
 
@@ -227,9 +233,18 @@ func (loop *eventLoop) processNextEvent() (more bool, err error) { // nolint[fun
 
 		_, errUnauthorized := errors.Cause(err).(*pmapi.ErrUnauthorized)
 
+		if err == nil {
+			loop.errCounter = 0
+		}
 		// All errors except Invalid Token (which is not possible to recover from) are ignored.
 		if err != nil && !errUnauthorized && errors.Cause(err) != pmapi.ErrInvalidToken {
-			l.WithError(err).Error("Error skipped")
+			l.WithError(err).WithField("errors", loop.errCounter).Error("Error skipped")
+			loop.errCounter++
+			if loop.errCounter == errMaxSentry {
+				if sentryErr := loop.store.sentryReporter.ReportMessage("Warning: event loop issues: " + err.Error() + ", " + loop.currentEventID); sentryErr != nil {
+					l.WithError(sentryErr).Error("Failed to report error to sentry")
+				}
+			}
 			err = nil
 		}
 	}()
@@ -283,6 +298,10 @@ func (loop *eventLoop) processEvent(event *pmapi.Event) (err error) {
 		eventLog.Info("Processing refresh event")
 		loop.store.triggerSync()
 
+		if sentryErr := loop.store.sentryReporter.ReportMessage("Warning: refresh occurred, " + loop.currentEventID); sentryErr != nil {
+			loop.log.WithError(sentryErr).Error("Failed to report refresh to sentry")
+		}
+
 		return
 	}
 

internal/store/store.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/ProtonMail/proton-bridge/pkg/listener"
 	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
+	"github.com/ProtonMail/proton-bridge/pkg/sentry"
 	"github.com/hashicorp/go-multierror"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -95,6 +96,7 @@ var (
 
 // Store is local user storage, which handles the synchronization between IMAP and PM API.
 type Store struct {
+	sentryReporter *sentry.Reporter
 	panicHandler   PanicHandler
 	eventLoop      *eventLoop
 	user           BridgeUser
@@ -115,7 +117,8 @@ type Store struct {
 }
 
 // New creates or opens a store for the given `user`.
-func New(
+func New( // nolint[funlen]
+	sentryReporter *sentry.Reporter,
 	panicHandler PanicHandler,
 	user BridgeUser,
 	clientManager ClientManager,
@@ -145,6 +148,7 @@ func New(
 	}
 
 	store = &Store{
+		sentryReporter: sentryReporter,
 		panicHandler:   panicHandler,
 		clientManager:  clientManager,
 		user:           user,

internal/store/store_test.go

@@ -125,6 +125,7 @@ func (mocks *mocksForStore) newStoreNoEvents(combinedMode bool, msgs ...*pmapi.M
 
 	var err error
 	mocks.store, err = New(
+		nil, // Sentry reporter is not used under unit tests.
 		mocks.panicHandler,
 		mocks.user,
 		mocks.clientManager,

internal/users/users_test.go

@@ -31,6 +31,7 @@ import (
 	usersmocks "github.com/ProtonMail/proton-bridge/internal/users/mocks"
 	"github.com/ProtonMail/proton-bridge/pkg/pmapi"
 	pmapimocks "github.com/ProtonMail/proton-bridge/pkg/pmapi/mocks"
+	"github.com/ProtonMail/proton-bridge/pkg/sentry"
 	gomock "github.com/golang/mock/gomock"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
@@ -185,9 +186,10 @@ func initMocks(t *testing.T) mocks {
 
 	// Set up store factory.
 	m.storeMaker.EXPECT().New(gomock.Any()).DoAndReturn(func(user store.BridgeUser) (*store.Store, error) {
+		var sentryReporter *sentry.Reporter // Sentry reporter is not used under unit tests.
 		dbFile, err := ioutil.TempFile("", "bridge-store-db-*.db")
 		require.NoError(t, err, "could not get temporary file for store db")
-		return store.New(m.PanicHandler, user, m.clientManager, m.eventListener, dbFile.Name(), m.storeCache)
+		return store.New(sentryReporter, m.PanicHandler, user, m.clientManager, m.eventListener, dbFile.Name(), m.storeCache)
 	}).AnyTimes()
 	m.storeMaker.EXPECT().Remove(gomock.Any()).AnyTimes()
 

test/context/bridge.go

@@ -21,8 +21,10 @@ import (
 	"time"
 
 	"github.com/ProtonMail/proton-bridge/internal/bridge"
+	"github.com/ProtonMail/proton-bridge/internal/constants"
 	"github.com/ProtonMail/proton-bridge/internal/users"
 	"github.com/ProtonMail/proton-bridge/pkg/listener"
+	"github.com/ProtonMail/proton-bridge/pkg/sentry"
 )
 
 // GetBridge returns bridge instance.
@@ -67,8 +69,9 @@ func newBridgeInstance(
 	eventListener listener.Listener,
 	clientManager users.ClientManager,
 ) *bridge.Bridge {
+	sentryReporter := sentry.NewReporter("bridge", constants.Version)
 	panicHandler := &panicHandler{t: t}
 	updater := newFakeUpdater()
 	versioner := newFakeVersioner()
-	return bridge.New(locations, cache, settings, panicHandler, eventListener, clientManager, credStore, updater, versioner)
+	return bridge.New(locations, cache, settings, sentryReporter, panicHandler, eventListener, clientManager, credStore, updater, versioner)
 }