feat(BRIDGE-150): Observability service modification; user distinction utility & heartbeat; various observbility metrics & relevant integration tests

internal/services/imapservice/observabilitymetrics/evtloopmsgevents/metrics.go

@@ -0,0 +1,67 @@
+// Copyright (c) 2024 Proton AG
+//
+// This file is part of Proton Mail Bridge.
+//
+// Proton Mail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Proton Mail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Proton Mail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package evtloopmsgevents
+
+import (
+	"time"
+
+	"github.com/ProtonMail/go-proton-api"
+)
+
+const (
+	messageEventErrorCaseSchemaName    = "bridge_event_loop_message_event_failures_total"
+	messageEventErrorCaseSchemaVersion = 1
+)
+
+func generateMessageEventFailureObservabilityMetric(eventType string) proton.ObservabilityMetric {
+	return proton.ObservabilityMetric{
+		Name:      messageEventErrorCaseSchemaName,
+		Version:   messageEventErrorCaseSchemaVersion,
+		Timestamp: time.Now().Unix(),
+		Data: map[string]interface{}{
+			"Value": 1,
+			"Labels": map[string]string{
+				"eventType": eventType,
+			},
+		},
+	}
+}
+
+func GenerateMessageEventFailureCreateMessageMetric() proton.ObservabilityMetric {
+	return generateMessageEventFailureObservabilityMetric("createMessageEvent")
+}
+
+func GenerateMessageEventFailureDeleteMessageMetric() proton.ObservabilityMetric {
+	return generateMessageEventFailureObservabilityMetric("deleteMessageEvent")
+}
+
+func GenerateMessageEventFailureUpdateMetric() proton.ObservabilityMetric {
+	return generateMessageEventFailureObservabilityMetric("updateEvent")
+}
+
+func GenerateMessageEventFailedToBuildMessage() proton.ObservabilityMetric {
+	return generateMessageEventFailureObservabilityMetric("failedToBuildMessage")
+}
+
+func GenerateMessageEventFailedToBuildDraft() proton.ObservabilityMetric {
+	return generateMessageEventFailureObservabilityMetric("failedToBuildDraft")
+}
+
+func GenerateMessageEventUpdateChannelDoesNotExist() proton.ObservabilityMetric {
+	return generateMessageEventFailureObservabilityMetric("messageUpdateChannelDoesNotExist")
+}

internal/services/imapservice/observabilitymetrics/syncmsgevents/metrics.go

@@ -0,0 +1,51 @@
+// Copyright (c) 2024 Proton AG
+//
+// This file is part of Proton Mail Bridge.
+//
+// Proton Mail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Proton Mail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Proton Mail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package syncmsgevents
+
+import (
+	"time"
+
+	"github.com/ProtonMail/go-proton-api"
+)
+
+const (
+	syncEventErrorCaseSchemaName    = "bridge_sync_message_event_failures_total"
+	syncEventErrorCaseSchemaVersion = 1
+)
+
+func generateSyncEventFailureObservabilityMetric(eventType string) proton.ObservabilityMetric {
+	return proton.ObservabilityMetric{
+		Name:      syncEventErrorCaseSchemaName,
+		Version:   syncEventErrorCaseSchemaVersion,
+		Timestamp: time.Now().Unix(),
+		Data: map[string]interface{}{
+			"Value": 1,
+			"Labels": map[string]string{
+				"eventType": eventType,
+			},
+		},
+	}
+}
+
+func GenerateSyncFailureCreateMessageEventMetric() proton.ObservabilityMetric {
+	return generateSyncEventFailureObservabilityMetric("createMessageEvent")
+}
+
+func GenerateSyncFailureDeleteMessageEventMetric() proton.ObservabilityMetric {
+	return generateSyncEventFailureObservabilityMetric("deleteMessageEvent")
+}

internal/services/imapservice/service.go

@@ -30,6 +30,7 @@ import (
 	"github.com/ProtonMail/gluon/watcher"
 	"github.com/ProtonMail/go-proton-api"
 	"github.com/ProtonMail/proton-bridge/v3/internal/events"
+	"github.com/ProtonMail/proton-bridge/v3/internal/services/observability"
 	"github.com/ProtonMail/proton-bridge/v3/internal/services/orderedtasks"
 	"github.com/ProtonMail/proton-bridge/v3/internal/services/sendrecorder"
 	"github.com/ProtonMail/proton-bridge/v3/internal/services/syncservice"
@@ -96,6 +97,8 @@ type Service struct {
 	syncConfigPath     string
 	lastHandledEventID string
 	isSyncing          atomic.Bool
+
+	observabilitySender observability.Sender
 }
 
 func NewService(
@@ -116,6 +119,7 @@ func NewService(
 	syncConfigDir string,
 	maxSyncMemory uint64,
 	showAllMail bool,
+	observabilitySender observability.Sender,
 ) *Service {
 	subscriberName := fmt.Sprintf("imap-%v", identityState.User.ID)
 
@@ -160,6 +164,8 @@ func NewService(
 		syncMessageBuilder: syncMessageBuilder,
 		syncReporter:       syncReporter,
 		syncConfigPath:     GetSyncConfigPath(syncConfigDir, identityState.User.ID),
+
+		observabilitySender: observabilitySender,
 	}
 }
 

internal/services/imapservice/service_message_events.go

@@ -26,11 +26,11 @@ import (
 
 	"github.com/ProtonMail/gluon"
 	"github.com/ProtonMail/gluon/imap"
-	"github.com/ProtonMail/gluon/reporter"
 	"github.com/ProtonMail/go-proton-api"
 	"github.com/ProtonMail/gopenpgp/v2/crypto"
-	"github.com/ProtonMail/proton-bridge/v3/internal"
 	"github.com/ProtonMail/proton-bridge/v3/internal/logging"
+	obsMetrics "github.com/ProtonMail/proton-bridge/v3/internal/services/imapservice/observabilitymetrics/evtloopmsgevents"
+	"github.com/ProtonMail/proton-bridge/v3/internal/services/observability"
 	"github.com/ProtonMail/proton-bridge/v3/internal/usertypes"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/exp/maps"
@@ -46,7 +46,7 @@ func (s *Service) HandleMessageEvents(ctx context.Context, events []proton.Messa
 		case proton.EventCreate:
 			updates, err := onMessageCreated(logging.WithLogrusField(ctx, "action", "create message"), s, event.Message, false)
 			if err != nil {
-				reportError(s.reporter, s.log, "Failed to apply create message event", err)
+				s.observabilitySender.AddDistinctMetrics(observability.EventLoopError, obsMetrics.GenerateMessageEventFailureCreateMessageMetric())
 				return fmt.Errorf("failed to handle create message event: %w", err)
 			}
 
@@ -64,7 +64,7 @@ func (s *Service) HandleMessageEvents(ctx context.Context, events []proton.Messa
 					event,
 				)
 				if err != nil {
-					reportError(s.reporter, s.log, "Failed to apply update draft message event", err)
+					s.observabilitySender.AddDistinctMetrics(observability.EventLoopError, obsMetrics.GenerateMessageEventFailureUpdateMetric())
 					return fmt.Errorf("failed to handle update draft event: %w", err)
 				}
 
@@ -85,7 +85,7 @@ func (s *Service) HandleMessageEvents(ctx context.Context, events []proton.Messa
 				event.Message,
 			)
 			if err != nil {
-				reportError(s.reporter, s.log, "Failed to apply update message event", err)
+				s.observabilitySender.AddDistinctMetrics(observability.EventLoopError, obsMetrics.GenerateMessageEventFailureUpdateMetric())
 				return fmt.Errorf("failed to handle update message event: %w", err)
 			}
 
@@ -113,6 +113,7 @@ func (s *Service) HandleMessageEvents(ctx context.Context, events []proton.Messa
 			)
 
 			if err := waitOnIMAPUpdates(ctx, updates); err != nil {
+				s.observabilitySender.AddDistinctMetrics(observability.EventLoopError, obsMetrics.GenerateMessageEventFailureDeleteMessageMetric())
 				return fmt.Errorf("failed to handle delete message event in gluon: %w", err)
 			}
 		}
@@ -158,8 +159,7 @@ func onMessageCreated(
 				s.log.WithError(err).Error("Failed to add failed message ID to vault")
 			}
 
-			reportErrorAndMessageID(s.reporter, s.log, "Failed to build message (event create)", res.err, res.messageID)
-
+			s.observabilitySender.AddDistinctMetrics(observability.EventLoopError, obsMetrics.GenerateMessageEventFailedToBuildMessage())
 			return nil
 		}
 
@@ -221,8 +221,7 @@ func onMessageUpdateDraftOrSent(ctx context.Context, s *Service, event proton.Me
 				s.log.WithError(err).Error("Failed to add failed message ID to vault")
 			}
 
-			reportErrorAndMessageID(s.reporter, s.log, "Failed to build draft message (event update)", res.err, res.messageID)
-
+			s.observabilitySender.AddDistinctMetrics(observability.EventLoopError, obsMetrics.GenerateMessageEventFailedToBuildDraft())
 			return nil
 		}
 
@@ -299,24 +298,6 @@ func onMessageDeleted(ctx context.Context, s *Service, event proton.MessageEvent
 	return updates
 }
 
-func reportError(r reporter.Reporter, entry *logrus.Entry, title string, err error) {
-	reportErrorNoContextCancel(r, entry, title, err, reporter.Context{})
-}
-
-func reportErrorAndMessageID(r reporter.Reporter, entry *logrus.Entry, title string, err error, messgeID string) {
-	reportErrorNoContextCancel(r, entry, title, err, reporter.Context{"messageID": messgeID})
-}
-
-func reportErrorNoContextCancel(r reporter.Reporter, entry *logrus.Entry, title string, err error, reportContext reporter.Context) {
-	if !errors.Is(err, context.Canceled) {
-		reportContext["error"] = err
-		reportContext["error_type"] = internal.ErrCauseType(err)
-		if rerr := r.ReportMessageWithContext(title, reportContext); rerr != nil {
-			entry.WithError(err).WithField("title", title).Error("Failed to report message")
-		}
-	}
-}
-
 // safePublishMessageUpdate handles the rare case where the address' update channel may have been deleted in the same
 // event. This rare case can take place if in the same event fetch request there is an update for delete address and
 // create/update message.
@@ -341,7 +322,7 @@ func safePublishMessageUpdate(ctx context.Context, s *Service, addressID string,
 		}
 
 		logrus.Warnf("Update channel not found for address %v, it may have been already deleted", addressID)
-		_ = s.reporter.ReportMessage("Message Update channel does not exist")
+		s.observabilitySender.AddDistinctMetrics(observability.EventLoopError, obsMetrics.GenerateMessageEventUpdateChannelDoesNotExist())
 
 		return false, nil
 	}

internal/services/imapservice/service_sync_events.go

@@ -23,6 +23,8 @@ import (
 
 	"github.com/ProtonMail/go-proton-api"
 	"github.com/ProtonMail/proton-bridge/v3/internal/logging"
+	obsMetrics "github.com/ProtonMail/proton-bridge/v3/internal/services/imapservice/observabilitymetrics/syncmsgevents"
+	"github.com/ProtonMail/proton-bridge/v3/internal/services/observability"
 	"github.com/ProtonMail/proton-bridge/v3/internal/services/userevents"
 )
 
@@ -55,7 +57,7 @@ func (s syncMessageEventHandler) HandleMessageEvents(ctx context.Context, events
 				true,
 			)
 			if err != nil {
-				reportError(s.service.reporter, s.service.log, "Failed to apply create message event", err)
+				s.service.observabilitySender.AddDistinctMetrics(observability.SyncError, obsMetrics.GenerateSyncFailureCreateMessageEventMetric())
 				return fmt.Errorf("failed to handle create message event: %w", err)
 			}
 
@@ -71,6 +73,7 @@ func (s syncMessageEventHandler) HandleMessageEvents(ctx context.Context, events
 			)
 
 			if err := waitOnIMAPUpdates(ctx, updates); err != nil {
+				s.service.observabilitySender.AddDistinctMetrics(observability.SyncError, obsMetrics.GenerateSyncFailureDeleteMessageEventMetric())
 				return fmt.Errorf("failed to handle delete message event in gluon: %w", err)
 			}
 		default: