feat(BRIDGE-218): observability adapter; gluon observability metrics and tests;

go.mod

@@ -7,7 +7,7 @@ toolchain go1.21.9
 require (
 	github.com/0xAX/notificator v0.0.0-20220220101646-ee9b8921e557
 	github.com/Masterminds/semver/v3 v3.2.0
-	github.com/ProtonMail/gluon v0.17.1-0.20240923151549-d23b4bec3602
+	github.com/ProtonMail/gluon v0.17.1-0.20241008123701-ddf4a459d0b4
 	github.com/ProtonMail/go-autostart v0.0.0-20210130080809-00ed301c8e9a
 	github.com/ProtonMail/go-proton-api v0.4.1-0.20240918100656-b4860af56d47
 	github.com/ProtonMail/gopenpgp/v2 v2.7.4-proton

go.sum

@@ -35,6 +35,14 @@ github.com/ProtonMail/gluon v0.17.1-0.20240923094038-e319bf6047c5 h1:LzaUpUj6M2P
 github.com/ProtonMail/gluon v0.17.1-0.20240923094038-e319bf6047c5/go.mod h1:0/c03TzZPNiSgY5UDJK1iRDkjlDPwWugxTT6et2qDu8=
 github.com/ProtonMail/gluon v0.17.1-0.20240923151549-d23b4bec3602 h1:EoMjWlC32tg46L/07hWoiZfLkqJyxVMcsq4Cyn+Ofqc=
 github.com/ProtonMail/gluon v0.17.1-0.20240923151549-d23b4bec3602/go.mod h1:0/c03TzZPNiSgY5UDJK1iRDkjlDPwWugxTT6et2qDu8=
+github.com/ProtonMail/gluon v0.17.1-0.20241002092751-3bbeea9053af h1:iMxTQUg2cB47cXqpMev3cZmQoGBOef3cSUjBbdEl33M=
+github.com/ProtonMail/gluon v0.17.1-0.20241002092751-3bbeea9053af/go.mod h1:0/c03TzZPNiSgY5UDJK1iRDkjlDPwWugxTT6et2qDu8=
+github.com/ProtonMail/gluon v0.17.1-0.20241002111651-173859b80060 h1:dcu3tT84GjoXb++n7crv8UJeG8eRwogjTYdkoJ+MjQI=
+github.com/ProtonMail/gluon v0.17.1-0.20241002111651-173859b80060/go.mod h1:0/c03TzZPNiSgY5UDJK1iRDkjlDPwWugxTT6et2qDu8=
+github.com/ProtonMail/gluon v0.17.1-0.20241002142736-ef4153d156d8 h1:YxPHSJUA87i1hc6s1YrW89++V7HpcR7LSFQ6XM0TsAE=
+github.com/ProtonMail/gluon v0.17.1-0.20241002142736-ef4153d156d8/go.mod h1:0/c03TzZPNiSgY5UDJK1iRDkjlDPwWugxTT6et2qDu8=
+github.com/ProtonMail/gluon v0.17.1-0.20241008123701-ddf4a459d0b4 h1:xE+V17O9HIttMpVymNCORQILk9OKpSekrrPbX7YGnF8=
+github.com/ProtonMail/gluon v0.17.1-0.20241008123701-ddf4a459d0b4/go.mod h1:0/c03TzZPNiSgY5UDJK1iRDkjlDPwWugxTT6et2qDu8=
 github.com/ProtonMail/go-autostart v0.0.0-20210130080809-00ed301c8e9a h1:D+aZah+k14Gn6kmL7eKxoo/4Dr/lK3ChBcwce2+SQP4=
 github.com/ProtonMail/go-autostart v0.0.0-20210130080809-00ed301c8e9a/go.mod h1:oTGdE7/DlWIr23G0IKW3OXK9wZ5Hw1GGiaJFccTvZi4=
 github.com/ProtonMail/go-crypto v0.0.0-20230321155629-9a39f2531310/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE=

internal/bridge/bridge.go

@@ -325,6 +325,7 @@ func newBridge(
 		reporter,
 		uidValidityGenerator,
 		&bridgeIMAPSMTPTelemetry{b: bridge},
+		observabilityService,
 	)
 
 	// Check whether username has changed and correct (macOS only)

internal/services/imapsmtpserver/imap.go

@@ -36,6 +36,7 @@ import (
 	"github.com/ProtonMail/proton-bridge/v3/internal/constants"
 	"github.com/ProtonMail/proton-bridge/v3/internal/files"
 	"github.com/ProtonMail/proton-bridge/v3/internal/logging"
+	"github.com/ProtonMail/proton-bridge/v3/internal/services/observability"
 	"github.com/sirupsen/logrus"
 )
 
@@ -77,6 +78,7 @@ func newIMAPServer(
 	tasks *async.Group,
 	uidValidityGenerator imap.UIDValidityGenerator,
 	panicHandler async.PanicHandler,
+	observabilitySender observability.Sender,
 ) (*gluon.Server, error) {
 	gluonCacheDir = ApplyGluonCachePathSuffix(gluonCacheDir)
 	gluonConfigDir = ApplyGluonConfigPathSuffix(gluonConfigDir)
@@ -121,6 +123,7 @@ func newIMAPServer(
 		gluon.WithReporter(reporter),
 		gluon.WithUIDValidityGenerator(uidValidityGenerator),
 		gluon.WithPanicHandler(panicHandler),
+		gluon.WithObservabilitySender(observability.NewAdapter(observabilitySender), int(observability.GluonImapError), int(observability.GluonMessageError), int(observability.GluonOtherError)),
 	)
 	if err != nil {
 		return nil, err

internal/services/imapsmtpserver/service.go

@@ -31,6 +31,7 @@ import (
 	"github.com/ProtonMail/gluon/reporter"
 	"github.com/ProtonMail/proton-bridge/v3/internal/events"
 	"github.com/ProtonMail/proton-bridge/v3/internal/services/imapservice"
+	"github.com/ProtonMail/proton-bridge/v3/internal/services/observability"
 	bridgesmtp "github.com/ProtonMail/proton-bridge/v3/internal/services/smtp"
 	"github.com/ProtonMail/proton-bridge/v3/internal/services/syncservice"
 	"github.com/ProtonMail/proton-bridge/v3/pkg/cpc"
@@ -60,6 +61,8 @@ type Service struct {
 
 	uidValidityGenerator imap.UIDValidityGenerator
 	telemetry            Telemetry
+
+	observabilitySender observability.Sender
 }
 
 func NewService(
@@ -71,6 +74,7 @@ func NewService(
 	reporter reporter.Reporter,
 	uidValidityGenerator imap.UIDValidityGenerator,
 	telemetry Telemetry,
+	observabilitySender observability.Sender,
 ) *Service {
 	return &Service{
 		requests:     cpc.NewCPC(),
@@ -85,6 +89,8 @@ func NewService(
 		tasks:                async.NewGroup(ctx, panicHandler),
 		uidValidityGenerator: uidValidityGenerator,
 		telemetry:            telemetry,
+
+		observabilitySender: observabilitySender,
 	}
 }
 
@@ -449,6 +455,7 @@ func (sm *Service) createIMAPServer(ctx context.Context) (*gluon.Server, error)
 		sm.tasks,
 		sm.uidValidityGenerator,
 		sm.panicHandler,
+		sm.observabilitySender,
 	)
 	if err == nil {
 		sm.eventPublisher.PublishEvent(ctx, events.IMAPServerCreated{})

internal/services/observability/adapter.go

@@ -0,0 +1,93 @@
+// Copyright (c) 2024 Proton AG
+//
+// This file is part of Proton Mail Bridge.
+//
+// Proton Mail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Proton Mail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Proton Mail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package observability
+
+import (
+	"github.com/ProtonMail/go-proton-api"
+)
+
+type Adapter struct {
+	sender Sender
+}
+
+func NewAdapter(sender Sender) *Adapter {
+	return &Adapter{sender: sender}
+}
+
+// VerifyAndParseGenericMetrics parses a metric provided as an interface into a proton.ObservabilityMetric type.
+// It's exported as it is also used in integration tests.
+func VerifyAndParseGenericMetrics(metric map[string]interface{}) (bool, proton.ObservabilityMetric) {
+	name, ok := metric["Name"].(string)
+	if !ok {
+		return false, proton.ObservabilityMetric{}
+	}
+
+	version, ok := metric["Version"].(int)
+	if !ok {
+		return false, proton.ObservabilityMetric{}
+	}
+
+	timestamp, ok := metric["Timestamp"].(int64)
+	if !ok {
+		return false, proton.ObservabilityMetric{}
+	}
+
+	data, ok := metric["Data"]
+	if !ok {
+		return false, proton.ObservabilityMetric{}
+	}
+
+	return true, proton.ObservabilityMetric{
+		Name:      name,
+		Version:   version,
+		Timestamp: timestamp,
+		Data:      data,
+	}
+}
+
+func (adapter *Adapter) AddMetrics(metrics ...map[string]interface{}) {
+	var typedMetrics []proton.ObservabilityMetric
+
+	for _, metric := range metrics {
+		if ok, m := VerifyAndParseGenericMetrics(metric); ok {
+			typedMetrics = append(typedMetrics, m)
+		}
+	}
+
+	if len(typedMetrics) > 0 {
+		adapter.sender.AddMetrics(typedMetrics...)
+	}
+}
+
+func (adapter *Adapter) AddDistinctMetrics(errType interface{}, metrics ...map[string]interface{}) {
+	errTypeInt, ok := errType.(int)
+	if !ok {
+		return
+	}
+
+	var typedMetrics []proton.ObservabilityMetric
+	for _, metric := range metrics {
+		if ok, m := VerifyAndParseGenericMetrics(metric); ok {
+			typedMetrics = append(typedMetrics, m)
+		}
+	}
+
+	if len(typedMetrics) > 0 {
+		adapter.sender.AddDistinctMetrics(DistinctionErrorTypeEnum(errTypeInt), typedMetrics...)
+	}
+}

internal/services/observability/adapter_test.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2024 Proton AG
+//
+// This file is part of Proton Mail Bridge.
+//
+// Proton Mail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Proton Mail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Proton Mail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package observability
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func Test_AdapterCustomMetrics(t *testing.T) {
+	customMetric := map[string]interface{}{
+		"Name":      "name",
+		"Version":   1,
+		"Timestamp": time.Now().Unix(),
+		"Data": map[string]interface{}{
+			"Value": 1,
+			"Labels": map[string]string{
+				"error": "customError",
+			},
+		},
+	}
+
+	ok, metric := VerifyAndParseGenericMetrics(customMetric)
+	require.True(t, ok)
+
+	require.Equal(t, metric.Name, customMetric["Name"])
+	require.Equal(t, metric.Timestamp, customMetric["Timestamp"])
+	require.Equal(t, metric.Version, customMetric["Version"])
+	require.Equal(t, metric.Data, customMetric["Data"])
+}
+
+func Test_AdapterGluonMetrics(t *testing.T) {
+	metrics := GenerateAllGluonMetrics()
+
+	for _, metric := range metrics {
+		ok, m := VerifyAndParseGenericMetrics(metric)
+		fmt.Println(m)
+		require.True(t, ok)
+	}
+}

internal/services/observability/distinction_error_types.go

@@ -25,13 +25,19 @@ type DistinctionErrorTypeEnum int
 
 const (
 	SyncError DistinctionErrorTypeEnum = iota
-	EventLoopError
+	GluonImapError
+	GluonMessageError
+	GluonOtherError
+	EventLoopError // EventLoopError - should always be kept last when inserting new keys.
 )
 
 // errorSchemaMap - maps between the DistinctionErrorTypeEnum and the relevant schema name.
 var errorSchemaMap = map[DistinctionErrorTypeEnum]string{ //nolint:gochecknoglobals
-	SyncError:      "bridge_sync_errors_users_total",
+	SyncError:         "bridge_sync_errors_users_total",
-	EventLoopError: "bridge_event_loop_events_errors_users_total",
+	EventLoopError:    "bridge_event_loop_events_errors_users_total",
+	GluonImapError:    "bridge_gluon_imap_errors_users_total",
+	GluonMessageError: "bridge_gluon_message_errors_users_total",
+	GluonOtherError:   "bridge_gluon_other_errors_users_total",
 }
 
 // createLastSentMap - needs to be updated whenever we make changes to the enum.

internal/services/observability/heartbeat.go

@@ -26,17 +26,20 @@ import (
 )
 
 const genericHeartbeatSchemaName = "bridge_generic_user_heartbeat_total"
+const genericHeartbeatVersion = 2
 
 type heartbeatData struct {
 	receivedSyncError      bool
 	receivedEventLoopError bool
 	receivedOtherError     bool
+	receivedGluonError     bool
 }
 
 func (d *distinctionUtility) resetHeartbeatData() {
 	d.heartbeatData.receivedSyncError = false
 	d.heartbeatData.receivedOtherError = false
 	d.heartbeatData.receivedEventLoopError = false
+	d.heartbeatData.receivedGluonError = false
 }
 
 func (d *distinctionUtility) updateHeartbeatData(errType DistinctionErrorTypeEnum) {
@@ -46,6 +49,8 @@ func (d *distinctionUtility) updateHeartbeatData(errType DistinctionErrorTypeEnu
 			d.heartbeatData.receivedSyncError = true
 		case EventLoopError:
 			d.heartbeatData.receivedEventLoopError = true
+		case GluonMessageError, GluonImapError, GluonOtherError:
+			d.heartbeatData.receivedGluonError = true
 		}
 	})
 }
@@ -95,13 +100,14 @@ func (d *distinctionUtility) generateHeartbeatUserMetric() proton.ObservabilityM
 		formatBool(d.heartbeatData.receivedOtherError),
 		formatBool(d.heartbeatData.receivedSyncError),
 		formatBool(d.heartbeatData.receivedEventLoopError),
+		formatBool(d.heartbeatData.receivedGluonError),
 	)
 }
 
-func generateHeartbeatMetric(plan, mailClient, dohEnabled, betaAccess, otherError, syncError, eventLoopError string) proton.ObservabilityMetric {
+func generateHeartbeatMetric(plan, mailClient, dohEnabled, betaAccess, otherError, syncError, eventLoopError, gluonError string) proton.ObservabilityMetric {
 	return proton.ObservabilityMetric{
 		Name:      genericHeartbeatSchemaName,
-		Version:   1,
+		Version:   genericHeartbeatVersion,
 		Timestamp: time.Now().Unix(),
 		Data: map[string]interface{}{
 			"Value": 1,
@@ -113,6 +119,7 @@ func generateHeartbeatMetric(plan, mailClient, dohEnabled, betaAccess, otherErro
 				"receivedOtherError":     otherError,
 				"receivedSyncError":      syncError,
 				"receivedEventLoopError": eventLoopError,
+				"receivedGluonError":     gluonError,
 			},
 		},
 	}

internal/services/observability/test_utils.go

@@ -18,6 +18,7 @@
 package observability
 
 import (
+	gluonMetrics "github.com/ProtonMail/gluon/observability/metrics"
 	"github.com/ProtonMail/go-proton-api"
 )
 
@@ -85,16 +86,19 @@ func GenerateAllHeartbeatMetricPermutations() []proton.ObservabilityMetric {
 					for _, receivedOtherError := range trueFalseValues {
 						for _, receivedSyncError := range trueFalseValues {
 							for _, receivedEventLoopError := range trueFalseValues {
-								metrics = append(metrics,
+								for _, receivedGluonError := range trueFalseValues {
-									generateHeartbeatMetric(plan,
+									metrics = append(metrics,
-										mailClient,
+										generateHeartbeatMetric(plan,
-										dohEnabled,
+											mailClient,
-										betaAccess,
+											dohEnabled,
-										receivedOtherError,
+											betaAccess,
-										receivedSyncError,
+											receivedOtherError,
-										receivedEventLoopError,
+											receivedSyncError,
-									),
+											receivedEventLoopError,
-								)
+											receivedGluonError,
+										),
+									)
+								}
 							}
 						}
 					}
@@ -104,3 +108,19 @@ func GenerateAllHeartbeatMetricPermutations() []proton.ObservabilityMetric {
 	}
 	return metrics
 }
+
+func GenerateAllGluonMetrics() []map[string]interface{} {
+	var metrics []map[string]interface{}
+	metrics = append(metrics,
+		gluonMetrics.GenerateFailedParseIMAPCommandMetric(),
+		gluonMetrics.GenerateFailedToCreateMailbox(),
+		gluonMetrics.GenerateFailedToDeleteMailboxMetric(),
+		gluonMetrics.GenerateFailedToCopyMessagesMetric(),
+		gluonMetrics.GenerateFailedToMoveMessagesFromMailboxMetric(),
+		gluonMetrics.GenerateFailedToRemoveDeletedMessagesMetric(),
+		gluonMetrics.GenerateFailedToCommitDatabaseTransactionMetric(),
+		gluonMetrics.GenerateAppendToDraftsMustNotReturnExistingRemoteID(),
+		gluonMetrics.GenerateDatabaseMigrationFailed(),
+	)
+	return metrics
+}

tests/features/observability/gluon_metrics.feature

@@ -0,0 +1,11 @@
+Feature: Bridge send remote notification observability metrics
+  Background:
+    Given there exists an account with username "[user:user1]" and password "password"
+    Then it succeeds
+    When bridge starts
+    Then it succeeds
+
+  Scenario: Test all possible gluon error observability metrics
+    When the user logs in with username "[user:user1]" and password "password"
+    And the user with username "[user:user1]" sends all possible gluon error observability metrics
+    Then it succeeds

tests/observability_test.go

@@ -19,6 +19,7 @@ package tests
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/ProtonMail/go-proton-api"
 	"github.com/ProtonMail/proton-bridge/v3/internal/services/imapservice/observabilitymetrics/evtloopmsgevents"
@@ -27,18 +28,35 @@ import (
 	"github.com/ProtonMail/proton-bridge/v3/internal/services/syncservice/observabilitymetrics"
 )
 
-// userHeartbeatPermutationsObservability - corresponds to bridge_generic_user_heartbeat_total_v1.schema.json.
+// userHeartbeatPermutationsObservability corresponds to bridge_generic_user_heartbeat_total_v1.schema.json.
 func (s *scenario) userHeartbeatPermutationsObservability(username string) error {
+	const batchSize = 1000
 	metrics := observability.GenerateAllHeartbeatMetricPermutations()
+	metricLen := len(metrics)
+
 	return s.t.withClientPass(context.Background(), username, s.t.getUserByName(username).userPass, func(ctx context.Context, c *proton.Client) error {
-		batch := proton.ObservabilityBatch{Metrics: metrics}
+		for i := 0; i < len(metrics); i += batchSize {
-		return c.SendObservabilityBatch(ctx, batch)
+			end := i + batchSize
+			if end > metricLen {
+				end = metricLen
+			}
+
+			batch := proton.ObservabilityBatch{Metrics: metrics[i:end]}
+			if err := c.SendObservabilityBatch(ctx, batch); err != nil {
+				return err
+			}
+		}
+
+		return nil
 	})
 }
 
-// userDistinctionMetricsPermutationsObservability - corresponds to:
+// userDistinctionMetricsPermutationsObservability corresponds to:
-// bridge_sync_errors_users_total_v1.schema.json
+//   - bridge_sync_errors_users_total_v1.schema.json
-// bridge_event_loop_events_errors_users_total_v1.schema.json.
+//   - bridge_gluon_imap_errors_users_total_v1.schema.json
+//   - bridge_gluon_message_errors_users_total_v1.schema.json
+//   - bridge_gluon_other_errors_users_total_v1.schema.json
+//   - bridge_event_loop_events_errors_users_total_v1.schema.json.
 func (s *scenario) userDistinctionMetricsPermutationsObservability(username string) error {
 	batch := proton.ObservabilityBatch{
 		Metrics: observability.GenerateAllUsedDistinctionMetricPermutations()}
@@ -48,7 +66,7 @@ func (s *scenario) userDistinctionMetricsPermutationsObservability(username stri
 	})
 }
 
-// syncFailureMessageEventsObservability - corresponds to bridge_sync_message_event_failures_total_v1.schema.json.
+// syncFailureMessageEventsObservability corresponds to bridge_sync_message_event_failures_total_v1.schema.json.
 func (s *scenario) syncFailureMessageEventsObservability(username string) error {
 	batch := proton.ObservabilityBatch{
 		Metrics: []proton.ObservabilityMetric{
@@ -62,7 +80,7 @@ func (s *scenario) syncFailureMessageEventsObservability(username string) error
 	})
 }
 
-// eventLoopFailureMessageEventsObservability - corresponds to bridge_event_loop_message_event_failures_total_v1.schema.json.
+// eventLoopFailureMessageEventsObservability corresponds to bridge_event_loop_message_event_failures_total_v1.schema.json.
 func (s *scenario) eventLoopFailureMessageEventsObservability(username string) error {
 	batch := proton.ObservabilityBatch{
 		Metrics: []proton.ObservabilityMetric{
@@ -81,7 +99,7 @@ func (s *scenario) eventLoopFailureMessageEventsObservability(username string) e
 	})
 }
 
-// syncFailureMessageBuiltObservability - corresponds to bridge_sync_message_event_failures_total_v1.schema.json.
+// syncFailureMessageBuiltObservability corresponds to bridge_sync_message_event_failures_total_v1.schema.json.
 func (s *scenario) syncFailureMessageBuiltObservability(username string) error {
 	batch := proton.ObservabilityBatch{
 		Metrics: []proton.ObservabilityMetric{
@@ -96,7 +114,7 @@ func (s *scenario) syncFailureMessageBuiltObservability(username string) error {
 	})
 }
 
-// syncSuccessMessageBuiltObservability - corresponds to bridge_sync_message_build_success_total_v1.schema.json.
+// syncSuccessMessageBuiltObservability corresponds to bridge_sync_message_build_success_total_v1.schema.json.
 func (s *scenario) syncSuccessMessageBuiltObservability(username string) error {
 	batch := proton.ObservabilityBatch{
 		Metrics: []proton.ObservabilityMetric{
@@ -109,3 +127,24 @@ func (s *scenario) syncSuccessMessageBuiltObservability(username string) error {
 		return err
 	})
 }
+
+// testGluonErrorObservabilityMetrics corresponds to bridge_gluon_errors_total_v1.schema.json.
+func (s *scenario) testGluonErrorObservabilityMetrics(username string) error {
+	allMetrics := observability.GenerateAllGluonMetrics()
+
+	parsedMetrics := []proton.ObservabilityMetric{}
+	for _, el := range allMetrics {
+		ok, parsedMetric := observability.VerifyAndParseGenericMetrics(el)
+		if !ok {
+			return fmt.Errorf("failed to parse generic gluon metric")
+		}
+		parsedMetrics = append(parsedMetrics, parsedMetric)
+	}
+
+	batch := proton.ObservabilityBatch{Metrics: parsedMetrics}
+
+	return s.t.withClientPass(context.Background(), username, s.t.getUserByName(username).userPass, func(ctx context.Context, c *proton.Client) error {
+		err := c.SendObservabilityBatch(ctx, batch)
+		return err
+	})
+}

tests/steps_test.go

@@ -226,4 +226,6 @@ func (s *scenario) steps(ctx *godog.ScenarioContext) {
 	ctx.Step(`^the user with username "([^"]*)" sends all possible event loop message events observability metrics$`, s.eventLoopFailureMessageEventsObservability)
 	ctx.Step(`^the user with username "([^"]*)" sends all possible sync message building failure observability metrics$`, s.syncFailureMessageBuiltObservability)
 	ctx.Step(`^the user with username "([^"]*)" sends all possible sync message building success observability metrics$`, s.syncSuccessMessageBuiltObservability)
+	// Gluon related metrics
+	ctx.Step(`^the user with username "([^"]*)" sends all possible gluon error observability metrics$`, s.testGluonErrorObservabilityMetrics)
 }