diff --git a/go.mod b/go.mod index d72b651c..d6e58b47 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.21.9 require ( github.com/0xAX/notificator v0.0.0-20220220101646-ee9b8921e557 github.com/Masterminds/semver/v3 v3.2.0 - github.com/ProtonMail/gluon v0.17.1-0.20240923151549-d23b4bec3602 + github.com/ProtonMail/gluon v0.17.1-0.20241008123701-ddf4a459d0b4 github.com/ProtonMail/go-autostart v0.0.0-20210130080809-00ed301c8e9a github.com/ProtonMail/go-proton-api v0.4.1-0.20240918100656-b4860af56d47 github.com/ProtonMail/gopenpgp/v2 v2.7.4-proton diff --git a/go.sum b/go.sum index 6b77452a..21b4e928 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,14 @@ github.com/ProtonMail/gluon v0.17.1-0.20240923094038-e319bf6047c5 h1:LzaUpUj6M2P github.com/ProtonMail/gluon v0.17.1-0.20240923094038-e319bf6047c5/go.mod h1:0/c03TzZPNiSgY5UDJK1iRDkjlDPwWugxTT6et2qDu8= github.com/ProtonMail/gluon v0.17.1-0.20240923151549-d23b4bec3602 h1:EoMjWlC32tg46L/07hWoiZfLkqJyxVMcsq4Cyn+Ofqc= github.com/ProtonMail/gluon v0.17.1-0.20240923151549-d23b4bec3602/go.mod h1:0/c03TzZPNiSgY5UDJK1iRDkjlDPwWugxTT6et2qDu8= +github.com/ProtonMail/gluon v0.17.1-0.20241002092751-3bbeea9053af h1:iMxTQUg2cB47cXqpMev3cZmQoGBOef3cSUjBbdEl33M= +github.com/ProtonMail/gluon v0.17.1-0.20241002092751-3bbeea9053af/go.mod h1:0/c03TzZPNiSgY5UDJK1iRDkjlDPwWugxTT6et2qDu8= +github.com/ProtonMail/gluon v0.17.1-0.20241002111651-173859b80060 h1:dcu3tT84GjoXb++n7crv8UJeG8eRwogjTYdkoJ+MjQI= +github.com/ProtonMail/gluon v0.17.1-0.20241002111651-173859b80060/go.mod h1:0/c03TzZPNiSgY5UDJK1iRDkjlDPwWugxTT6et2qDu8= +github.com/ProtonMail/gluon v0.17.1-0.20241002142736-ef4153d156d8 h1:YxPHSJUA87i1hc6s1YrW89++V7HpcR7LSFQ6XM0TsAE= +github.com/ProtonMail/gluon v0.17.1-0.20241002142736-ef4153d156d8/go.mod h1:0/c03TzZPNiSgY5UDJK1iRDkjlDPwWugxTT6et2qDu8= +github.com/ProtonMail/gluon v0.17.1-0.20241008123701-ddf4a459d0b4 h1:xE+V17O9HIttMpVymNCORQILk9OKpSekrrPbX7YGnF8= +github.com/ProtonMail/gluon v0.17.1-0.20241008123701-ddf4a459d0b4/go.mod h1:0/c03TzZPNiSgY5UDJK1iRDkjlDPwWugxTT6et2qDu8= github.com/ProtonMail/go-autostart v0.0.0-20210130080809-00ed301c8e9a h1:D+aZah+k14Gn6kmL7eKxoo/4Dr/lK3ChBcwce2+SQP4= github.com/ProtonMail/go-autostart v0.0.0-20210130080809-00ed301c8e9a/go.mod h1:oTGdE7/DlWIr23G0IKW3OXK9wZ5Hw1GGiaJFccTvZi4= github.com/ProtonMail/go-crypto v0.0.0-20230321155629-9a39f2531310/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE= diff --git a/internal/bridge/bridge.go b/internal/bridge/bridge.go index 9c4465ce..3d6dca97 100644 --- a/internal/bridge/bridge.go +++ b/internal/bridge/bridge.go @@ -325,6 +325,7 @@ func newBridge( reporter, uidValidityGenerator, &bridgeIMAPSMTPTelemetry{b: bridge}, + observabilityService, ) // Check whether username has changed and correct (macOS only) diff --git a/internal/services/imapsmtpserver/imap.go b/internal/services/imapsmtpserver/imap.go index d1d8d9c4..5783231b 100644 --- a/internal/services/imapsmtpserver/imap.go +++ b/internal/services/imapsmtpserver/imap.go @@ -36,6 +36,7 @@ import ( "github.com/ProtonMail/proton-bridge/v3/internal/constants" "github.com/ProtonMail/proton-bridge/v3/internal/files" "github.com/ProtonMail/proton-bridge/v3/internal/logging" + "github.com/ProtonMail/proton-bridge/v3/internal/services/observability" "github.com/sirupsen/logrus" ) @@ -77,6 +78,7 @@ func newIMAPServer( tasks *async.Group, uidValidityGenerator imap.UIDValidityGenerator, panicHandler async.PanicHandler, + observabilitySender observability.Sender, ) (*gluon.Server, error) { gluonCacheDir = ApplyGluonCachePathSuffix(gluonCacheDir) gluonConfigDir = ApplyGluonConfigPathSuffix(gluonConfigDir) @@ -121,6 +123,7 @@ func newIMAPServer( gluon.WithReporter(reporter), gluon.WithUIDValidityGenerator(uidValidityGenerator), gluon.WithPanicHandler(panicHandler), + gluon.WithObservabilitySender(observability.NewAdapter(observabilitySender), int(observability.GluonImapError), int(observability.GluonMessageError), int(observability.GluonOtherError)), ) if err != nil { return nil, err diff --git a/internal/services/imapsmtpserver/service.go b/internal/services/imapsmtpserver/service.go index c603479c..f56e7275 100644 --- a/internal/services/imapsmtpserver/service.go +++ b/internal/services/imapsmtpserver/service.go @@ -31,6 +31,7 @@ import ( "github.com/ProtonMail/gluon/reporter" "github.com/ProtonMail/proton-bridge/v3/internal/events" "github.com/ProtonMail/proton-bridge/v3/internal/services/imapservice" + "github.com/ProtonMail/proton-bridge/v3/internal/services/observability" bridgesmtp "github.com/ProtonMail/proton-bridge/v3/internal/services/smtp" "github.com/ProtonMail/proton-bridge/v3/internal/services/syncservice" "github.com/ProtonMail/proton-bridge/v3/pkg/cpc" @@ -60,6 +61,8 @@ type Service struct { uidValidityGenerator imap.UIDValidityGenerator telemetry Telemetry + + observabilitySender observability.Sender } func NewService( @@ -71,6 +74,7 @@ func NewService( reporter reporter.Reporter, uidValidityGenerator imap.UIDValidityGenerator, telemetry Telemetry, + observabilitySender observability.Sender, ) *Service { return &Service{ requests: cpc.NewCPC(), @@ -85,6 +89,8 @@ func NewService( tasks: async.NewGroup(ctx, panicHandler), uidValidityGenerator: uidValidityGenerator, telemetry: telemetry, + + observabilitySender: observabilitySender, } } @@ -449,6 +455,7 @@ func (sm *Service) createIMAPServer(ctx context.Context) (*gluon.Server, error) sm.tasks, sm.uidValidityGenerator, sm.panicHandler, + sm.observabilitySender, ) if err == nil { sm.eventPublisher.PublishEvent(ctx, events.IMAPServerCreated{}) diff --git a/internal/services/observability/adapter.go b/internal/services/observability/adapter.go new file mode 100644 index 00000000..0bc88fd0 --- /dev/null +++ b/internal/services/observability/adapter.go @@ -0,0 +1,93 @@ +// Copyright (c) 2024 Proton AG +// +// This file is part of Proton Mail Bridge. +// +// Proton Mail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Proton Mail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Proton Mail Bridge. If not, see . + +package observability + +import ( + "github.com/ProtonMail/go-proton-api" +) + +type Adapter struct { + sender Sender +} + +func NewAdapter(sender Sender) *Adapter { + return &Adapter{sender: sender} +} + +// VerifyAndParseGenericMetrics parses a metric provided as an interface into a proton.ObservabilityMetric type. +// It's exported as it is also used in integration tests. +func VerifyAndParseGenericMetrics(metric map[string]interface{}) (bool, proton.ObservabilityMetric) { + name, ok := metric["Name"].(string) + if !ok { + return false, proton.ObservabilityMetric{} + } + + version, ok := metric["Version"].(int) + if !ok { + return false, proton.ObservabilityMetric{} + } + + timestamp, ok := metric["Timestamp"].(int64) + if !ok { + return false, proton.ObservabilityMetric{} + } + + data, ok := metric["Data"] + if !ok { + return false, proton.ObservabilityMetric{} + } + + return true, proton.ObservabilityMetric{ + Name: name, + Version: version, + Timestamp: timestamp, + Data: data, + } +} + +func (adapter *Adapter) AddMetrics(metrics ...map[string]interface{}) { + var typedMetrics []proton.ObservabilityMetric + + for _, metric := range metrics { + if ok, m := VerifyAndParseGenericMetrics(metric); ok { + typedMetrics = append(typedMetrics, m) + } + } + + if len(typedMetrics) > 0 { + adapter.sender.AddMetrics(typedMetrics...) + } +} + +func (adapter *Adapter) AddDistinctMetrics(errType interface{}, metrics ...map[string]interface{}) { + errTypeInt, ok := errType.(int) + if !ok { + return + } + + var typedMetrics []proton.ObservabilityMetric + for _, metric := range metrics { + if ok, m := VerifyAndParseGenericMetrics(metric); ok { + typedMetrics = append(typedMetrics, m) + } + } + + if len(typedMetrics) > 0 { + adapter.sender.AddDistinctMetrics(DistinctionErrorTypeEnum(errTypeInt), typedMetrics...) + } +} diff --git a/internal/services/observability/adapter_test.go b/internal/services/observability/adapter_test.go new file mode 100644 index 00000000..23197204 --- /dev/null +++ b/internal/services/observability/adapter_test.go @@ -0,0 +1,58 @@ +// Copyright (c) 2024 Proton AG +// +// This file is part of Proton Mail Bridge. +// +// Proton Mail Bridge is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Proton Mail Bridge is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Proton Mail Bridge. If not, see . + +package observability + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func Test_AdapterCustomMetrics(t *testing.T) { + customMetric := map[string]interface{}{ + "Name": "name", + "Version": 1, + "Timestamp": time.Now().Unix(), + "Data": map[string]interface{}{ + "Value": 1, + "Labels": map[string]string{ + "error": "customError", + }, + }, + } + + ok, metric := VerifyAndParseGenericMetrics(customMetric) + require.True(t, ok) + + require.Equal(t, metric.Name, customMetric["Name"]) + require.Equal(t, metric.Timestamp, customMetric["Timestamp"]) + require.Equal(t, metric.Version, customMetric["Version"]) + require.Equal(t, metric.Data, customMetric["Data"]) +} + +func Test_AdapterGluonMetrics(t *testing.T) { + metrics := GenerateAllGluonMetrics() + + for _, metric := range metrics { + ok, m := VerifyAndParseGenericMetrics(metric) + fmt.Println(m) + require.True(t, ok) + } +} diff --git a/internal/services/observability/distinction_error_types.go b/internal/services/observability/distinction_error_types.go index 9cc8526d..b1a56e13 100644 --- a/internal/services/observability/distinction_error_types.go +++ b/internal/services/observability/distinction_error_types.go @@ -25,13 +25,19 @@ type DistinctionErrorTypeEnum int const ( SyncError DistinctionErrorTypeEnum = iota - EventLoopError + GluonImapError + GluonMessageError + GluonOtherError + EventLoopError // EventLoopError - should always be kept last when inserting new keys. ) // errorSchemaMap - maps between the DistinctionErrorTypeEnum and the relevant schema name. var errorSchemaMap = map[DistinctionErrorTypeEnum]string{ //nolint:gochecknoglobals - SyncError: "bridge_sync_errors_users_total", - EventLoopError: "bridge_event_loop_events_errors_users_total", + SyncError: "bridge_sync_errors_users_total", + EventLoopError: "bridge_event_loop_events_errors_users_total", + GluonImapError: "bridge_gluon_imap_errors_users_total", + GluonMessageError: "bridge_gluon_message_errors_users_total", + GluonOtherError: "bridge_gluon_other_errors_users_total", } // createLastSentMap - needs to be updated whenever we make changes to the enum. diff --git a/internal/services/observability/heartbeat.go b/internal/services/observability/heartbeat.go index 6d2c6b49..af26f7e6 100644 --- a/internal/services/observability/heartbeat.go +++ b/internal/services/observability/heartbeat.go @@ -26,17 +26,20 @@ import ( ) const genericHeartbeatSchemaName = "bridge_generic_user_heartbeat_total" +const genericHeartbeatVersion = 2 type heartbeatData struct { receivedSyncError bool receivedEventLoopError bool receivedOtherError bool + receivedGluonError bool } func (d *distinctionUtility) resetHeartbeatData() { d.heartbeatData.receivedSyncError = false d.heartbeatData.receivedOtherError = false d.heartbeatData.receivedEventLoopError = false + d.heartbeatData.receivedGluonError = false } func (d *distinctionUtility) updateHeartbeatData(errType DistinctionErrorTypeEnum) { @@ -46,6 +49,8 @@ func (d *distinctionUtility) updateHeartbeatData(errType DistinctionErrorTypeEnu d.heartbeatData.receivedSyncError = true case EventLoopError: d.heartbeatData.receivedEventLoopError = true + case GluonMessageError, GluonImapError, GluonOtherError: + d.heartbeatData.receivedGluonError = true } }) } @@ -95,13 +100,14 @@ func (d *distinctionUtility) generateHeartbeatUserMetric() proton.ObservabilityM formatBool(d.heartbeatData.receivedOtherError), formatBool(d.heartbeatData.receivedSyncError), formatBool(d.heartbeatData.receivedEventLoopError), + formatBool(d.heartbeatData.receivedGluonError), ) } -func generateHeartbeatMetric(plan, mailClient, dohEnabled, betaAccess, otherError, syncError, eventLoopError string) proton.ObservabilityMetric { +func generateHeartbeatMetric(plan, mailClient, dohEnabled, betaAccess, otherError, syncError, eventLoopError, gluonError string) proton.ObservabilityMetric { return proton.ObservabilityMetric{ Name: genericHeartbeatSchemaName, - Version: 1, + Version: genericHeartbeatVersion, Timestamp: time.Now().Unix(), Data: map[string]interface{}{ "Value": 1, @@ -113,6 +119,7 @@ func generateHeartbeatMetric(plan, mailClient, dohEnabled, betaAccess, otherErro "receivedOtherError": otherError, "receivedSyncError": syncError, "receivedEventLoopError": eventLoopError, + "receivedGluonError": gluonError, }, }, } diff --git a/internal/services/observability/test_utils.go b/internal/services/observability/test_utils.go index c12d4936..f1ca97f5 100644 --- a/internal/services/observability/test_utils.go +++ b/internal/services/observability/test_utils.go @@ -18,6 +18,7 @@ package observability import ( + gluonMetrics "github.com/ProtonMail/gluon/observability/metrics" "github.com/ProtonMail/go-proton-api" ) @@ -85,16 +86,19 @@ func GenerateAllHeartbeatMetricPermutations() []proton.ObservabilityMetric { for _, receivedOtherError := range trueFalseValues { for _, receivedSyncError := range trueFalseValues { for _, receivedEventLoopError := range trueFalseValues { - metrics = append(metrics, - generateHeartbeatMetric(plan, - mailClient, - dohEnabled, - betaAccess, - receivedOtherError, - receivedSyncError, - receivedEventLoopError, - ), - ) + for _, receivedGluonError := range trueFalseValues { + metrics = append(metrics, + generateHeartbeatMetric(plan, + mailClient, + dohEnabled, + betaAccess, + receivedOtherError, + receivedSyncError, + receivedEventLoopError, + receivedGluonError, + ), + ) + } } } } @@ -104,3 +108,19 @@ func GenerateAllHeartbeatMetricPermutations() []proton.ObservabilityMetric { } return metrics } + +func GenerateAllGluonMetrics() []map[string]interface{} { + var metrics []map[string]interface{} + metrics = append(metrics, + gluonMetrics.GenerateFailedParseIMAPCommandMetric(), + gluonMetrics.GenerateFailedToCreateMailbox(), + gluonMetrics.GenerateFailedToDeleteMailboxMetric(), + gluonMetrics.GenerateFailedToCopyMessagesMetric(), + gluonMetrics.GenerateFailedToMoveMessagesFromMailboxMetric(), + gluonMetrics.GenerateFailedToRemoveDeletedMessagesMetric(), + gluonMetrics.GenerateFailedToCommitDatabaseTransactionMetric(), + gluonMetrics.GenerateAppendToDraftsMustNotReturnExistingRemoteID(), + gluonMetrics.GenerateDatabaseMigrationFailed(), + ) + return metrics +} diff --git a/tests/features/observability/gluon_metrics.feature b/tests/features/observability/gluon_metrics.feature new file mode 100644 index 00000000..d10d063e --- /dev/null +++ b/tests/features/observability/gluon_metrics.feature @@ -0,0 +1,11 @@ +Feature: Bridge send remote notification observability metrics + Background: + Given there exists an account with username "[user:user1]" and password "password" + Then it succeeds + When bridge starts + Then it succeeds + + Scenario: Test all possible gluon error observability metrics + When the user logs in with username "[user:user1]" and password "password" + And the user with username "[user:user1]" sends all possible gluon error observability metrics + Then it succeeds diff --git a/tests/observability_test.go b/tests/observability_test.go index 9777f52d..4de56ec8 100644 --- a/tests/observability_test.go +++ b/tests/observability_test.go @@ -19,6 +19,7 @@ package tests import ( "context" + "fmt" "github.com/ProtonMail/go-proton-api" "github.com/ProtonMail/proton-bridge/v3/internal/services/imapservice/observabilitymetrics/evtloopmsgevents" @@ -27,18 +28,35 @@ import ( "github.com/ProtonMail/proton-bridge/v3/internal/services/syncservice/observabilitymetrics" ) -// userHeartbeatPermutationsObservability - corresponds to bridge_generic_user_heartbeat_total_v1.schema.json. +// userHeartbeatPermutationsObservability corresponds to bridge_generic_user_heartbeat_total_v1.schema.json. func (s *scenario) userHeartbeatPermutationsObservability(username string) error { + const batchSize = 1000 metrics := observability.GenerateAllHeartbeatMetricPermutations() + metricLen := len(metrics) + return s.t.withClientPass(context.Background(), username, s.t.getUserByName(username).userPass, func(ctx context.Context, c *proton.Client) error { - batch := proton.ObservabilityBatch{Metrics: metrics} - return c.SendObservabilityBatch(ctx, batch) + for i := 0; i < len(metrics); i += batchSize { + end := i + batchSize + if end > metricLen { + end = metricLen + } + + batch := proton.ObservabilityBatch{Metrics: metrics[i:end]} + if err := c.SendObservabilityBatch(ctx, batch); err != nil { + return err + } + } + + return nil }) } -// userDistinctionMetricsPermutationsObservability - corresponds to: -// bridge_sync_errors_users_total_v1.schema.json -// bridge_event_loop_events_errors_users_total_v1.schema.json. +// userDistinctionMetricsPermutationsObservability corresponds to: +// - bridge_sync_errors_users_total_v1.schema.json +// - bridge_gluon_imap_errors_users_total_v1.schema.json +// - bridge_gluon_message_errors_users_total_v1.schema.json +// - bridge_gluon_other_errors_users_total_v1.schema.json +// - bridge_event_loop_events_errors_users_total_v1.schema.json. func (s *scenario) userDistinctionMetricsPermutationsObservability(username string) error { batch := proton.ObservabilityBatch{ Metrics: observability.GenerateAllUsedDistinctionMetricPermutations()} @@ -48,7 +66,7 @@ func (s *scenario) userDistinctionMetricsPermutationsObservability(username stri }) } -// syncFailureMessageEventsObservability - corresponds to bridge_sync_message_event_failures_total_v1.schema.json. +// syncFailureMessageEventsObservability corresponds to bridge_sync_message_event_failures_total_v1.schema.json. func (s *scenario) syncFailureMessageEventsObservability(username string) error { batch := proton.ObservabilityBatch{ Metrics: []proton.ObservabilityMetric{ @@ -62,7 +80,7 @@ func (s *scenario) syncFailureMessageEventsObservability(username string) error }) } -// eventLoopFailureMessageEventsObservability - corresponds to bridge_event_loop_message_event_failures_total_v1.schema.json. +// eventLoopFailureMessageEventsObservability corresponds to bridge_event_loop_message_event_failures_total_v1.schema.json. func (s *scenario) eventLoopFailureMessageEventsObservability(username string) error { batch := proton.ObservabilityBatch{ Metrics: []proton.ObservabilityMetric{ @@ -81,7 +99,7 @@ func (s *scenario) eventLoopFailureMessageEventsObservability(username string) e }) } -// syncFailureMessageBuiltObservability - corresponds to bridge_sync_message_event_failures_total_v1.schema.json. +// syncFailureMessageBuiltObservability corresponds to bridge_sync_message_event_failures_total_v1.schema.json. func (s *scenario) syncFailureMessageBuiltObservability(username string) error { batch := proton.ObservabilityBatch{ Metrics: []proton.ObservabilityMetric{ @@ -96,7 +114,7 @@ func (s *scenario) syncFailureMessageBuiltObservability(username string) error { }) } -// syncSuccessMessageBuiltObservability - corresponds to bridge_sync_message_build_success_total_v1.schema.json. +// syncSuccessMessageBuiltObservability corresponds to bridge_sync_message_build_success_total_v1.schema.json. func (s *scenario) syncSuccessMessageBuiltObservability(username string) error { batch := proton.ObservabilityBatch{ Metrics: []proton.ObservabilityMetric{ @@ -109,3 +127,24 @@ func (s *scenario) syncSuccessMessageBuiltObservability(username string) error { return err }) } + +// testGluonErrorObservabilityMetrics corresponds to bridge_gluon_errors_total_v1.schema.json. +func (s *scenario) testGluonErrorObservabilityMetrics(username string) error { + allMetrics := observability.GenerateAllGluonMetrics() + + parsedMetrics := []proton.ObservabilityMetric{} + for _, el := range allMetrics { + ok, parsedMetric := observability.VerifyAndParseGenericMetrics(el) + if !ok { + return fmt.Errorf("failed to parse generic gluon metric") + } + parsedMetrics = append(parsedMetrics, parsedMetric) + } + + batch := proton.ObservabilityBatch{Metrics: parsedMetrics} + + return s.t.withClientPass(context.Background(), username, s.t.getUserByName(username).userPass, func(ctx context.Context, c *proton.Client) error { + err := c.SendObservabilityBatch(ctx, batch) + return err + }) +} diff --git a/tests/steps_test.go b/tests/steps_test.go index caa3fba9..091769d9 100644 --- a/tests/steps_test.go +++ b/tests/steps_test.go @@ -226,4 +226,6 @@ func (s *scenario) steps(ctx *godog.ScenarioContext) { ctx.Step(`^the user with username "([^"]*)" sends all possible event loop message events observability metrics$`, s.eventLoopFailureMessageEventsObservability) ctx.Step(`^the user with username "([^"]*)" sends all possible sync message building failure observability metrics$`, s.syncFailureMessageBuiltObservability) ctx.Step(`^the user with username "([^"]*)" sends all possible sync message building success observability metrics$`, s.syncSuccessMessageBuiltObservability) + // Gluon related metrics + ctx.Step(`^the user with username "([^"]*)" sends all possible gluon error observability metrics$`, s.testGluonErrorObservabilityMetrics) }