yesBad/proton-bridge
1
0
Fork 0
forked from Silverfish/proton-bridge
Code Actions Activity

feat(BRIDGE-218): observability adapter; gluon observability metrics and tests;

Browse Source
This commit is contained in:
Atanas Janeshliev
2024-10-08 13:13:07 +00:00
parent 3710dff0cd
commit 040d887aae
13 changed files with 281 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -7,7 +7,7 @@ toolchain go1.21.9
require (
github.com/0xAX/notificator v0.0.0-20220220101646-ee9b8921e557
github.com/Masterminds/semver/v3 v3.2.0
github.com/ProtonMail/gluon v0.17.1-0.20240923151549-d23b4bec3602
github.com/ProtonMail/gluon v0.17.1-0.20241008123701-ddf4a459d0b4
github.com/ProtonMail/go-autostart v0.0.0-20210130080809-00ed301c8e9a
github.com/ProtonMail/go-proton-api v0.4.1-0.20240918100656-b4860af56d47
github.com/ProtonMail/gopenpgp/v2 v2.7.4-proton

8
go.sum
View File

@ -35,6 +35,14 @@ github.com/ProtonMail/gluon v0.17.1-0.20240923094038-e319bf6047c5 h1:LzaUpUj6M2P
github.com/ProtonMail/gluon v0.17.1-0.20240923094038-e319bf6047c5/go.mod h1:0/c03TzZPNiSgY5UDJK1iRDkjlDPwWugxTT6et2qDu8=
github.com/ProtonMail/gluon v0.17.1-0.20240923151549-d23b4bec3602 h1:EoMjWlC32tg46L/07hWoiZfLkqJyxVMcsq4Cyn+Ofqc=
github.com/ProtonMail/gluon v0.17.1-0.20240923151549-d23b4bec3602/go.mod h1:0/c03TzZPNiSgY5UDJK1iRDkjlDPwWugxTT6et2qDu8=
github.com/ProtonMail/gluon v0.17.1-0.20241002092751-3bbeea9053af h1:iMxTQUg2cB47cXqpMev3cZmQoGBOef3cSUjBbdEl33M=
github.com/ProtonMail/gluon v0.17.1-0.20241002092751-3bbeea9053af/go.mod h1:0/c03TzZPNiSgY5UDJK1iRDkjlDPwWugxTT6et2qDu8=
github.com/ProtonMail/gluon v0.17.1-0.20241002111651-173859b80060 h1:dcu3tT84GjoXb++n7crv8UJeG8eRwogjTYdkoJ+MjQI=
github.com/ProtonMail/gluon v0.17.1-0.20241002111651-173859b80060/go.mod h1:0/c03TzZPNiSgY5UDJK1iRDkjlDPwWugxTT6et2qDu8=
github.com/ProtonMail/gluon v0.17.1-0.20241002142736-ef4153d156d8 h1:YxPHSJUA87i1hc6s1YrW89++V7HpcR7LSFQ6XM0TsAE=
github.com/ProtonMail/gluon v0.17.1-0.20241002142736-ef4153d156d8/go.mod h1:0/c03TzZPNiSgY5UDJK1iRDkjlDPwWugxTT6et2qDu8=
github.com/ProtonMail/gluon v0.17.1-0.20241008123701-ddf4a459d0b4 h1:xE+V17O9HIttMpVymNCORQILk9OKpSekrrPbX7YGnF8=
github.com/ProtonMail/gluon v0.17.1-0.20241008123701-ddf4a459d0b4/go.mod h1:0/c03TzZPNiSgY5UDJK1iRDkjlDPwWugxTT6et2qDu8=
github.com/ProtonMail/go-autostart v0.0.0-20210130080809-00ed301c8e9a h1:D+aZah+k14Gn6kmL7eKxoo/4Dr/lK3ChBcwce2+SQP4=
github.com/ProtonMail/go-autostart v0.0.0-20210130080809-00ed301c8e9a/go.mod h1:oTGdE7/DlWIr23G0IKW3OXK9wZ5Hw1GGiaJFccTvZi4=
github.com/ProtonMail/go-crypto v0.0.0-20230321155629-9a39f2531310/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE=

1
internal/bridge/bridge.go
View File

@ -325,6 +325,7 @@ func newBridge(
reporter,
uidValidityGenerator,
&bridgeIMAPSMTPTelemetry{b: bridge},
observabilityService,
)
// Check whether username has changed and correct (macOS only)

3
internal/services/imapsmtpserver/imap.go
View File

@ -36,6 +36,7 @@ import (
"github.com/ProtonMail/proton-bridge/v3/internal/constants"
"github.com/ProtonMail/proton-bridge/v3/internal/files"
"github.com/ProtonMail/proton-bridge/v3/internal/logging"
"github.com/ProtonMail/proton-bridge/v3/internal/services/observability"
"github.com/sirupsen/logrus"
)
@ -77,6 +78,7 @@ func newIMAPServer(
tasks *async.Group,
uidValidityGenerator imap.UIDValidityGenerator,
panicHandler async.PanicHandler,
observabilitySender observability.Sender,
) (*gluon.Server, error) {
gluonCacheDir = ApplyGluonCachePathSuffix(gluonCacheDir)
gluonConfigDir = ApplyGluonConfigPathSuffix(gluonConfigDir)
@ -121,6 +123,7 @@ func newIMAPServer(
gluon.WithReporter(reporter),
gluon.WithUIDValidityGenerator(uidValidityGenerator),
gluon.WithPanicHandler(panicHandler),
gluon.WithObservabilitySender(observability.NewAdapter(observabilitySender), int(observability.GluonImapError), int(observability.GluonMessageError), int(observability.GluonOtherError)),
)
if err != nil {
return nil, err

7
internal/services/imapsmtpserver/service.go
View File

@ -31,6 +31,7 @@ import (
"github.com/ProtonMail/gluon/reporter"
"github.com/ProtonMail/proton-bridge/v3/internal/events"
"github.com/ProtonMail/proton-bridge/v3/internal/services/imapservice"
"github.com/ProtonMail/proton-bridge/v3/internal/services/observability"
bridgesmtp "github.com/ProtonMail/proton-bridge/v3/internal/services/smtp"
"github.com/ProtonMail/proton-bridge/v3/internal/services/syncservice"
"github.com/ProtonMail/proton-bridge/v3/pkg/cpc"
@ -60,6 +61,8 @@ type Service struct {
uidValidityGenerator imap.UIDValidityGenerator
telemetry Telemetry
observabilitySender observability.Sender
}
func NewService(
@ -71,6 +74,7 @@ func NewService(
reporter reporter.Reporter,
uidValidityGenerator imap.UIDValidityGenerator,
telemetry Telemetry,
observabilitySender observability.Sender,
) *Service {
return &Service{
requests: cpc.NewCPC(),
@ -85,6 +89,8 @@ func NewService(
tasks: async.NewGroup(ctx, panicHandler),
uidValidityGenerator: uidValidityGenerator,
telemetry: telemetry,
observabilitySender: observabilitySender,
}
}
@ -449,6 +455,7 @@ func (sm *Service) createIMAPServer(ctx context.Context) (*gluon.Server, error)
sm.tasks,
sm.uidValidityGenerator,
sm.panicHandler,
sm.observabilitySender,
)
if err == nil {
sm.eventPublisher.PublishEvent(ctx, events.IMAPServerCreated{})

93
internal/services/observability/adapter.go Normal file
View File

@ -0,0 +1,93 @@
// Copyright (c) 2024 Proton AG
//
// This file is part of Proton Mail Bridge.
//
// Proton Mail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Proton Mail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Proton Mail Bridge. If not, see <https://www.gnu.org/licenses/>.
package observability
import (
"github.com/ProtonMail/go-proton-api"
)
type Adapter struct {
sender Sender
}
func NewAdapter(sender Sender) *Adapter {
return &Adapter{sender: sender}
}
// VerifyAndParseGenericMetrics parses a metric provided as an interface into a proton.ObservabilityMetric type.
// It's exported as it is also used in integration tests.
func VerifyAndParseGenericMetrics(metric map[string]interface{}) (bool, proton.ObservabilityMetric) {
name, ok := metric["Name"].(string)
if !ok {
return false, proton.ObservabilityMetric{}
}
version, ok := metric["Version"].(int)
if !ok {
return false, proton.ObservabilityMetric{}
}
timestamp, ok := metric["Timestamp"].(int64)
if !ok {
return false, proton.ObservabilityMetric{}
}
data, ok := metric["Data"]
if !ok {
return false, proton.ObservabilityMetric{}
}
return true, proton.ObservabilityMetric{
Name: name,
Version: version,
Timestamp: timestamp,
Data: data,
}
}
func (adapter *Adapter) AddMetrics(metrics ...map[string]interface{}) {
var typedMetrics []proton.ObservabilityMetric
for _, metric := range metrics {
if ok, m := VerifyAndParseGenericMetrics(metric); ok {
typedMetrics = append(typedMetrics, m)
}
}
if len(typedMetrics) > 0 {
adapter.sender.AddMetrics(typedMetrics...)
}
}
func (adapter *Adapter) AddDistinctMetrics(errType interface{}, metrics ...map[string]interface{}) {
errTypeInt, ok := errType.(int)
if !ok {
return
}
var typedMetrics []proton.ObservabilityMetric
for _, metric := range metrics {
if ok, m := VerifyAndParseGenericMetrics(metric); ok {
typedMetrics = append(typedMetrics, m)
}
}
if len(typedMetrics) > 0 {
adapter.sender.AddDistinctMetrics(DistinctionErrorTypeEnum(errTypeInt), typedMetrics...)
}
}

58
internal/services/observability/adapter_test.go Normal file
View File

@ -0,0 +1,58 @@
// Copyright (c) 2024 Proton AG
//
// This file is part of Proton Mail Bridge.
//
// Proton Mail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Proton Mail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Proton Mail Bridge. If not, see <https://www.gnu.org/licenses/>.
package observability
import (
"fmt"
"testing"
"time"
"github.com/stretchr/testify/require"
)
func Test_AdapterCustomMetrics(t *testing.T) {
customMetric := map[string]interface{}{
"Name": "name",
"Version": 1,
"Timestamp": time.Now().Unix(),
"Data": map[string]interface{}{
"Value": 1,
"Labels": map[string]string{
"error": "customError",
},
},
}
ok, metric := VerifyAndParseGenericMetrics(customMetric)
require.True(t, ok)
require.Equal(t, metric.Name, customMetric["Name"])
require.Equal(t, metric.Timestamp, customMetric["Timestamp"])
require.Equal(t, metric.Version, customMetric["Version"])
require.Equal(t, metric.Data, customMetric["Data"])
}
func Test_AdapterGluonMetrics(t *testing.T) {
metrics := GenerateAllGluonMetrics()
for _, metric := range metrics {
ok, m := VerifyAndParseGenericMetrics(metric)
fmt.Println(m)
require.True(t, ok)
}
}

12
internal/services/observability/distinction_error_types.go
View File

@ -25,13 +25,19 @@ type DistinctionErrorTypeEnum int
const (
SyncError DistinctionErrorTypeEnum = iota
EventLoopError
GluonImapError
GluonMessageError
GluonOtherError
EventLoopError // EventLoopError - should always be kept last when inserting new keys.
)
// errorSchemaMap - maps between the DistinctionErrorTypeEnum and the relevant schema name.
var errorSchemaMap = map[DistinctionErrorTypeEnum]string{ //nolint:gochecknoglobals
SyncError: "bridge_sync_errors_users_total",
EventLoopError: "bridge_event_loop_events_errors_users_total",
SyncError: "bridge_sync_errors_users_total",
EventLoopError: "bridge_event_loop_events_errors_users_total",
GluonImapError: "bridge_gluon_imap_errors_users_total",
GluonMessageError: "bridge_gluon_message_errors_users_total",
GluonOtherError: "bridge_gluon_other_errors_users_total",
}
// createLastSentMap - needs to be updated whenever we make changes to the enum.

11
internal/services/observability/heartbeat.go
View File

@ -26,17 +26,20 @@ import (
)
const genericHeartbeatSchemaName = "bridge_generic_user_heartbeat_total"
const genericHeartbeatVersion = 2
type heartbeatData struct {
receivedSyncError bool
receivedEventLoopError bool
receivedOtherError bool
receivedGluonError bool
}
func (d *distinctionUtility) resetHeartbeatData() {
d.heartbeatData.receivedSyncError = false
d.heartbeatData.receivedOtherError = false
d.heartbeatData.receivedEventLoopError = false
d.heartbeatData.receivedGluonError = false
}
func (d *distinctionUtility) updateHeartbeatData(errType DistinctionErrorTypeEnum) {
@ -46,6 +49,8 @@ func (d *distinctionUtility) updateHeartbeatData(errType DistinctionErrorTypeEnu
d.heartbeatData.receivedSyncError = true
case EventLoopError:
d.heartbeatData.receivedEventLoopError = true
case GluonMessageError, GluonImapError, GluonOtherError:
d.heartbeatData.receivedGluonError = true
}
})
}
@ -95,13 +100,14 @@ func (d *distinctionUtility) generateHeartbeatUserMetric() proton.ObservabilityM
formatBool(d.heartbeatData.receivedOtherError),
formatBool(d.heartbeatData.receivedSyncError),
formatBool(d.heartbeatData.receivedEventLoopError),
formatBool(d.heartbeatData.receivedGluonError),
)
}
func generateHeartbeatMetric(plan, mailClient, dohEnabled, betaAccess, otherError, syncError, eventLoopError string) proton.ObservabilityMetric {
func generateHeartbeatMetric(plan, mailClient, dohEnabled, betaAccess, otherError, syncError, eventLoopError, gluonError string) proton.ObservabilityMetric {
return proton.ObservabilityMetric{
Name: genericHeartbeatSchemaName,
Version: 1,
Version: genericHeartbeatVersion,
Timestamp: time.Now().Unix(),
Data: map[string]interface{}{
"Value": 1,
@ -113,6 +119,7 @@ func generateHeartbeatMetric(plan, mailClient, dohEnabled, betaAccess, otherErro
"receivedOtherError": otherError,
"receivedSyncError": syncError,
"receivedEventLoopError": eventLoopError,
"receivedGluonError": gluonError,
},
},
}

40
internal/services/observability/test_utils.go
View File

@ -18,6 +18,7 @@
package observability
import (
gluonMetrics "github.com/ProtonMail/gluon/observability/metrics"
"github.com/ProtonMail/go-proton-api"
)
@ -85,16 +86,19 @@ func GenerateAllHeartbeatMetricPermutations() []proton.ObservabilityMetric {
for _, receivedOtherError := range trueFalseValues {
for _, receivedSyncError := range trueFalseValues {
for _, receivedEventLoopError := range trueFalseValues {
metrics = append(metrics,
generateHeartbeatMetric(plan,
mailClient,
dohEnabled,
betaAccess,
receivedOtherError,
receivedSyncError,
receivedEventLoopError,
),
)
for _, receivedGluonError := range trueFalseValues {
metrics = append(metrics,
generateHeartbeatMetric(plan,
mailClient,
dohEnabled,
betaAccess,
receivedOtherError,
receivedSyncError,
receivedEventLoopError,
receivedGluonError,
),
)
}
}
}
}
@ -104,3 +108,19 @@ func GenerateAllHeartbeatMetricPermutations() []proton.ObservabilityMetric {
}
return metrics
}
func GenerateAllGluonMetrics() []map[string]interface{} {
var metrics []map[string]interface{}
metrics = append(metrics,
gluonMetrics.GenerateFailedParseIMAPCommandMetric(),
gluonMetrics.GenerateFailedToCreateMailbox(),
gluonMetrics.GenerateFailedToDeleteMailboxMetric(),
gluonMetrics.GenerateFailedToCopyMessagesMetric(),
gluonMetrics.GenerateFailedToMoveMessagesFromMailboxMetric(),
gluonMetrics.GenerateFailedToRemoveDeletedMessagesMetric(),
gluonMetrics.GenerateFailedToCommitDatabaseTransactionMetric(),
gluonMetrics.GenerateAppendToDraftsMustNotReturnExistingRemoteID(),
gluonMetrics.GenerateDatabaseMigrationFailed(),
)
return metrics
}

11
tests/features/observability/gluon_metrics.feature Normal file
View File

@ -0,0 +1,11 @@
Feature: Bridge send remote notification observability metrics
Background:
Given there exists an account with username "[user:user1]" and password "password"
Then it succeeds
When bridge starts
Then it succeeds
Scenario: Test all possible gluon error observability metrics
When the user logs in with username "[user:user1]" and password "password"
And the user with username "[user:user1]" sends all possible gluon error observability metrics
Then it succeeds

59
tests/observability_test.go
View File

@ -19,6 +19,7 @@ package tests
import (
"context"
"fmt"
"github.com/ProtonMail/go-proton-api"
"github.com/ProtonMail/proton-bridge/v3/internal/services/imapservice/observabilitymetrics/evtloopmsgevents"
@ -27,18 +28,35 @@ import (
"github.com/ProtonMail/proton-bridge/v3/internal/services/syncservice/observabilitymetrics"
)
// userHeartbeatPermutationsObservability - corresponds to bridge_generic_user_heartbeat_total_v1.schema.json.
// userHeartbeatPermutationsObservability corresponds to bridge_generic_user_heartbeat_total_v1.schema.json.
func (s *scenario) userHeartbeatPermutationsObservability(username string) error {
const batchSize = 1000
metrics := observability.GenerateAllHeartbeatMetricPermutations()
metricLen := len(metrics)
return s.t.withClientPass(context.Background(), username, s.t.getUserByName(username).userPass, func(ctx context.Context, c *proton.Client) error {
batch := proton.ObservabilityBatch{Metrics: metrics}
return c.SendObservabilityBatch(ctx, batch)
for i := 0; i < len(metrics); i += batchSize {
end := i + batchSize
if end > metricLen {
end = metricLen
}
batch := proton.ObservabilityBatch{Metrics: metrics[i:end]}
if err := c.SendObservabilityBatch(ctx, batch); err != nil {
return err
}
}
return nil
})
}
// userDistinctionMetricsPermutationsObservability - corresponds to:
// bridge_sync_errors_users_total_v1.schema.json
// bridge_event_loop_events_errors_users_total_v1.schema.json.
// userDistinctionMetricsPermutationsObservability corresponds to:
// - bridge_sync_errors_users_total_v1.schema.json
// - bridge_gluon_imap_errors_users_total_v1.schema.json
// - bridge_gluon_message_errors_users_total_v1.schema.json
// - bridge_gluon_other_errors_users_total_v1.schema.json
// - bridge_event_loop_events_errors_users_total_v1.schema.json.
func (s *scenario) userDistinctionMetricsPermutationsObservability(username string) error {
batch := proton.ObservabilityBatch{
Metrics: observability.GenerateAllUsedDistinctionMetricPermutations()}
@ -48,7 +66,7 @@ func (s *scenario) userDistinctionMetricsPermutationsObservability(username stri
})
}
// syncFailureMessageEventsObservability - corresponds to bridge_sync_message_event_failures_total_v1.schema.json.
// syncFailureMessageEventsObservability corresponds to bridge_sync_message_event_failures_total_v1.schema.json.
func (s *scenario) syncFailureMessageEventsObservability(username string) error {
batch := proton.ObservabilityBatch{
Metrics: []proton.ObservabilityMetric{
@ -62,7 +80,7 @@ func (s *scenario) syncFailureMessageEventsObservability(username string) error
})
}
// eventLoopFailureMessageEventsObservability - corresponds to bridge_event_loop_message_event_failures_total_v1.schema.json.
// eventLoopFailureMessageEventsObservability corresponds to bridge_event_loop_message_event_failures_total_v1.schema.json.
func (s *scenario) eventLoopFailureMessageEventsObservability(username string) error {
batch := proton.ObservabilityBatch{
Metrics: []proton.ObservabilityMetric{
@ -81,7 +99,7 @@ func (s *scenario) eventLoopFailureMessageEventsObservability(username string) e
})
}
// syncFailureMessageBuiltObservability - corresponds to bridge_sync_message_event_failures_total_v1.schema.json.
// syncFailureMessageBuiltObservability corresponds to bridge_sync_message_event_failures_total_v1.schema.json.
func (s *scenario) syncFailureMessageBuiltObservability(username string) error {
batch := proton.ObservabilityBatch{
Metrics: []proton.ObservabilityMetric{
@ -96,7 +114,7 @@ func (s *scenario) syncFailureMessageBuiltObservability(username string) error {
})
}
// syncSuccessMessageBuiltObservability - corresponds to bridge_sync_message_build_success_total_v1.schema.json.
// syncSuccessMessageBuiltObservability corresponds to bridge_sync_message_build_success_total_v1.schema.json.
func (s *scenario) syncSuccessMessageBuiltObservability(username string) error {
batch := proton.ObservabilityBatch{
Metrics: []proton.ObservabilityMetric{
@ -109,3 +127,24 @@ func (s *scenario) syncSuccessMessageBuiltObservability(username string) error {
return err
})
}
// testGluonErrorObservabilityMetrics corresponds to bridge_gluon_errors_total_v1.schema.json.
func (s *scenario) testGluonErrorObservabilityMetrics(username string) error {
allMetrics := observability.GenerateAllGluonMetrics()
parsedMetrics := []proton.ObservabilityMetric{}
for _, el := range allMetrics {
ok, parsedMetric := observability.VerifyAndParseGenericMetrics(el)
if !ok {
return fmt.Errorf("failed to parse generic gluon metric")
}
parsedMetrics = append(parsedMetrics, parsedMetric)
}
batch := proton.ObservabilityBatch{Metrics: parsedMetrics}
return s.t.withClientPass(context.Background(), username, s.t.getUserByName(username).userPass, func(ctx context.Context, c *proton.Client) error {
err := c.SendObservabilityBatch(ctx, batch)
return err
})
}

2
tests/steps_test.go
View File

@ -226,4 +226,6 @@ func (s *scenario) steps(ctx *godog.ScenarioContext) {
ctx.Step(`^the user with username "([^"]*)" sends all possible event loop message events observability metrics$`, s.eventLoopFailureMessageEventsObservability)
ctx.Step(`^the user with username "([^"]*)" sends all possible sync message building failure observability metrics$`, s.syncFailureMessageBuiltObservability)
ctx.Step(`^the user with username "([^"]*)" sends all possible sync message building success observability metrics$`, s.syncSuccessMessageBuiltObservability)
// Gluon related metrics
ctx.Step(`^the user with username "([^"]*)" sends all possible gluon error observability metrics$`, s.testGluonErrorObservabilityMetrics)
}