From 4352154b8483a1c5747bb907e3d2da87c178058d Mon Sep 17 00:00:00 2001 From: Leander Beernaert Date: Thu, 29 Jun 2023 12:24:23 +0200 Subject: [PATCH] test: Force sync limits to minimum with env variable Set `BRIDGE_SYNC_FORCE_MINIMUM_SPEC` as environment variable to force all the sync limits to minimum spec. This is enabled for windows builds. --- .gitlab-ci.yml | 1 + internal/user/sync.go | 107 +++++++++++++++++++++++++++--------------- 2 files changed, 71 insertions(+), 37 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1122ba2d..8effd6b4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,6 +50,7 @@ stages: # ENV .env-windows: before_script: + - export BRIDGE_SYNC_FORCE_MINIMUM_SPEC=1 - export GOROOT=/c/Go1.20/ - export PATH=$GOROOT/bin:$PATH - export GOARCH=amd64 diff --git a/internal/user/sync.go b/internal/user/sync.go index 2cfe820e..9fb59342 100644 --- a/internal/user/sync.go +++ b/internal/user/sync.go @@ -21,6 +21,7 @@ import ( "bytes" "context" "fmt" + "os" "runtime" "strings" "time" @@ -243,6 +244,46 @@ func toMB(v uint64) float64 { return float64(v) / float64(Megabyte) } +type syncLimits struct { + MaxDownloadRequestMem uint64 + MinDownloadRequestMem uint64 + MaxMessageBuildingMem uint64 + MinMessageBuildingMem uint64 + MaxSyncMemory uint64 + MaxParallelDownloads int +} + +func newSyncLimits(maxSyncMemory uint64) syncLimits { + limits := syncLimits{ + // There's no point in using more than 128MB of download data per stage, after that we reach a point of diminishing + // returns as we can't keep the pipeline fed fast enough. + MaxDownloadRequestMem: 128 * Megabyte, + + // Any lower than this and we may fail to download messages. + MinDownloadRequestMem: 40 * Megabyte, + + // This value can be increased to your hearts content. The more system memory the user has, the more messages + // we can build in parallel. + MaxMessageBuildingMem: 128 * Megabyte, + MinMessageBuildingMem: 64 * Megabyte, + + // Maximum recommend value for parallel downloads by the API team. + MaxParallelDownloads: 20, + + MaxSyncMemory: maxSyncMemory, + } + + if _, ok := os.LookupEnv("BRIDGE_SYNC_FORCE_MINIMUM_SPEC"); ok { + logrus.Warn("Sync specs forced to minimum") + limits.MaxDownloadRequestMem = 50 * Megabyte + limits.MaxMessageBuildingMem = 80 * Megabyte + limits.MaxParallelDownloads = 2 + limits.MaxSyncMemory = 800 * Megabyte + } + + return limits +} + // nolint:gocyclo func (user *User) syncMessages( ctx context.Context, @@ -255,7 +296,7 @@ func (user *User) syncMessages( addrKRs map[string]*crypto.KeyRing, updateCh map[string]*async.QueuedChannel[imap.Update], eventCh *async.QueuedChannel[events.Event], - maxSyncMemory uint64, + cfgMaxSyncMemory uint64, ) error { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -278,59 +319,51 @@ func (user *User) syncMessages( // Expected mem usage for this whole process should be the sum of MaxMessageBuildingMem and MaxDownloadRequestMem // times x due to pipeline and all additional memory used by network requests and compression+io. - // There's no point in using more than 128MB of download data per stage, after that we reach a point of diminishing - // returns as we can't keep the pipeline fed fast enough. - const MaxDownloadRequestMem = 128 * Megabyte - - // Any lower than this and we may fail to download messages. - const MinDownloadRequestMem = 40 * Megabyte - - // This value can be increased to your hearts content. The more system memory the user has, the more messages - // we can build in parallel. - const MaxMessageBuildingMem = 128 * Megabyte - const MinMessageBuildingMem = 64 * Megabyte - - // Maximum recommend value for parallel downloads by the API team. - const maxParallelDownloads = 20 - totalMemory := memory.TotalMemory() - if maxSyncMemory >= totalMemory/2 { + syncLimits := newSyncLimits(cfgMaxSyncMemory) + + if syncLimits.MaxSyncMemory >= totalMemory/2 { logrus.Warnf("Requested max sync memory of %v MB is greater than half of system memory (%v MB), forcing to half of system memory", - maxSyncMemory, toMB(totalMemory/2)) - maxSyncMemory = totalMemory / 2 + toMB(syncLimits.MaxSyncMemory), toMB(totalMemory/2)) + syncLimits.MaxSyncMemory = totalMemory / 2 } - if maxSyncMemory < 800*Megabyte { - logrus.Warnf("Requested max sync memory of %v MB, but minimum recommended is 800 MB, forcing max syncMemory to 800MB", toMB(maxSyncMemory)) - maxSyncMemory = 800 * Megabyte + if syncLimits.MaxSyncMemory < 800*Megabyte { + logrus.Warnf("Requested max sync memory of %v MB, but minimum recommended is 800 MB, forcing max syncMemory to 800MB", toMB(syncLimits.MaxSyncMemory)) + syncLimits.MaxSyncMemory = 800 * Megabyte } logrus.Debugf("Total System Memory: %v", toMB(totalMemory)) - syncMaxDownloadRequestMem := MaxDownloadRequestMem - syncMaxMessageBuildingMem := MaxMessageBuildingMem + // Linter says it's not used. This is a lie. + //nolint: staticcheck + syncMaxDownloadRequestMem := syncLimits.MaxDownloadRequestMem + + // Linter says it's not used. This is a lie. + //nolint: staticcheck + syncMaxMessageBuildingMem := syncLimits.MaxMessageBuildingMem // If less than 2GB available try and limit max memory to 512 MB switch { - case maxSyncMemory < 2*Gigabyte: - if maxSyncMemory < 800*Megabyte { + case syncLimits.MaxSyncMemory < 2*Gigabyte: + if syncLimits.MaxSyncMemory < 800*Megabyte { logrus.Warnf("System has less than 800MB of memory, you may experience issues sycing large mailboxes") } - syncMaxDownloadRequestMem = MinDownloadRequestMem - syncMaxMessageBuildingMem = MinMessageBuildingMem - case maxSyncMemory == 2*Gigabyte: + syncMaxDownloadRequestMem = syncLimits.MinDownloadRequestMem + syncMaxMessageBuildingMem = syncLimits.MinMessageBuildingMem + case syncLimits.MaxSyncMemory == 2*Gigabyte: // Increasing the max download capacity has very little effect on sync speed. We could increase the download // memory but the user would see less sync notifications. A smaller value here leads to more frequent // updates. Additionally, most of ot sync time is spent in the message building. - syncMaxDownloadRequestMem = MaxDownloadRequestMem + syncMaxDownloadRequestMem = syncLimits.MaxDownloadRequestMem // Currently limited so that if a user has multiple accounts active it also doesn't cause excessive memory usage. - syncMaxMessageBuildingMem = MaxMessageBuildingMem + syncMaxMessageBuildingMem = syncLimits.MaxMessageBuildingMem default: // Divide by 8 as download stage and build stage will use aprox. 4x the specified memory. - remainingMemory := (maxSyncMemory - 2*Gigabyte) / 8 - syncMaxDownloadRequestMem = MaxDownloadRequestMem + remainingMemory - syncMaxMessageBuildingMem = MaxMessageBuildingMem + remainingMemory + remainingMemory := (syncLimits.MaxSyncMemory - 2*Gigabyte) / 8 + syncMaxDownloadRequestMem = syncLimits.MaxDownloadRequestMem + remainingMemory + syncMaxMessageBuildingMem = syncLimits.MaxMessageBuildingMem + remainingMemory } logrus.Debugf("Max memory usage for sync Download=%vMB Building=%vMB Predicted Max Total=%vMB", @@ -369,7 +402,7 @@ func (user *User) syncMessages( flushUpdateCh := make(chan flushUpdate) - errorCh := make(chan error, maxParallelDownloads*4) + errorCh := make(chan error, syncLimits.MaxParallelDownloads*4) // Go routine in charge of downloading message metadata async.GoAnnotated(ctx, user.panicHandler, func(ctx context.Context) { @@ -443,7 +476,7 @@ func (user *User) syncMessages( logrus.Debugf("sync downloader exit") }() - attachmentDownloader := user.newAttachmentDownloader(ctx, client, maxParallelDownloads) + attachmentDownloader := user.newAttachmentDownloader(ctx, client, syncLimits.MaxParallelDownloads) defer attachmentDownloader.close() for request := range downloadCh { @@ -458,7 +491,7 @@ func (user *User) syncMessages( return } - result, err := parallel.MapContext(ctx, maxParallelDownloads, request.ids, func(ctx context.Context, id string) (proton.FullMessage, error) { + result, err := parallel.MapContext(ctx, syncLimits.MaxParallelDownloads, request.ids, func(ctx context.Context, id string) (proton.FullMessage, error) { defer async.HandlePanic(user.panicHandler) var result proton.FullMessage