yesBad/proton-bridge
1
0
Fork 0
forked from Silverfish/proton-bridge
Code Actions Activity

GODT-2224: Refactor bridge sync to use less memory

Browse Source 
Updates go-proton-api and Gluon to includes memory reduction changes and
modify the sync process to take into account how much memory is used
during the sync stage.

The sync process now has an extra stage which first download the message
metada to ensure that we only download up to `syncMaxDownloadRequesMem`
messages or 250 messages total. This allows for scaling the download
request automatically to accommodate many small or few very large
messages.

The IDs are then sent to a download go-routine which downloads the
message and its attachments. The result is then forwarded to another
go-routine which builds the actual message. This stage tries to ensure
that we don't use more than `syncMaxMessageBuildingMem` to build these
messages.

Finally the result is sent to a last go-routine which applies the
changes to Gluon and waits for them to be completed.

The new process is currently limited to 2GB. Dynamic scaling will be
implemented in a follow up. For systems with less than 2GB of memory we
limit the values to a set of values that is known to work.
This commit is contained in:
Leander Beernaert
2023-01-24 13:41:03 +01:00
parent d7ff54d679
commit e464e11ab9
24 changed files with 481 additions and 303 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
internal/vault/settings.go
View File

@ -189,27 +189,3 @@ func (vault *Vault) SetFirstStart(firstStart bool) error {
data.Settings.FirstStart = firstStart
})
}
// SyncWorkers returns the number of workers to use for syncing.
func (vault *Vault) SyncWorkers() int {
return vault.get().Settings.SyncWorkers
}
// SetSyncWorkers sets the number of workers to use for syncing.
func (vault *Vault) SetSyncWorkers(workers int) error {
return vault.mod(func(data *Data) {
data.Settings.SyncWorkers = workers
})
}
// SyncAttPool returns the size of the attachment pool.
func (vault *Vault) SyncAttPool() int {
return vault.get().Settings.SyncAttPool
}
// SetSyncAttPool sets the size of the attachment pool.
func (vault *Vault) SetSyncAttPool(pool int) error {
return vault.mod(func(data *Data) {
data.Settings.SyncAttPool = pool
})
}

9
internal/vault/settings_test.go
View File

@ -202,12 +202,3 @@ func TestVault_Settings_FirstStart(t *testing.T) {
// Check the new first start value.
require.Equal(t, false, s.GetFirstStart())
}
func TestVault_Settings_SyncWorkers(t *testing.T) {
// create a new test vault.
s := newVault(t)
syncWorkers := vault.GetDefaultSyncWorkerCount()
require.Equal(t, syncWorkers, s.SyncWorkers())
require.Equal(t, syncWorkers, s.SyncAttPool())
}

21
internal/vault/types_settings.go
View File

@ -19,7 +19,6 @@ package vault
import (
"math/rand"
"runtime"
"github.com/ProtonMail/proton-bridge/v3/internal/updater"
)
@ -43,26 +42,9 @@ type Settings struct {
LastVersion string
FirstStart bool
SyncWorkers int
SyncAttPool int
}
func GetDefaultSyncWorkerCount() int {
const minSyncWorkers = 16
syncWorkers := runtime.NumCPU() * 4
if syncWorkers < minSyncWorkers {
syncWorkers = minSyncWorkers
}
return syncWorkers
}
func newDefaultSettings(gluonDir string) Settings {
syncWorkers := GetDefaultSyncWorkerCount()
return Settings{
GluonDir: gluonDir,
@ -82,8 +64,5 @@ func newDefaultSettings(gluonDir string) Settings {
LastVersion: "0.0.0",
FirstStart: true,
SyncWorkers: syncWorkers,
SyncAttPool: syncWorkers,
}
}