GODT-2224: Refactor bridge sync to use less memory

Updates go-proton-api and Gluon to includes memory reduction changes and
modify the sync process to take into account how much memory is used
during the sync stage.

The sync process now has an extra stage which first download the message
metada to ensure that we only download up to `syncMaxDownloadRequesMem`
messages or 250 messages total. This allows for scaling the download
request automatically to accommodate many small or few very large
messages.

The IDs are then sent to a download go-routine which downloads the
message and its attachments. The result is then forwarded to another
go-routine which builds the actual message. This stage tries to ensure
that we don't use more than `syncMaxMessageBuildingMem` to build these
messages.

Finally the result is sent to a last go-routine which applies the
changes to Gluon and waits for them to be completed.

The new process is currently limited to 2GB. Dynamic scaling will be
implemented in a follow up. For systems with less than 2GB of memory we
limit the values to a set of values that is known to work.

internal/app/app.go

@@ -205,7 +205,7 @@ func run(c *cli.Context) error { //nolint:funlen
 						// Ensure we are the only instance running.
 						return withSingleInstance(locations, version, func() error {
 							// Unlock the encrypted vault.
-							return WithVault(locations, func(vault *vault.Vault, insecure, corrupt bool) error {
+							return WithVault(locations, func(v *vault.Vault, insecure, corrupt bool) error {
 								// Report insecure vault.
 								if insecure {
 									_ = reporter.ReportMessageWithContext("Vault is insecure", map[string]interface{}{})
@@ -216,27 +216,39 @@ func run(c *cli.Context) error { //nolint:funlen
 									_ = reporter.ReportMessageWithContext("Vault is corrupt", map[string]interface{}{})
 								}
 
-								if !vault.Migrated() {
+								// Force re-sync if last version <= 3.0.12 due to chances in the gluon cache format.
+								if lastVersion := v.GetLastVersion(); lastVersion != nil {
+									versionWithLZ4Cache := semver.MustParse("3.0.13")
+									if lastVersion.LessThan(versionWithLZ4Cache) {
+										if err := v.ForUser(1, func(user *vault.User) error {
+											return user.ClearSyncStatus()
+										}); err != nil {
+											logrus.WithError(err).Error("Failed to force resync on user")
+										}
+									}
+								}
+
+								if !v.Migrated() {
 									// Migrate old settings into the vault.
-									if err := migrateOldSettings(vault); err != nil {
+									if err := migrateOldSettings(v); err != nil {
 										logrus.WithError(err).Error("Failed to migrate old settings")
 									}
 
 									// Migrate old accounts into the vault.
-									if err := migrateOldAccounts(locations, vault); err != nil {
+									if err := migrateOldAccounts(locations, v); err != nil {
 										logrus.WithError(err).Error("Failed to migrate old accounts")
 									}
 
 									// The vault has been migrated.
-									if err := vault.SetMigrated(); err != nil {
+									if err := v.SetMigrated(); err != nil {
 										logrus.WithError(err).Error("Failed to mark vault as migrated")
 									}
 								}
 
 								// Load the cookies from the vault.
-								return withCookieJar(vault, func(cookieJar http.CookieJar) error {
+								return withCookieJar(v, func(cookieJar http.CookieJar) error {
 									// Create a new bridge instance.
-									return withBridge(c, exe, locations, version, identifier, crashHandler, reporter, vault, cookieJar, func(b *bridge.Bridge, eventCh <-chan events.Event) error {
+									return withBridge(c, exe, locations, version, identifier, crashHandler, reporter, v, cookieJar, func(b *bridge.Bridge, eventCh <-chan events.Event) error {
 										if insecure {
 											logrus.Warn("The vault key could not be retrieved; the vault will not be encrypted")
 											b.PushError(bridge.ErrVaultInsecure)

internal/app/migration.go

@@ -265,14 +265,6 @@ func migratePrefsToVault(vault *vault.Vault, b []byte) error {
 		errs = multierror.Append(errs, fmt.Errorf("failed to migrate show all mail: %w", err))
 	}
 
-	if err := vault.SetSyncWorkers(prefs.FetchWorkers); err != nil {
-		errs = multierror.Append(errs, fmt.Errorf("failed to migrate sync workers: %w", err))
-	}
-
-	if err := vault.SetSyncAttPool(prefs.AttachmentWorkers); err != nil {
-		errs = multierror.Append(errs, fmt.Errorf("failed to migrate sync attachment pool: %w", err))
-	}
-
 	if err := vault.SetCookies([]byte(prefs.Cookies)); err != nil {
 		errs = multierror.Append(errs, fmt.Errorf("failed to migrate cookies: %w", err))
 	}

internal/app/migration_test.go

@@ -68,8 +68,6 @@ func TestMigratePrefsToVault(t *testing.T) {
 	require.True(t, vault.GetAutostart())
 
 	// Check that the other app settings have been migrated.
-	require.Equal(t, 16, vault.SyncWorkers())
-	require.Equal(t, 16, vault.SyncAttPool())
 	require.False(t, vault.GetProxyAllowed())
 	require.False(t, vault.GetShowAllMail())