Optimalization of RebuildMailboxes

Changelog.md

@@ -6,6 +6,7 @@ Changelog [format](http://keepachangelog.com/en/1.0.0/)
 
 
 ### Changed
+* GODT-165 Optimalization of RebuildMailboxes
 * Adding DSN Sentry as build time parameter
 
 ## [v1.2.6] Donghai - beta (2020-03-XXX)

go.sum

@@ -169,6 +169,8 @@ github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/therecipe/qt v0.0.0-20200126204426-5074eb6d8c41 h1:yBVcrpbaQYJBdKT2pxTdlL4hBE/eM4UPcyj9YpyvSok=
 github.com/therecipe/qt v0.0.0-20200126204426-5074eb6d8c41/go.mod h1:SUUR2j3aE1z6/g76SdD6NwACEpvCxb3fvG82eKbD6us=
+github.com/therecipe/qt/internal/binding/files/docs/5.12.0 v0.0.0-20200126204426-5074eb6d8c41/go.mod h1:7m8PDYDEtEVqfjoUQc2UrFqhG0CDmoVJjRlQxexndFc=
+github.com/therecipe/qt/internal/binding/files/docs/5.13.0 v0.0.0-20200126204426-5074eb6d8c41/go.mod h1:mH55Ek7AZcdns5KPp99O0bg+78el64YCYWHiQKrOdt4=
 github.com/twinj/uuid v1.0.0 h1:fzz7COZnDrXGTAOHGuUGYd6sG+JMq+AoE7+Jlu0przk=
 github.com/twinj/uuid v1.0.0/go.mod h1:mMgcE1RHFUFqe5AfiwlINXisXfDGro23fWdPUfOMjRY=
 github.com/urfave/cli v1.22.3 h1:FpNT6zq26xNpHZy08emi755QwzLPs6Pukqjlc7RfOMU=

internal/bridge/user_credentials_test.go

@@ -71,7 +71,6 @@ func TestUserSwitchAddressMode(t *testing.T) {
 		m.credentialsStore.EXPECT().SwitchAddressMode("user").Return(nil),
 		m.credentialsStore.EXPECT().Get("user").Return(testCredentialsSplit, nil),
 	)
-	m.pmapiClient.EXPECT().ListMessages(gomock.Any()).Return([]*pmapi.Message{}, 0, nil)
 
 	assert.NoError(t, user.SwitchAddressMode())
 	assert.False(t, user.store.IsCombinedMode())

internal/store/mailbox.go

@@ -115,7 +115,7 @@ func syncDraftsIfNecssary(mb *Mailbox) { //nolint[funlen]
 				}
 				for _, msgLabelID := range msg.LabelIDs {
 					if msgLabelID == pmapi.DraftLabel {
-						log.WithField("id", msg.ID).Debug("Drafts mailbox created: syncing draft locally")
+						log.WithField("id", msg.ID).Trace("Drafts mailbox created: syncing draft locally")
 						_ = mb.txCreateOrUpdateMessages(tx, []*pmapi.Message{msg})
 						break
 					}

internal/store/user_address.go

@@ -69,8 +69,7 @@ func (store *Store) RebuildMailboxes() (err error) {
 	}
 
 	log.WithField("user", store.UserID()).Trace("Rebuilding mailboxes")
-	store.triggerSync()
+	return store.initMailboxesBucket()
-	return nil
 }
 
 // createOrDeleteAddressesEvent creates address objects in the store for each necessary address
@@ -186,31 +185,48 @@ func (store *Store) truncateMailboxesBucket() (err error) {
 }
 
 // initMailboxesBucket recreates the mailboxes bucket from the metadata bucket.
-func (store *Store) initMailboxesBucket() error { //nolint[unused]
+func (store *Store) initMailboxesBucket() error {
+	return store.db.Update(func(tx *bolt.Tx) error {
 		i := 0
-	tx := func(tx *bolt.Tx) error {
+		msgs := []*pmapi.Message{}
-		return tx.Bucket(metadataBucket).ForEach(func(k, v []byte) error {
+
+		err := tx.Bucket(metadataBucket).ForEach(func(k, v []byte) error {
 			msg := &pmapi.Message{}
 
 			if err := json.Unmarshal(v, msg); err != nil {
 				return err
 			}
+			msgs = append(msgs, msg)
+
+			// Calling txCreateOrUpdateMessages does some overhead by iterating
+			// all mailboxes, accessing buckets and so on. It's better to do in
+			// batches instead of one by one (seconds vs hours for huge accounts).
+			// Average size of metadata is 1k bytes, sometimes up to 2k bytes.
+			// 10k messages will take about 20 MB of memory.
+			i++
+			if i%10000 == 0 {
+				store.log.WithField("i", i).Debug("Init mboxes heartbeat")
 
 				for _, a := range store.addresses {
-				if err := a.txCreateOrUpdateMessages(tx, []*pmapi.Message{msg}); err != nil {
+					if err := a.txCreateOrUpdateMessages(tx, msgs); err != nil {
 						return err
 					}
 				}
-
+				msgs = []*pmapi.Message{}
-			i++
-			if i%100 == 0 {
-				store.log.WithField("i", i).
-					Trace("Init mboxes heartbeat")
 			}
 
 			return nil
 		})
+		if err != nil {
+			return err
 		}
 
-	return store.db.Update(tx)
+		for _, a := range store.addresses {
+			if err := a.txCreateOrUpdateMessages(tx, msgs); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
 }