GODT-2085: Revise sync algorithm

Revise syncing work distribution. Sync time can be reduced by up to 50%.

Rework the sync so that it pipelines better with bigger batch counts at
each stage. We now use 3 separate stages: Download, Updates and Sync.

The Download stage downloads messages in maxBatchSize intervals using
1.5x syncWorkers. Once the current batch has finished downloading it's
forwarded to the Updates stage and we proceed to download the next
batch.

The Update stage converts everything into gluon updates and prepares a
collection of noops that the sync stage can wait on for termination.

Finally the sync stage waits until the updates have been applied in
Gluon so that the vault information can be updated. We allow up to 4
pending wait operations to be queued currently to not block the
pipeline.
This commit is contained in:
Leander Beernaert
2022-11-22 11:11:52 +01:00
parent 1395f1c990
commit a2ab5df7ce
2 changed files with 99 additions and 61 deletions

View File

@ -18,9 +18,6 @@
package user
import (
"context"
"sync"
"github.com/ProtonMail/gluon/imap"
"github.com/ProtonMail/gluon/queue"
)
@ -31,8 +28,6 @@ type flusher struct {
maxUpdateSize int
curChunkSize int
pushLock sync.Mutex
}
func newFlusher(updateCh *queue.QueuedChannel[imap.Update], maxUpdateSize int) *flusher {
@ -42,28 +37,18 @@ func newFlusher(updateCh *queue.QueuedChannel[imap.Update], maxUpdateSize int) *
}
}
func (f *flusher) push(ctx context.Context, update *imap.MessageCreated) {
f.pushLock.Lock()
defer f.pushLock.Unlock()
func (f *flusher) push(update *imap.MessageCreated) {
f.updates = append(f.updates, update)
if f.curChunkSize += len(update.Literal); f.curChunkSize >= f.maxUpdateSize {
f.flush(ctx, false)
f.flush()
}
}
func (f *flusher) flush(ctx context.Context, wait bool) {
func (f *flusher) flush() {
if len(f.updates) > 0 {
f.updateCh.Enqueue(imap.NewMessagesCreated(f.updates...))
f.updates = nil
f.curChunkSize = 0
}
if wait {
update := imap.NewNoop()
defer update.WaitContext(ctx)
f.updateCh.Enqueue(update)
}
}