GODT-1158: Store full messages bodies on disk

- GODT-1158: simple on-disk cache in store
- GODT-1158: better member naming in event loop
- GODT-1158: create on-disk cache during bridge setup
- GODT-1158: better job options
- GODT-1158: rename GetLiteral to GetRFC822
- GODT-1158: rename events -> currentEvents
- GODT-1158: unlock cache per-user
- GODT-1158: clean up cache after logout
- GODT-1158: randomized encrypted cache passphrase
- GODT-1158: Opt out of on-disk cache in settings
- GODT-1158: free space in cache
- GODT-1158: make tests compile
- GODT-1158: optional compression
- GODT-1158: cache custom location
- GODT-1158: basic capacity checker
- GODT-1158: cache free space config
- GODT-1158: only unlock cache if pmapi client is unlocked as well
- GODT-1158: simple background sync worker
- GODT-1158: set size/bodystructure when caching message
- GODT-1158: limit store db update blocking with semaphore
- GODT-1158: dumb 10-semaphore
- GODT-1158: properly handle delete; remove bad bodystructure handling
- GODT-1158: hacky fix for caching after logout... baaaaad
- GODT-1158: cache worker
- GODT-1158: compute body structure lazily
- GODT-1158: cache size in store
- GODT-1158: notify cacher when adding to store
- GODT-1158: 15 second store cache watcher
- GODT-1158: enable cacher
- GODT-1158: better cache worker starting/stopping
- GODT-1158: limit cacher to less concurrency than disk cache
- GODT-1158: message builder prio + pchan pkg
- GODT-1158: fix pchan, use in message builder
- GODT-1158: no sem in cacher (rely on message builder prio)
- GODT-1158: raise priority of existing jobs when requested
- GODT-1158: pending messages in on-disk cache
- GODT-1158: WIP just a note about deleting messages from disk cache
- GODT-1158: pending wait when trying to write
- GODT-1158: pending.add to return bool
- GODT-1225: Headers in bodystructure are stored as bytes.
- GODT-1158: fixing header caching
- GODT-1158: don't cache in background
- GODT-1158: all concurrency set in settings
- GODT-1158: worker pools inside message builder
- GODT-1158: fix linter issues
- GODT-1158: remove completed builds from builder
- GODT-1158: remove builder pool
- GODT-1158: cacher defer job done properly
- GODT-1158: fix linter
- GODT-1299: Continue with bodystructure build if deserialization failed
- GODT-1324: Delete messages from the cache when they are deleted on the server
- GODT-1158: refactor cache tests
- GODT-1158: move builder to app/bridge
- GODT-1306: Migrate cache on disk when location is changed (and delete when disabled)
This commit is contained in:
James Houlahan
2021-07-30 12:20:38 +02:00
committed by Jakub
parent 5cb893fc1b
commit 6bd0739013
79 changed files with 2911 additions and 1387 deletions

73
internal/store/cache/cache_test.go vendored Normal file
View File

@ -0,0 +1,73 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package cache
import (
"runtime"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestOnDiskCacheNoCompression(t *testing.T) {
cache, err := NewOnDiskCache(t.TempDir(), &NoopCompressor{}, Options{ConcurrentRead: runtime.NumCPU(), ConcurrentWrite: runtime.NumCPU()})
require.NoError(t, err)
testCache(t, cache)
}
func TestOnDiskCacheGZipCompression(t *testing.T) {
cache, err := NewOnDiskCache(t.TempDir(), &GZipCompressor{}, Options{ConcurrentRead: runtime.NumCPU(), ConcurrentWrite: runtime.NumCPU()})
require.NoError(t, err)
testCache(t, cache)
}
func TestInMemoryCache(t *testing.T) {
testCache(t, NewInMemoryCache(1<<20))
}
func testCache(t *testing.T, cache Cache) {
assert.NoError(t, cache.Unlock("userID1", []byte("my secret passphrase")))
assert.NoError(t, cache.Unlock("userID2", []byte("my other passphrase")))
getSetCachedMessage(t, cache, "userID1", "messageID1", "some secret")
assert.True(t, cache.Has("userID1", "messageID1"))
getSetCachedMessage(t, cache, "userID2", "messageID2", "some other secret")
assert.True(t, cache.Has("userID2", "messageID2"))
assert.NoError(t, cache.Rem("userID1", "messageID1"))
assert.False(t, cache.Has("userID1", "messageID1"))
assert.NoError(t, cache.Rem("userID2", "messageID2"))
assert.False(t, cache.Has("userID2", "messageID2"))
assert.NoError(t, cache.Delete("userID1"))
assert.NoError(t, cache.Delete("userID2"))
}
func getSetCachedMessage(t *testing.T, cache Cache, userID, messageID, secret string) {
assert.NoError(t, cache.Set(userID, messageID, []byte(secret)))
data, err := cache.Get(userID, messageID)
assert.NoError(t, err)
assert.Equal(t, []byte(secret), data)
}

33
internal/store/cache/compressor.go vendored Normal file
View File

@ -0,0 +1,33 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package cache
type Compressor interface {
Compress([]byte) ([]byte, error)
Decompress([]byte) ([]byte, error)
}
type NoopCompressor struct{}
func (NoopCompressor) Compress(dec []byte) ([]byte, error) {
return dec, nil
}
func (NoopCompressor) Decompress(cmp []byte) ([]byte, error) {
return cmp, nil
}

60
internal/store/cache/compressor_gzip.go vendored Normal file
View File

@ -0,0 +1,60 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package cache
import (
"bytes"
"compress/gzip"
)
type GZipCompressor struct{}
func (GZipCompressor) Compress(dec []byte) ([]byte, error) {
buf := new(bytes.Buffer)
zw := gzip.NewWriter(buf)
if _, err := zw.Write(dec); err != nil {
return nil, err
}
if err := zw.Close(); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func (GZipCompressor) Decompress(cmp []byte) ([]byte, error) {
zr, err := gzip.NewReader(bytes.NewReader(cmp))
if err != nil {
return nil, err
}
buf := new(bytes.Buffer)
if _, err := buf.ReadFrom(zr); err != nil {
return nil, err
}
if err := zr.Close(); err != nil {
return nil, err
}
return buf.Bytes(), nil
}

244
internal/store/cache/disk.go vendored Normal file
View File

@ -0,0 +1,244 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package cache
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"crypto/sha256"
"errors"
"io/ioutil"
"os"
"path/filepath"
"sync"
"github.com/ProtonMail/proton-bridge/pkg/semaphore"
"github.com/ricochet2200/go-disk-usage/du"
)
var ErrLowSpace = errors.New("not enough free space left on device")
type onDiskCache struct {
path string
opts Options
gcm map[string]cipher.AEAD
cmp Compressor
rsem, wsem semaphore.Semaphore
pending *pending
diskSize uint64
diskFree uint64
once *sync.Once
lock sync.Mutex
}
func NewOnDiskCache(path string, cmp Compressor, opts Options) (Cache, error) {
if err := os.MkdirAll(path, 0700); err != nil {
return nil, err
}
usage := du.NewDiskUsage(path)
// NOTE(GODT-1158): use Available() or Free()?
return &onDiskCache{
path: path,
opts: opts,
gcm: make(map[string]cipher.AEAD),
cmp: cmp,
rsem: semaphore.New(opts.ConcurrentRead),
wsem: semaphore.New(opts.ConcurrentWrite),
pending: newPending(),
diskSize: usage.Size(),
diskFree: usage.Available(),
once: &sync.Once{},
}, nil
}
func (c *onDiskCache) Unlock(userID string, passphrase []byte) error {
hash := sha256.New()
if _, err := hash.Write(passphrase); err != nil {
return err
}
aes, err := aes.NewCipher(hash.Sum(nil))
if err != nil {
return err
}
gcm, err := cipher.NewGCM(aes)
if err != nil {
return err
}
if err := os.MkdirAll(c.getUserPath(userID), 0700); err != nil {
return err
}
c.gcm[userID] = gcm
return nil
}
func (c *onDiskCache) Delete(userID string) error {
defer c.update()
return os.RemoveAll(c.getUserPath(userID))
}
// Has returns whether the given message exists in the cache.
func (c *onDiskCache) Has(userID, messageID string) bool {
c.pending.wait(c.getMessagePath(userID, messageID))
c.rsem.Lock()
defer c.rsem.Unlock()
_, err := os.Stat(c.getMessagePath(userID, messageID))
switch {
case err == nil:
return true
case os.IsNotExist(err):
return false
default:
panic(err)
}
}
func (c *onDiskCache) Get(userID, messageID string) ([]byte, error) {
enc, err := c.readFile(c.getMessagePath(userID, messageID))
if err != nil {
return nil, err
}
cmp, err := c.gcm[userID].Open(nil, enc[:c.gcm[userID].NonceSize()], enc[c.gcm[userID].NonceSize():], nil)
if err != nil {
return nil, err
}
return c.cmp.Decompress(cmp)
}
func (c *onDiskCache) Set(userID, messageID string, literal []byte) error {
nonce := make([]byte, c.gcm[userID].NonceSize())
if _, err := rand.Read(nonce); err != nil {
return err
}
cmp, err := c.cmp.Compress(literal)
if err != nil {
return err
}
// NOTE(GODT-1158): How to properly handle low space? Don't return error, that's bad. Instead send event?
if !c.hasSpace(len(cmp)) {
return nil
}
return c.writeFile(c.getMessagePath(userID, messageID), c.gcm[userID].Seal(nonce, nonce, cmp, nil))
}
func (c *onDiskCache) Rem(userID, messageID string) error {
defer c.update()
return os.Remove(c.getMessagePath(userID, messageID))
}
func (c *onDiskCache) readFile(path string) ([]byte, error) {
c.rsem.Lock()
defer c.rsem.Unlock()
// Wait before reading in case the file is currently being written.
c.pending.wait(path)
return ioutil.ReadFile(filepath.Clean(path))
}
func (c *onDiskCache) writeFile(path string, b []byte) error {
c.wsem.Lock()
defer c.wsem.Unlock()
// Mark the file as currently being written.
// If it's already being written, wait for it to be done and return nil.
// NOTE(GODT-1158): Let's hope it succeeded...
if ok := c.pending.add(path); !ok {
c.pending.wait(path)
return nil
}
defer c.pending.done(path)
// Reduce the approximate free space (update it exactly later).
c.lock.Lock()
c.diskFree -= uint64(len(b))
c.lock.Unlock()
// Update the diskFree eventually.
defer c.update()
// NOTE(GODT-1158): What happens when this fails? Should be fixed eventually.
return ioutil.WriteFile(filepath.Clean(path), b, 0600)
}
func (c *onDiskCache) hasSpace(size int) bool {
c.lock.Lock()
defer c.lock.Unlock()
if c.opts.MinFreeAbs > 0 {
if c.diskFree-uint64(size) < c.opts.MinFreeAbs {
return false
}
}
if c.opts.MinFreeRat > 0 {
if float64(c.diskFree-uint64(size))/float64(c.diskSize) < c.opts.MinFreeRat {
return false
}
}
return true
}
func (c *onDiskCache) update() {
go func() {
c.once.Do(func() {
c.lock.Lock()
defer c.lock.Unlock()
// Update the free space.
c.diskFree = du.NewDiskUsage(c.path).Available()
// Reset the Once object (so we can update again).
c.once = &sync.Once{}
})
}()
}
func (c *onDiskCache) getUserPath(userID string) string {
return filepath.Join(c.path, getHash(userID))
}
func (c *onDiskCache) getMessagePath(userID, messageID string) string {
return filepath.Join(c.getUserPath(userID), getHash(messageID))
}

33
internal/store/cache/hash.go vendored Normal file
View File

@ -0,0 +1,33 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package cache
import (
"crypto/sha256"
"encoding/hex"
)
func getHash(name string) string {
hash := sha256.New()
if _, err := hash.Write([]byte(name)); err != nil {
panic(err)
}
return hex.EncodeToString(hash.Sum(nil))
}

104
internal/store/cache/memory.go vendored Normal file
View File

@ -0,0 +1,104 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package cache
import (
"errors"
"sync"
)
type inMemoryCache struct {
lock sync.RWMutex
data map[string]map[string][]byte
size, limit int
}
// NewInMemoryCache creates a new in memory cache which stores up to the given number of bytes of cached data.
// NOTE(GODT-1158): Make this threadsafe.
func NewInMemoryCache(limit int) Cache {
return &inMemoryCache{
data: make(map[string]map[string][]byte),
limit: limit,
}
}
func (c *inMemoryCache) Unlock(userID string, passphrase []byte) error {
c.data[userID] = make(map[string][]byte)
return nil
}
func (c *inMemoryCache) Delete(userID string) error {
c.lock.Lock()
defer c.lock.Unlock()
for _, message := range c.data[userID] {
c.size -= len(message)
}
delete(c.data, userID)
return nil
}
// Has returns whether the given message exists in the cache.
func (c *inMemoryCache) Has(userID, messageID string) bool {
if _, err := c.Get(userID, messageID); err != nil {
return false
}
return true
}
func (c *inMemoryCache) Get(userID, messageID string) ([]byte, error) {
c.lock.RLock()
defer c.lock.RUnlock()
literal, ok := c.data[userID][messageID]
if !ok {
return nil, errors.New("no such message in cache")
}
return literal, nil
}
// NOTE(GODT-1158): What to actually do when memory limit is reached? Replace something existing? Return error? Drop silently?
// NOTE(GODT-1158): Pull in cache-rotating feature from old IMAP cache.
func (c *inMemoryCache) Set(userID, messageID string, literal []byte) error {
c.lock.Lock()
defer c.lock.Unlock()
if c.size+len(literal) > c.limit {
return nil
}
c.size += len(literal)
c.data[userID][messageID] = literal
return nil
}
func (c *inMemoryCache) Rem(userID, messageID string) error {
c.lock.Lock()
defer c.lock.Unlock()
c.size -= len(c.data[userID][messageID])
delete(c.data[userID], messageID)
return nil
}

25
internal/store/cache/options.go vendored Normal file
View File

@ -0,0 +1,25 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package cache
type Options struct {
MinFreeAbs uint64
MinFreeRat float64
ConcurrentRead int
ConcurrentWrite int
}

61
internal/store/cache/pending.go vendored Normal file
View File

@ -0,0 +1,61 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package cache
import "sync"
type pending struct {
lock sync.Mutex
path map[string]chan struct{}
}
func newPending() *pending {
return &pending{path: make(map[string]chan struct{})}
}
func (p *pending) add(path string) bool {
p.lock.Lock()
defer p.lock.Unlock()
if _, ok := p.path[path]; ok {
return false
}
p.path[path] = make(chan struct{})
return true
}
func (p *pending) wait(path string) {
p.lock.Lock()
ch, ok := p.path[path]
p.lock.Unlock()
if ok {
<-ch
}
}
func (p *pending) done(path string) {
p.lock.Lock()
defer p.lock.Unlock()
defer close(p.path[path])
delete(p.path, path)
}

51
internal/store/cache/pending_test.go vendored Normal file
View File

@ -0,0 +1,51 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package cache
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestPending(t *testing.T) {
pending := newPending()
pending.add("1")
pending.add("2")
pending.add("3")
resCh := make(chan string)
go func() { pending.wait("1"); resCh <- "1" }()
go func() { pending.wait("2"); resCh <- "2" }()
go func() { pending.wait("3"); resCh <- "3" }()
pending.done("1")
assert.Equal(t, "1", <-resCh)
pending.done("2")
assert.Equal(t, "2", <-resCh)
pending.done("3")
assert.Equal(t, "3", <-resCh)
}
func TestPendingUnknown(t *testing.T) {
newPending().wait("this is not currently being waited")
}

28
internal/store/cache/types.go vendored Normal file
View File

@ -0,0 +1,28 @@
// Copyright (c) 2021 Proton Technologies AG
//
// This file is part of ProtonMail Bridge.
//
// ProtonMail Bridge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// ProtonMail Bridge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with ProtonMail Bridge. If not, see <https://www.gnu.org/licenses/>.
package cache
type Cache interface {
Unlock(userID string, passphrase []byte) error
Delete(userID string) error
Has(userID, messageID string) bool
Get(userID, messageID string) ([]byte, error)
Set(userID, messageID string, literal []byte) error
Rem(userID, messageID string) error
}