mirror of
https://github.com/ProtonMail/proton-bridge.git
synced 2025-12-22 18:16:43 +00:00
GODT-1158: Store full messages bodies on disk
- GODT-1158: simple on-disk cache in store - GODT-1158: better member naming in event loop - GODT-1158: create on-disk cache during bridge setup - GODT-1158: better job options - GODT-1158: rename GetLiteral to GetRFC822 - GODT-1158: rename events -> currentEvents - GODT-1158: unlock cache per-user - GODT-1158: clean up cache after logout - GODT-1158: randomized encrypted cache passphrase - GODT-1158: Opt out of on-disk cache in settings - GODT-1158: free space in cache - GODT-1158: make tests compile - GODT-1158: optional compression - GODT-1158: cache custom location - GODT-1158: basic capacity checker - GODT-1158: cache free space config - GODT-1158: only unlock cache if pmapi client is unlocked as well - GODT-1158: simple background sync worker - GODT-1158: set size/bodystructure when caching message - GODT-1158: limit store db update blocking with semaphore - GODT-1158: dumb 10-semaphore - GODT-1158: properly handle delete; remove bad bodystructure handling - GODT-1158: hacky fix for caching after logout... baaaaad - GODT-1158: cache worker - GODT-1158: compute body structure lazily - GODT-1158: cache size in store - GODT-1158: notify cacher when adding to store - GODT-1158: 15 second store cache watcher - GODT-1158: enable cacher - GODT-1158: better cache worker starting/stopping - GODT-1158: limit cacher to less concurrency than disk cache - GODT-1158: message builder prio + pchan pkg - GODT-1158: fix pchan, use in message builder - GODT-1158: no sem in cacher (rely on message builder prio) - GODT-1158: raise priority of existing jobs when requested - GODT-1158: pending messages in on-disk cache - GODT-1158: WIP just a note about deleting messages from disk cache - GODT-1158: pending wait when trying to write - GODT-1158: pending.add to return bool - GODT-1225: Headers in bodystructure are stored as bytes. - GODT-1158: fixing header caching - GODT-1158: don't cache in background - GODT-1158: all concurrency set in settings - GODT-1158: worker pools inside message builder - GODT-1158: fix linter issues - GODT-1158: remove completed builds from builder - GODT-1158: remove builder pool - GODT-1158: cacher defer job done properly - GODT-1158: fix linter - GODT-1299: Continue with bodystructure build if deserialization failed - GODT-1324: Delete messages from the cache when they are deleted on the server - GODT-1158: refactor cache tests - GODT-1158: move builder to app/bridge - GODT-1306: Migrate cache on disk when location is changed (and delete when disabled)
This commit is contained in:
James Houlahan
@ -18,99 +18,113 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/ProtonMail/gopenpgp/v2/crypto"
|
||||
"github.com/ProtonMail/proton-bridge/pkg/message"
|
||||
"github.com/sirupsen/logrus"
|
||||
bolt "go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
// Cache caches the last event IDs for all accounts (there should be only one instance).
|
||||
type Cache struct {
|
||||
// cache is map from userID => key (such as last event) => value (such as event ID).
|
||||
cache map[string]map[string]string
|
||||
path string
|
||||
lock *sync.RWMutex
|
||||
}
|
||||
const passphraseKey = "passphrase"
|
||||
|
||||
// NewCache constructs a new cache at the given path.
|
||||
func NewCache(path string) *Cache {
|
||||
return &Cache{
|
||||
path: path,
|
||||
lock: &sync.RWMutex{},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cache) getEventID(userID string) string {
|
||||
c.lock.Lock()
|
||||
defer c.lock.Unlock()
|
||||
|
||||
if err := c.loadCache(); err != nil {
|
||||
log.WithError(err).Warn("Problem to load store cache")
|
||||
}
|
||||
|
||||
if c.cache == nil {
|
||||
c.cache = map[string]map[string]string{}
|
||||
}
|
||||
if c.cache[userID] == nil {
|
||||
c.cache[userID] = map[string]string{}
|
||||
}
|
||||
|
||||
return c.cache[userID]["events"]
|
||||
}
|
||||
|
||||
func (c *Cache) setEventID(userID, eventID string) error {
|
||||
c.lock.Lock()
|
||||
defer c.lock.Unlock()
|
||||
|
||||
if c.cache[userID] == nil {
|
||||
c.cache[userID] = map[string]string{}
|
||||
}
|
||||
c.cache[userID]["events"] = eventID
|
||||
|
||||
return c.saveCache()
|
||||
}
|
||||
|
||||
func (c *Cache) loadCache() error {
|
||||
if c.cache != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
f, err := os.Open(c.path)
|
||||
// UnlockCache unlocks the cache for the user with the given keyring.
|
||||
func (store *Store) UnlockCache(kr *crypto.KeyRing) error {
|
||||
passphrase, err := store.getCachePassphrase()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close() //nolint[errcheck]
|
||||
|
||||
return json.NewDecoder(f).Decode(&c.cache)
|
||||
}
|
||||
if passphrase == nil {
|
||||
if passphrase, err = crypto.RandomToken(32); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Cache) saveCache() error {
|
||||
if c.cache == nil {
|
||||
return errors.New("events: cannot save cache: cache is nil")
|
||||
enc, err := kr.Encrypt(crypto.NewPlainMessage(passphrase), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := store.setCachePassphrase(enc.GetBinary()); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
dec, err := kr.Decrypt(crypto.NewPGPMessage(passphrase), nil, crypto.GetUnixTime())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
passphrase = dec.GetBinary()
|
||||
}
|
||||
|
||||
f, err := os.Create(c.path)
|
||||
if err := store.cache.Unlock(store.user.ID(), passphrase); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
store.cacher.start()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *Store) getCachePassphrase() ([]byte, error) {
|
||||
var passphrase []byte
|
||||
|
||||
if err := store.db.View(func(tx *bolt.Tx) error {
|
||||
passphrase = tx.Bucket(cachePassphraseBucket).Get([]byte(passphraseKey))
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return passphrase, nil
|
||||
}
|
||||
|
||||
func (store *Store) setCachePassphrase(passphrase []byte) error {
|
||||
return store.db.Update(func(tx *bolt.Tx) error {
|
||||
return tx.Bucket(cachePassphraseBucket).Put([]byte(passphraseKey), passphrase)
|
||||
})
|
||||
}
|
||||
|
||||
func (store *Store) clearCachePassphrase() error {
|
||||
return store.db.Update(func(tx *bolt.Tx) error {
|
||||
return tx.Bucket(cachePassphraseBucket).Delete([]byte(passphraseKey))
|
||||
})
|
||||
}
|
||||
|
||||
func (store *Store) getCachedMessage(messageID string) ([]byte, error) {
|
||||
if store.cache.Has(store.user.ID(), messageID) {
|
||||
return store.cache.Get(store.user.ID(), messageID)
|
||||
}
|
||||
|
||||
job, done := store.newBuildJob(messageID, message.ForegroundPriority)
|
||||
defer done()
|
||||
|
||||
literal, err := job.GetResult()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// NOTE(GODT-1158): No need to block until cache has been set; do this async?
|
||||
if err := store.cache.Set(store.user.ID(), messageID, literal); err != nil {
|
||||
logrus.WithError(err).Error("Failed to cache message")
|
||||
}
|
||||
|
||||
return literal, nil
|
||||
}
|
||||
|
||||
// IsCached returns whether the given message already exists in the cache.
|
||||
func (store *Store) IsCached(messageID string) bool {
|
||||
return store.cache.Has(store.user.ID(), messageID)
|
||||
}
|
||||
|
||||
// BuildAndCacheMessage builds the given message (with background priority) and puts it in the cache.
|
||||
// It builds with background priority.
|
||||
func (store *Store) BuildAndCacheMessage(messageID string) error {
|
||||
job, done := store.newBuildJob(messageID, message.BackgroundPriority)
|
||||
defer done()
|
||||
|
||||
literal, err := job.GetResult()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close() //nolint[errcheck]
|
||||
|
||||
return json.NewEncoder(f).Encode(c.cache)
|
||||
}
|
||||
|
||||
func (c *Cache) clearCacheUser(userID string) error {
|
||||
c.lock.Lock()
|
||||
defer c.lock.Unlock()
|
||||
|
||||
if c.cache == nil {
|
||||
log.WithField("user", userID).Warning("Cannot clear user from cache: cache is nil")
|
||||
return nil
|
||||
}
|
||||
|
||||
log.WithField("user", userID).Trace("Removing user from event loop cache")
|
||||
|
||||
delete(c.cache, userID)
|
||||
|
||||
return c.saveCache()
|
||||
return store.cache.Set(store.user.ID(), messageID, literal)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user