GODT-1351: Cache and update of space bytes in user object.

This commit is contained in:
Jakub
2021-09-21 14:54:15 +02:00
parent 41e15db442
commit 2899e7bb15
27 changed files with 243 additions and 49 deletions

View File

@ -42,7 +42,7 @@ Item {
function spaceWithUnits(bytes){
if (bytes*1 !== bytes ) return "0 kB"
var units = ['B',"kB", "MB", "TB"];
var units = ['B',"kB", "MB", "GB", "TB"];
var i = parseInt(Math.floor(Math.log(bytes)/Math.log(1024)));
return Math.round(bytes*10 / Math.pow(1024, i))/10 + " " + units[i]

View File

@ -287,8 +287,8 @@ func (qu *QMLUser) update(user types.User) {
qu.SetLoggedIn(user.IsConnected())
qu.SetSplitMode(!user.IsCombinedAddressMode())
qu.SetSetupGuideSeen(false)
qu.SetUsedBytes(1.0) // TODO
qu.SetTotalBytes(10000.0) // TODO
qu.SetUsedBytes(float32(user.UsedBytes()))
qu.SetTotalBytes(float32(user.TotalBytes()))
qu.SetPassword(user.GetBridgePassword())
qu.SetAddresses(user.GetAddresses())
}

View File

@ -60,6 +60,8 @@ type UserManager interface {
// User is an interface of user needed by frontend.
type User interface {
ID() string
UsedBytes() int64
TotalBytes() int64
Username() string
IsConnected() bool
IsCombinedAddressMode() bool

View File

@ -31,7 +31,7 @@ import (
type storeUserProvider interface {
UserID() string
GetSpace() (usedSpace, maxSpace uint, err error)
GetSpaceKB() (usedSpace, maxSpace uint32, err error)
GetMaxUpload() (int64, error)
GetAddress(addressID string) (storeAddressProvider, error)

View File

@ -225,7 +225,7 @@ func (iu *imapUser) GetQuota(name string) (*imapquota.Status, error) {
// Called from go-imap in goroutines - we need to handle panics for each function.
defer iu.panicHandler.HandlePanic()
usedSpace, maxSpace, err := iu.storeUser.GetSpace()
usedSpace, maxSpace, err := iu.storeUser.GetSpaceKB()
if err != nil {
log.Error("Failed getting quota: ", err)
return nil, err
@ -233,9 +233,8 @@ func (iu *imapUser) GetQuota(name string) (*imapquota.Status, error) {
resources := make(map[string][2]uint32)
var list [2]uint32
// Quota is "in units of 1024 octets" (or KB) and PM returns bytes.
list[0] = uint32(usedSpace / 1024)
list[1] = uint32(maxSpace / 1024)
list[0] = usedSpace
list[1] = maxSpace
resources[imapquota.ResourceStorage] = list
status := &imapquota.Status{
Name: "",

View File

@ -337,6 +337,11 @@ func (loop *eventLoop) processEvent(event *pmapi.Event) (err error) {
}
}
if event.User != nil {
loop.user.UpdateSpace(event.User)
loop.listener.Emit(bridgeEvents.UserRefreshEvent, loop.user.ID())
}
// One would expect that every event would contain MessageCount as part of
// the event.Messages, but this is apparently not the case.
// MessageCounts are served on an irregular basis, so we should update and

View File

@ -207,6 +207,18 @@ func (mr *MockBridgeUserMockRecorder) Logout() *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Logout", reflect.TypeOf((*MockBridgeUser)(nil).Logout))
}
// UpdateSpace mocks base method.
func (m *MockBridgeUser) UpdateSpace(arg0 *pmapi.User) {
m.ctrl.T.Helper()
m.ctrl.Call(m, "UpdateSpace", arg0)
}
// UpdateSpace indicates an expected call of UpdateSpace.
func (mr *MockBridgeUserMockRecorder) UpdateSpace(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateSpace", reflect.TypeOf((*MockBridgeUser)(nil).UpdateSpace), arg0)
}
// UpdateUser mocks base method.
func (m *MockBridgeUser) UpdateUser(arg0 context.Context) error {
m.ctrl.T.Helper()

View File

@ -37,6 +37,7 @@ type BridgeUser interface {
GetStoreAddresses() []string
GetClient() pmapi.Client
UpdateUser(context.Context) error
UpdateSpace(*pmapi.User)
CloseAllConnections()
CloseConnection(string)
Logout() error

View File

@ -17,18 +17,47 @@
package store
import "math"
// UserID returns user ID.
func (store *Store) UserID() string {
return store.user.ID()
}
// GetSpace returns used and total space in bytes.
func (store *Store) GetSpace() (usedSpace, maxSpace uint, err error) {
// GetSpaceKB returns used and total space in kilo bytes (needed for IMAP
// Quota. Quota is "in units of 1024 octets" (or KB) and PM returns bytes.
func (store *Store) GetSpaceKB() (usedSpace, maxSpace uint32, err error) {
apiUser, err := store.client().CurrentUser(exposeContextForIMAP())
if err != nil {
return 0, 0, err
}
return uint(apiUser.UsedSpace), uint(apiUser.MaxSpace), nil
if apiUser.UsedSpace != nil {
usedSpace = store.toKBandLimit(*apiUser.UsedSpace, usedSpaceType)
}
if apiUser.MaxSpace != nil {
maxSpace = store.toKBandLimit(*apiUser.MaxSpace, maxSpaceType)
}
return
}
type spaceType string
const (
usedSpaceType = spaceType("used")
maxSpaceType = spaceType("max")
)
func (store *Store) toKBandLimit(n int64, space spaceType) uint32 {
if n < 0 {
log.WithField("space", space).Warning("negative number of bytes")
return uint32(0)
}
n /= 1024
if n > math.MaxUint32 {
log.WithField("space", space).Warning("too large number of bytes")
return uint32(math.MaxUint32)
}
return uint32(n)
}
// GetMaxUpload returns max size of message + all attachments in bytes.

View File

@ -49,6 +49,8 @@ type User struct {
userID string
creds *credentials.Credentials
usedBytes, totalBytes int64
lock sync.RWMutex
}
@ -122,6 +124,8 @@ func (u *User) connect(client pmapi.Client, creds *credentials.Credentials) erro
u.store.StartWatcher()
}
u.UpdateSpace(nil)
return nil
}
@ -201,6 +205,40 @@ func (u *User) ID() string {
return u.userID
}
// UsedBytes returns number of bytes used on server.
func (u *User) UsedBytes() int64 {
return u.usedBytes
}
// TotalBytes returns number of bytes available on server.
func (u *User) TotalBytes() int64 {
return u.totalBytes
}
// UpdateSpace will update TotalBytes and UsedBytes values from API user. If
// pointer is nill it will get fresh user from API. API user can come from
// update event which means it doesn't contain all data. Therefore only
// positive values will be updated.
func (u *User) UpdateSpace(apiUser *pmapi.User) {
// If missing get latest pmapi.User from API instead of using cached
// values from client.CurrentUser()
if apiUser == nil {
var err error
apiUser, err = u.client.GetUser(context.Background())
if err != nil {
u.log.WithError(err).Warning("Cannot update user space")
return
}
}
if apiUser.UsedSpace != nil {
u.usedBytes = *apiUser.UsedSpace
}
if apiUser.MaxSpace != nil {
u.totalBytes = *apiUser.MaxSpace
}
}
// Username returns the user's username as found in the user's credentials.
func (u *User) Username() string {
u.lock.RLock()
@ -351,7 +389,7 @@ func (u *User) UpdateUser(ctx context.Context) error {
defer u.lock.Unlock()
defer u.listener.Emit(events.UserRefreshEvent, u.userID)
_, err := u.client.UpdateUser(ctx)
user, err := u.client.UpdateUser(ctx)
if err != nil {
return err
}
@ -367,6 +405,8 @@ func (u *User) UpdateUser(ctx context.Context) error {
u.creds = creds
u.UpdateSpace(user)
return nil
}

View File

@ -36,7 +36,7 @@ func TestUpdateUser(t *testing.T) {
defer cleanUpUserData(user)
gomock.InOrder(
m.pmapiClient.EXPECT().UpdateUser(gomock.Any()).Return(nil, nil),
m.pmapiClient.EXPECT().UpdateUser(gomock.Any()).Return(testPMAPIUser, nil),
m.pmapiClient.EXPECT().ReloadKeys(gomock.Any(), testCredentials.MailboxPassword).Return(nil),
m.pmapiClient.EXPECT().Addresses().Return([]*pmapi.Address{testPMAPIAddress}),

View File

@ -115,6 +115,10 @@ func (u *Users) watchEvents() {
log.WithError(err).Error("Failed to load store after reconnecting")
}
}
if user.totalBytes == 0 {
user.UpdateSpace(nil)
}
}
}
}

View File

@ -116,9 +116,14 @@ var (
IsCombinedAddressMode: false,
}
usedSpace = int64(1048576)
maxSpace = int64(10485760)
testPMAPIUser = &pmapi.User{ //nolint[gochecknoglobals]
ID: "user",
Name: "username",
ID: "user",
Name: "username",
UsedSpace: &usedSpace,
MaxSpace: &maxSpace,
}
testPMAPIUserDisconnected = &pmapi.User{ //nolint[gochecknoglobals]
@ -297,6 +302,7 @@ func mockInitConnectedUser(t *testing.T, m mocks) {
// Mock of user initialisation.
m.pmapiClient.EXPECT().AddAuthRefreshHandler(gomock.Any())
m.pmapiClient.EXPECT().IsUnlocked().Return(true).AnyTimes()
m.pmapiClient.EXPECT().GetUser(gomock.Any()).Return(testPMAPIUser, nil) // load connected user
// Mock of store initialisation.
gomock.InOrder(

View File

@ -32,6 +32,7 @@ type Client interface {
AuthDelete(context.Context) error
AddAuthRefreshHandler(AuthRefreshHandler)
GetUser(ctx context.Context) (*User, error)
CurrentUser(ctx context.Context) (*User, error)
UpdateUser(ctx context.Context) (*User, error)
Unlock(ctx context.Context, passphrase []byte) (err error)

View File

@ -40,7 +40,7 @@ type Event struct {
// Changes applied to labels.
Labels []*EventLabel
// Current user status.
User User
User *User
// Changes to addresses.
Addresses []*EventAddress
// Messages to show to the user.
@ -198,17 +198,32 @@ func (c *client) getEvent(ctx context.Context, eventID string, numberOfMergedEve
// mergeEvents combines an old events and a new events object.
// This is not as simple as just blindly joining the two because some things should only be taken from the new events.
func mergeEvents(eventsOld *Event, eventsNew *Event) (mergedEvents *Event) {
mergedEvents = &Event{
return &Event{
EventID: eventsNew.EventID,
Refresh: eventsOld.Refresh | eventsNew.Refresh,
More: eventsNew.More,
Messages: append(eventsOld.Messages, eventsNew.Messages...),
MessageCounts: append(eventsOld.MessageCounts, eventsNew.MessageCounts...),
Labels: append(eventsOld.Labels, eventsNew.Labels...),
User: eventsNew.User,
User: mergeUserEvents(eventsOld.User, eventsNew.User),
Addresses: append(eventsOld.Addresses, eventsNew.Addresses...),
Notices: append(eventsOld.Notices, eventsNew.Notices...),
}
return
}
func mergeUserEvents(userOld, userNew *User) *User {
if userNew == nil {
return userOld
}
if userOld != nil {
if userNew.MaxSpace == nil {
userNew.MaxSpace = userOld.MaxSpace
}
if userNew.UsedSpace == nil {
userNew.UsedSpace = userOld.UsedSpace
}
}
return userNew
}

View File

@ -221,10 +221,11 @@ var (
},
},
},
User: User{
User: &User{
ID: "userID1",
Name: "user",
UsedSpace: 23456,
UsedSpace: &usedSpace,
MaxSpace: &maxSpace,
},
Addresses: []*EventAddress{
{
@ -464,7 +465,8 @@ const (
"User": {
"ID": "userID1",
"Name": "user",
"UsedSpace": 12345
"UsedSpace": 12345,
"MaxSpace": 12345678
},
"Addresses": [
{

View File

@ -362,6 +362,21 @@ func (mr *MockClientMockRecorder) GetPublicKeysForEmail(arg0, arg1 interface{})
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPublicKeysForEmail", reflect.TypeOf((*MockClient)(nil).GetPublicKeysForEmail), arg0, arg1)
}
// GetUser mocks base method.
func (m *MockClient) GetUser(arg0 context.Context) (*pmapi.User, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetUser", arg0)
ret0, _ := ret[0].(*pmapi.User)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetUser indicates an expected call of GetUser.
func (mr *MockClientMockRecorder) GetUser(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUser", reflect.TypeOf((*MockClient)(nil).GetUser), arg0)
}
// GetUserKeyRing mocks base method.
func (m *MockClient) GetUserKeyRing() (*crypto.KeyRing, error) {
m.ctrl.T.Helper()

View File

@ -3,10 +3,10 @@
"User": {
"ID": "MJLke8kWh1BBvG95JBIrZvzpgsZ94hNNgjNHVyhXMiv4g9cn6SgvqiIFR5cigpml2LD_iUk_3DkV29oojTt3eA==",
"Name": "jason",
"UsedSpace": 96691332,
"UsedSpace": 23456,
"Currency": "USD",
"Credit": 0,
"MaxSpace": 10737418240,
"MaxSpace": 12345678,
"MaxUpload": 26214400,
"Role": 2,
"Private": 1,

View File

@ -63,10 +63,10 @@ const (
type User struct {
ID string
Name string
UsedSpace int64
UsedSpace *int64
Currency string
Credit int
MaxSpace int64
MaxSpace *int64
MaxUpload int64
Role int
Private int
@ -85,7 +85,7 @@ type User struct {
}
}
func (c *client) getUser(ctx context.Context) (user *User, err error) {
func (c *client) GetUser(ctx context.Context) (user *User, err error) {
var res struct {
User *User
}
@ -114,7 +114,7 @@ func (c *client) unlockUser(passphrase []byte) (err error) {
// UpdateUser retrieves details about user and loads its addresses.
func (c *client) UpdateUser(ctx context.Context) (*User, error) {
user, err := c.getUser(ctx)
user, err := c.GetUser(ctx)
if err != nil {
return nil, err
}

View File

@ -28,19 +28,24 @@ import (
r "github.com/stretchr/testify/require"
)
var testCurrentUser = &User{
ID: "MJLke8kWh1BBvG95JBIrZvzpgsZ94hNNgjNHVyhXMiv4g9cn6SgvqiIFR5cigpml2LD_iUk_3DkV29oojTt3eA==",
Name: "jason",
UsedSpace: 96691332,
Currency: "USD",
Role: 2,
Subscribed: 1,
Services: 1,
MaxSpace: 10737418240,
MaxUpload: 26214400,
Private: 1,
Keys: *loadPMKeys(readTestFile("keyring_userKey_JSON", false)),
}
var (
usedSpace = int64(23456)
maxSpace = int64(12345678)
testCurrentUser = &User{
ID: "MJLke8kWh1BBvG95JBIrZvzpgsZ94hNNgjNHVyhXMiv4g9cn6SgvqiIFR5cigpml2LD_iUk_3DkV29oojTt3eA==",
Name: "jason",
UsedSpace: &usedSpace,
Currency: "USD",
Role: 2,
Subscribed: 1,
Services: 1,
MaxSpace: &maxSpace,
MaxUpload: 26214400,
Private: 1,
Keys: *loadPMKeys(readTestFile("keyring_userKey_JSON", false)),
}
)
func routeGetUsers(tb testing.TB, w http.ResponseWriter, req *http.Request) string {
r.NoError(tb, checkMethodAndPath(req, "GET", "/users"))

View File

@ -3,27 +3,37 @@
"user": {
"ID": "1",
"Name": "user",
"MaxUpload": 26214400
"MaxUpload": 26214400,
"UsedSpace": 1048576,
"MaxSpace": 10485760
},
"user2fa": {
"ID": "2",
"Name": "user2fa",
"MaxUpload": 26214400
"MaxUpload": 26214400,
"UsedSpace": 1048576,
"MaxSpace": 10485760
},
"userAddressWithCapitalLetter": {
"ID": "3",
"Name": "userAddressWithCapitalLetter",
"MaxUpload": 26214400
"MaxUpload": 26214400,
"UsedSpace": 1048576,
"MaxSpace": 10485760
},
"userMoreAddresses": {
"ID": "4",
"Name": "userMoreAddresses",
"MaxUpload": 26214400
"MaxUpload": 26214400,
"UsedSpace": 1048576,
"MaxSpace": 10485760
},
"userDisabledPrimaryAddress": {
"ID": "5",
"Name": "userDisabledPrimaryAddress",
"MaxUpload": 26214400
"MaxUpload": 26214400,
"UsedSpace": 1048576,
"MaxSpace": 10485760
}
},
"addresses": {

View File

@ -85,6 +85,14 @@ func (api *FakePMAPI) UpdateUser(context.Context) (*pmapi.User, error) {
return api.user, nil
}
func (api *FakePMAPI) GetUser(ctx context.Context) (*pmapi.User, error) {
if err := api.checkAndRecordCall(GET, "/users", nil); err != nil {
return nil, err
}
return api.user, nil
}
func (api *FakePMAPI) GetAddresses(context.Context) (pmapi.AddressList, error) {
if err := api.checkAndRecordCall(GET, "/addresses", nil); err != nil {
return nil, err

View File

@ -6,6 +6,7 @@ Feature: Start bridge
Then "user" is connected
And "user" has loaded store
And "user" has running event loop
And "user" has non-zero space
Scenario: Start with connected user, database file and no internet connection
Given there is user "user" which just logged in
@ -15,6 +16,7 @@ Feature: Start bridge
Then "user" is connected
And "user" has loaded store
And "user" has running event loop
And "user" has zero space
Scenario: Start with connected user, no database file and internet connection
Given there is user "user" which just logged in
@ -23,6 +25,7 @@ Feature: Start bridge
Then "user" is connected
And "user" has loaded store
And "user" has running event loop
And "user" has non-zero space
Scenario: Start with connected user, no database file and no internet connection
Given there is user "user" which just logged in
@ -37,6 +40,7 @@ Feature: Start bridge
Then "user" is connected
And "user" has loaded store
And "user" has running event loop
And "user" has non-zero space
Scenario: Start with disconnected user, database file and internet connection
Given there is disconnected user "user"
@ -45,6 +49,7 @@ Feature: Start bridge
Then "user" is disconnected
And "user" has loaded store
And "user" does not have running event loop
And "user" has zero space
Scenario: Start with disconnected user, database file and no internet connection
Given there is disconnected user "user"
@ -54,6 +59,7 @@ Feature: Start bridge
Then "user" is disconnected
And "user" has loaded store
And "user" does not have running event loop
And "user" has zero space
Scenario: Start with disconnected user, no database file and internet connection
Given there is disconnected user "user"
@ -62,6 +68,7 @@ Feature: Start bridge
Then "user" is disconnected
And "user" does not have loaded store
And "user" does not have running event loop
And "user" has zero space
Scenario: Start with disconnected user, no database file and no internet connection
Given there is disconnected user "user"
@ -71,3 +78,4 @@ Feature: Start bridge
Then "user" is disconnected
And "user" does not have loaded store
And "user" does not have running event loop
And "user" has zero space

View File

@ -6,6 +6,7 @@ Feature: Login for the first time
And "user" is connected
And "user" has database file
And "user" has running event loop
And "user" has non-zero space
@ignore-live
Scenario: Login with bad username
@ -31,6 +32,7 @@ Feature: Login for the first time
And "user2fa" is connected
And "user2fa" has database file
And "user2fa" has running event loop
And "user2fa" has non-zero space
Scenario: Login user with capital letters in address
Given there is user "userAddressWithCapitalLetter"
@ -39,6 +41,7 @@ Feature: Login for the first time
And "userAddressWithCapitalLetter" is connected
And "userAddressWithCapitalLetter" has database file
And "userAddressWithCapitalLetter" has running event loop
And "userAddressWithCapitalLetter" has non-zero space
Scenario: Login user with more addresses
Given there is user "userMoreAddresses"
@ -47,6 +50,7 @@ Feature: Login for the first time
And "userMoreAddresses" is connected
And "userMoreAddresses" has database file
And "userMoreAddresses" has running event loop
And "userMoreAddresses" has non-zero space
@ignore-live
Scenario: Login user with disabled primary address
@ -56,6 +60,7 @@ Feature: Login for the first time
And "userDisabledPrimaryAddress" is connected
And "userDisabledPrimaryAddress" has database file
And "userDisabledPrimaryAddress" has running event loop
And "userDisabledPrimaryAddress" has non-zero space
Scenario: Login two users
Given there is user "user"

View File

@ -6,6 +6,7 @@ Feature: Re-login
Then last response is "failed to finish login: user is already connected"
And "user" is connected
And "user" has running event loop
And "user" has non-zero space
@ignore
Scenario: Re-login with connected user and no database file
@ -24,6 +25,7 @@ Feature: Re-login
Then last response is "OK"
And "user" is connected
And "user" has running event loop
And "user" has non-zero space
Scenario: Re-login with disconnected user and no database file
Given there is disconnected user "user"
@ -33,3 +35,4 @@ Feature: Re-login
And "user" is connected
And "user" has database file
And "user" has running event loop
And "user" has non-zero space

View File

@ -21,7 +21,6 @@ import (
"context"
"fmt"
"math/rand"
"os"
"sync"
"github.com/ProtonMail/go-srp"
@ -84,7 +83,7 @@ func (pc *persistentClient) GetEvent(ctx context.Context, eventID string) (*pmap
}
func SetupPersistentClients() {
app := os.Getenv("TEST_APP")
app := "bridge"
persistentClients.manager = pmapi.New(pmapi.NewConfig(app, constants.Version))
persistentClients.manager.SetLogging(logrus.WithField("pkg", "liveapi"), logrus.GetLevel() == logrus.TraceLevel)

View File

@ -34,6 +34,8 @@ func UsersChecksFeatureContext(s *godog.ScenarioContext) {
s.Step(`^"([^"]*)" does not have loaded store$`, userDoesNotHaveLoadedStore)
s.Step(`^"([^"]*)" has running event loop$`, userHasRunningEventLoop)
s.Step(`^"([^"]*)" does not have running event loop$`, userDoesNotHaveRunningEventLoop)
s.Step(`^"([^"]*)" has non-zero space$`, userHasNonZeroSpace)
s.Step(`^"([^"]*)" has zero space$`, userHasZeroSpace)
}
func userHasAddressModeInMode(bddUserID, wantAddressMode string) error {
@ -162,3 +164,26 @@ func userDoesNotHaveRunningEventLoop(bddUserID string) error {
}, 5*time.Second, 10*time.Millisecond)
return ctx.GetTestingError()
}
func userHasSpace(bddUserID string, wantNonZero bool) error {
account := ctx.GetTestAccount(bddUserID)
if account == nil {
return godog.ErrPending
}
user, err := ctx.GetUser(account.Username())
if err != nil {
return internalError(err, "getting user %s", account.Username())
}
// We should only test `user.TotalBytes()>0` and not test
// `user.UsedBytes()>0`. The later value can be always zero when
// account is empty even when user object had already cached the value
// from server.
a.Equal(ctx.GetTestingT(), wantNonZero, user.TotalBytes() > 0)
return ctx.GetTestingError()
}
func userHasNonZeroSpace(bddUserID string) error { return userHasSpace(bddUserID, true) }
func userHasZeroSpace(bddUserID string) error { return userHasSpace(bddUserID, false) }