Launcher, app/base, sentry, update service

internal/logging/clear.go

@@ -0,0 +1,85 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package logging
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+
+	"github.com/sirupsen/logrus"
+)
+
+func clearLogs(logDir string, maxLogs int) error {
+	files, err := ioutil.ReadDir(logDir)
+	if err != nil {
+		return err
+	}
+
+	var logsWithPrefix []string
+	var crashesWithPrefix []string
+
+	for _, file := range files {
+		if matchLogName(file.Name()) {
+			if matchStackTraceName(file.Name()) {
+				crashesWithPrefix = append(crashesWithPrefix, file.Name())
+			} else {
+				logsWithPrefix = append(logsWithPrefix, file.Name())
+			}
+		} else {
+			// Older versions of Bridge stored logs in subfolders for each version.
+			// That also has to be cleared and the functionality can be removed after some time.
+			if file.IsDir() {
+				if err := clearLogs(filepath.Join(logDir, file.Name()), maxLogs); err != nil {
+					return err
+				}
+			} else {
+				removeLog(logDir, file.Name())
+			}
+		}
+	}
+
+	removeOldLogs(logDir, logsWithPrefix, maxLogs)
+	removeOldLogs(logDir, crashesWithPrefix, maxLogs)
+
+	return nil
+}
+
+func removeOldLogs(logDir string, filenames []string, maxLogs int) {
+	count := len(filenames)
+	if count <= maxLogs {
+		return
+	}
+
+	sort.Strings(filenames) // Sorted by timestamp: oldest first.
+	for _, filename := range filenames[:count-maxLogs] {
+		removeLog(logDir, filename)
+	}
+}
+
+func removeLog(logDir, filename string) {
+	// We need to be sure to delete only log files.
+	// Directory with logs can also contain other files.
+	if !matchLogName(filename) {
+		return
+	}
+	if err := os.RemoveAll(filepath.Join(logDir, filename)); err != nil {
+		logrus.WithError(err).Error("Failed to remove old logs")
+	}
+}

internal/logging/crash.go

@@ -0,0 +1,62 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package logging
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"runtime/pprof"
+	"time"
+
+	"github.com/ProtonMail/proton-bridge/internal/constants"
+	"github.com/ProtonMail/proton-bridge/internal/crash"
+	"github.com/sirupsen/logrus"
+)
+
+func DumpStackTrace(logsPath string) crash.RecoveryAction {
+	return func(r interface{}) error {
+		file := filepath.Join(logsPath, getStackTraceName(constants.Version, constants.Revision))
+
+		f, err := os.OpenFile(file, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600)
+		if err != nil {
+			return err
+		}
+
+		if _, err := f.WriteString(fmt.Sprintf("Recover: %v", r)); err != nil {
+			return err
+		}
+
+		if err := pprof.Lookup("goroutine").WriteTo(f, 2); err != nil {
+			return err
+		}
+
+		logrus.WithField("file", file).Warn("Saved crash report")
+
+		return nil
+	}
+}
+
+func getStackTraceName(version, revision string) string {
+	return fmt.Sprintf("v%v_%v_crash_%v.log", version, revision, time.Now().Unix())
+}
+
+func matchStackTraceName(name string) bool {
+	return regexp.MustCompile(`^v.*_crash_.*\.log$`).MatchString(name)
+}

internal/logging/logging.go

@@ -0,0 +1,90 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package logging
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"regexp"
+	"time"
+
+	"github.com/ProtonMail/proton-bridge/internal/constants"
+	"github.com/sirupsen/logrus"
+	"github.com/sirupsen/logrus/hooks/writer"
+)
+
+const (
+	// MaxLogSize defines the maximum log size we should permit.
+	// Zendesk has a file size limit of 20MB. When the last N log files are zipped,
+	// it should fit under 20MB. So here we permit up to 10MB (most files are a few hundred kB).
+	MaxLogSize = 10 * 2 << 20
+
+	// MaxLogs defines how many old log files should be kept.
+	MaxLogs = 3
+)
+
+func Init(logsPath string) error {
+	logrus.SetFormatter(&logrus.TextFormatter{
+		FullTimestamp:   true,
+		TimestampFormat: time.StampMilli,
+	})
+
+	rotator, err := NewRotator(MaxLogSize, func() (io.WriteCloser, error) {
+		if err := clearLogs(logsPath, MaxLogs); err != nil {
+			return nil, err
+		}
+
+		return os.Create(filepath.Join(logsPath, getLogName(constants.Version, constants.Revision)))
+	})
+	if err != nil {
+		return err
+	}
+
+	logrus.SetOutput(rotator)
+
+	logrus.AddHook(&writer.Hook{
+		Writer: os.Stderr,
+		LogLevels: []logrus.Level{
+			logrus.PanicLevel,
+			logrus.FatalLevel,
+			logrus.ErrorLevel,
+		},
+	})
+
+	return nil
+}
+
+func SetLevel(level string) {
+	if lvl, err := logrus.ParseLevel(level); err == nil {
+		logrus.SetLevel(lvl)
+	}
+
+	if logrus.GetLevel() == logrus.DebugLevel || logrus.GetLevel() == logrus.TraceLevel {
+		logrus.SetOutput(os.Stderr)
+	}
+}
+
+func getLogName(version, revision string) string {
+	return fmt.Sprintf("v%v_%v_%v.log", version, revision, time.Now().Unix())
+}
+
+func matchLogName(name string) bool {
+	return regexp.MustCompile(`^v.*\.log$`).MatchString(name)
+}

internal/logging/logging_test.go

@@ -0,0 +1,69 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package logging
+
+import (
+	"io/ioutil"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// TestClearLogs tests that cearLogs removes only bridge old log files keeping last three of them.
+func TestClearLogs(t *testing.T) {
+	dir, err := ioutil.TempDir("", "clear-logs-test")
+	require.NoError(t, err)
+
+	require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "other.log"), []byte("Hello"), 0755))
+	require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "v1_10.log"), []byte("Hello"), 0755))
+	require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "v1_11.log"), []byte("Hello"), 0755))
+	require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "v2_12.log"), []byte("Hello"), 0755))
+	require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "v2_13.log"), []byte("Hello"), 0755))
+
+	require.NoError(t, clearLogs(dir, 3))
+	checkFileNames(t, dir, []string{
+		"other.log",
+		"v1_11.log",
+		"v2_12.log",
+		"v2_13.log",
+	})
+}
+
+func checkFileNames(t *testing.T, dir string, expectedFileNames []string) {
+	fileNames := getFileNames(t, dir)
+	require.Equal(t, expectedFileNames, fileNames)
+}
+
+func getFileNames(t *testing.T, dir string) []string {
+	files, err := ioutil.ReadDir(dir)
+	require.NoError(t, err)
+
+	fileNames := []string{}
+	for _, file := range files {
+		fileNames = append(fileNames, file.Name())
+		if file.IsDir() {
+			subDir := filepath.Join(dir, file.Name())
+			subFileNames := getFileNames(t, subDir)
+			for _, subFileName := range subFileNames {
+				fileNames = append(fileNames, file.Name()+"/"+subFileName)
+			}
+		}
+	}
+	return fileNames
+}

internal/logging/rotator.go

@@ -0,0 +1,75 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package logging
+
+import "io"
+
+type Rotator struct {
+	getFile FileProvider
+	wc      io.WriteCloser
+	size    int
+	maxSize int
+}
+
+type FileProvider func() (io.WriteCloser, error)
+
+func NewRotator(maxSize int, getFile FileProvider) (*Rotator, error) {
+	r := &Rotator{
+		getFile: getFile,
+		maxSize: maxSize,
+	}
+
+	if err := r.rotate(); err != nil {
+		return nil, err
+	}
+
+	return r, nil
+}
+
+func (r *Rotator) Write(p []byte) (int, error) {
+	if r.size+len(p) > r.maxSize {
+		if err := r.rotate(); err != nil {
+			return 0, err
+		}
+	}
+
+	n, err := r.wc.Write(p)
+	if err != nil {
+		return n, err
+	}
+
+	r.size += n
+
+	return n, nil
+}
+
+func (r *Rotator) rotate() error {
+	if r.wc != nil {
+		_ = r.wc.Close()
+	}
+
+	wc, err := r.getFile()
+	if err != nil {
+		return err
+	}
+
+	r.wc = wc
+	r.size = 0
+
+	return nil
+}

internal/logging/rotator_test.go

@@ -0,0 +1,131 @@
+// Copyright (c) 2020 Proton Technologies AG
+//
+// This file is part of ProtonMail Bridge.
+//
+// ProtonMail Bridge is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// ProtonMail Bridge is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with ProtonMail Bridge.  If not, see <https://www.gnu.org/licenses/>.
+
+package logging
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type WriteCloser struct {
+	bytes.Buffer
+}
+
+func (c *WriteCloser) Close() error {
+	return nil
+}
+
+func TestRotator(t *testing.T) {
+	n := 0
+
+	getFile := func() (io.WriteCloser, error) {
+		n++
+		return &WriteCloser{}, nil
+	}
+
+	r, err := NewRotator(10, getFile)
+	require.NoError(t, err)
+
+	_, err = r.Write([]byte("12345"))
+	require.NoError(t, err)
+	assert.Equal(t, 1, n)
+
+	_, err = r.Write([]byte("12345"))
+	require.NoError(t, err)
+	assert.Equal(t, 1, n)
+
+	_, err = r.Write([]byte("01234"))
+	require.NoError(t, err)
+	assert.Equal(t, 2, n)
+
+	_, err = r.Write([]byte("01234"))
+	require.NoError(t, err)
+	assert.Equal(t, 2, n)
+
+	_, err = r.Write([]byte("01234"))
+	require.NoError(t, err)
+	assert.Equal(t, 3, n)
+
+	_, err = r.Write([]byte("01234"))
+	require.NoError(t, err)
+	assert.Equal(t, 3, n)
+
+	_, err = r.Write([]byte("01234"))
+	require.NoError(t, err)
+	assert.Equal(t, 4, n)
+}
+
+func BenchmarkRotateRAMFile(b *testing.B) {
+	dir, err := ioutil.TempDir("", "rotate-benchmark")
+	require.NoError(b, err)
+	defer os.RemoveAll(dir) // nolint[errcheck]
+
+	benchRotate(b, MaxLogSize, getTestFile(b, dir, MaxLogSize-1))
+}
+
+func BenchmarkRotateDiskFile(b *testing.B) {
+	cache, err := os.UserCacheDir()
+	require.NoError(b, err)
+
+	dir, err := ioutil.TempDir(cache, "rotate-benchmark")
+	require.NoError(b, err)
+	defer os.RemoveAll(dir) // nolint[errcheck]
+
+	benchRotate(b, MaxLogSize, getTestFile(b, dir, MaxLogSize-1))
+}
+
+func benchRotate(b *testing.B, logSize int, getFile func() (io.WriteCloser, error)) {
+	r, err := NewRotator(logSize, getFile)
+	require.NoError(b, err)
+
+	for n := 0; n < b.N; n++ {
+		require.NoError(b, r.rotate())
+
+		f, ok := r.wc.(*os.File)
+		require.True(b, ok)
+		require.NoError(b, os.Remove(f.Name()))
+	}
+}
+
+func getTestFile(b *testing.B, dir string, length int) func() (io.WriteCloser, error) {
+	return func() (io.WriteCloser, error) {
+		b.StopTimer()
+		defer b.StartTimer()
+
+		f, err := ioutil.TempFile(dir, "log")
+		if err != nil {
+			return nil, err
+		}
+
+		if _, err := f.Write(make([]byte, length)); err != nil {
+			return nil, err
+		}
+
+		if err := f.Sync(); err != nil {
+			return nil, err
+		}
+
+		return f, nil
+	}
+}