Silverfish/proton-bridge
1
0
Fork 1
mirror of https://github.com/ProtonMail/proton-bridge.git synced 2025-12-10 04:36:43 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Import/Export backend

Browse Source
This commit is contained in:
Michal Horejsek
2020-06-17 15:29:41 +02:00
parent 49316a935c
commit 1c10cc5065
107 changed files with 2869 additions and 743 deletions
Download Patch File Download Diff File Expand all files Collapse all files

214
cmd/Desktop-Bridge/main.go
View File

@ -35,18 +35,13 @@ package main
*/
import (
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"runtime"
"runtime/pprof"
"strconv"
"strings"
"github.com/ProtonMail/proton-bridge/internal/api"
"github.com/ProtonMail/proton-bridge/internal/bridge"
"github.com/ProtonMail/proton-bridge/internal/cmd"
"github.com/ProtonMail/proton-bridge/internal/cookies"
"github.com/ProtonMail/proton-bridge/internal/events"
"github.com/ProtonMail/proton-bridge/internal/frontend"
@ -54,104 +49,42 @@ import (
"github.com/ProtonMail/proton-bridge/internal/preferences"
"github.com/ProtonMail/proton-bridge/internal/smtp"
"github.com/ProtonMail/proton-bridge/internal/users/credentials"
"github.com/ProtonMail/proton-bridge/pkg/args"
"github.com/ProtonMail/proton-bridge/pkg/config"
"github.com/ProtonMail/proton-bridge/pkg/constants"
"github.com/ProtonMail/proton-bridge/pkg/listener"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/ProtonMail/proton-bridge/pkg/updates"
"github.com/allan-simon/go-singleinstance"
"github.com/getsentry/raven-go"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
)
// cacheVersion is used for cache files such as lock, events, preferences, user_info, db files.
// Different number will drop old files and create new ones.
const cacheVersion = "c11"
const (
// cacheVersion is used for cache files such as lock, events, preferences, user_info, db files.
// Different number will drop old files and create new ones.
cacheVersion = "c11"
appName = "bridge"
)
var (
log = logrus.WithField("pkg", "main") //nolint[gochecknoglobals]
// How many crashes in a row.
numberOfCrashes = 0 //nolint[gochecknoglobals]
// After how many crashes bridge gives up starting.
maxAllowedCrashes = 10 //nolint[gochecknoglobals]
)
func main() {
if err := raven.SetDSN(constants.DSNSentry); err != nil {
log.WithError(err).Errorln("Can not setup sentry DSN")
}
raven.SetRelease(constants.Revision)
args.FilterProcessSerialNumberFromArgs()
filterRestartNumberFromArgs()
app := cli.NewApp()
app.Name = "Protonmail Bridge"
app.Version = constants.BuildVersion
app.Flags = []cli.Flag{
cli.StringFlag{
Name: "log-level, l",
Usage: "Set the log level (one of panic, fatal, error, warn, info, debug, debug-client, debug-server)"},
cli.BoolFlag{
Name: "no-window",
Usage: "Don't show window after start"},
cli.BoolFlag{
Name: "cli, c",
Usage: "Use command line interface"},
cli.BoolFlag{
Name: "noninteractive",
Usage: "Start Bridge entirely noninteractively"},
cli.StringFlag{
Name: "version-json, g",
Usage: "Generate json version file"},
cli.BoolFlag{
Name: "mem-prof, m",
Usage: "Generate memory profile"},
cli.BoolFlag{
Name: "cpu-prof, p",
Usage: "Generate CPU profile"},
}
app.Usage = "ProtonMail IMAP and SMTP Bridge"
app.Action = run
// Always log the basic info about current bridge.
logrus.SetLevel(logrus.InfoLevel)
log.WithField("version", constants.Version).
WithField("revision", constants.Revision).
WithField("runtime", runtime.GOOS).
WithField("build", constants.BuildTime).
WithField("args", os.Args).
WithField("appLong", app.Name).
WithField("appShort", constants.AppShortName).
Info("Run app")
if err := app.Run(os.Args); err != nil {
log.Error("Program exited with error: ", err)
}
}
type panicHandler struct {
cfg *config.Config
err *error // Pointer to error of cli action.
}
func (ph *panicHandler) HandlePanic() {
r := recover()
if r == nil {
return
}
config.HandlePanic(ph.cfg, fmt.Sprintf("Recover: %v", r))
frontend.HandlePanic("ProtonMail Bridge")
*ph.err = cli.NewExitError("Panic and restart", 255)
numberOfCrashes++
log.Error("Restarting after panic")
restartApp()
os.Exit(255)
cmd.Main(
"ProtonMail Bridge",
"ProtonMail IMAP and SMTP Bridge",
[]cli.Flag{
cli.BoolFlag{
Name: "no-window",
Usage: "Don't show window after start"},
cli.BoolFlag{
Name: "noninteractive",
Usage: "Start Bridge entirely noninteractively"},
},
run,
)
}
// run initializes and starts everything in a precise order.
@ -159,13 +92,17 @@ func (ph *panicHandler) HandlePanic() {
// IMPORTANT: ***Read the comments before CHANGING the order ***
func run(context *cli.Context) (contextError error) { // nolint[funlen]
// We need to have config instance to setup a logs, panic handler, etc ...
cfg := config.New(constants.AppShortName, constants.Version, constants.Revision, cacheVersion)
cfg := config.New(appName, constants.Version, constants.Revision, cacheVersion)
// We want to know about any problem. Our PanicHandler calls sentry which is
// not dependent on anything else. If that fails, it tries to create crash
// report which will not be possible if no folder can be created. That's the
// only problem we will not be notified about in any way.
panicHandler := &panicHandler{cfg, &contextError}
panicHandler := &cmd.PanicHandler{
AppName: "ProtonMail Bridge",
Config: cfg,
Err: &contextError,
}
defer panicHandler.HandlePanic()
// First we need config and create necessary folder; it's dependency for everything.
@ -177,13 +114,6 @@ func run(context *cli.Context) (contextError error) { // nolint[funlen]
logLevel := context.GlobalString("log-level")
debugClient, debugServer := config.SetupLog(cfg, logLevel)
// Should be called after logs are configured but before preferences are created.
migratePreferencesFromC10(cfg)
if err := cfg.ClearOldData(); err != nil {
log.Error("Cannot clear old data: ", err)
}
// Doesn't make sense to continue when Bridge was invoked with wrong arguments.
// We should tell that to the user before we do anything else.
if context.Args().First() != "" {
@ -193,21 +123,16 @@ func run(context *cli.Context) (contextError error) { // nolint[funlen]
// It's safe to get version JSON file even when other instance is running.
// (thus we put it before check of presence of other Bridge instance).
updates := updates.New(
constants.AppShortName,
constants.Version,
constants.Revision,
constants.BuildTime,
bridge.ReleaseNotes,
bridge.ReleaseFixedBugs,
cfg.GetUpdateDir(),
)
updates := updates.NewBridge(cfg.GetUpdateDir())
if dir := context.GlobalString("version-json"); dir != "" {
generateVersionFiles(updates, dir)
cmd.GenerateVersionFiles(updates, dir)
return nil
}
// Should be called after logs are configured but before preferences are created.
migratePreferencesFromC10(cfg)
// ClearOldData before starting new bridge to do a proper setup.
//
// IMPORTANT: If you the change position of this you will need to wait
@ -234,7 +159,7 @@ func run(context *cli.Context) (contextError error) { // nolint[funlen]
if err != nil {
log.Warn("Bridge is already running")
if err := api.CheckOtherInstanceAndFocus(pref.GetInt(preferences.APIPortKey), tls); err != nil {
numberOfCrashes = maxAllowedCrashes
cmd.DisableRestart()
log.Error("Second instance: ", err)
}
return cli.NewExitError("Bridge is already running.", 3)
@ -254,7 +179,7 @@ func run(context *cli.Context) (contextError error) { // nolint[funlen]
}
if doMemoryProfile := context.GlobalBool("mem-prof"); doMemoryProfile {
defer makeMemoryProfile()
defer cmd.MakeMemoryProfile()
}
// Now we initialize all Bridge parts.
@ -262,7 +187,7 @@ func run(context *cli.Context) (contextError error) { // nolint[funlen]
eventListener := listener.New()
events.SetupEvents(eventListener)
credentialsStore, credentialsError := credentials.NewStore("bridge")
credentialsStore, credentialsError := credentials.NewStore(appName)
if credentialsError != nil {
log.Error("Could not get credentials store: ", credentialsError)
}
@ -338,7 +263,7 @@ func run(context *cli.Context) (contextError error) { // nolint[funlen]
}
if frontend.IsAppRestarting() {
restartApp()
cmd.RestartApp()
}
return nil
@ -348,7 +273,7 @@ func run(context *cli.Context) (contextError error) { // nolint[funlen]
// It will happen only when c10/prefs.json exists and c11/prefs.json not.
// No configuration changed between c10 and c11 versions.
func migratePreferencesFromC10(cfg *config.Config) {
pref10Path := config.New(constants.AppShortName, constants.Version, constants.Revision, "c10").GetPreferencesPath()
pref10Path := config.New(appName, constants.Version, constants.Revision, "c10").GetPreferencesPath()
if _, err := os.Stat(pref10Path); os.IsNotExist(err) {
log.WithField("path", pref10Path).Trace("Old preferences does not exist, migration skipped")
return
@ -374,68 +299,3 @@ func migratePreferencesFromC10(cfg *config.Config) {
log.Info("Preferences migrated")
}
// generateVersionFiles writes a JSON file with details about current build.
// Those files are used for upgrading the app.
func generateVersionFiles(updates *updates.Updates, dir string) {
log.Info("Generating version files")
for _, goos := range []string{"windows", "darwin", "linux"} {
log.Debug("Generating JSON for ", goos)
if err := updates.CreateJSONAndSign(dir, goos); err != nil {
log.Error(err)
}
}
}
func makeMemoryProfile() {
name := "./mem.pprof"
f, err := os.Create(name)
if err != nil {
log.Error("Could not create memory profile: ", err)
}
if abs, err := filepath.Abs(name); err == nil {
name = abs
}
log.Info("Writing memory profile to ", name)
runtime.GC() // get up-to-date statistics
if err := pprof.WriteHeapProfile(f); err != nil {
log.Error("Could not write memory profile: ", err)
}
_ = f.Close()
}
// filterRestartNumberFromArgs removes flag with a number how many restart we already did.
// See restartApp how that number is used.
func filterRestartNumberFromArgs() {
tmp := os.Args[:0]
for i, arg := range os.Args {
if !strings.HasPrefix(arg, "--restart_") {
tmp = append(tmp, arg)
continue
}
var err error
numberOfCrashes, err = strconv.Atoi(os.Args[i][10:])
if err != nil {
numberOfCrashes = maxAllowedCrashes
}
}
os.Args = tmp
}
// restartApp starts a new instance in background.
func restartApp() {
if numberOfCrashes >= maxAllowedCrashes {
log.Error("Too many crashes")
return
}
if exeFile, err := os.Executable(); err == nil {
arguments := append(os.Args[1:], fmt.Sprintf("--restart_%d", numberOfCrashes))
cmd := exec.Command(exeFile, arguments...) //nolint[gosec]
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Stdin = os.Stdin
if err := cmd.Start(); err != nil {
log.Error("Restart failed: ", err)
}
}
}

194
cmd/Import-Export/main.go
View File

@ -18,108 +18,40 @@
package main
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"runtime"
"runtime/pprof"
"strconv"
"strings"
"github.com/ProtonMail/proton-bridge/internal/cmd"
"github.com/ProtonMail/proton-bridge/internal/events"
"github.com/ProtonMail/proton-bridge/internal/frontend"
"github.com/ProtonMail/proton-bridge/internal/importexport"
"github.com/ProtonMail/proton-bridge/internal/users/credentials"
"github.com/ProtonMail/proton-bridge/pkg/args"
"github.com/ProtonMail/proton-bridge/pkg/config"
"github.com/ProtonMail/proton-bridge/pkg/constants"
"github.com/ProtonMail/proton-bridge/pkg/listener"
"github.com/ProtonMail/proton-bridge/pkg/pmapi"
"github.com/ProtonMail/proton-bridge/pkg/updates"
"github.com/getsentry/raven-go"
"github.com/allan-simon/go-singleinstance"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
)
const (
appName = "importExport"
appNameDash = "import-export"
)
var (
log = logrus.WithField("pkg", "main") //nolint[gochecknoglobals]
// How many crashes in a row.
numberOfCrashes = 0 //nolint[gochecknoglobals]
// After how many crashes import/export gives up starting.
maxAllowedCrashes = 10 //nolint[gochecknoglobals]
)
func main() {
constants.AppShortName = "importExport" //TODO
if err := raven.SetDSN(constants.DSNSentry); err != nil {
log.WithError(err).Errorln("Can not setup sentry DSN")
}
raven.SetRelease(constants.Revision)
args.FilterProcessSerialNumberFromArgs()
filterRestartNumberFromArgs()
app := cli.NewApp()
app.Name = "Protonmail Import/Export"
app.Version = constants.BuildVersion
app.Flags = []cli.Flag{
cli.StringFlag{
Name: "log-level, l",
Usage: "Set the log level (one of panic, fatal, error, warn, info, debug, debug-client, debug-server)"},
cli.BoolFlag{
Name: "cli, c",
Usage: "Use command line interface"},
cli.StringFlag{
Name: "version-json, g",
Usage: "Generate json version file"},
cli.BoolFlag{
Name: "mem-prof, m",
Usage: "Generate memory profile"},
cli.BoolFlag{
Name: "cpu-prof, p",
Usage: "Generate CPU profile"},
}
app.Usage = "ProtonMail Import/Export"
app.Action = run
// Always log the basic info about current import/export.
logrus.SetLevel(logrus.InfoLevel)
log.WithField("version", constants.Version).
WithField("revision", constants.Revision).
WithField("runtime", runtime.GOOS).
WithField("build", constants.BuildTime).
WithField("args", os.Args).
WithField("appLong", app.Name).
WithField("appShort", constants.AppShortName).
Info("Run app")
if err := app.Run(os.Args); err != nil {
log.Error("Program exited with error: ", err)
}
}
type panicHandler struct {
cfg *config.Config
err *error // Pointer to error of cli action.
}
func (ph *panicHandler) HandlePanic() {
r := recover()
if r == nil {
return
}
config.HandlePanic(ph.cfg, fmt.Sprintf("Recover: %v", r))
frontend.HandlePanic("ProtonMail Import-Export")
*ph.err = cli.NewExitError("Panic and restart", 255)
numberOfCrashes++
log.Error("Restarting after panic")
restartApp()
os.Exit(255)
cmd.Main(
"ProtonMail Import/Export",
"ProtonMail Import/Export tool",
nil,
run,
)
}
// run initializes and starts everything in a precise order.
@ -127,13 +59,17 @@ func (ph *panicHandler) HandlePanic() {
// IMPORTANT: ***Read the comments before CHANGING the order ***
func run(context *cli.Context) (contextError error) { // nolint[funlen]
// We need to have config instance to setup a logs, panic handler, etc ...
cfg := config.New(constants.AppShortName, constants.Version, constants.Revision, "")
cfg := config.New(appName, constants.Version, constants.Revision, "")
// We want to know about any problem. Our PanicHandler calls sentry which is
// not dependent on anything else. If that fails, it tries to create crash
// report which will not be possible if no folder can be created. That's the
// only problem we will not be notified about in any way.
panicHandler := &panicHandler{cfg, &contextError}
panicHandler := &cmd.PanicHandler{
AppName: "ProtonMail Import/Export",
Config: cfg,
Err: &contextError,
}
defer panicHandler.HandlePanic()
// First we need config and create necessary folder; it's dependency for everything.
@ -154,21 +90,22 @@ func run(context *cli.Context) (contextError error) { // nolint[funlen]
// It's safe to get version JSON file even when other instance is running.
// (thus we put it before check of presence of other Import/Export instance).
updates := updates.New(
constants.AppShortName,
constants.Version,
constants.Revision,
constants.BuildTime,
importexport.ReleaseNotes,
importexport.ReleaseFixedBugs,
cfg.GetUpdateDir(),
)
updates := updates.NewImportExport(cfg.GetUpdateDir())
if dir := context.GlobalString("version-json"); dir != "" {
generateVersionFiles(updates, dir)
cmd.GenerateVersionFiles(updates, dir)
return nil
}
// Now we can try to proceed with starting the import/export. First we need to ensure
// this is the only instance. If not, we will end and focus the existing one.
lock, err := singleinstance.CreateLockFile(cfg.GetLockPath())
if err != nil {
log.Warn("Import/Export is already running")
return cli.NewExitError("Import/Export is already running.", 3)
}
defer lock.Close() //nolint[errcheck]
// In case user wants to do CPU or memory profiles...
if doCPUProfile := context.GlobalBool("cpu-prof"); doCPUProfile {
f, err := os.Create("cpu.pprof")
@ -182,7 +119,7 @@ func run(context *cli.Context) (contextError error) { // nolint[funlen]
}
if doMemoryProfile := context.GlobalBool("mem-prof"); doMemoryProfile {
defer makeMemoryProfile()
defer cmd.MakeMemoryProfile()
}
// Now we initialize all Import/Export parts.
@ -190,7 +127,7 @@ func run(context *cli.Context) (contextError error) { // nolint[funlen]
eventListener := listener.New()
events.SetupEvents(eventListener)
credentialsStore, credentialsError := credentials.NewStore("import-export")
credentialsStore, credentialsError := credentials.NewStore(appNameDash)
if credentialsError != nil {
log.Error("Could not get credentials store: ", credentialsError)
}
@ -224,73 +161,8 @@ func run(context *cli.Context) (contextError error) { // nolint[funlen]
}
if frontend.IsAppRestarting() {
restartApp()
cmd.RestartApp()
}
return nil
}
// generateVersionFiles writes a JSON file with details about current build.
// Those files are used for upgrading the app.
func generateVersionFiles(updates *updates.Updates, dir string) {
log.Info("Generating version files")
for _, goos := range []string{"windows", "darwin", "linux"} {
log.Debug("Generating JSON for ", goos)
if err := updates.CreateJSONAndSign(dir, goos); err != nil {
log.Error(err)
}
}
}
func makeMemoryProfile() {
name := "./mem.pprof"
f, err := os.Create(name)
if err != nil {
log.Error("Could not create memory profile: ", err)
}
if abs, err := filepath.Abs(name); err == nil {
name = abs
}
log.Info("Writing memory profile to ", name)
runtime.GC() // get up-to-date statistics
if err := pprof.WriteHeapProfile(f); err != nil {
log.Error("Could not write memory profile: ", err)
}
_ = f.Close()
}
// filterRestartNumberFromArgs removes flag with a number how many restart we already did.
// See restartApp how that number is used.
func filterRestartNumberFromArgs() {
tmp := os.Args[:0]
for i, arg := range os.Args {
if !strings.HasPrefix(arg, "--restart_") {
tmp = append(tmp, arg)
continue
}
var err error
numberOfCrashes, err = strconv.Atoi(os.Args[i][10:])
if err != nil {
numberOfCrashes = maxAllowedCrashes
}
}
os.Args = tmp
}
// restartApp starts a new instance in background.
func restartApp() {
if numberOfCrashes >= maxAllowedCrashes {
log.Error("Too many crashes")
return
}
if exeFile, err := os.Executable(); err == nil {
arguments := append(os.Args[1:], fmt.Sprintf("--restart_%d", numberOfCrashes))
cmd := exec.Command(exeFile, arguments...) //nolint[gosec]
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Stdin = os.Stdin
if err := cmd.Start(); err != nil {
log.Error("Restart failed: ", err)
}
}
}