From 1286e57b6371bb9ffdbe53343b9e043b47f3b712 Mon Sep 17 00:00:00 2001 From: Michal Horejsek Date: Thu, 22 Oct 2020 13:46:03 +0200 Subject: [PATCH] Support Apple Mail MBOX export format --- Changelog.md | 1 + internal/transfer/provider_mbox.go | 26 ++++++++++++++++- internal/transfer/provider_mbox_source.go | 8 +++++- internal/transfer/provider_mbox_test.go | 28 ++++++++++++++++++- .../mbox-applemail/All Mail.mbox/mbox | 16 +++++++++++ .../testdata/mbox-applemail/Inbox.mbox/.keep | 0 internal/transfer/utils.go | 19 +++++++++++-- internal/transfer/utils_test.go | 13 ++++++++- 8 files changed, 105 insertions(+), 6 deletions(-) create mode 100644 internal/transfer/testdata/mbox-applemail/All Mail.mbox/mbox create mode 100644 internal/transfer/testdata/mbox-applemail/Inbox.mbox/.keep diff --git a/Changelog.md b/Changelog.md index 3803ad4c..a6973edc 100644 --- a/Changelog.md +++ b/Changelog.md @@ -7,6 +7,7 @@ Changelog [format](http://keepachangelog.com/en/1.0.0/) ### Added * GODT-763 Detect Gmail labels from All Mail mbox export (using X-Gmail-Label header). * GODT-834 Info about tags in BUILDS.md and link to Import-Export page in README.md. +* GODT-777 Support Apple Mail MBOX export format. ### Fixed * GODT-677 Windows IE: global import settings not fit in window. diff --git a/internal/transfer/provider_mbox.go b/internal/transfer/provider_mbox.go index 1d1b9727..8b0d19b5 100644 --- a/internal/transfer/provider_mbox.go +++ b/internal/transfer/provider_mbox.go @@ -18,8 +18,11 @@ package transfer import ( + "os" "path/filepath" "strings" + + "github.com/pkg/errors" ) // MBOXProvider implements import and export to/from MBOX structure. @@ -44,7 +47,7 @@ func (p *MBOXProvider) ID() string { // In case the same folder name is used more than once (for example root/a/foo // and root/b/foo), it's treated as the same folder. func (p *MBOXProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, error) { - filePaths, err := getFilePathsWithSuffix(p.root, "mbox") + filePaths, err := getAllPathsWithSuffix(p.root, ".mbox") if err != nil { return nil, err } @@ -52,6 +55,12 @@ func (p *MBOXProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, mailboxNames := map[string]bool{} for _, filePath := range filePaths { fileName := filepath.Base(filePath) + filePath, err := p.handleAppleMailMBOXStructure(filePath) + if err != nil { + log.WithError(err).Warn("Failed to handle MBOX structure") + continue + } + mailboxName := strings.TrimSuffix(fileName, ".mbox") mailboxNames[mailboxName] = true @@ -76,3 +85,18 @@ func (p *MBOXProvider) Mailboxes(includeEmpty, includeAllMail bool) ([]Mailbox, } return mailboxes, nil } + +// handleAppleMailMBOXStructure changes the path of mailbox directory to +// the path of mbox file. Apple Mail MBOX exports has this structure: +// `Folder.mbox` directory with `mbox` file inside. +// Example: `Folder.mbox/mbox` (and this function converts `Folder.mbox` +// to `Folder.mbox/mbox`). +func (p *MBOXProvider) handleAppleMailMBOXStructure(filePath string) (string, error) { + if info, err := os.Stat(filepath.Join(p.root, filePath)); err == nil && info.IsDir() { + if _, err := os.Stat(filepath.Join(p.root, filePath, "mbox")); err != nil { + return "", errors.Wrap(err, "wrong mbox structure") + } + return filepath.Join(filePath, "mbox"), nil + } + return filePath, nil +} diff --git a/internal/transfer/provider_mbox_source.go b/internal/transfer/provider_mbox_source.go index 59ed267b..5b11ef17 100644 --- a/internal/transfer/provider_mbox_source.go +++ b/internal/transfer/provider_mbox_source.go @@ -67,7 +67,7 @@ func (p *MBOXProvider) TransferTo(rules transferRules, progress *Progress, ch ch } func (p *MBOXProvider) getFilePathsPerFolder() (map[string][]string, error) { - filePaths, err := getFilePathsWithSuffix(p.root, ".mbox") + filePaths, err := getAllPathsWithSuffix(p.root, ".mbox") if err != nil { return nil, err } @@ -75,6 +75,12 @@ func (p *MBOXProvider) getFilePathsPerFolder() (map[string][]string, error) { filePathsMap := map[string][]string{} for _, filePath := range filePaths { fileName := filepath.Base(filePath) + filePath, err := p.handleAppleMailMBOXStructure(filePath) + // Skip unsupported MBOX structures. It was already filtered out in configuration step. + if err != nil { + continue + } + folder := strings.TrimSuffix(fileName, ".mbox") filePathsMap[folder] = append(filePathsMap[folder], filePath) } diff --git a/internal/transfer/provider_mbox_test.go b/internal/transfer/provider_mbox_test.go index 56aa196d..acd54406 100644 --- a/internal/transfer/provider_mbox_test.go +++ b/internal/transfer/provider_mbox_test.go @@ -52,6 +52,11 @@ func TestMBOXProviderMailboxes(t *testing.T) { {Name: "Bar"}, {Name: "Inbox"}, }}, + {newTestMBOXProvider("testdata/mbox-applemail"), true, []Mailbox{ + {Name: "All Mail"}, + {Name: "Foo"}, + {Name: "Bar"}, + }}, } for _, tc := range tests { tc := tc @@ -88,6 +93,27 @@ func TestMBOXProviderTransferTo(t *testing.T) { }, got) } +func TestMBOXProviderTransferToAppleMail(t *testing.T) { + provider := newTestMBOXProvider("testdata/mbox-applemail") + + rules, rulesClose := newTestRules(t) + defer rulesClose() + setupMBOXRules(rules) + + msgs := testTransferTo(t, rules, provider, []string{ + "All Mail.mbox/mbox:1", + "All Mail.mbox/mbox:2", + }) + got := map[string][]string{} + for _, msg := range msgs { + got[msg.ID] = msg.targetNames() + } + r.Equal(t, map[string][]string{ + "All Mail.mbox/mbox:1": {"Archive", "Foo"}, // Bar is not in rules. + "All Mail.mbox/mbox:2": {"Archive", "Foo"}, + }, got) +} + func TestMBOXProviderTransferFrom(t *testing.T) { dir, err := ioutil.TempDir("", "mbox") r.NoError(t, err) @@ -168,7 +194,7 @@ func setupMBOXRules(rules transferRules) { } func checkMBOXFileStructure(t *testing.T, root string, expectedFiles []string) { - files, err := getFilePathsWithSuffix(root, ".mbox") + files, err := getAllPathsWithSuffix(root, ".mbox") r.NoError(t, err) r.Equal(t, expectedFiles, files) } diff --git a/internal/transfer/testdata/mbox-applemail/All Mail.mbox/mbox b/internal/transfer/testdata/mbox-applemail/All Mail.mbox/mbox new file mode 100644 index 00000000..2e758e86 --- /dev/null +++ b/internal/transfer/testdata/mbox-applemail/All Mail.mbox/mbox @@ -0,0 +1,16 @@ +From - Mon May 4 16:40:31 2020 +From: Bridge Test +To: Bridge Test +Subject: Test 1 +X-Gmail-Labels: Foo,Bar + +hello + + +From - Mon May 4 16:40:31 2020 +From: Bridge Test +To: Bridge Test +Subject: Test 2 +X-Gmail-Labels: Foo + +hello diff --git a/internal/transfer/testdata/mbox-applemail/Inbox.mbox/.keep b/internal/transfer/testdata/mbox-applemail/Inbox.mbox/.keep new file mode 100644 index 00000000..e69de29b diff --git a/internal/transfer/utils.go b/internal/transfer/utils.go index afa71c7d..aa7f67b6 100644 --- a/internal/transfer/utils.go +++ b/internal/transfer/utils.go @@ -82,7 +82,7 @@ func getFolderNamesWithFileSuffix(root, fileSuffix string) ([]string, error) { // getFilePathsWithSuffix collects all file names with `suffix` under `root`. // File names will be with relative path based to `root`. func getFilePathsWithSuffix(root, suffix string) ([]string, error) { - fileNames, err := getFilePathsWithSuffixInner("", root, suffix) + fileNames, err := getFilePathsWithSuffixInner("", root, suffix, false) if err != nil { return nil, err } @@ -90,7 +90,18 @@ func getFilePathsWithSuffix(root, suffix string) ([]string, error) { return fileNames, err } -func getFilePathsWithSuffixInner(prefix, root, suffix string) ([]string, error) { +// getAllPathsWithSuffix is the same as getFilePathsWithSuffix but includes +// also directories. +func getAllPathsWithSuffix(root, suffix string) ([]string, error) { + fileNames, err := getFilePathsWithSuffixInner("", root, suffix, true) + if err != nil { + return nil, err + } + sort.Strings(fileNames) + return fileNames, err +} + +func getFilePathsWithSuffixInner(prefix, root, suffix string, includeDir bool) ([]string, error) { fileNames := []string{} files, err := ioutil.ReadDir(root) @@ -104,10 +115,14 @@ func getFilePathsWithSuffixInner(prefix, root, suffix string) ([]string, error) fileNames = append(fileNames, filepath.Join(prefix, file.Name())) } } else { + if includeDir && strings.HasSuffix(file.Name(), suffix) { + fileNames = append(fileNames, filepath.Join(prefix, file.Name())) + } subfolderFileNames, err := getFilePathsWithSuffixInner( filepath.Join(prefix, file.Name()), filepath.Join(root, file.Name()), suffix, + includeDir, ) if err != nil { return nil, err diff --git a/internal/transfer/utils_test.go b/internal/transfer/utils_test.go index 1fa841ce..27045bcb 100644 --- a/internal/transfer/utils_test.go +++ b/internal/transfer/utils_test.go @@ -39,6 +39,7 @@ func TestGetFolderNames(t *testing.T) { "", []string{ "bar", + "bar.mbox", "baz", filepath.Base(root), "foo", @@ -95,6 +96,13 @@ func TestGetFilePathsWithSuffix(t *testing.T) { "test/foo/msg9.eml", }, }, + { + ".mbox", + []string{ + "bar.mbox", + "foo.mbox", + }, + }, { ".txt", []string{ @@ -109,7 +117,7 @@ func TestGetFilePathsWithSuffix(t *testing.T) { for _, tc := range tests { tc := tc t.Run(tc.suffix, func(t *testing.T) { - paths, err := getFilePathsWithSuffix(root, tc.suffix) + paths, err := getAllPathsWithSuffix(root, tc.suffix) r.NoError(t, err) r.Equal(t, tc.wantPaths, paths) }) @@ -125,6 +133,7 @@ func createTestingFolderStructure(t *testing.T) (string, func()) { "foo/baz", "test/foo", "qwerty", + "bar.mbox", } { err = os.MkdirAll(filepath.Join(root, path), os.ModePerm) r.NoError(t, err) @@ -142,6 +151,8 @@ func createTestingFolderStructure(t *testing.T) (string, func()) { "test/foo/msg9.eml", "msg10.eml", "info.txt", + "foo.mbox", + "bar.mbox/mbox", // Apple Mail mbox export format. } { f, err := os.Create(filepath.Join(root, path)) r.NoError(t, err)