GODT-1158: Store full messages bodies on disk

- GODT-1158: simple on-disk cache in store
- GODT-1158: better member naming in event loop
- GODT-1158: create on-disk cache during bridge setup
- GODT-1158: better job options
- GODT-1158: rename GetLiteral to GetRFC822
- GODT-1158: rename events -> currentEvents
- GODT-1158: unlock cache per-user
- GODT-1158: clean up cache after logout
- GODT-1158: randomized encrypted cache passphrase
- GODT-1158: Opt out of on-disk cache in settings
- GODT-1158: free space in cache
- GODT-1158: make tests compile
- GODT-1158: optional compression
- GODT-1158: cache custom location
- GODT-1158: basic capacity checker
- GODT-1158: cache free space config
- GODT-1158: only unlock cache if pmapi client is unlocked as well
- GODT-1158: simple background sync worker
- GODT-1158: set size/bodystructure when caching message
- GODT-1158: limit store db update blocking with semaphore
- GODT-1158: dumb 10-semaphore
- GODT-1158: properly handle delete; remove bad bodystructure handling
- GODT-1158: hacky fix for caching after logout... baaaaad
- GODT-1158: cache worker
- GODT-1158: compute body structure lazily
- GODT-1158: cache size in store
- GODT-1158: notify cacher when adding to store
- GODT-1158: 15 second store cache watcher
- GODT-1158: enable cacher
- GODT-1158: better cache worker starting/stopping
- GODT-1158: limit cacher to less concurrency than disk cache
- GODT-1158: message builder prio + pchan pkg
- GODT-1158: fix pchan, use in message builder
- GODT-1158: no sem in cacher (rely on message builder prio)
- GODT-1158: raise priority of existing jobs when requested
- GODT-1158: pending messages in on-disk cache
- GODT-1158: WIP just a note about deleting messages from disk cache
- GODT-1158: pending wait when trying to write
- GODT-1158: pending.add to return bool
- GODT-1225: Headers in bodystructure are stored as bytes.
- GODT-1158: fixing header caching
- GODT-1158: don't cache in background
- GODT-1158: all concurrency set in settings
- GODT-1158: worker pools inside message builder
- GODT-1158: fix linter issues
- GODT-1158: remove completed builds from builder
- GODT-1158: remove builder pool
- GODT-1158: cacher defer job done properly
- GODT-1158: fix linter
- GODT-1299: Continue with bodystructure build if deserialization failed
- GODT-1324: Delete messages from the cache when they are deleted on the server
- GODT-1158: refactor cache tests
- GODT-1158: move builder to app/bridge
- GODT-1306: Migrate cache on disk when location is changed (and delete when disabled)

pkg/message/section_test.go

@@ -21,7 +21,6 @@ import (
 	"bytes"
 	"fmt"
 	"io/ioutil"
-	"net/textproto"
 	"path/filepath"
 	"runtime"
 	"sort"
@@ -71,7 +70,9 @@ func TestParseBodyStructure(t *testing.T) {
 	debug("%10s: %-50s %5s %5s %5s %5s", "section", "type", "start", "size", "bsize", "lines")
 	for _, path := range paths {
 		sec := (*bs)[path]
-		contentType := (*bs)[path].Header.Get("Content-Type")
+		header, err := sec.GetMIMEHeader()
+		require.NoError(t, err)
+		contentType := header.Get("Content-Type")
 		debug("%10s: %-50s %5d %5d %5d %5d", path, contentType, sec.Start, sec.Size, sec.BSize, sec.Lines)
 		require.Equal(t, expectedStructure[path], contentType)
 	}
@@ -100,7 +101,9 @@ func TestParseBodyStructurePGP(t *testing.T) {
 
 	haveStructure := map[string]string{}
 	for path := range *bs {
-		haveStructure[path] = (*bs)[path].Header.Get("Content-Type")
+		header, err := (*bs)[path].GetMIMEHeader()
+		require.NoError(t, err)
+		haveStructure[path] = header.Get("Content-Type")
 	}
 
 	require.Equal(t, expectedStructure, haveStructure)
@@ -192,7 +195,7 @@ Content-Type: plain/text
 		r.NoError(err, debug(wantPath, info, haveBody))
 		r.Equal(wantBody, string(haveBody), debug(wantPath, info, haveBody))
 
-		haveHeader, err := bs.GetSectionHeaderBytes(strings.NewReader(wantMail), wantPath)
+		haveHeader, err := bs.GetSectionHeaderBytes(wantPath)
 		r.NoError(err, debug(wantPath, info, haveHeader))
 		r.Equal(wantHeader, string(haveHeader), debug(wantPath, info, haveHeader))
 	}
@@ -211,7 +214,7 @@ Content-Type: multipart/mixed; boundary="0000MAIN"
 	bs, err := NewBodyStructure(structReader)
 	require.NoError(t, err)
 
-	haveHeader, err := bs.GetMailHeaderBytes(strings.NewReader(sampleMail))
+	haveHeader, err := bs.GetMailHeaderBytes()
 	require.NoError(t, err)
 	require.Equal(t, wantHeader, haveHeader)
 }
@@ -533,18 +536,14 @@ func TestBodyStructureSerialize(t *testing.T) {
 	r := require.New(t)
 	want := &BodyStructure{
 		"1": {
-			Header: textproto.MIMEHeader{
-				"Content": []string{"type"},
-			},
-			Start: 1,
-			Size:  2,
-			BSize: 3,
-			Lines: 4,
+			Header: []byte("Content: type"),
+			Start:  1,
+			Size:   2,
+			BSize:  3,
+			Lines:  4,
 		},
 		"1.1.1": {
-			Header: textproto.MIMEHeader{
-				"X-Pm-Key": []string{"id"},
-			},
+			Header: []byte("X-Pm-Key: id"),
 			Start:  11,
 			Size:   12,
 			BSize:  13,
@@ -562,3 +561,32 @@ func TestBodyStructureSerialize(t *testing.T) {
 	(*want)["1.1.1"].reader = nil
 	r.Equal(want, have)
 }
+
+func TestSectionInfoReadHeader(t *testing.T) {
+	r := require.New(t)
+
+	testData := []struct {
+		wantHeader, mail string
+	}{
+		{
+			"key1: val1\nkey2: val2\n\n",
+			"key1: val1\nkey2: val2\n\nbody is here\n\nand it is not confused",
+		},
+		{
+			"key1:\n val1\n\n",
+			"key1:\n val1\n\nbody is here",
+		},
+		{
+			"key1: val1\r\nkey2: val2\r\n\r\n",
+			"key1: val1\r\nkey2: val2\r\n\r\nbody is here\r\n\r\nand it is not confused",
+		},
+	}
+
+	for _, td := range testData {
+		bs, err := NewBodyStructure(strings.NewReader(td.mail))
+		r.NoError(err, "case %q", td.mail)
+		haveHeader, err := bs.GetMailHeaderBytes()
+		r.NoError(err, "case %q", td.mail)
+		r.Equal(td.wantHeader, string(haveHeader), "case %q", td.mail)
+	}
+}