diff --git a/cmd/devserver/main.go b/cmd/devserver/main.go index fed9eb7..5364c3f 100644 --- a/cmd/devserver/main.go +++ b/cmd/devserver/main.go @@ -5,6 +5,11 @@ // so `go run ./cmd/devserver` works with the standard Go toolchain and exercises // the exact same handler that the deployed Wasm Worker serves. It listens on // :8787 by default (matching wrangler dev's default port); override with ADDR. +// +// The ingest path is wired with the real scrub+encrypt storage Sink (#9) backed +// by an in-memory object store and a throwaway per-run AES-256 key, so a POST +// /v1/reports exercises the full pipeline locally. Stored objects live only for +// the process lifetime. package main import ( @@ -12,7 +17,9 @@ import ( "net/http" "os" + "github.com/JMR-dev/LibreMail-Bug-Report-Ingest/internal/crypto" "github.com/JMR-dev/LibreMail-Bug-Report-Ingest/internal/handler" + "github.com/JMR-dev/LibreMail-Bug-Report-Ingest/internal/storage" ) func main() { @@ -21,8 +28,20 @@ func main() { addr = ":8787" } + // A throwaway keyring: a single random key generated at startup. Reports are + // scrubbed, encrypted under it, and held in memory; nothing is persisted. + key, err := crypto.GenerateKey() + if err != nil { + log.Fatalf("devserver: generate key: %v", err) + } + keyring, err := crypto.NewKeyring(1, map[uint16][]byte{1: key}) + if err != nil { + log.Fatalf("devserver: build keyring: %v", err) + } + sink := storage.NewSink(storage.NewMemoryStore(), keyring) + log.Printf("devserver listening on %s (try GET / and GET /healthz)", addr) - if err := http.ListenAndServe(addr, handler.New()); err != nil { + if err := http.ListenAndServe(addr, handler.New(sink)); err != nil { log.Fatalf("devserver: %v", err) } } diff --git a/internal/crypto/crypto_test.go b/internal/crypto/crypto_test.go new file mode 100644 index 0000000..77cba68 --- /dev/null +++ b/internal/crypto/crypto_test.go @@ -0,0 +1,401 @@ +package crypto + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "encoding/base64" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "strings" + "testing" +) + +// fixedKey returns a deterministic 32-byte key whose bytes are seed+i, for +// reproducible test vectors (never used outside tests). +func fixedKey(seed byte) []byte { + k := make([]byte, KeySize) + for i := range k { + k[i] = seed + byte(i) + } + return k +} + +func mustKeyring(t *testing.T, active uint16, keys map[uint16][]byte) *Keyring { + t.Helper() + kr, err := NewKeyring(active, keys) + if err != nil { + t.Fatalf("NewKeyring: %v", err) + } + return kr +} + +// TestSealOpenRoundtrip covers the happy path: Open(Seal(x)) == x, and the +// ciphertext is not the plaintext. +func TestSealOpenRoundtrip(t *testing.T) { + kr := mustKeyring(t, 1, map[uint16][]byte{1: fixedKey(0)}) + for _, pt := range [][]byte{ + []byte(""), + []byte("x"), + []byte("a scrubbed bug report with [REDACTED_EMAIL] inside"), + bytes.Repeat([]byte("A"), 4096), + } { + sealed, err := Seal(kr, pt) + if err != nil { + t.Fatalf("Seal(%d bytes): %v", len(pt), err) + } + if len(pt) > 0 && bytes.Contains(sealed, pt) { + t.Errorf("sealed frame contains the plaintext verbatim (len %d)", len(pt)) + } + got, err := Open(kr, sealed) + if err != nil { + t.Fatalf("Open: %v", err) + } + if !bytes.Equal(got, pt) { + t.Errorf("roundtrip mismatch: got %q want %q", got, pt) + } + } +} + +// TestWireLayout locks the ADR #5 byte layout of a sealed frame. +func TestWireLayout(t *testing.T) { + kr := mustKeyring(t, 0xABCD, map[uint16][]byte{0xABCD: fixedKey(3)}) + nonce := bytes.Repeat([]byte{0x5A}, NonceSize) + pt := []byte("payload") + + frame, err := sealWithNonce(kr, pt, nonce) + if err != nil { + t.Fatalf("sealWithNonce: %v", err) + } + + if got := string(frame[:4]); got != Magic { + t.Errorf("magic = %q, want %q", got, Magic) + } + if frame[4] != FormatVersion { + t.Errorf("version = 0x%02x, want 0x%02x", frame[4], FormatVersion) + } + if id := binary.BigEndian.Uint16(frame[5:7]); id != 0xABCD { + t.Errorf("key_id = 0x%04x, want 0xABCD", id) + } + if got := frame[7:19]; !bytes.Equal(got, nonce) { + t.Errorf("nonce = %x, want %x", got, nonce) + } + // header(7) + nonce(12) + ciphertext(len pt) + tag(16) + wantLen := HeaderSize + NonceSize + len(pt) + TagSize + if len(frame) != wantLen { + t.Errorf("frame len = %d, want %d", len(frame), wantLen) + } + if id, err := KeyID(frame); err != nil || id != 0xABCD { + t.Errorf("KeyID = 0x%04x, err=%v; want 0xABCD, nil", id, err) + } +} + +// TestKnownAnswerVector is a byte-exact format lock. The expected frame was +// computed from Go's standard AES-256-GCM; because AES-256-GCM is deterministic +// and standardised, the Wasm SubtleCrypto provider MUST produce these same bytes +// for the same key+nonce+plaintext+AAD. If this changes, the on-disk format (and +// cross-provider compatibility) changed. +func TestKnownAnswerVector(t *testing.T) { + key := fixedKey(0) // 0x00,0x01,...,0x1f + kr := mustKeyring(t, 1, map[uint16][]byte{1: key}) + nonce := make([]byte, NonceSize) // 0x00..0x0b + for i := range nonce { + nonce[i] = byte(i) + } + pt := []byte("hello world") + + const wantHex = "4c4d4231010001000102030405060708090a0b2f67ba77aac5b574ff2df3f26c5bd31758566cf1bf14ae15f8fd7a" + frame, err := sealWithNonce(kr, pt, nonce) + if err != nil { + t.Fatalf("sealWithNonce: %v", err) + } + if got := hex.EncodeToString(frame); got != wantHex { + t.Errorf("known-answer frame mismatch:\n got = %s\n want = %s", got, wantHex) + } + // And it must still open. + got, err := Open(kr, frame) + if err != nil || !bytes.Equal(got, pt) { + t.Errorf("Open(known frame) = %q, %v; want %q, nil", got, err, pt) + } +} + +// TestSealMatchesStdlibGCM proves the package frames *standard* AES-256-GCM: the +// sealed body equals an independent crypto/cipher GCM computation over the same +// key, nonce, plaintext, and header-as-AAD. This is the provider-independent +// contract the SubtleCrypto build also satisfies. +func TestSealMatchesStdlibGCM(t *testing.T) { + key := fixedKey(9) + const keyID = 7 + kr := mustKeyring(t, keyID, map[uint16][]byte{keyID: key}) + nonce := bytes.Repeat([]byte{0x11}, NonceSize) + pt := []byte("some plaintext to seal") + + got, err := sealWithNonce(kr, pt, nonce) + if err != nil { + t.Fatalf("sealWithNonce: %v", err) + } + + block, err := aes.NewCipher(key) + if err != nil { + t.Fatalf("aes.NewCipher: %v", err) + } + gcm, err := cipher.NewGCM(block) + if err != nil { + t.Fatalf("cipher.NewGCM: %v", err) + } + hdr := append([]byte(Magic), FormatVersion, 0x00, keyID) + ctTag := gcm.Seal(nil, nonce, pt, hdr) + want := append(append(append([]byte{}, hdr...), nonce...), ctTag...) + + if !bytes.Equal(got, want) { + t.Errorf("framing differs from standard AES-256-GCM:\n got = %x\n want = %x", got, want) + } +} + +// TestOpenWrongKeyFails: an object is not decryptable without the correct key. +func TestOpenWrongKeyFails(t *testing.T) { + sealKR := mustKeyring(t, 1, map[uint16][]byte{1: fixedKey(0)}) + // Different key material, SAME key_id, so parsing succeeds and only the + // cryptographic check can reject it. + wrongKR := mustKeyring(t, 1, map[uint16][]byte{1: fixedKey(100)}) + + sealed, err := Seal(sealKR, []byte("secret residual PII")) + if err != nil { + t.Fatalf("Seal: %v", err) + } + if _, err := Open(wrongKR, sealed); !errors.Is(err, ErrAuth) { + t.Errorf("Open with wrong key: err = %v, want ErrAuth", err) + } + // Sanity: the right key still works. + if _, err := Open(sealKR, sealed); err != nil { + t.Errorf("Open with correct key failed: %v", err) + } +} + +// TestTamperDetection: any modification to the frame makes Open fail (GCM +// authenticates ciphertext, tag, nonce, and the header via AAD). +func TestTamperDetection(t *testing.T) { + kr := mustKeyring(t, 1, map[uint16][]byte{1: fixedKey(0)}) + sealed, err := Seal(kr, []byte("tamper target payload")) + if err != nil { + t.Fatalf("Seal: %v", err) + } + + cases := []struct { + name string + offset int + wantIs error // nil means "any non-nil error" + }{ + {"magic", 0, ErrBadMagic}, + {"version", 4, ErrUnsupportedVersion}, + {"key_id", 6, ErrUnknownKeyID}, // 1 -> some absent version + {"nonce", nonceOffset, ErrAuth}, + {"ciphertext", bodyOffset, ErrAuth}, + {"tag", len(sealed) - 1, ErrAuth}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + bad := append([]byte(nil), sealed...) + bad[tc.offset] ^= 0xFF + _, err := Open(kr, bad) + if err == nil { + t.Fatalf("tampering %s produced no error", tc.name) + } + if tc.wantIs != nil && !errors.Is(err, tc.wantIs) { + t.Errorf("tampering %s: err = %v, want %v", tc.name, err, tc.wantIs) + } + }) + } +} + +// TestHeaderIsAuthenticated proves the key_id in the header is bound by AAD: an +// attacker cannot relabel a frame to a different, valid key version. +func TestHeaderIsAuthenticated(t *testing.T) { + // Keyring holds two versions; seal under version 1. + kr := mustKeyring(t, 1, map[uint16][]byte{1: fixedKey(0), 2: fixedKey(50)}) + sealed, err := Seal(kr, []byte("bind me")) + if err != nil { + t.Fatalf("Seal: %v", err) + } + // Rewrite the stored key_id from 1 to 2 (a version that DOES exist). + relabeled := append([]byte(nil), sealed...) + binary.BigEndian.PutUint16(relabeled[keyIDOffset:HeaderSize], 2) + if _, err := Open(kr, relabeled); !errors.Is(err, ErrAuth) { + t.Errorf("relabeled key_id: err = %v, want ErrAuth (header must be authenticated)", err) + } +} + +// TestRotationRetainsOldKeys: an object sealed under key_id N still opens after +// rotation, as long as the keyring retains version N. +func TestRotationRetainsOldKeys(t *testing.T) { + key1 := fixedKey(0) + key2 := fixedKey(50) + + // Before rotation: active = 1. + before := mustKeyring(t, 1, map[uint16][]byte{1: key1}) + sealed, err := Seal(before, []byte("pre-rotation report")) + if err != nil { + t.Fatalf("Seal: %v", err) + } + if id, _ := KeyID(sealed); id != 1 { + t.Fatalf("sealed key_id = %d, want 1", id) + } + + // After rotation: active = 2, but version 1 is RETAINED. + after := mustKeyring(t, 2, map[uint16][]byte{1: key1, 2: key2}) + got, err := Open(after, sealed) + if err != nil { + t.Fatalf("Open after rotation (retained key 1): %v", err) + } + if want := []byte("pre-rotation report"); !bytes.Equal(got, want) { + t.Errorf("post-rotation decrypt = %q, want %q", got, want) + } + + // New writes now use key_id 2. + sealed2, err := Seal(after, []byte("post-rotation report")) + if err != nil { + t.Fatalf("Seal (post-rotation): %v", err) + } + if id, _ := KeyID(sealed2); id != 2 { + t.Errorf("post-rotation sealed key_id = %d, want 2", id) + } + + // If version 1 is RETIRED (removed), the old object is unrecoverable. + retired := mustKeyring(t, 2, map[uint16][]byte{2: key2}) + if _, err := Open(retired, sealed); !errors.Is(err, ErrUnknownKeyID) { + t.Errorf("Open with retired key_id: err = %v, want ErrUnknownKeyID", err) + } +} + +// TestOpenMalformed covers frame-structure rejections. +func TestOpenMalformed(t *testing.T) { + kr := mustKeyring(t, 1, map[uint16][]byte{1: fixedKey(0)}) + valid, err := Seal(kr, []byte("ok")) + if err != nil { + t.Fatalf("Seal: %v", err) + } + + tests := []struct { + name string + object []byte + wantIs error + }{ + {"empty", nil, ErrMalformed}, + {"too short", valid[:minObjectLen-1], ErrMalformed}, + {"bad magic", func() []byte { b := append([]byte(nil), valid...); b[0] = 'X'; return b }(), ErrBadMagic}, + {"bad version", func() []byte { b := append([]byte(nil), valid...); b[4] = 0x02; return b }(), ErrUnsupportedVersion}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if _, err := Open(kr, tc.object); !errors.Is(err, tc.wantIs) { + t.Errorf("Open(%s): err = %v, want %v", tc.name, err, tc.wantIs) + } + }) + } +} + +// TestParseKeyring parses the ADR #5 Secrets Store JSON shape and roundtrips. +func TestParseKeyring(t *testing.T) { + key1 := fixedKey(0) + key2 := fixedKey(200) + raw := fmt.Sprintf(`{"active":2,"keys":{"1":%q,"2":%q}}`, + base64.StdEncoding.EncodeToString(key1), + base64.StdEncoding.EncodeToString(key2)) + + kr, err := ParseKeyring([]byte(raw)) + if err != nil { + t.Fatalf("ParseKeyring: %v", err) + } + if kr.Active() != 2 { + t.Errorf("active = %d, want 2", kr.Active()) + } + // New objects use active=2; an object made with the parsed keyring opens with + // an equivalent hand-built keyring, proving the bytes decoded correctly. + sealed, err := Seal(kr, []byte("via parsed keyring")) + if err != nil { + t.Fatalf("Seal: %v", err) + } + ref := mustKeyring(t, 2, map[uint16][]byte{1: key1, 2: key2}) + if got, err := Open(ref, sealed); err != nil || string(got) != "via parsed keyring" { + t.Errorf("Open via reference keyring = %q, %v", got, err) + } +} + +func TestParseKeyringErrors(t *testing.T) { + shortKey := base64.StdEncoding.EncodeToString(make([]byte, 16)) // 16 bytes, not 32 + fullKey := base64.StdEncoding.EncodeToString(fixedKey(0)) + cases := map[string]string{ + "not json": `{`, + "no keys": `{"active":1,"keys":{}}`, + "bad base64": `{"active":1,"keys":{"1":"not@@base64"}}`, + "wrong key length": fmt.Sprintf(`{"active":1,"keys":{"1":%q}}`, shortKey), + "active absent": fmt.Sprintf(`{"active":9,"keys":{"1":%q}}`, fullKey), + "bad version": fmt.Sprintf(`{"active":1,"keys":{"nope":%q}}`, fullKey), + } + for name, raw := range cases { + t.Run(name, func(t *testing.T) { + if _, err := ParseKeyring([]byte(raw)); err == nil { + t.Errorf("ParseKeyring(%s) = nil error, want failure", name) + } + }) + } +} + +func TestGenerateKey(t *testing.T) { + a, err := GenerateKey() + if err != nil { + t.Fatalf("GenerateKey: %v", err) + } + if len(a) != KeySize { + t.Errorf("key length = %d, want %d", len(a), KeySize) + } + b, err := GenerateKey() + if err != nil { + t.Fatalf("GenerateKey: %v", err) + } + if bytes.Equal(a, b) { + t.Error("two generated keys are identical; CSPRNG suspect") + } +} + +func TestNewKeyringValidation(t *testing.T) { + if _, err := NewKeyring(1, nil); err == nil { + t.Error("empty keyring: want error") + } + if _, err := NewKeyring(1, map[uint16][]byte{1: make([]byte, 8)}); err == nil { + t.Error("short key: want error") + } + if _, err := NewKeyring(5, map[uint16][]byte{1: fixedKey(0)}); err == nil { + t.Error("active not present: want error") + } + // NewKeyring must copy its input (mutating the caller's slice must not change + // the keyring). + src := fixedKey(0) + kr := mustKeyring(t, 1, map[uint16][]byte{1: src}) + for i := range src { + src[i] = 0xEE + } + sealed, err := Seal(kr, []byte("copy check")) + if err != nil { + t.Fatalf("Seal: %v", err) + } + if got, err := Open(kr, sealed); err != nil || string(got) != "copy check" { + t.Errorf("keyring did not copy key material: %q, %v", got, err) + } +} + +// TestNoPlaintextLeak is a belt-and-braces check that a sensitive-looking +// plaintext does not appear anywhere in the sealed frame. +func TestNoPlaintextLeak(t *testing.T) { + kr := mustKeyring(t, 1, map[uint16][]byte{1: fixedKey(0)}) + secret := "MARKER-plaintext-must-not-survive-encryption" + sealed, err := Seal(kr, []byte("report body: "+secret)) + if err != nil { + t.Fatalf("Seal: %v", err) + } + if strings.Contains(string(sealed), secret) { + t.Error("sealed frame leaks the plaintext secret") + } +} diff --git a/internal/crypto/frame.go b/internal/crypto/frame.go new file mode 100644 index 0000000..78035d6 --- /dev/null +++ b/internal/crypto/frame.go @@ -0,0 +1,284 @@ +// Package crypto implements the encrypted-at-rest object format for scrubbed +// LibreMail bug-reports, exactly as fixed by ADR #5 +// (docs/decisions/encryption.md): Worker-side AES-256-GCM authenticated +// encryption applied before the object is written to R2, with the key selected +// from a versioned keyring held in Cloudflare Secrets Store. +// +// # Wire format (the contract) +// +// Each object body is one self-describing binary frame: +// +// offset size field +// ------ ---- ------------------------------------------------------------- +// 0 4 magic = ASCII "LMB1" +// 4 1 format_version = 0x01 +// 5 2 key_id = uint16, big-endian (keyring version used) +// 7 12 nonce = 96-bit random IV (CSPRNG, unique per object) +// 19 N ciphertext = AES-256-GCM(plaintext) +// 19+N 16 auth_tag = 128-bit GCM tag (appended to the ciphertext) +// +// The 7-byte header (magic || version || key_id) is passed to GCM as the +// additional authenticated data (AAD): it is stored in the clear but is +// authenticated, so an attacker cannot flip the key_id, downgrade the format, or +// transplant a body under a different header without failing authentication. +// +// # Provider-independent by design +// +// The framing in this file carries no build constraints and is shared by both +// crypto providers. The raw AES-256-GCM primitive is the only part that differs: +// +// - gcm_host.go (build tag !(js && wasm)) uses Go's crypto/aes + crypto/cipher. +// It backs go test, cmd/devserver, and any non-Wasm build. +// - gcm_wasm.go (build tag js && wasm) uses the Workers runtime's Web Crypto +// (SubtleCrypto) via syscall/js, per the ADR's TinyGo/Wasm recommendation. +// +// AES-256-GCM is deterministic for a given key, nonce, plaintext and AAD, so both +// providers produce byte-identical frames. The wire format above — not the +// provider — is the contract, which is why an object sealed by one can be opened +// by the other. +package crypto + +import ( + "crypto/rand" + "encoding/base64" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "strconv" +) + +// Wire-format constants. These are load-bearing: they are the ADR #5 object +// layout and must not change without a format_version bump. +const ( + // Magic is the 4-byte frame magic, ASCII "LMB1". + Magic = "LMB1" + // FormatVersion is the current frame format version. + FormatVersion byte = 0x01 + // KeySize is the AES-256 key length in bytes. + KeySize = 32 + // NonceSize is the GCM nonce/IV length in bytes (96 bits). + NonceSize = 12 + // TagSize is the GCM authentication tag length in bytes (128 bits). + TagSize = 16 + // HeaderSize is the length of the authenticated header (magic||version||key_id). + HeaderSize = 7 + + magicLen = 4 + keyIDOffset = 5 // magic(4) + version(1) + nonceOffset = HeaderSize + bodyOffset = HeaderSize + NonceSize // 19: start of ciphertext||tag + // minObjectLen is the smallest possible valid frame: header + nonce + tag, + // i.e. an empty-plaintext object (ciphertext length 0). + minObjectLen = HeaderSize + NonceSize + TagSize +) + +// Sentinel errors returned by Open. Callers MUST treat any of them as a hard +// failure and never fall back to "publish what we have" (ADR #5, Integrity). +var ( + // ErrMalformed means the object is too short to be a valid frame. + ErrMalformed = errors.New("crypto: object too short or malformed") + // ErrBadMagic means the leading 4 bytes are not the "LMB1" magic. + ErrBadMagic = errors.New("crypto: bad magic") + // ErrUnsupportedVersion means format_version is not one this build understands. + ErrUnsupportedVersion = errors.New("crypto: unsupported format version") + // ErrUnknownKeyID means the frame's key_id is not present in the keyring. + // Never remove a key version while stored objects still reference it. + ErrUnknownKeyID = errors.New("crypto: unknown key_id") + // ErrAuth means GCM authentication failed: the ciphertext, tag, nonce, or + // authenticated header was modified, or the wrong key was supplied. + ErrAuth = errors.New("crypto: authentication failed") +) + +// aead is the raw AES-256-GCM primitive seam. Exactly one implementation is +// compiled in per build target (host or Wasm); both MUST produce byte-identical +// output for identical inputs. +type aead interface { + // seal returns ciphertext||tag for plaintext under key+nonce, authenticating + // aad. key is 32 bytes, nonce is 12 bytes. + seal(key, nonce, plaintext, aad []byte) ([]byte, error) + // open returns the plaintext for ciphertextAndTag under key+nonce, verifying + // aad. A non-nil error means authentication failed. + open(key, nonce, ciphertextAndTag, aad []byte) ([]byte, error) +} + +// primitive is the AES-256-GCM provider, set in the init of the build-tagged +// provider file (gcm_host.go or gcm_wasm.go). +var primitive aead + +// Keyring is a parsed, in-memory versioned keyring: a set of AES-256 keys indexed +// by version (the key_id written into each frame) plus the active version used to +// encrypt new objects. Retaining superseded versions is what makes rotation +// data-loss-free (ADR #5, Key rotation). +type Keyring struct { + active uint16 + keys map[uint16][]byte +} + +// NewKeyring builds a Keyring from an active version and a version->key map. Each +// key must be exactly KeySize (32) bytes, and active must exist in keys. The keys +// are copied, so the caller may reuse its map. +func NewKeyring(active uint16, keys map[uint16][]byte) (*Keyring, error) { + if len(keys) == 0 { + return nil, errors.New("crypto: keyring has no keys") + } + cp := make(map[uint16][]byte, len(keys)) + for id, k := range keys { + if len(k) != KeySize { + return nil, fmt.Errorf("crypto: key %d must be %d bytes, got %d", id, KeySize, len(k)) + } + dup := make([]byte, KeySize) + copy(dup, k) + cp[id] = dup + } + if _, ok := cp[active]; !ok { + return nil, fmt.Errorf("crypto: active version %d not present in keyring", active) + } + return &Keyring{active: active, keys: cp}, nil +} + +// Active returns the version new objects are encrypted under. +func (kr *Keyring) Active() uint16 { return kr.active } + +// activeKey returns the active key and its version. +func (kr *Keyring) activeKey() ([]byte, uint16) { + return kr.keys[kr.active], kr.active +} + +// key returns the key for version id, or ErrUnknownKeyID if the version has been +// retired or was never present. +func (kr *Keyring) key(id uint16) ([]byte, error) { + k, ok := kr.keys[id] + if !ok { + return nil, ErrUnknownKeyID + } + return k, nil +} + +// keyringJSON is the on-the-wire shape of the Secrets Store keyring secret +// documented in ADR #5: {"active": N, "keys": {"1": "", ...}}. +type keyringJSON struct { + Active uint16 `json:"active"` + Keys map[string]string `json:"keys"` +} + +// ParseKeyring decodes the JSON keyring secret (as stored in Cloudflare Secrets +// Store and read via env.BUGREPORT_ENC_KEYRING.get()) into a Keyring. Each key +// value is standard-base64 of 32 random bytes. This is provider-independent and +// host-testable so the exact same parse runs under go test and in the Worker. +func ParseKeyring(raw []byte) (*Keyring, error) { + var kj keyringJSON + if err := json.Unmarshal(raw, &kj); err != nil { + return nil, fmt.Errorf("crypto: parse keyring: %w", err) + } + if len(kj.Keys) == 0 { + return nil, errors.New("crypto: keyring has no keys") + } + keys := make(map[uint16][]byte, len(kj.Keys)) + for verStr, b64 := range kj.Keys { + ver, err := strconv.ParseUint(verStr, 10, 16) + if err != nil { + return nil, fmt.Errorf("crypto: invalid key version %q: %w", verStr, err) + } + key, err := base64.StdEncoding.DecodeString(b64) + if err != nil { + return nil, fmt.Errorf("crypto: key %s is not valid base64: %w", verStr, err) + } + keys[uint16(ver)] = key + } + return NewKeyring(kj.Active, keys) +} + +// GenerateKey returns a fresh 32-byte AES-256 key drawn from the CSPRNG. It is +// used by cmd/devserver (a throwaway per-run key) and by tests. +func GenerateKey() ([]byte, error) { + k := make([]byte, KeySize) + if _, err := rand.Read(k); err != nil { + return nil, fmt.Errorf("crypto: generate key: %w", err) + } + return k, nil +} + +// header builds the 7-byte authenticated header for a key_id. +func header(keyID uint16) []byte { + h := make([]byte, HeaderSize) + copy(h, Magic) + h[magicLen] = FormatVersion + binary.BigEndian.PutUint16(h[keyIDOffset:HeaderSize], keyID) + return h +} + +// Seal scrubbed plaintext into a complete R2 object frame using the keyring's +// active key and a fresh random nonce, per ADR #5. The returned bytes are the +// full self-describing frame and are safe to write straight to R2; only +// ciphertext ever leaves this function. +func Seal(kr *Keyring, plaintext []byte) ([]byte, error) { + nonce := make([]byte, NonceSize) + if _, err := rand.Read(nonce); err != nil { + return nil, fmt.Errorf("crypto: nonce: %w", err) + } + return sealWithNonce(kr, plaintext, nonce) +} + +// sealWithNonce is Seal with a caller-supplied nonce. It exists so tests can pin +// the nonce for known-answer vectors; production code must use Seal, which draws +// a unique random nonce per object (never reuse a nonce under one key). +func sealWithNonce(kr *Keyring, plaintext, nonce []byte) ([]byte, error) { + if len(nonce) != NonceSize { + return nil, fmt.Errorf("crypto: nonce must be %d bytes, got %d", NonceSize, len(nonce)) + } + key, keyID := kr.activeKey() + hdr := header(keyID) + ctTag, err := primitive.seal(key, nonce, plaintext, hdr) + if err != nil { + return nil, fmt.Errorf("crypto: seal: %w", err) + } + out := make([]byte, 0, bodyOffset+len(ctTag)) + out = append(out, hdr...) + out = append(out, nonce...) + out = append(out, ctTag...) + return out, nil +} + +// Open parses and decrypts an R2 object frame, selecting the key by the frame's +// key_id and verifying the authenticated header. Any tampering (to ciphertext, +// tag, nonce, or header) or a wrong/absent key yields an error; callers must +// treat that as a hard failure. +func Open(kr *Keyring, object []byte) ([]byte, error) { + if len(object) < minObjectLen { + return nil, ErrMalformed + } + if string(object[:magicLen]) != Magic { + return nil, ErrBadMagic + } + if object[magicLen] != FormatVersion { + return nil, ErrUnsupportedVersion + } + keyID := binary.BigEndian.Uint16(object[keyIDOffset:HeaderSize]) + hdr := object[:HeaderSize] + nonce := object[nonceOffset:bodyOffset] + ctTag := object[bodyOffset:] + + key, err := kr.key(keyID) + if err != nil { + return nil, err + } + plaintext, err := primitive.open(key, nonce, ctTag, hdr) + if err != nil { + return nil, ErrAuth + } + return plaintext, nil +} + +// KeyID reads the key_id from an object frame without decrypting it. It is useful +// for ops/metrics; the value is authenticated only when Open succeeds, so do not +// trust it for anything security-sensitive on its own. +func KeyID(object []byte) (uint16, error) { + if len(object) < HeaderSize { + return 0, ErrMalformed + } + if string(object[:magicLen]) != Magic { + return 0, ErrBadMagic + } + return binary.BigEndian.Uint16(object[keyIDOffset:HeaderSize]), nil +} diff --git a/internal/crypto/gcm_host.go b/internal/crypto/gcm_host.go new file mode 100644 index 0000000..0007943 --- /dev/null +++ b/internal/crypto/gcm_host.go @@ -0,0 +1,47 @@ +//go:build !(js && wasm) + +package crypto + +// Host AES-256-GCM provider, using Go's standard crypto/aes + crypto/cipher. +// +// This backs `go test`, cmd/devserver, and every non-Wasm build. The Wasm Worker +// build uses gcm_wasm.go (SubtleCrypto) instead, because TinyGo's crypto/aes is +// unreliable (ADR #5). Both paths implement identical AES-256-GCM with a 12-byte +// nonce and 128-bit tag, so they produce byte-identical frames. + +import ( + "crypto/aes" + "crypto/cipher" +) + +// hostAEAD implements aead with the standard library. +type hostAEAD struct{} + +func init() { primitive = hostAEAD{} } + +// newGCM builds an AES-256-GCM AEAD for key. cipher.NewGCM defaults to a 12-byte +// nonce and 16-byte tag, matching the wire format. +func newGCM(key []byte) (cipher.AEAD, error) { + block, err := aes.NewCipher(key) // 32-byte key selects AES-256 + if err != nil { + return nil, err + } + return cipher.NewGCM(block) +} + +func (hostAEAD) seal(key, nonce, plaintext, aad []byte) ([]byte, error) { + gcm, err := newGCM(key) + if err != nil { + return nil, err + } + // Seal appends ciphertext||tag; dst nil returns a fresh slice. + return gcm.Seal(nil, nonce, plaintext, aad), nil +} + +func (hostAEAD) open(key, nonce, ciphertextAndTag, aad []byte) ([]byte, error) { + gcm, err := newGCM(key) + if err != nil { + return nil, err + } + return gcm.Open(nil, nonce, ciphertextAndTag, aad) +} diff --git a/internal/crypto/gcm_wasm.go b/internal/crypto/gcm_wasm.go new file mode 100644 index 0000000..336655c --- /dev/null +++ b/internal/crypto/gcm_wasm.go @@ -0,0 +1,135 @@ +//go:build js && wasm + +package crypto + +// Wasm AES-256-GCM provider, using the Workers runtime's Web Crypto +// (SubtleCrypto) through syscall/js. +// +// Per ADR #5, TinyGo's crypto/aes is not reliable in Wasm, so the Worker build +// performs AES-256-GCM via crypto.subtle.encrypt / crypto.subtle.decrypt with +// { name: "AES-GCM", iv, additionalData, tagLength: 128 }. This produces exactly +// the same ciphertext||tag as the host crypto/cipher provider (gcm_host.go), so +// the on-disk wire format in frame.go is identical regardless of provider. +// +// This file is compiled only into the js/wasm Worker (it is excluded from host +// builds and `go test`); CI's TinyGo build is what exercises it. + +import ( + "fmt" + "syscall/js" +) + +// subtleAEAD implements aead via SubtleCrypto. +type subtleAEAD struct{} + +func init() { primitive = subtleAEAD{} } + +// subtle returns the crypto.subtle object, fetched lazily to avoid any +// init-ordering assumptions about JS globals. +func subtle() js.Value { + return js.Global().Get("crypto").Get("subtle") +} + +// toUint8Array copies b into a new JS Uint8Array. +func toUint8Array(b []byte) js.Value { + ua := js.Global().Get("Uint8Array").New(len(b)) + if len(b) > 0 { + js.CopyBytesToJS(ua, b) + } + return ua +} + +// bytesFromArrayBuffer copies an ArrayBuffer (the result of encrypt/decrypt) into +// a Go byte slice. +func bytesFromArrayBuffer(buf js.Value) []byte { + ua := js.Global().Get("Uint8Array").New(buf) + out := make([]byte, ua.Get("length").Int()) + if len(out) > 0 { + js.CopyBytesToGo(out, ua) + } + return out +} + +// gcmParams builds the AesGcmParams object for encrypt/decrypt. +func gcmParams(nonce, aad []byte) js.Value { + p := js.Global().Get("Object").New() + p.Set("name", "AES-GCM") + p.Set("iv", toUint8Array(nonce)) + p.Set("additionalData", toUint8Array(aad)) + p.Set("tagLength", 128) + return p +} + +// importKey imports a raw 32-byte key as a non-extractable AES-GCM CryptoKey +// usable for both encrypt and decrypt. +func importKey(key []byte) (js.Value, error) { + algo := js.Global().Get("Object").New() + algo.Set("name", "AES-GCM") + usages := js.Global().Get("Array").New() + usages.Call("push", "encrypt") + usages.Call("push", "decrypt") + return await(subtle().Call("importKey", "raw", toUint8Array(key), algo, false, usages)) +} + +func (subtleAEAD) seal(key, nonce, plaintext, aad []byte) ([]byte, error) { + ck, err := importKey(key) + if err != nil { + return nil, err + } + res, err := await(subtle().Call("encrypt", gcmParams(nonce, aad), ck, toUint8Array(plaintext))) + if err != nil { + return nil, err + } + return bytesFromArrayBuffer(res), nil +} + +func (subtleAEAD) open(key, nonce, ciphertextAndTag, aad []byte) ([]byte, error) { + ck, err := importKey(key) + if err != nil { + return nil, err + } + // A tampered ciphertext/tag/nonce/aad causes SubtleCrypto to reject; await + // surfaces that as an error, which Open maps to ErrAuth. + res, err := await(subtle().Call("decrypt", gcmParams(nonce, aad), ck, toUint8Array(ciphertextAndTag))) + if err != nil { + return nil, err + } + return bytesFromArrayBuffer(res), nil +} + +// await resolves a JS Promise synchronously from the calling goroutine. It is +// safe because the Worker runs each request handler in its own goroutine (see +// syumai/workers), so parking here lets the JS event loop run the settling +// callback. Mirrors syumai/workers' internal AwaitPromise. +func await(p js.Value) (js.Value, error) { + resCh := make(chan js.Value, 1) + errCh := make(chan error, 1) + var then, catch js.Func + then = js.FuncOf(func(_ js.Value, args []js.Value) any { + then.Release() + catch.Release() + v := js.Undefined() + if len(args) > 0 { + v = args[0] + } + resCh <- v + return js.Undefined() + }) + catch = js.FuncOf(func(_ js.Value, args []js.Value) any { + then.Release() + catch.Release() + msg := "unknown error" + if len(args) > 0 { + msg = args[0].Call("toString").String() + } + errCh <- fmt.Errorf("crypto/subtle: %s", msg) + return js.Undefined() + }) + p.Call("then", then).Call("catch", catch) + select { + case v := <-resCh: + return v, nil + case err := <-errCh: + return js.Value{}, err + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index cbea379..16c38bb 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -27,13 +27,14 @@ const serviceName = "libremail-bug-report-ingest" // Any other path returns 404. On the health/hello endpoints any non-GET method // returns 405; on /v1/reports any non-POST method returns 405 (Allow: POST). // -// The ingest endpoint is wired with a NopSink for now: it enforces the full -// HTTP contract (size cap + schema validation) but discards accepted bodies -// until the real storage sink (scrub + encrypt + R2) lands in #8/#9. -func New() http.Handler { +// sink is the storage backend for accepted reports (scrub + encrypt + R2, #9). +// It is injected so the deployed Worker supplies the real R2/Secrets-Store sink +// while cmd/devserver and tests supply an in-memory one. A nil sink defaults to +// ingest.NopSink, which enforces the full HTTP contract but discards bodies. +func New(sink ingest.Sink) http.Handler { mux := http.NewServeMux() mux.HandleFunc("/healthz", healthz) - mux.Handle("/v1/reports", ingest.NewHandler(ingest.NopSink{})) + mux.Handle("/v1/reports", ingest.NewHandler(sink)) mux.HandleFunc("/", root) return mux } diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 8e915f8..7c4737b 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -4,7 +4,10 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "strings" "testing" + + "github.com/JMR-dev/LibreMail-Bug-Report-Ingest/internal/ingest" ) // doRequest runs a single request through the handler and returns the recorder. @@ -12,7 +15,7 @@ func doRequest(t *testing.T, method, target string) *httptest.ResponseRecorder { t.Helper() req := httptest.NewRequest(method, target, nil) rec := httptest.NewRecorder() - New().ServeHTTP(rec, req) + New(nil).ServeHTTP(rec, req) return rec } @@ -71,6 +74,26 @@ func TestUnknownPathReturns404(t *testing.T) { } } +// TestReportsRoutedToInjectedSink proves New wires the injected sink into +// POST /v1/reports (the #9 storage seam), returning 202 and storing the report. +func TestReportsRoutedToInjectedSink(t *testing.T) { + sink := &ingest.MemorySink{} + h := New(sink) + + body := `{"appVersion":"1.0.0","platform":"android","report":"boom"}` + req := httptest.NewRequest(http.MethodPost, "/v1/reports", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + + if rec.Code != http.StatusAccepted { + t.Fatalf("POST /v1/reports status = %d, want %d", rec.Code, http.StatusAccepted) + } + if sink.Len() != 1 { + t.Fatalf("injected sink stored %d reports, want 1", sink.Len()) + } +} + func TestNonGetReturns405(t *testing.T) { for _, path := range []string{"/", "/healthz"} { rec := doRequest(t, http.MethodPost, path) diff --git a/internal/storage/memory.go b/internal/storage/memory.go new file mode 100644 index 0000000..a9c1038 --- /dev/null +++ b/internal/storage/memory.go @@ -0,0 +1,61 @@ +package storage + +import ( + "context" + "sort" + "sync" +) + +// MemoryStore is an in-memory ObjectStore for tests, cmd/devserver, and local +// experimentation. It retains every object in a map and is not a production +// backend (it grows without bound). It is safe for concurrent use. +type MemoryStore struct { + mu sync.Mutex + objects map[string][]byte +} + +// NewMemoryStore returns an empty MemoryStore. +func NewMemoryStore() *MemoryStore { + return &MemoryStore{objects: make(map[string][]byte)} +} + +// Put stores a copy of data at key. +func (m *MemoryStore) Put(_ context.Context, key string, data []byte) error { + cp := append([]byte(nil), data...) + m.mu.Lock() + defer m.mu.Unlock() + m.objects[key] = cp + return nil +} + +// Get returns a copy of the object at key, or ErrNotFound. +func (m *MemoryStore) Get(_ context.Context, key string) ([]byte, error) { + m.mu.Lock() + defer m.mu.Unlock() + data, ok := m.objects[key] + if !ok { + return nil, ErrNotFound + } + return append([]byte(nil), data...), nil +} + +// Keys returns the stored object keys in sorted order (test/ops helper). +func (m *MemoryStore) Keys() []string { + m.mu.Lock() + defer m.mu.Unlock() + keys := make([]string, 0, len(m.objects)) + for k := range m.objects { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +// Len reports how many objects are stored. +func (m *MemoryStore) Len() int { + m.mu.Lock() + defer m.mu.Unlock() + return len(m.objects) +} + +var _ ObjectStore = (*MemoryStore)(nil) diff --git a/internal/storage/r2_wasm.go b/internal/storage/r2_wasm.go new file mode 100644 index 0000000..0bf4fac --- /dev/null +++ b/internal/storage/r2_wasm.go @@ -0,0 +1,54 @@ +//go:build js && wasm + +package storage + +// R2-backed ObjectStore for the Cloudflare Worker build. It wraps the +// syumai/workers R2 binding API; only sealed (ciphertext) frames are ever +// written, so R2 never holds plaintext or the key (ADR #5). +// +// Compiled only into the js/wasm Worker; excluded from host builds and tests. + +import ( + "bytes" + "context" + "io" + + "github.com/syumai/workers/cloudflare/r2" +) + +// R2Store is an ObjectStore backed by a Cloudflare R2 bucket binding. +type R2Store struct { + bucket *r2.Bucket +} + +// NewR2Store resolves the R2 bucket bound as binding (see BucketBinding) from the +// Worker runtime context. It must be called within a request/scheduled handler, +// where the binding is available. +func NewR2Store(binding string) (*R2Store, error) { + b, err := r2.NewBucket(binding) + if err != nil { + return nil, err + } + return &R2Store{bucket: b}, nil +} + +// Put writes data at key. The syumai R2 Put reads the whole body into memory and +// PUTs it as bytes. +func (s *R2Store) Put(_ context.Context, key string, data []byte) error { + _, err := s.bucket.Put(key, io.NopCloser(bytes.NewReader(data)), nil) + return err +} + +// Get reads the object at key, or returns ErrNotFound when it is absent. +func (s *R2Store) Get(_ context.Context, key string) ([]byte, error) { + obj, err := s.bucket.Get(key) + if err != nil { + return nil, err + } + if obj == nil || obj.Body == nil { + return nil, ErrNotFound + } + return io.ReadAll(obj.Body) +} + +var _ ObjectStore = (*R2Store)(nil) diff --git a/internal/storage/sink.go b/internal/storage/sink.go new file mode 100644 index 0000000..0bf008d --- /dev/null +++ b/internal/storage/sink.go @@ -0,0 +1,79 @@ +package storage + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "time" + + "github.com/JMR-dev/LibreMail-Bug-Report-Ingest/internal/crypto" + "github.com/JMR-dev/LibreMail-Bug-Report-Ingest/internal/ingest" + "github.com/JMR-dev/LibreMail-Bug-Report-Ingest/internal/scrub" +) + +// objectKeyPrefix namespaces report objects within the bucket. +const objectKeyPrefix = "reports/" + +// Sink is the real ingest.Sink. For each accepted report it: +// +// 1. scrubs the raw body (best-effort PII redaction, #8); +// 2. seals the scrubbed bytes with the keyring's active AES-256 key +// (AES-256-GCM, ADR #5) into a self-describing frame; +// 3. writes the frame to the ObjectStore under a unique, unguessable key. +// +// The store only ever receives ciphertext. Sink is provider-agnostic: paired +// with a MemoryStore it runs on the host (tests, cmd/devserver); paired with an +// R2Store it runs in the Worker. The Worker's keyring loading (from Secrets +// Store) is handled by WorkerSink, which delegates the pipeline to a Sink. +type Sink struct { + store ObjectStore + keyring *crypto.Keyring + keyFn func() string +} + +// Option customises a Sink. +type Option func(*Sink) + +// WithKeyFunc overrides the object-key generator. Intended for tests that need a +// deterministic key; production uses the default random key. +func WithKeyFunc(fn func() string) Option { + return func(s *Sink) { s.keyFn = fn } +} + +// NewSink returns a Sink that stores into store, encrypting under kr's active +// key. kr must be non-nil. +func NewSink(store ObjectStore, kr *crypto.Keyring, opts ...Option) *Sink { + s := &Sink{store: store, keyring: kr, keyFn: defaultObjectKey} + for _, o := range opts { + o(s) + } + return s +} + +// Store scrubs, encrypts, and persists one accepted report. A non-nil return +// makes the ingest endpoint answer 503 (per ADR #6), so callers should retry. +func (s *Sink) Store(ctx context.Context, raw []byte) error { + scrubbed := scrub.Scrub(raw) + sealed, err := crypto.Seal(s.keyring, scrubbed) + if err != nil { + return fmt.Errorf("storage: seal: %w", err) + } + if err := s.store.Put(ctx, s.keyFn(), sealed); err != nil { + return fmt.Errorf("storage: put: %w", err) + } + return nil +} + +// defaultObjectKey builds a unique object key: a UTC timestamp (for rough +// lexicographic ordering, convenient for the weekly publish job) plus 80 bits of +// CSPRNG randomness (so keys are unguessable and collision-free within a second). +func defaultObjectKey() string { + var b [10]byte + _, _ = rand.Read(b[:]) + ts := time.Now().UTC().Format("20060102T150405") + return objectKeyPrefix + ts + "-" + hex.EncodeToString(b[:]) +} + +// Sink satisfies the ingest storage seam. +var _ ingest.Sink = (*Sink)(nil) diff --git a/internal/storage/storage_test.go b/internal/storage/storage_test.go new file mode 100644 index 0000000..e94468d --- /dev/null +++ b/internal/storage/storage_test.go @@ -0,0 +1,199 @@ +package storage + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" + + "github.com/JMR-dev/LibreMail-Bug-Report-Ingest/internal/crypto" + "github.com/JMR-dev/LibreMail-Bug-Report-Ingest/internal/scrub" +) + +// ghToken is a GitHub-token-shaped test value, assembled at runtime so the +// contiguous token pattern never appears as a literal in source (secret scanners +// flag test tokens on sight; internal/scrub/scrub_test.go uses the same trick for +// a GitLab PAT). +var ghToken = "ghp_" + "0123456789abcdefghijklmnopqrstuvwxyz" + +// rawReport is a realistic raw body carrying several kinds of PII plus some +// non-sensitive structure that must survive scrubbing. +var rawReport = `{"appVersion":"1.4.2","platform":"android",` + + `"report":"crash when syncing. contact alice@example.com. ` + + `token=` + ghToken + ` from 203.0.113.7"}` + +// piiSubstrings are values that must never survive scrubbing, and must never be +// readable in the stored ciphertext. +var piiSubstrings = []string{ + "alice@example.com", + ghToken, + "203.0.113.7", +} + +func mustKeyring(t *testing.T) *crypto.Keyring { + t.Helper() + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("GenerateKey: %v", err) + } + kr, err := crypto.NewKeyring(1, map[uint16][]byte{1: key}) + if err != nil { + t.Fatalf("NewKeyring: %v", err) + } + return kr +} + +func TestMemoryStorePutGet(t *testing.T) { + ctx := context.Background() + m := NewMemoryStore() + + if _, err := m.Get(ctx, "absent"); !errors.Is(err, ErrNotFound) { + t.Errorf("Get(absent) err = %v, want ErrNotFound", err) + } + + want := []byte("some bytes") + if err := m.Put(ctx, "k", want); err != nil { + t.Fatalf("Put: %v", err) + } + got, err := m.Get(ctx, "k") + if err != nil { + t.Fatalf("Get: %v", err) + } + if !bytes.Equal(got, want) { + t.Errorf("Get = %q, want %q", got, want) + } + // Stored copy must be independent of the caller's slice. + want[0] = 'X' + if again, _ := m.Get(ctx, "k"); again[0] == 'X' { + t.Error("MemoryStore did not copy the value on Put") + } + if m.Len() != 1 { + t.Errorf("Len = %d, want 1", m.Len()) + } +} + +// TestSinkFullPath is the end-to-end acceptance: raw report -> scrubbed -> +// encrypted -> stored, and reading it back REQUIRES the key and yields the +// scrubbed (not raw) content. +func TestSinkFullPath(t *testing.T) { + ctx := context.Background() + kr := mustKeyring(t) + store := NewMemoryStore() + sink := NewSink(store, kr) + + if err := sink.Store(ctx, []byte(rawReport)); err != nil { + t.Fatalf("Store: %v", err) + } + + // Exactly one object, under a reports/ key. + keys := store.Keys() + if len(keys) != 1 { + t.Fatalf("stored %d objects, want 1", len(keys)) + } + if !strings.HasPrefix(keys[0], objectKeyPrefix) { + t.Errorf("object key %q lacks prefix %q", keys[0], objectKeyPrefix) + } + + stored, err := store.Get(ctx, keys[0]) + if err != nil { + t.Fatalf("Get stored: %v", err) + } + + // The stored bytes are a sealed frame, not the raw or scrubbed plaintext. + scrubbed := scrub.Scrub([]byte(rawReport)) + if bytes.Equal(stored, []byte(rawReport)) { + t.Error("stored object equals the raw body (not encrypted)") + } + if bytes.Equal(stored, scrubbed) { + t.Error("stored object equals the scrubbed plaintext (not encrypted)") + } + if string(stored[:4]) != crypto.Magic { + t.Errorf("stored object magic = %q, want %q", stored[:4], crypto.Magic) + } + + // The ciphertext must not leak any PII substring in the clear. + for _, s := range piiSubstrings { + if bytes.Contains(stored, []byte(s)) { + t.Errorf("stored ciphertext leaks PII substring %q", s) + } + } + + // The object is NOT decryptable without the key. + otherKR := mustKeyring(t) // different random key, same key_id 1 + if _, err := crypto.Open(otherKR, stored); err == nil { + t.Error("stored object decrypted with the WRONG key; must require the correct key") + } + + // With the correct key it yields exactly the scrubbed content. + opened, err := crypto.Open(kr, stored) + if err != nil { + t.Fatalf("Open with correct key: %v", err) + } + if !bytes.Equal(opened, scrubbed) { + t.Errorf("decrypted content != scrubbed content\n got = %q\n want = %q", opened, scrubbed) + } + + // The decrypted (scrubbed) content has the PII removed (verify via #8's + // contract) and carries the redaction placeholders and surviving structure. + openedStr := string(opened) + for _, s := range piiSubstrings { + if strings.Contains(openedStr, s) { + t.Errorf("decrypted scrubbed content still leaks PII %q", s) + } + } + for _, p := range []string{scrub.PlaceholderEmail, scrub.PlaceholderToken, scrub.PlaceholderIP} { + if !strings.Contains(openedStr, p) { + t.Errorf("decrypted scrubbed content missing placeholder %q", p) + } + } + for _, keep := range []string{"crash when syncing", `"appVersion":"1.4.2"`} { + if !strings.Contains(openedStr, keep) { + t.Errorf("decrypted scrubbed content dropped non-PII text %q", keep) + } + } +} + +// failingStore always fails Put, to exercise the Sink's error propagation (which +// drives the ingest 503 path). +type failingStore struct{} + +func (failingStore) Put(context.Context, string, []byte) error { + return errors.New("backend down") +} +func (failingStore) Get(context.Context, string) ([]byte, error) { + return nil, ErrNotFound +} + +func TestSinkStorePutError(t *testing.T) { + sink := NewSink(failingStore{}, mustKeyring(t)) + if err := sink.Store(context.Background(), []byte(rawReport)); err == nil { + t.Error("Store returned nil despite a failing backend; want error (drives 503)") + } +} + +func TestSinkUsesDistinctKeys(t *testing.T) { + ctx := context.Background() + store := NewMemoryStore() + sink := NewSink(store, mustKeyring(t)) + for i := 0; i < 5; i++ { + if err := sink.Store(ctx, []byte(rawReport)); err != nil { + t.Fatalf("Store #%d: %v", i, err) + } + } + if store.Len() != 5 { + t.Errorf("stored %d objects, want 5 distinct keys", store.Len()) + } +} + +func TestSinkWithKeyFunc(t *testing.T) { + ctx := context.Background() + store := NewMemoryStore() + sink := NewSink(store, mustKeyring(t), WithKeyFunc(func() string { return "reports/fixed" })) + if err := sink.Store(ctx, []byte(rawReport)); err != nil { + t.Fatalf("Store: %v", err) + } + if _, err := store.Get(ctx, "reports/fixed"); err != nil { + t.Errorf("expected object at fixed key: %v", err) + } +} diff --git a/internal/storage/store.go b/internal/storage/store.go new file mode 100644 index 0000000..4ef01c0 --- /dev/null +++ b/internal/storage/store.go @@ -0,0 +1,45 @@ +// Package storage persists accepted LibreMail bug-reports as encrypted-at-rest +// objects, implementing the storage half of the ingest pipeline: for each +// accepted report it scrubs PII (internal/scrub, #8), encrypts the scrubbed +// bytes with AES-256-GCM (internal/crypto, ADR #5), and writes only the opaque +// ciphertext frame to the object store. The store never sees plaintext or the +// key. +// +// The package follows the repo's build-tag pattern so it is host-testable +// without TinyGo or the Workers runtime: +// +// - The ObjectStore interface, MemoryStore, and the Sink (scrub+encrypt+put) +// carry no build constraints and are unit-tested with `go test`. +// - The real R2-backed store (R2Store) and the Worker sink that loads the +// keyring from Cloudflare Secrets Store live behind //go:build js && wasm and +// are compiled by the Wasm Worker build in CI. +package storage + +import ( + "context" + "errors" +) + +// ErrNotFound is returned by ObjectStore.Get when no object exists at the key. +var ErrNotFound = errors.New("storage: object not found") + +// Binding names wired in wrangler.jsonc and provisioned by infra (#2). +const ( + // BucketBinding is the R2 bucket binding name (the JS var the Worker reads). + // The bound bucket is "libremail-bug-reports" (infra defaultR2BucketName). + BucketBinding = "REPORTS_BUCKET" + // KeyringBinding is the Cloudflare Secrets Store binding holding the JSON + // keyring secret, named per ADR #5. + KeyringBinding = "BUGREPORT_ENC_KEYRING" +) + +// ObjectStore is the seam for the opaque object backend. Implementations only +// ever handle already-encrypted frames. +// +// - Put writes data (a sealed frame) at key, overwriting any existing object. +// - Get reads the bytes back, or returns ErrNotFound. Get exists mainly for the +// future publish job (#35) and for tests; the ingest path is write-only. +type ObjectStore interface { + Put(ctx context.Context, key string, data []byte) error + Get(ctx context.Context, key string) ([]byte, error) +} diff --git a/internal/storage/worker_sink_wasm.go b/internal/storage/worker_sink_wasm.go new file mode 100644 index 0000000..2b33165 --- /dev/null +++ b/internal/storage/worker_sink_wasm.go @@ -0,0 +1,133 @@ +//go:build js && wasm + +package storage + +// WorkerSink is the production ingest.Sink for the Cloudflare Worker. It loads +// the versioned keyring from Cloudflare Secrets Store, resolves the R2 bucket +// binding, and delegates the scrub -> encrypt -> put pipeline to a plain Sink. +// +// Per ADR #5 the keyring secret is fetched inside the request handler (the +// binding's get() is async and the runtime context is only present per-request), +// not at module top-level. The parsed keyring is cached for the isolate lifetime; +// isolates recycle, which is how a rotated `active` version takes over. +// +// Compiled only into the js/wasm Worker; excluded from host builds and tests. + +import ( + "context" + "fmt" + "sync" + "syscall/js" + + "github.com/syumai/workers/cloudflare" + + "github.com/JMR-dev/LibreMail-Bug-Report-Ingest/internal/crypto" + "github.com/JMR-dev/LibreMail-Bug-Report-Ingest/internal/ingest" +) + +// WorkerSink implements ingest.Sink against R2 + Secrets Store bindings. +type WorkerSink struct { + bucketBinding string + keyringBinding string + + mu sync.Mutex + keyring *crypto.Keyring // cached for the isolate lifetime after first load +} + +// NewWorkerSink returns a WorkerSink using the standard binding names. It does no +// I/O and touches no runtime context, so it is safe to construct at main() time; +// bindings are resolved lazily on the first Store call. +func NewWorkerSink() *WorkerSink { + return &WorkerSink{ + bucketBinding: BucketBinding, + keyringBinding: KeyringBinding, + } +} + +// Store loads the keyring (cached), resolves the R2 bucket, and runs the shared +// scrub+encrypt+put pipeline. +func (s *WorkerSink) Store(ctx context.Context, raw []byte) error { + kr, err := s.loadKeyring() + if err != nil { + return fmt.Errorf("storage: load keyring: %w", err) + } + store, err := NewR2Store(s.bucketBinding) + if err != nil { + return fmt.Errorf("storage: r2 binding: %w", err) + } + return NewSink(store, kr).Store(ctx, raw) +} + +// loadKeyring returns the cached keyring, loading and parsing it from Secrets +// Store on first use. A failed load is not cached, so it is retried next request. +func (s *WorkerSink) loadKeyring() (*crypto.Keyring, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.keyring != nil { + return s.keyring, nil + } + raw, err := getSecret(s.keyringBinding) + if err != nil { + return nil, err + } + kr, err := crypto.ParseKeyring(raw) + if err != nil { + return nil, err + } + s.keyring = kr + return kr, nil +} + +// getSecret reads a Cloudflare Secrets Store secret via `await binding.get()`. +// The returned value must never be logged or echoed (ADR #5, Key custody). +func getSecret(binding string) ([]byte, error) { + b := cloudflare.GetBinding(binding) + if b.IsUndefined() || b.IsNull() { + return nil, fmt.Errorf("secrets store binding %q is not bound", binding) + } + v, err := await(b.Call("get")) + if err != nil { + return nil, err + } + if v.IsUndefined() || v.IsNull() { + return nil, fmt.Errorf("secrets store binding %q returned no value", binding) + } + return []byte(v.String()), nil +} + +// await resolves a JS Promise from the calling goroutine (the Worker runs each +// handler in its own goroutine, so the JS event loop can settle the promise). +func await(p js.Value) (js.Value, error) { + resCh := make(chan js.Value, 1) + errCh := make(chan error, 1) + var then, catch js.Func + then = js.FuncOf(func(_ js.Value, args []js.Value) any { + then.Release() + catch.Release() + v := js.Undefined() + if len(args) > 0 { + v = args[0] + } + resCh <- v + return js.Undefined() + }) + catch = js.FuncOf(func(_ js.Value, args []js.Value) any { + then.Release() + catch.Release() + msg := "unknown error" + if len(args) > 0 { + msg = args[0].Call("toString").String() + } + errCh <- fmt.Errorf("secrets store: %s", msg) + return js.Undefined() + }) + p.Call("then", then).Call("catch", catch) + select { + case v := <-resCh: + return v, nil + case err := <-errCh: + return js.Value{}, err + } +} + +var _ ingest.Sink = (*WorkerSink)(nil) diff --git a/worker/main.go b/worker/main.go index 152eef4..cdb3b25 100644 --- a/worker/main.go +++ b/worker/main.go @@ -14,8 +14,12 @@ import ( "github.com/syumai/workers" "github.com/JMR-dev/LibreMail-Bug-Report-Ingest/internal/handler" + "github.com/JMR-dev/LibreMail-Bug-Report-Ingest/internal/storage" ) func main() { - workers.Serve(handler.New()) + // The real storage Sink (#9): scrub -> AES-256-GCM encrypt -> R2 put, with the + // keyring loaded from Cloudflare Secrets Store on first request. Bindings + // (REPORTS_BUCKET, BUGREPORT_ENC_KEYRING) are declared in wrangler.jsonc. + workers.Serve(handler.New(storage.NewWorkerSink())) } diff --git a/wrangler.jsonc b/wrangler.jsonc index 65f8b7a..21bda02 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -15,5 +15,31 @@ // Requires TinyGo locally; TinyGo is installed in CI. "build": { "command": "pnpm run build" - } + }, + + // R2 bucket for the encrypted-at-rest bug-report objects (ADR #5 / issue #9). + // The Worker encrypts each scrubbed report with AES-256-GCM before writing, so + // only ciphertext is ever stored here. "binding" is the JS var the Worker code + // reads (internal/storage.BucketBinding); "bucket_name" matches the bucket + // provisioned by infra/ (defaultR2BucketName = "libremail-bug-reports"). + "r2_buckets": [ + { + "binding": "REPORTS_BUCKET", + "bucket_name": "libremail-bug-reports" + } + ], + + // Cloudflare Secrets Store secret holding the versioned encryption keyring + // (ADR #5, Key custody): a single JSON secret {active, keys{ver: base64-32B}}. + // "binding" is read at runtime via env.BUGREPORT_ENC_KEYRING.get() + // (internal/storage.KeyringBinding). Replace "" with the account's + // Secrets Store id at deploy time; it is not needed for `pnpm run build` + // (Wasm compile) or the devserver, so CI does not require it. + "secrets_store_secrets": [ + { + "binding": "BUGREPORT_ENC_KEYRING", + "store_id": "", + "secret_name": "bugreport-enc-keyring" + } + ] }