#9 Encrypted-at-rest R2 storage for scrubbed reports
Implement the storage path: for each accepted report, scrub PII (#8), encrypt with AES-256-GCM (ADR #5), and write only ciphertext to R2, wired in as the real ingest Sink replacing NopSink. - internal/crypto: AES-256-GCM in the exact ADR #5 wire format (magic "LMB1" || version || key_id BE16 || nonce(12) || ct || tag(16); the 7-byte header is the GCM AAD). Provider-independent framing shared by a host crypto/aes+crypto/cipher impl (tests, devserver) and a Wasm SubtleCrypto impl (syscall/js, //go:build js && wasm) per the TinyGo constraint; both produce byte-identical frames. Versioned keyring with key_id rotation; ParseKeyring reads the Secrets Store JSON secret. - internal/storage: ObjectStore interface with an in-memory fake (tests, devserver) and a Wasm R2Store (syumai/workers R2 binding). Sink ties scrub -> Seal -> Put under a unique reports/<ts>-<rand> key. WorkerSink loads the keyring from Secrets Store (BUGREPORT_ENC_KEYRING), cached for the isolate lifetime. - handler.New now takes an injectable ingest.Sink; the Worker uses the real R2/Secrets-Store sink, the devserver a memory + throwaway-key sink. - wrangler.jsonc: add REPORTS_BUCKET (R2) and BUGREPORT_ENC_KEYRING (Secrets Store) bindings. Tests (host, no TinyGo): encrypt/decrypt roundtrip; ciphertext != plaintext; wrong key + tamper (ct/tag/nonce/header-AAD) fail; exact wire layout plus a known-answer vector; key_id rotation with retained keys; full sink path (PII scrubbed then encrypted, readback requires the key and yields the scrubbed content). Existing ingest/handler behavior preserved (202 on valid POST). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
committed by
Jason Ross
Jason Ross
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
3d320a8cea
commit
bd7fe21d97
+20
-1
@@ -5,6 +5,11 @@
|
||||
// so `go run ./cmd/devserver` works with the standard Go toolchain and exercises
|
||||
// the exact same handler that the deployed Wasm Worker serves. It listens on
|
||||
// :8787 by default (matching wrangler dev's default port); override with ADDR.
|
||||
//
|
||||
// The ingest path is wired with the real scrub+encrypt storage Sink (#9) backed
|
||||
// by an in-memory object store and a throwaway per-run AES-256 key, so a POST
|
||||
// /v1/reports exercises the full pipeline locally. Stored objects live only for
|
||||
// the process lifetime.
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -12,7 +17,9 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
"github.com/JMR-dev/LibreMail-Bug-Report-Ingest/internal/crypto"
|
||||
"github.com/JMR-dev/LibreMail-Bug-Report-Ingest/internal/handler"
|
||||
"github.com/JMR-dev/LibreMail-Bug-Report-Ingest/internal/storage"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -21,8 +28,20 @@ func main() {
|
||||
addr = ":8787"
|
||||
}
|
||||
|
||||
// A throwaway keyring: a single random key generated at startup. Reports are
|
||||
// scrubbed, encrypted under it, and held in memory; nothing is persisted.
|
||||
key, err := crypto.GenerateKey()
|
||||
if err != nil {
|
||||
log.Fatalf("devserver: generate key: %v", err)
|
||||
}
|
||||
keyring, err := crypto.NewKeyring(1, map[uint16][]byte{1: key})
|
||||
if err != nil {
|
||||
log.Fatalf("devserver: build keyring: %v", err)
|
||||
}
|
||||
sink := storage.NewSink(storage.NewMemoryStore(), keyring)
|
||||
|
||||
log.Printf("devserver listening on %s (try GET / and GET /healthz)", addr)
|
||||
if err := http.ListenAndServe(addr, handler.New()); err != nil {
|
||||
if err := http.ListenAndServe(addr, handler.New(sink)); err != nil {
|
||||
log.Fatalf("devserver: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,401 @@
|
||||
package crypto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// fixedKey returns a deterministic 32-byte key whose bytes are seed+i, for
|
||||
// reproducible test vectors (never used outside tests).
|
||||
func fixedKey(seed byte) []byte {
|
||||
k := make([]byte, KeySize)
|
||||
for i := range k {
|
||||
k[i] = seed + byte(i)
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
func mustKeyring(t *testing.T, active uint16, keys map[uint16][]byte) *Keyring {
|
||||
t.Helper()
|
||||
kr, err := NewKeyring(active, keys)
|
||||
if err != nil {
|
||||
t.Fatalf("NewKeyring: %v", err)
|
||||
}
|
||||
return kr
|
||||
}
|
||||
|
||||
// TestSealOpenRoundtrip covers the happy path: Open(Seal(x)) == x, and the
|
||||
// ciphertext is not the plaintext.
|
||||
func TestSealOpenRoundtrip(t *testing.T) {
|
||||
kr := mustKeyring(t, 1, map[uint16][]byte{1: fixedKey(0)})
|
||||
for _, pt := range [][]byte{
|
||||
[]byte(""),
|
||||
[]byte("x"),
|
||||
[]byte("a scrubbed bug report with [REDACTED_EMAIL] inside"),
|
||||
bytes.Repeat([]byte("A"), 4096),
|
||||
} {
|
||||
sealed, err := Seal(kr, pt)
|
||||
if err != nil {
|
||||
t.Fatalf("Seal(%d bytes): %v", len(pt), err)
|
||||
}
|
||||
if len(pt) > 0 && bytes.Contains(sealed, pt) {
|
||||
t.Errorf("sealed frame contains the plaintext verbatim (len %d)", len(pt))
|
||||
}
|
||||
got, err := Open(kr, sealed)
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
if !bytes.Equal(got, pt) {
|
||||
t.Errorf("roundtrip mismatch: got %q want %q", got, pt)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestWireLayout locks the ADR #5 byte layout of a sealed frame.
|
||||
func TestWireLayout(t *testing.T) {
|
||||
kr := mustKeyring(t, 0xABCD, map[uint16][]byte{0xABCD: fixedKey(3)})
|
||||
nonce := bytes.Repeat([]byte{0x5A}, NonceSize)
|
||||
pt := []byte("payload")
|
||||
|
||||
frame, err := sealWithNonce(kr, pt, nonce)
|
||||
if err != nil {
|
||||
t.Fatalf("sealWithNonce: %v", err)
|
||||
}
|
||||
|
||||
if got := string(frame[:4]); got != Magic {
|
||||
t.Errorf("magic = %q, want %q", got, Magic)
|
||||
}
|
||||
if frame[4] != FormatVersion {
|
||||
t.Errorf("version = 0x%02x, want 0x%02x", frame[4], FormatVersion)
|
||||
}
|
||||
if id := binary.BigEndian.Uint16(frame[5:7]); id != 0xABCD {
|
||||
t.Errorf("key_id = 0x%04x, want 0xABCD", id)
|
||||
}
|
||||
if got := frame[7:19]; !bytes.Equal(got, nonce) {
|
||||
t.Errorf("nonce = %x, want %x", got, nonce)
|
||||
}
|
||||
// header(7) + nonce(12) + ciphertext(len pt) + tag(16)
|
||||
wantLen := HeaderSize + NonceSize + len(pt) + TagSize
|
||||
if len(frame) != wantLen {
|
||||
t.Errorf("frame len = %d, want %d", len(frame), wantLen)
|
||||
}
|
||||
if id, err := KeyID(frame); err != nil || id != 0xABCD {
|
||||
t.Errorf("KeyID = 0x%04x, err=%v; want 0xABCD, nil", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestKnownAnswerVector is a byte-exact format lock. The expected frame was
|
||||
// computed from Go's standard AES-256-GCM; because AES-256-GCM is deterministic
|
||||
// and standardised, the Wasm SubtleCrypto provider MUST produce these same bytes
|
||||
// for the same key+nonce+plaintext+AAD. If this changes, the on-disk format (and
|
||||
// cross-provider compatibility) changed.
|
||||
func TestKnownAnswerVector(t *testing.T) {
|
||||
key := fixedKey(0) // 0x00,0x01,...,0x1f
|
||||
kr := mustKeyring(t, 1, map[uint16][]byte{1: key})
|
||||
nonce := make([]byte, NonceSize) // 0x00..0x0b
|
||||
for i := range nonce {
|
||||
nonce[i] = byte(i)
|
||||
}
|
||||
pt := []byte("hello world")
|
||||
|
||||
const wantHex = "4c4d4231010001000102030405060708090a0b2f67ba77aac5b574ff2df3f26c5bd31758566cf1bf14ae15f8fd7a"
|
||||
frame, err := sealWithNonce(kr, pt, nonce)
|
||||
if err != nil {
|
||||
t.Fatalf("sealWithNonce: %v", err)
|
||||
}
|
||||
if got := hex.EncodeToString(frame); got != wantHex {
|
||||
t.Errorf("known-answer frame mismatch:\n got = %s\n want = %s", got, wantHex)
|
||||
}
|
||||
// And it must still open.
|
||||
got, err := Open(kr, frame)
|
||||
if err != nil || !bytes.Equal(got, pt) {
|
||||
t.Errorf("Open(known frame) = %q, %v; want %q, nil", got, err, pt)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSealMatchesStdlibGCM proves the package frames *standard* AES-256-GCM: the
|
||||
// sealed body equals an independent crypto/cipher GCM computation over the same
|
||||
// key, nonce, plaintext, and header-as-AAD. This is the provider-independent
|
||||
// contract the SubtleCrypto build also satisfies.
|
||||
func TestSealMatchesStdlibGCM(t *testing.T) {
|
||||
key := fixedKey(9)
|
||||
const keyID = 7
|
||||
kr := mustKeyring(t, keyID, map[uint16][]byte{keyID: key})
|
||||
nonce := bytes.Repeat([]byte{0x11}, NonceSize)
|
||||
pt := []byte("some plaintext to seal")
|
||||
|
||||
got, err := sealWithNonce(kr, pt, nonce)
|
||||
if err != nil {
|
||||
t.Fatalf("sealWithNonce: %v", err)
|
||||
}
|
||||
|
||||
block, err := aes.NewCipher(key)
|
||||
if err != nil {
|
||||
t.Fatalf("aes.NewCipher: %v", err)
|
||||
}
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
t.Fatalf("cipher.NewGCM: %v", err)
|
||||
}
|
||||
hdr := append([]byte(Magic), FormatVersion, 0x00, keyID)
|
||||
ctTag := gcm.Seal(nil, nonce, pt, hdr)
|
||||
want := append(append(append([]byte{}, hdr...), nonce...), ctTag...)
|
||||
|
||||
if !bytes.Equal(got, want) {
|
||||
t.Errorf("framing differs from standard AES-256-GCM:\n got = %x\n want = %x", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenWrongKeyFails: an object is not decryptable without the correct key.
|
||||
func TestOpenWrongKeyFails(t *testing.T) {
|
||||
sealKR := mustKeyring(t, 1, map[uint16][]byte{1: fixedKey(0)})
|
||||
// Different key material, SAME key_id, so parsing succeeds and only the
|
||||
// cryptographic check can reject it.
|
||||
wrongKR := mustKeyring(t, 1, map[uint16][]byte{1: fixedKey(100)})
|
||||
|
||||
sealed, err := Seal(sealKR, []byte("secret residual PII"))
|
||||
if err != nil {
|
||||
t.Fatalf("Seal: %v", err)
|
||||
}
|
||||
if _, err := Open(wrongKR, sealed); !errors.Is(err, ErrAuth) {
|
||||
t.Errorf("Open with wrong key: err = %v, want ErrAuth", err)
|
||||
}
|
||||
// Sanity: the right key still works.
|
||||
if _, err := Open(sealKR, sealed); err != nil {
|
||||
t.Errorf("Open with correct key failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTamperDetection: any modification to the frame makes Open fail (GCM
|
||||
// authenticates ciphertext, tag, nonce, and the header via AAD).
|
||||
func TestTamperDetection(t *testing.T) {
|
||||
kr := mustKeyring(t, 1, map[uint16][]byte{1: fixedKey(0)})
|
||||
sealed, err := Seal(kr, []byte("tamper target payload"))
|
||||
if err != nil {
|
||||
t.Fatalf("Seal: %v", err)
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
offset int
|
||||
wantIs error // nil means "any non-nil error"
|
||||
}{
|
||||
{"magic", 0, ErrBadMagic},
|
||||
{"version", 4, ErrUnsupportedVersion},
|
||||
{"key_id", 6, ErrUnknownKeyID}, // 1 -> some absent version
|
||||
{"nonce", nonceOffset, ErrAuth},
|
||||
{"ciphertext", bodyOffset, ErrAuth},
|
||||
{"tag", len(sealed) - 1, ErrAuth},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
bad := append([]byte(nil), sealed...)
|
||||
bad[tc.offset] ^= 0xFF
|
||||
_, err := Open(kr, bad)
|
||||
if err == nil {
|
||||
t.Fatalf("tampering %s produced no error", tc.name)
|
||||
}
|
||||
if tc.wantIs != nil && !errors.Is(err, tc.wantIs) {
|
||||
t.Errorf("tampering %s: err = %v, want %v", tc.name, err, tc.wantIs)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestHeaderIsAuthenticated proves the key_id in the header is bound by AAD: an
|
||||
// attacker cannot relabel a frame to a different, valid key version.
|
||||
func TestHeaderIsAuthenticated(t *testing.T) {
|
||||
// Keyring holds two versions; seal under version 1.
|
||||
kr := mustKeyring(t, 1, map[uint16][]byte{1: fixedKey(0), 2: fixedKey(50)})
|
||||
sealed, err := Seal(kr, []byte("bind me"))
|
||||
if err != nil {
|
||||
t.Fatalf("Seal: %v", err)
|
||||
}
|
||||
// Rewrite the stored key_id from 1 to 2 (a version that DOES exist).
|
||||
relabeled := append([]byte(nil), sealed...)
|
||||
binary.BigEndian.PutUint16(relabeled[keyIDOffset:HeaderSize], 2)
|
||||
if _, err := Open(kr, relabeled); !errors.Is(err, ErrAuth) {
|
||||
t.Errorf("relabeled key_id: err = %v, want ErrAuth (header must be authenticated)", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRotationRetainsOldKeys: an object sealed under key_id N still opens after
|
||||
// rotation, as long as the keyring retains version N.
|
||||
func TestRotationRetainsOldKeys(t *testing.T) {
|
||||
key1 := fixedKey(0)
|
||||
key2 := fixedKey(50)
|
||||
|
||||
// Before rotation: active = 1.
|
||||
before := mustKeyring(t, 1, map[uint16][]byte{1: key1})
|
||||
sealed, err := Seal(before, []byte("pre-rotation report"))
|
||||
if err != nil {
|
||||
t.Fatalf("Seal: %v", err)
|
||||
}
|
||||
if id, _ := KeyID(sealed); id != 1 {
|
||||
t.Fatalf("sealed key_id = %d, want 1", id)
|
||||
}
|
||||
|
||||
// After rotation: active = 2, but version 1 is RETAINED.
|
||||
after := mustKeyring(t, 2, map[uint16][]byte{1: key1, 2: key2})
|
||||
got, err := Open(after, sealed)
|
||||
if err != nil {
|
||||
t.Fatalf("Open after rotation (retained key 1): %v", err)
|
||||
}
|
||||
if want := []byte("pre-rotation report"); !bytes.Equal(got, want) {
|
||||
t.Errorf("post-rotation decrypt = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
// New writes now use key_id 2.
|
||||
sealed2, err := Seal(after, []byte("post-rotation report"))
|
||||
if err != nil {
|
||||
t.Fatalf("Seal (post-rotation): %v", err)
|
||||
}
|
||||
if id, _ := KeyID(sealed2); id != 2 {
|
||||
t.Errorf("post-rotation sealed key_id = %d, want 2", id)
|
||||
}
|
||||
|
||||
// If version 1 is RETIRED (removed), the old object is unrecoverable.
|
||||
retired := mustKeyring(t, 2, map[uint16][]byte{2: key2})
|
||||
if _, err := Open(retired, sealed); !errors.Is(err, ErrUnknownKeyID) {
|
||||
t.Errorf("Open with retired key_id: err = %v, want ErrUnknownKeyID", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenMalformed covers frame-structure rejections.
|
||||
func TestOpenMalformed(t *testing.T) {
|
||||
kr := mustKeyring(t, 1, map[uint16][]byte{1: fixedKey(0)})
|
||||
valid, err := Seal(kr, []byte("ok"))
|
||||
if err != nil {
|
||||
t.Fatalf("Seal: %v", err)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
object []byte
|
||||
wantIs error
|
||||
}{
|
||||
{"empty", nil, ErrMalformed},
|
||||
{"too short", valid[:minObjectLen-1], ErrMalformed},
|
||||
{"bad magic", func() []byte { b := append([]byte(nil), valid...); b[0] = 'X'; return b }(), ErrBadMagic},
|
||||
{"bad version", func() []byte { b := append([]byte(nil), valid...); b[4] = 0x02; return b }(), ErrUnsupportedVersion},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if _, err := Open(kr, tc.object); !errors.Is(err, tc.wantIs) {
|
||||
t.Errorf("Open(%s): err = %v, want %v", tc.name, err, tc.wantIs)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseKeyring parses the ADR #5 Secrets Store JSON shape and roundtrips.
|
||||
func TestParseKeyring(t *testing.T) {
|
||||
key1 := fixedKey(0)
|
||||
key2 := fixedKey(200)
|
||||
raw := fmt.Sprintf(`{"active":2,"keys":{"1":%q,"2":%q}}`,
|
||||
base64.StdEncoding.EncodeToString(key1),
|
||||
base64.StdEncoding.EncodeToString(key2))
|
||||
|
||||
kr, err := ParseKeyring([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseKeyring: %v", err)
|
||||
}
|
||||
if kr.Active() != 2 {
|
||||
t.Errorf("active = %d, want 2", kr.Active())
|
||||
}
|
||||
// New objects use active=2; an object made with the parsed keyring opens with
|
||||
// an equivalent hand-built keyring, proving the bytes decoded correctly.
|
||||
sealed, err := Seal(kr, []byte("via parsed keyring"))
|
||||
if err != nil {
|
||||
t.Fatalf("Seal: %v", err)
|
||||
}
|
||||
ref := mustKeyring(t, 2, map[uint16][]byte{1: key1, 2: key2})
|
||||
if got, err := Open(ref, sealed); err != nil || string(got) != "via parsed keyring" {
|
||||
t.Errorf("Open via reference keyring = %q, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseKeyringErrors(t *testing.T) {
|
||||
shortKey := base64.StdEncoding.EncodeToString(make([]byte, 16)) // 16 bytes, not 32
|
||||
fullKey := base64.StdEncoding.EncodeToString(fixedKey(0))
|
||||
cases := map[string]string{
|
||||
"not json": `{`,
|
||||
"no keys": `{"active":1,"keys":{}}`,
|
||||
"bad base64": `{"active":1,"keys":{"1":"not@@base64"}}`,
|
||||
"wrong key length": fmt.Sprintf(`{"active":1,"keys":{"1":%q}}`, shortKey),
|
||||
"active absent": fmt.Sprintf(`{"active":9,"keys":{"1":%q}}`, fullKey),
|
||||
"bad version": fmt.Sprintf(`{"active":1,"keys":{"nope":%q}}`, fullKey),
|
||||
}
|
||||
for name, raw := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if _, err := ParseKeyring([]byte(raw)); err == nil {
|
||||
t.Errorf("ParseKeyring(%s) = nil error, want failure", name)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateKey(t *testing.T) {
|
||||
a, err := GenerateKey()
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateKey: %v", err)
|
||||
}
|
||||
if len(a) != KeySize {
|
||||
t.Errorf("key length = %d, want %d", len(a), KeySize)
|
||||
}
|
||||
b, err := GenerateKey()
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateKey: %v", err)
|
||||
}
|
||||
if bytes.Equal(a, b) {
|
||||
t.Error("two generated keys are identical; CSPRNG suspect")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewKeyringValidation(t *testing.T) {
|
||||
if _, err := NewKeyring(1, nil); err == nil {
|
||||
t.Error("empty keyring: want error")
|
||||
}
|
||||
if _, err := NewKeyring(1, map[uint16][]byte{1: make([]byte, 8)}); err == nil {
|
||||
t.Error("short key: want error")
|
||||
}
|
||||
if _, err := NewKeyring(5, map[uint16][]byte{1: fixedKey(0)}); err == nil {
|
||||
t.Error("active not present: want error")
|
||||
}
|
||||
// NewKeyring must copy its input (mutating the caller's slice must not change
|
||||
// the keyring).
|
||||
src := fixedKey(0)
|
||||
kr := mustKeyring(t, 1, map[uint16][]byte{1: src})
|
||||
for i := range src {
|
||||
src[i] = 0xEE
|
||||
}
|
||||
sealed, err := Seal(kr, []byte("copy check"))
|
||||
if err != nil {
|
||||
t.Fatalf("Seal: %v", err)
|
||||
}
|
||||
if got, err := Open(kr, sealed); err != nil || string(got) != "copy check" {
|
||||
t.Errorf("keyring did not copy key material: %q, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNoPlaintextLeak is a belt-and-braces check that a sensitive-looking
|
||||
// plaintext does not appear anywhere in the sealed frame.
|
||||
func TestNoPlaintextLeak(t *testing.T) {
|
||||
kr := mustKeyring(t, 1, map[uint16][]byte{1: fixedKey(0)})
|
||||
secret := "MARKER-plaintext-must-not-survive-encryption"
|
||||
sealed, err := Seal(kr, []byte("report body: "+secret))
|
||||
if err != nil {
|
||||
t.Fatalf("Seal: %v", err)
|
||||
}
|
||||
if strings.Contains(string(sealed), secret) {
|
||||
t.Error("sealed frame leaks the plaintext secret")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
// Package crypto implements the encrypted-at-rest object format for scrubbed
|
||||
// LibreMail bug-reports, exactly as fixed by ADR #5
|
||||
// (docs/decisions/encryption.md): Worker-side AES-256-GCM authenticated
|
||||
// encryption applied before the object is written to R2, with the key selected
|
||||
// from a versioned keyring held in Cloudflare Secrets Store.
|
||||
//
|
||||
// # Wire format (the contract)
|
||||
//
|
||||
// Each object body is one self-describing binary frame:
|
||||
//
|
||||
// offset size field
|
||||
// ------ ---- -------------------------------------------------------------
|
||||
// 0 4 magic = ASCII "LMB1"
|
||||
// 4 1 format_version = 0x01
|
||||
// 5 2 key_id = uint16, big-endian (keyring version used)
|
||||
// 7 12 nonce = 96-bit random IV (CSPRNG, unique per object)
|
||||
// 19 N ciphertext = AES-256-GCM(plaintext)
|
||||
// 19+N 16 auth_tag = 128-bit GCM tag (appended to the ciphertext)
|
||||
//
|
||||
// The 7-byte header (magic || version || key_id) is passed to GCM as the
|
||||
// additional authenticated data (AAD): it is stored in the clear but is
|
||||
// authenticated, so an attacker cannot flip the key_id, downgrade the format, or
|
||||
// transplant a body under a different header without failing authentication.
|
||||
//
|
||||
// # Provider-independent by design
|
||||
//
|
||||
// The framing in this file carries no build constraints and is shared by both
|
||||
// crypto providers. The raw AES-256-GCM primitive is the only part that differs:
|
||||
//
|
||||
// - gcm_host.go (build tag !(js && wasm)) uses Go's crypto/aes + crypto/cipher.
|
||||
// It backs go test, cmd/devserver, and any non-Wasm build.
|
||||
// - gcm_wasm.go (build tag js && wasm) uses the Workers runtime's Web Crypto
|
||||
// (SubtleCrypto) via syscall/js, per the ADR's TinyGo/Wasm recommendation.
|
||||
//
|
||||
// AES-256-GCM is deterministic for a given key, nonce, plaintext and AAD, so both
|
||||
// providers produce byte-identical frames. The wire format above — not the
|
||||
// provider — is the contract, which is why an object sealed by one can be opened
|
||||
// by the other.
|
||||
package crypto
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// Wire-format constants. These are load-bearing: they are the ADR #5 object
|
||||
// layout and must not change without a format_version bump.
|
||||
const (
|
||||
// Magic is the 4-byte frame magic, ASCII "LMB1".
|
||||
Magic = "LMB1"
|
||||
// FormatVersion is the current frame format version.
|
||||
FormatVersion byte = 0x01
|
||||
// KeySize is the AES-256 key length in bytes.
|
||||
KeySize = 32
|
||||
// NonceSize is the GCM nonce/IV length in bytes (96 bits).
|
||||
NonceSize = 12
|
||||
// TagSize is the GCM authentication tag length in bytes (128 bits).
|
||||
TagSize = 16
|
||||
// HeaderSize is the length of the authenticated header (magic||version||key_id).
|
||||
HeaderSize = 7
|
||||
|
||||
magicLen = 4
|
||||
keyIDOffset = 5 // magic(4) + version(1)
|
||||
nonceOffset = HeaderSize
|
||||
bodyOffset = HeaderSize + NonceSize // 19: start of ciphertext||tag
|
||||
// minObjectLen is the smallest possible valid frame: header + nonce + tag,
|
||||
// i.e. an empty-plaintext object (ciphertext length 0).
|
||||
minObjectLen = HeaderSize + NonceSize + TagSize
|
||||
)
|
||||
|
||||
// Sentinel errors returned by Open. Callers MUST treat any of them as a hard
|
||||
// failure and never fall back to "publish what we have" (ADR #5, Integrity).
|
||||
var (
|
||||
// ErrMalformed means the object is too short to be a valid frame.
|
||||
ErrMalformed = errors.New("crypto: object too short or malformed")
|
||||
// ErrBadMagic means the leading 4 bytes are not the "LMB1" magic.
|
||||
ErrBadMagic = errors.New("crypto: bad magic")
|
||||
// ErrUnsupportedVersion means format_version is not one this build understands.
|
||||
ErrUnsupportedVersion = errors.New("crypto: unsupported format version")
|
||||
// ErrUnknownKeyID means the frame's key_id is not present in the keyring.
|
||||
// Never remove a key version while stored objects still reference it.
|
||||
ErrUnknownKeyID = errors.New("crypto: unknown key_id")
|
||||
// ErrAuth means GCM authentication failed: the ciphertext, tag, nonce, or
|
||||
// authenticated header was modified, or the wrong key was supplied.
|
||||
ErrAuth = errors.New("crypto: authentication failed")
|
||||
)
|
||||
|
||||
// aead is the raw AES-256-GCM primitive seam. Exactly one implementation is
|
||||
// compiled in per build target (host or Wasm); both MUST produce byte-identical
|
||||
// output for identical inputs.
|
||||
type aead interface {
|
||||
// seal returns ciphertext||tag for plaintext under key+nonce, authenticating
|
||||
// aad. key is 32 bytes, nonce is 12 bytes.
|
||||
seal(key, nonce, plaintext, aad []byte) ([]byte, error)
|
||||
// open returns the plaintext for ciphertextAndTag under key+nonce, verifying
|
||||
// aad. A non-nil error means authentication failed.
|
||||
open(key, nonce, ciphertextAndTag, aad []byte) ([]byte, error)
|
||||
}
|
||||
|
||||
// primitive is the AES-256-GCM provider, set in the init of the build-tagged
|
||||
// provider file (gcm_host.go or gcm_wasm.go).
|
||||
var primitive aead
|
||||
|
||||
// Keyring is a parsed, in-memory versioned keyring: a set of AES-256 keys indexed
|
||||
// by version (the key_id written into each frame) plus the active version used to
|
||||
// encrypt new objects. Retaining superseded versions is what makes rotation
|
||||
// data-loss-free (ADR #5, Key rotation).
|
||||
type Keyring struct {
|
||||
active uint16
|
||||
keys map[uint16][]byte
|
||||
}
|
||||
|
||||
// NewKeyring builds a Keyring from an active version and a version->key map. Each
|
||||
// key must be exactly KeySize (32) bytes, and active must exist in keys. The keys
|
||||
// are copied, so the caller may reuse its map.
|
||||
func NewKeyring(active uint16, keys map[uint16][]byte) (*Keyring, error) {
|
||||
if len(keys) == 0 {
|
||||
return nil, errors.New("crypto: keyring has no keys")
|
||||
}
|
||||
cp := make(map[uint16][]byte, len(keys))
|
||||
for id, k := range keys {
|
||||
if len(k) != KeySize {
|
||||
return nil, fmt.Errorf("crypto: key %d must be %d bytes, got %d", id, KeySize, len(k))
|
||||
}
|
||||
dup := make([]byte, KeySize)
|
||||
copy(dup, k)
|
||||
cp[id] = dup
|
||||
}
|
||||
if _, ok := cp[active]; !ok {
|
||||
return nil, fmt.Errorf("crypto: active version %d not present in keyring", active)
|
||||
}
|
||||
return &Keyring{active: active, keys: cp}, nil
|
||||
}
|
||||
|
||||
// Active returns the version new objects are encrypted under.
|
||||
func (kr *Keyring) Active() uint16 { return kr.active }
|
||||
|
||||
// activeKey returns the active key and its version.
|
||||
func (kr *Keyring) activeKey() ([]byte, uint16) {
|
||||
return kr.keys[kr.active], kr.active
|
||||
}
|
||||
|
||||
// key returns the key for version id, or ErrUnknownKeyID if the version has been
|
||||
// retired or was never present.
|
||||
func (kr *Keyring) key(id uint16) ([]byte, error) {
|
||||
k, ok := kr.keys[id]
|
||||
if !ok {
|
||||
return nil, ErrUnknownKeyID
|
||||
}
|
||||
return k, nil
|
||||
}
|
||||
|
||||
// keyringJSON is the on-the-wire shape of the Secrets Store keyring secret
|
||||
// documented in ADR #5: {"active": N, "keys": {"1": "<base64>", ...}}.
|
||||
type keyringJSON struct {
|
||||
Active uint16 `json:"active"`
|
||||
Keys map[string]string `json:"keys"`
|
||||
}
|
||||
|
||||
// ParseKeyring decodes the JSON keyring secret (as stored in Cloudflare Secrets
|
||||
// Store and read via env.BUGREPORT_ENC_KEYRING.get()) into a Keyring. Each key
|
||||
// value is standard-base64 of 32 random bytes. This is provider-independent and
|
||||
// host-testable so the exact same parse runs under go test and in the Worker.
|
||||
func ParseKeyring(raw []byte) (*Keyring, error) {
|
||||
var kj keyringJSON
|
||||
if err := json.Unmarshal(raw, &kj); err != nil {
|
||||
return nil, fmt.Errorf("crypto: parse keyring: %w", err)
|
||||
}
|
||||
if len(kj.Keys) == 0 {
|
||||
return nil, errors.New("crypto: keyring has no keys")
|
||||
}
|
||||
keys := make(map[uint16][]byte, len(kj.Keys))
|
||||
for verStr, b64 := range kj.Keys {
|
||||
ver, err := strconv.ParseUint(verStr, 10, 16)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("crypto: invalid key version %q: %w", verStr, err)
|
||||
}
|
||||
key, err := base64.StdEncoding.DecodeString(b64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("crypto: key %s is not valid base64: %w", verStr, err)
|
||||
}
|
||||
keys[uint16(ver)] = key
|
||||
}
|
||||
return NewKeyring(kj.Active, keys)
|
||||
}
|
||||
|
||||
// GenerateKey returns a fresh 32-byte AES-256 key drawn from the CSPRNG. It is
|
||||
// used by cmd/devserver (a throwaway per-run key) and by tests.
|
||||
func GenerateKey() ([]byte, error) {
|
||||
k := make([]byte, KeySize)
|
||||
if _, err := rand.Read(k); err != nil {
|
||||
return nil, fmt.Errorf("crypto: generate key: %w", err)
|
||||
}
|
||||
return k, nil
|
||||
}
|
||||
|
||||
// header builds the 7-byte authenticated header for a key_id.
|
||||
func header(keyID uint16) []byte {
|
||||
h := make([]byte, HeaderSize)
|
||||
copy(h, Magic)
|
||||
h[magicLen] = FormatVersion
|
||||
binary.BigEndian.PutUint16(h[keyIDOffset:HeaderSize], keyID)
|
||||
return h
|
||||
}
|
||||
|
||||
// Seal scrubbed plaintext into a complete R2 object frame using the keyring's
|
||||
// active key and a fresh random nonce, per ADR #5. The returned bytes are the
|
||||
// full self-describing frame and are safe to write straight to R2; only
|
||||
// ciphertext ever leaves this function.
|
||||
func Seal(kr *Keyring, plaintext []byte) ([]byte, error) {
|
||||
nonce := make([]byte, NonceSize)
|
||||
if _, err := rand.Read(nonce); err != nil {
|
||||
return nil, fmt.Errorf("crypto: nonce: %w", err)
|
||||
}
|
||||
return sealWithNonce(kr, plaintext, nonce)
|
||||
}
|
||||
|
||||
// sealWithNonce is Seal with a caller-supplied nonce. It exists so tests can pin
|
||||
// the nonce for known-answer vectors; production code must use Seal, which draws
|
||||
// a unique random nonce per object (never reuse a nonce under one key).
|
||||
func sealWithNonce(kr *Keyring, plaintext, nonce []byte) ([]byte, error) {
|
||||
if len(nonce) != NonceSize {
|
||||
return nil, fmt.Errorf("crypto: nonce must be %d bytes, got %d", NonceSize, len(nonce))
|
||||
}
|
||||
key, keyID := kr.activeKey()
|
||||
hdr := header(keyID)
|
||||
ctTag, err := primitive.seal(key, nonce, plaintext, hdr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("crypto: seal: %w", err)
|
||||
}
|
||||
out := make([]byte, 0, bodyOffset+len(ctTag))
|
||||
out = append(out, hdr...)
|
||||
out = append(out, nonce...)
|
||||
out = append(out, ctTag...)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Open parses and decrypts an R2 object frame, selecting the key by the frame's
|
||||
// key_id and verifying the authenticated header. Any tampering (to ciphertext,
|
||||
// tag, nonce, or header) or a wrong/absent key yields an error; callers must
|
||||
// treat that as a hard failure.
|
||||
func Open(kr *Keyring, object []byte) ([]byte, error) {
|
||||
if len(object) < minObjectLen {
|
||||
return nil, ErrMalformed
|
||||
}
|
||||
if string(object[:magicLen]) != Magic {
|
||||
return nil, ErrBadMagic
|
||||
}
|
||||
if object[magicLen] != FormatVersion {
|
||||
return nil, ErrUnsupportedVersion
|
||||
}
|
||||
keyID := binary.BigEndian.Uint16(object[keyIDOffset:HeaderSize])
|
||||
hdr := object[:HeaderSize]
|
||||
nonce := object[nonceOffset:bodyOffset]
|
||||
ctTag := object[bodyOffset:]
|
||||
|
||||
key, err := kr.key(keyID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plaintext, err := primitive.open(key, nonce, ctTag, hdr)
|
||||
if err != nil {
|
||||
return nil, ErrAuth
|
||||
}
|
||||
return plaintext, nil
|
||||
}
|
||||
|
||||
// KeyID reads the key_id from an object frame without decrypting it. It is useful
|
||||
// for ops/metrics; the value is authenticated only when Open succeeds, so do not
|
||||
// trust it for anything security-sensitive on its own.
|
||||
func KeyID(object []byte) (uint16, error) {
|
||||
if len(object) < HeaderSize {
|
||||
return 0, ErrMalformed
|
||||
}
|
||||
if string(object[:magicLen]) != Magic {
|
||||
return 0, ErrBadMagic
|
||||
}
|
||||
return binary.BigEndian.Uint16(object[keyIDOffset:HeaderSize]), nil
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
//go:build !(js && wasm)
|
||||
|
||||
package crypto
|
||||
|
||||
// Host AES-256-GCM provider, using Go's standard crypto/aes + crypto/cipher.
|
||||
//
|
||||
// This backs `go test`, cmd/devserver, and every non-Wasm build. The Wasm Worker
|
||||
// build uses gcm_wasm.go (SubtleCrypto) instead, because TinyGo's crypto/aes is
|
||||
// unreliable (ADR #5). Both paths implement identical AES-256-GCM with a 12-byte
|
||||
// nonce and 128-bit tag, so they produce byte-identical frames.
|
||||
|
||||
import (
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
)
|
||||
|
||||
// hostAEAD implements aead with the standard library.
|
||||
type hostAEAD struct{}
|
||||
|
||||
func init() { primitive = hostAEAD{} }
|
||||
|
||||
// newGCM builds an AES-256-GCM AEAD for key. cipher.NewGCM defaults to a 12-byte
|
||||
// nonce and 16-byte tag, matching the wire format.
|
||||
func newGCM(key []byte) (cipher.AEAD, error) {
|
||||
block, err := aes.NewCipher(key) // 32-byte key selects AES-256
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return cipher.NewGCM(block)
|
||||
}
|
||||
|
||||
func (hostAEAD) seal(key, nonce, plaintext, aad []byte) ([]byte, error) {
|
||||
gcm, err := newGCM(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Seal appends ciphertext||tag; dst nil returns a fresh slice.
|
||||
return gcm.Seal(nil, nonce, plaintext, aad), nil
|
||||
}
|
||||
|
||||
func (hostAEAD) open(key, nonce, ciphertextAndTag, aad []byte) ([]byte, error) {
|
||||
gcm, err := newGCM(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return gcm.Open(nil, nonce, ciphertextAndTag, aad)
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
//go:build js && wasm
|
||||
|
||||
package crypto
|
||||
|
||||
// Wasm AES-256-GCM provider, using the Workers runtime's Web Crypto
|
||||
// (SubtleCrypto) through syscall/js.
|
||||
//
|
||||
// Per ADR #5, TinyGo's crypto/aes is not reliable in Wasm, so the Worker build
|
||||
// performs AES-256-GCM via crypto.subtle.encrypt / crypto.subtle.decrypt with
|
||||
// { name: "AES-GCM", iv, additionalData, tagLength: 128 }. This produces exactly
|
||||
// the same ciphertext||tag as the host crypto/cipher provider (gcm_host.go), so
|
||||
// the on-disk wire format in frame.go is identical regardless of provider.
|
||||
//
|
||||
// This file is compiled only into the js/wasm Worker (it is excluded from host
|
||||
// builds and `go test`); CI's TinyGo build is what exercises it.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"syscall/js"
|
||||
)
|
||||
|
||||
// subtleAEAD implements aead via SubtleCrypto.
|
||||
type subtleAEAD struct{}
|
||||
|
||||
func init() { primitive = subtleAEAD{} }
|
||||
|
||||
// subtle returns the crypto.subtle object, fetched lazily to avoid any
|
||||
// init-ordering assumptions about JS globals.
|
||||
func subtle() js.Value {
|
||||
return js.Global().Get("crypto").Get("subtle")
|
||||
}
|
||||
|
||||
// toUint8Array copies b into a new JS Uint8Array.
|
||||
func toUint8Array(b []byte) js.Value {
|
||||
ua := js.Global().Get("Uint8Array").New(len(b))
|
||||
if len(b) > 0 {
|
||||
js.CopyBytesToJS(ua, b)
|
||||
}
|
||||
return ua
|
||||
}
|
||||
|
||||
// bytesFromArrayBuffer copies an ArrayBuffer (the result of encrypt/decrypt) into
|
||||
// a Go byte slice.
|
||||
func bytesFromArrayBuffer(buf js.Value) []byte {
|
||||
ua := js.Global().Get("Uint8Array").New(buf)
|
||||
out := make([]byte, ua.Get("length").Int())
|
||||
if len(out) > 0 {
|
||||
js.CopyBytesToGo(out, ua)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// gcmParams builds the AesGcmParams object for encrypt/decrypt.
|
||||
func gcmParams(nonce, aad []byte) js.Value {
|
||||
p := js.Global().Get("Object").New()
|
||||
p.Set("name", "AES-GCM")
|
||||
p.Set("iv", toUint8Array(nonce))
|
||||
p.Set("additionalData", toUint8Array(aad))
|
||||
p.Set("tagLength", 128)
|
||||
return p
|
||||
}
|
||||
|
||||
// importKey imports a raw 32-byte key as a non-extractable AES-GCM CryptoKey
|
||||
// usable for both encrypt and decrypt.
|
||||
func importKey(key []byte) (js.Value, error) {
|
||||
algo := js.Global().Get("Object").New()
|
||||
algo.Set("name", "AES-GCM")
|
||||
usages := js.Global().Get("Array").New()
|
||||
usages.Call("push", "encrypt")
|
||||
usages.Call("push", "decrypt")
|
||||
return await(subtle().Call("importKey", "raw", toUint8Array(key), algo, false, usages))
|
||||
}
|
||||
|
||||
func (subtleAEAD) seal(key, nonce, plaintext, aad []byte) ([]byte, error) {
|
||||
ck, err := importKey(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res, err := await(subtle().Call("encrypt", gcmParams(nonce, aad), ck, toUint8Array(plaintext)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return bytesFromArrayBuffer(res), nil
|
||||
}
|
||||
|
||||
func (subtleAEAD) open(key, nonce, ciphertextAndTag, aad []byte) ([]byte, error) {
|
||||
ck, err := importKey(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// A tampered ciphertext/tag/nonce/aad causes SubtleCrypto to reject; await
|
||||
// surfaces that as an error, which Open maps to ErrAuth.
|
||||
res, err := await(subtle().Call("decrypt", gcmParams(nonce, aad), ck, toUint8Array(ciphertextAndTag)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return bytesFromArrayBuffer(res), nil
|
||||
}
|
||||
|
||||
// await resolves a JS Promise synchronously from the calling goroutine. It is
|
||||
// safe because the Worker runs each request handler in its own goroutine (see
|
||||
// syumai/workers), so parking here lets the JS event loop run the settling
|
||||
// callback. Mirrors syumai/workers' internal AwaitPromise.
|
||||
func await(p js.Value) (js.Value, error) {
|
||||
resCh := make(chan js.Value, 1)
|
||||
errCh := make(chan error, 1)
|
||||
var then, catch js.Func
|
||||
then = js.FuncOf(func(_ js.Value, args []js.Value) any {
|
||||
then.Release()
|
||||
catch.Release()
|
||||
v := js.Undefined()
|
||||
if len(args) > 0 {
|
||||
v = args[0]
|
||||
}
|
||||
resCh <- v
|
||||
return js.Undefined()
|
||||
})
|
||||
catch = js.FuncOf(func(_ js.Value, args []js.Value) any {
|
||||
then.Release()
|
||||
catch.Release()
|
||||
msg := "unknown error"
|
||||
if len(args) > 0 {
|
||||
msg = args[0].Call("toString").String()
|
||||
}
|
||||
errCh <- fmt.Errorf("crypto/subtle: %s", msg)
|
||||
return js.Undefined()
|
||||
})
|
||||
p.Call("then", then).Call("catch", catch)
|
||||
select {
|
||||
case v := <-resCh:
|
||||
return v, nil
|
||||
case err := <-errCh:
|
||||
return js.Value{}, err
|
||||
}
|
||||
}
|
||||
@@ -27,13 +27,14 @@ const serviceName = "libremail-bug-report-ingest"
|
||||
// Any other path returns 404. On the health/hello endpoints any non-GET method
|
||||
// returns 405; on /v1/reports any non-POST method returns 405 (Allow: POST).
|
||||
//
|
||||
// The ingest endpoint is wired with a NopSink for now: it enforces the full
|
||||
// HTTP contract (size cap + schema validation) but discards accepted bodies
|
||||
// until the real storage sink (scrub + encrypt + R2) lands in #8/#9.
|
||||
func New() http.Handler {
|
||||
// sink is the storage backend for accepted reports (scrub + encrypt + R2, #9).
|
||||
// It is injected so the deployed Worker supplies the real R2/Secrets-Store sink
|
||||
// while cmd/devserver and tests supply an in-memory one. A nil sink defaults to
|
||||
// ingest.NopSink, which enforces the full HTTP contract but discards bodies.
|
||||
func New(sink ingest.Sink) http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/healthz", healthz)
|
||||
mux.Handle("/v1/reports", ingest.NewHandler(ingest.NopSink{}))
|
||||
mux.Handle("/v1/reports", ingest.NewHandler(sink))
|
||||
mux.HandleFunc("/", root)
|
||||
return mux
|
||||
}
|
||||
|
||||
@@ -4,7 +4,10 @@ import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/JMR-dev/LibreMail-Bug-Report-Ingest/internal/ingest"
|
||||
)
|
||||
|
||||
// doRequest runs a single request through the handler and returns the recorder.
|
||||
@@ -12,7 +15,7 @@ func doRequest(t *testing.T, method, target string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(method, target, nil)
|
||||
rec := httptest.NewRecorder()
|
||||
New().ServeHTTP(rec, req)
|
||||
New(nil).ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
@@ -71,6 +74,26 @@ func TestUnknownPathReturns404(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestReportsRoutedToInjectedSink proves New wires the injected sink into
|
||||
// POST /v1/reports (the #9 storage seam), returning 202 and storing the report.
|
||||
func TestReportsRoutedToInjectedSink(t *testing.T) {
|
||||
sink := &ingest.MemorySink{}
|
||||
h := New(sink)
|
||||
|
||||
body := `{"appVersion":"1.0.0","platform":"android","report":"boom"}`
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/reports", strings.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusAccepted {
|
||||
t.Fatalf("POST /v1/reports status = %d, want %d", rec.Code, http.StatusAccepted)
|
||||
}
|
||||
if sink.Len() != 1 {
|
||||
t.Fatalf("injected sink stored %d reports, want 1", sink.Len())
|
||||
}
|
||||
}
|
||||
|
||||
func TestNonGetReturns405(t *testing.T) {
|
||||
for _, path := range []string{"/", "/healthz"} {
|
||||
rec := doRequest(t, http.MethodPost, path)
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// MemoryStore is an in-memory ObjectStore for tests, cmd/devserver, and local
|
||||
// experimentation. It retains every object in a map and is not a production
|
||||
// backend (it grows without bound). It is safe for concurrent use.
|
||||
type MemoryStore struct {
|
||||
mu sync.Mutex
|
||||
objects map[string][]byte
|
||||
}
|
||||
|
||||
// NewMemoryStore returns an empty MemoryStore.
|
||||
func NewMemoryStore() *MemoryStore {
|
||||
return &MemoryStore{objects: make(map[string][]byte)}
|
||||
}
|
||||
|
||||
// Put stores a copy of data at key.
|
||||
func (m *MemoryStore) Put(_ context.Context, key string, data []byte) error {
|
||||
cp := append([]byte(nil), data...)
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.objects[key] = cp
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get returns a copy of the object at key, or ErrNotFound.
|
||||
func (m *MemoryStore) Get(_ context.Context, key string) ([]byte, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
data, ok := m.objects[key]
|
||||
if !ok {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return append([]byte(nil), data...), nil
|
||||
}
|
||||
|
||||
// Keys returns the stored object keys in sorted order (test/ops helper).
|
||||
func (m *MemoryStore) Keys() []string {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
keys := make([]string, 0, len(m.objects))
|
||||
for k := range m.objects {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
// Len reports how many objects are stored.
|
||||
func (m *MemoryStore) Len() int {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return len(m.objects)
|
||||
}
|
||||
|
||||
var _ ObjectStore = (*MemoryStore)(nil)
|
||||
@@ -0,0 +1,54 @@
|
||||
//go:build js && wasm
|
||||
|
||||
package storage
|
||||
|
||||
// R2-backed ObjectStore for the Cloudflare Worker build. It wraps the
|
||||
// syumai/workers R2 binding API; only sealed (ciphertext) frames are ever
|
||||
// written, so R2 never holds plaintext or the key (ADR #5).
|
||||
//
|
||||
// Compiled only into the js/wasm Worker; excluded from host builds and tests.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
|
||||
"github.com/syumai/workers/cloudflare/r2"
|
||||
)
|
||||
|
||||
// R2Store is an ObjectStore backed by a Cloudflare R2 bucket binding.
|
||||
type R2Store struct {
|
||||
bucket *r2.Bucket
|
||||
}
|
||||
|
||||
// NewR2Store resolves the R2 bucket bound as binding (see BucketBinding) from the
|
||||
// Worker runtime context. It must be called within a request/scheduled handler,
|
||||
// where the binding is available.
|
||||
func NewR2Store(binding string) (*R2Store, error) {
|
||||
b, err := r2.NewBucket(binding)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &R2Store{bucket: b}, nil
|
||||
}
|
||||
|
||||
// Put writes data at key. The syumai R2 Put reads the whole body into memory and
|
||||
// PUTs it as bytes.
|
||||
func (s *R2Store) Put(_ context.Context, key string, data []byte) error {
|
||||
_, err := s.bucket.Put(key, io.NopCloser(bytes.NewReader(data)), nil)
|
||||
return err
|
||||
}
|
||||
|
||||
// Get reads the object at key, or returns ErrNotFound when it is absent.
|
||||
func (s *R2Store) Get(_ context.Context, key string) ([]byte, error) {
|
||||
obj, err := s.bucket.Get(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if obj == nil || obj.Body == nil {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return io.ReadAll(obj.Body)
|
||||
}
|
||||
|
||||
var _ ObjectStore = (*R2Store)(nil)
|
||||
@@ -0,0 +1,79 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/JMR-dev/LibreMail-Bug-Report-Ingest/internal/crypto"
|
||||
"github.com/JMR-dev/LibreMail-Bug-Report-Ingest/internal/ingest"
|
||||
"github.com/JMR-dev/LibreMail-Bug-Report-Ingest/internal/scrub"
|
||||
)
|
||||
|
||||
// objectKeyPrefix namespaces report objects within the bucket.
|
||||
const objectKeyPrefix = "reports/"
|
||||
|
||||
// Sink is the real ingest.Sink. For each accepted report it:
|
||||
//
|
||||
// 1. scrubs the raw body (best-effort PII redaction, #8);
|
||||
// 2. seals the scrubbed bytes with the keyring's active AES-256 key
|
||||
// (AES-256-GCM, ADR #5) into a self-describing frame;
|
||||
// 3. writes the frame to the ObjectStore under a unique, unguessable key.
|
||||
//
|
||||
// The store only ever receives ciphertext. Sink is provider-agnostic: paired
|
||||
// with a MemoryStore it runs on the host (tests, cmd/devserver); paired with an
|
||||
// R2Store it runs in the Worker. The Worker's keyring loading (from Secrets
|
||||
// Store) is handled by WorkerSink, which delegates the pipeline to a Sink.
|
||||
type Sink struct {
|
||||
store ObjectStore
|
||||
keyring *crypto.Keyring
|
||||
keyFn func() string
|
||||
}
|
||||
|
||||
// Option customises a Sink.
|
||||
type Option func(*Sink)
|
||||
|
||||
// WithKeyFunc overrides the object-key generator. Intended for tests that need a
|
||||
// deterministic key; production uses the default random key.
|
||||
func WithKeyFunc(fn func() string) Option {
|
||||
return func(s *Sink) { s.keyFn = fn }
|
||||
}
|
||||
|
||||
// NewSink returns a Sink that stores into store, encrypting under kr's active
|
||||
// key. kr must be non-nil.
|
||||
func NewSink(store ObjectStore, kr *crypto.Keyring, opts ...Option) *Sink {
|
||||
s := &Sink{store: store, keyring: kr, keyFn: defaultObjectKey}
|
||||
for _, o := range opts {
|
||||
o(s)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Store scrubs, encrypts, and persists one accepted report. A non-nil return
|
||||
// makes the ingest endpoint answer 503 (per ADR #6), so callers should retry.
|
||||
func (s *Sink) Store(ctx context.Context, raw []byte) error {
|
||||
scrubbed := scrub.Scrub(raw)
|
||||
sealed, err := crypto.Seal(s.keyring, scrubbed)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: seal: %w", err)
|
||||
}
|
||||
if err := s.store.Put(ctx, s.keyFn(), sealed); err != nil {
|
||||
return fmt.Errorf("storage: put: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// defaultObjectKey builds a unique object key: a UTC timestamp (for rough
|
||||
// lexicographic ordering, convenient for the weekly publish job) plus 80 bits of
|
||||
// CSPRNG randomness (so keys are unguessable and collision-free within a second).
|
||||
func defaultObjectKey() string {
|
||||
var b [10]byte
|
||||
_, _ = rand.Read(b[:])
|
||||
ts := time.Now().UTC().Format("20060102T150405")
|
||||
return objectKeyPrefix + ts + "-" + hex.EncodeToString(b[:])
|
||||
}
|
||||
|
||||
// Sink satisfies the ingest storage seam.
|
||||
var _ ingest.Sink = (*Sink)(nil)
|
||||
@@ -0,0 +1,199 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/JMR-dev/LibreMail-Bug-Report-Ingest/internal/crypto"
|
||||
"github.com/JMR-dev/LibreMail-Bug-Report-Ingest/internal/scrub"
|
||||
)
|
||||
|
||||
// ghToken is a GitHub-token-shaped test value, assembled at runtime so the
|
||||
// contiguous token pattern never appears as a literal in source (secret scanners
|
||||
// flag test tokens on sight; internal/scrub/scrub_test.go uses the same trick for
|
||||
// a GitLab PAT).
|
||||
var ghToken = "ghp_" + "0123456789abcdefghijklmnopqrstuvwxyz"
|
||||
|
||||
// rawReport is a realistic raw body carrying several kinds of PII plus some
|
||||
// non-sensitive structure that must survive scrubbing.
|
||||
var rawReport = `{"appVersion":"1.4.2","platform":"android",` +
|
||||
`"report":"crash when syncing. contact alice@example.com. ` +
|
||||
`token=` + ghToken + ` from 203.0.113.7"}`
|
||||
|
||||
// piiSubstrings are values that must never survive scrubbing, and must never be
|
||||
// readable in the stored ciphertext.
|
||||
var piiSubstrings = []string{
|
||||
"alice@example.com",
|
||||
ghToken,
|
||||
"203.0.113.7",
|
||||
}
|
||||
|
||||
func mustKeyring(t *testing.T) *crypto.Keyring {
|
||||
t.Helper()
|
||||
key, err := crypto.GenerateKey()
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateKey: %v", err)
|
||||
}
|
||||
kr, err := crypto.NewKeyring(1, map[uint16][]byte{1: key})
|
||||
if err != nil {
|
||||
t.Fatalf("NewKeyring: %v", err)
|
||||
}
|
||||
return kr
|
||||
}
|
||||
|
||||
func TestMemoryStorePutGet(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
m := NewMemoryStore()
|
||||
|
||||
if _, err := m.Get(ctx, "absent"); !errors.Is(err, ErrNotFound) {
|
||||
t.Errorf("Get(absent) err = %v, want ErrNotFound", err)
|
||||
}
|
||||
|
||||
want := []byte("some bytes")
|
||||
if err := m.Put(ctx, "k", want); err != nil {
|
||||
t.Fatalf("Put: %v", err)
|
||||
}
|
||||
got, err := m.Get(ctx, "k")
|
||||
if err != nil {
|
||||
t.Fatalf("Get: %v", err)
|
||||
}
|
||||
if !bytes.Equal(got, want) {
|
||||
t.Errorf("Get = %q, want %q", got, want)
|
||||
}
|
||||
// Stored copy must be independent of the caller's slice.
|
||||
want[0] = 'X'
|
||||
if again, _ := m.Get(ctx, "k"); again[0] == 'X' {
|
||||
t.Error("MemoryStore did not copy the value on Put")
|
||||
}
|
||||
if m.Len() != 1 {
|
||||
t.Errorf("Len = %d, want 1", m.Len())
|
||||
}
|
||||
}
|
||||
|
||||
// TestSinkFullPath is the end-to-end acceptance: raw report -> scrubbed ->
|
||||
// encrypted -> stored, and reading it back REQUIRES the key and yields the
|
||||
// scrubbed (not raw) content.
|
||||
func TestSinkFullPath(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
kr := mustKeyring(t)
|
||||
store := NewMemoryStore()
|
||||
sink := NewSink(store, kr)
|
||||
|
||||
if err := sink.Store(ctx, []byte(rawReport)); err != nil {
|
||||
t.Fatalf("Store: %v", err)
|
||||
}
|
||||
|
||||
// Exactly one object, under a reports/ key.
|
||||
keys := store.Keys()
|
||||
if len(keys) != 1 {
|
||||
t.Fatalf("stored %d objects, want 1", len(keys))
|
||||
}
|
||||
if !strings.HasPrefix(keys[0], objectKeyPrefix) {
|
||||
t.Errorf("object key %q lacks prefix %q", keys[0], objectKeyPrefix)
|
||||
}
|
||||
|
||||
stored, err := store.Get(ctx, keys[0])
|
||||
if err != nil {
|
||||
t.Fatalf("Get stored: %v", err)
|
||||
}
|
||||
|
||||
// The stored bytes are a sealed frame, not the raw or scrubbed plaintext.
|
||||
scrubbed := scrub.Scrub([]byte(rawReport))
|
||||
if bytes.Equal(stored, []byte(rawReport)) {
|
||||
t.Error("stored object equals the raw body (not encrypted)")
|
||||
}
|
||||
if bytes.Equal(stored, scrubbed) {
|
||||
t.Error("stored object equals the scrubbed plaintext (not encrypted)")
|
||||
}
|
||||
if string(stored[:4]) != crypto.Magic {
|
||||
t.Errorf("stored object magic = %q, want %q", stored[:4], crypto.Magic)
|
||||
}
|
||||
|
||||
// The ciphertext must not leak any PII substring in the clear.
|
||||
for _, s := range piiSubstrings {
|
||||
if bytes.Contains(stored, []byte(s)) {
|
||||
t.Errorf("stored ciphertext leaks PII substring %q", s)
|
||||
}
|
||||
}
|
||||
|
||||
// The object is NOT decryptable without the key.
|
||||
otherKR := mustKeyring(t) // different random key, same key_id 1
|
||||
if _, err := crypto.Open(otherKR, stored); err == nil {
|
||||
t.Error("stored object decrypted with the WRONG key; must require the correct key")
|
||||
}
|
||||
|
||||
// With the correct key it yields exactly the scrubbed content.
|
||||
opened, err := crypto.Open(kr, stored)
|
||||
if err != nil {
|
||||
t.Fatalf("Open with correct key: %v", err)
|
||||
}
|
||||
if !bytes.Equal(opened, scrubbed) {
|
||||
t.Errorf("decrypted content != scrubbed content\n got = %q\n want = %q", opened, scrubbed)
|
||||
}
|
||||
|
||||
// The decrypted (scrubbed) content has the PII removed (verify via #8's
|
||||
// contract) and carries the redaction placeholders and surviving structure.
|
||||
openedStr := string(opened)
|
||||
for _, s := range piiSubstrings {
|
||||
if strings.Contains(openedStr, s) {
|
||||
t.Errorf("decrypted scrubbed content still leaks PII %q", s)
|
||||
}
|
||||
}
|
||||
for _, p := range []string{scrub.PlaceholderEmail, scrub.PlaceholderToken, scrub.PlaceholderIP} {
|
||||
if !strings.Contains(openedStr, p) {
|
||||
t.Errorf("decrypted scrubbed content missing placeholder %q", p)
|
||||
}
|
||||
}
|
||||
for _, keep := range []string{"crash when syncing", `"appVersion":"1.4.2"`} {
|
||||
if !strings.Contains(openedStr, keep) {
|
||||
t.Errorf("decrypted scrubbed content dropped non-PII text %q", keep)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// failingStore always fails Put, to exercise the Sink's error propagation (which
|
||||
// drives the ingest 503 path).
|
||||
type failingStore struct{}
|
||||
|
||||
func (failingStore) Put(context.Context, string, []byte) error {
|
||||
return errors.New("backend down")
|
||||
}
|
||||
func (failingStore) Get(context.Context, string) ([]byte, error) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
|
||||
func TestSinkStorePutError(t *testing.T) {
|
||||
sink := NewSink(failingStore{}, mustKeyring(t))
|
||||
if err := sink.Store(context.Background(), []byte(rawReport)); err == nil {
|
||||
t.Error("Store returned nil despite a failing backend; want error (drives 503)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSinkUsesDistinctKeys(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := NewMemoryStore()
|
||||
sink := NewSink(store, mustKeyring(t))
|
||||
for i := 0; i < 5; i++ {
|
||||
if err := sink.Store(ctx, []byte(rawReport)); err != nil {
|
||||
t.Fatalf("Store #%d: %v", i, err)
|
||||
}
|
||||
}
|
||||
if store.Len() != 5 {
|
||||
t.Errorf("stored %d objects, want 5 distinct keys", store.Len())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSinkWithKeyFunc(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := NewMemoryStore()
|
||||
sink := NewSink(store, mustKeyring(t), WithKeyFunc(func() string { return "reports/fixed" }))
|
||||
if err := sink.Store(ctx, []byte(rawReport)); err != nil {
|
||||
t.Fatalf("Store: %v", err)
|
||||
}
|
||||
if _, err := store.Get(ctx, "reports/fixed"); err != nil {
|
||||
t.Errorf("expected object at fixed key: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
// Package storage persists accepted LibreMail bug-reports as encrypted-at-rest
|
||||
// objects, implementing the storage half of the ingest pipeline: for each
|
||||
// accepted report it scrubs PII (internal/scrub, #8), encrypts the scrubbed
|
||||
// bytes with AES-256-GCM (internal/crypto, ADR #5), and writes only the opaque
|
||||
// ciphertext frame to the object store. The store never sees plaintext or the
|
||||
// key.
|
||||
//
|
||||
// The package follows the repo's build-tag pattern so it is host-testable
|
||||
// without TinyGo or the Workers runtime:
|
||||
//
|
||||
// - The ObjectStore interface, MemoryStore, and the Sink (scrub+encrypt+put)
|
||||
// carry no build constraints and are unit-tested with `go test`.
|
||||
// - The real R2-backed store (R2Store) and the Worker sink that loads the
|
||||
// keyring from Cloudflare Secrets Store live behind //go:build js && wasm and
|
||||
// are compiled by the Wasm Worker build in CI.
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
)
|
||||
|
||||
// ErrNotFound is returned by ObjectStore.Get when no object exists at the key.
|
||||
var ErrNotFound = errors.New("storage: object not found")
|
||||
|
||||
// Binding names wired in wrangler.jsonc and provisioned by infra (#2).
|
||||
const (
|
||||
// BucketBinding is the R2 bucket binding name (the JS var the Worker reads).
|
||||
// The bound bucket is "libremail-bug-reports" (infra defaultR2BucketName).
|
||||
BucketBinding = "REPORTS_BUCKET"
|
||||
// KeyringBinding is the Cloudflare Secrets Store binding holding the JSON
|
||||
// keyring secret, named per ADR #5.
|
||||
KeyringBinding = "BUGREPORT_ENC_KEYRING"
|
||||
)
|
||||
|
||||
// ObjectStore is the seam for the opaque object backend. Implementations only
|
||||
// ever handle already-encrypted frames.
|
||||
//
|
||||
// - Put writes data (a sealed frame) at key, overwriting any existing object.
|
||||
// - Get reads the bytes back, or returns ErrNotFound. Get exists mainly for the
|
||||
// future publish job (#35) and for tests; the ingest path is write-only.
|
||||
type ObjectStore interface {
|
||||
Put(ctx context.Context, key string, data []byte) error
|
||||
Get(ctx context.Context, key string) ([]byte, error)
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
//go:build js && wasm
|
||||
|
||||
package storage
|
||||
|
||||
// WorkerSink is the production ingest.Sink for the Cloudflare Worker. It loads
|
||||
// the versioned keyring from Cloudflare Secrets Store, resolves the R2 bucket
|
||||
// binding, and delegates the scrub -> encrypt -> put pipeline to a plain Sink.
|
||||
//
|
||||
// Per ADR #5 the keyring secret is fetched inside the request handler (the
|
||||
// binding's get() is async and the runtime context is only present per-request),
|
||||
// not at module top-level. The parsed keyring is cached for the isolate lifetime;
|
||||
// isolates recycle, which is how a rotated `active` version takes over.
|
||||
//
|
||||
// Compiled only into the js/wasm Worker; excluded from host builds and tests.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"syscall/js"
|
||||
|
||||
"github.com/syumai/workers/cloudflare"
|
||||
|
||||
"github.com/JMR-dev/LibreMail-Bug-Report-Ingest/internal/crypto"
|
||||
"github.com/JMR-dev/LibreMail-Bug-Report-Ingest/internal/ingest"
|
||||
)
|
||||
|
||||
// WorkerSink implements ingest.Sink against R2 + Secrets Store bindings.
|
||||
type WorkerSink struct {
|
||||
bucketBinding string
|
||||
keyringBinding string
|
||||
|
||||
mu sync.Mutex
|
||||
keyring *crypto.Keyring // cached for the isolate lifetime after first load
|
||||
}
|
||||
|
||||
// NewWorkerSink returns a WorkerSink using the standard binding names. It does no
|
||||
// I/O and touches no runtime context, so it is safe to construct at main() time;
|
||||
// bindings are resolved lazily on the first Store call.
|
||||
func NewWorkerSink() *WorkerSink {
|
||||
return &WorkerSink{
|
||||
bucketBinding: BucketBinding,
|
||||
keyringBinding: KeyringBinding,
|
||||
}
|
||||
}
|
||||
|
||||
// Store loads the keyring (cached), resolves the R2 bucket, and runs the shared
|
||||
// scrub+encrypt+put pipeline.
|
||||
func (s *WorkerSink) Store(ctx context.Context, raw []byte) error {
|
||||
kr, err := s.loadKeyring()
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: load keyring: %w", err)
|
||||
}
|
||||
store, err := NewR2Store(s.bucketBinding)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: r2 binding: %w", err)
|
||||
}
|
||||
return NewSink(store, kr).Store(ctx, raw)
|
||||
}
|
||||
|
||||
// loadKeyring returns the cached keyring, loading and parsing it from Secrets
|
||||
// Store on first use. A failed load is not cached, so it is retried next request.
|
||||
func (s *WorkerSink) loadKeyring() (*crypto.Keyring, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.keyring != nil {
|
||||
return s.keyring, nil
|
||||
}
|
||||
raw, err := getSecret(s.keyringBinding)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kr, err := crypto.ParseKeyring(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.keyring = kr
|
||||
return kr, nil
|
||||
}
|
||||
|
||||
// getSecret reads a Cloudflare Secrets Store secret via `await binding.get()`.
|
||||
// The returned value must never be logged or echoed (ADR #5, Key custody).
|
||||
func getSecret(binding string) ([]byte, error) {
|
||||
b := cloudflare.GetBinding(binding)
|
||||
if b.IsUndefined() || b.IsNull() {
|
||||
return nil, fmt.Errorf("secrets store binding %q is not bound", binding)
|
||||
}
|
||||
v, err := await(b.Call("get"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if v.IsUndefined() || v.IsNull() {
|
||||
return nil, fmt.Errorf("secrets store binding %q returned no value", binding)
|
||||
}
|
||||
return []byte(v.String()), nil
|
||||
}
|
||||
|
||||
// await resolves a JS Promise from the calling goroutine (the Worker runs each
|
||||
// handler in its own goroutine, so the JS event loop can settle the promise).
|
||||
func await(p js.Value) (js.Value, error) {
|
||||
resCh := make(chan js.Value, 1)
|
||||
errCh := make(chan error, 1)
|
||||
var then, catch js.Func
|
||||
then = js.FuncOf(func(_ js.Value, args []js.Value) any {
|
||||
then.Release()
|
||||
catch.Release()
|
||||
v := js.Undefined()
|
||||
if len(args) > 0 {
|
||||
v = args[0]
|
||||
}
|
||||
resCh <- v
|
||||
return js.Undefined()
|
||||
})
|
||||
catch = js.FuncOf(func(_ js.Value, args []js.Value) any {
|
||||
then.Release()
|
||||
catch.Release()
|
||||
msg := "unknown error"
|
||||
if len(args) > 0 {
|
||||
msg = args[0].Call("toString").String()
|
||||
}
|
||||
errCh <- fmt.Errorf("secrets store: %s", msg)
|
||||
return js.Undefined()
|
||||
})
|
||||
p.Call("then", then).Call("catch", catch)
|
||||
select {
|
||||
case v := <-resCh:
|
||||
return v, nil
|
||||
case err := <-errCh:
|
||||
return js.Value{}, err
|
||||
}
|
||||
}
|
||||
|
||||
var _ ingest.Sink = (*WorkerSink)(nil)
|
||||
+5
-1
@@ -14,8 +14,12 @@ import (
|
||||
"github.com/syumai/workers"
|
||||
|
||||
"github.com/JMR-dev/LibreMail-Bug-Report-Ingest/internal/handler"
|
||||
"github.com/JMR-dev/LibreMail-Bug-Report-Ingest/internal/storage"
|
||||
)
|
||||
|
||||
func main() {
|
||||
workers.Serve(handler.New())
|
||||
// The real storage Sink (#9): scrub -> AES-256-GCM encrypt -> R2 put, with the
|
||||
// keyring loaded from Cloudflare Secrets Store on first request. Bindings
|
||||
// (REPORTS_BUCKET, BUGREPORT_ENC_KEYRING) are declared in wrangler.jsonc.
|
||||
workers.Serve(handler.New(storage.NewWorkerSink()))
|
||||
}
|
||||
|
||||
+27
-1
@@ -15,5 +15,31 @@
|
||||
// Requires TinyGo locally; TinyGo is installed in CI.
|
||||
"build": {
|
||||
"command": "pnpm run build"
|
||||
}
|
||||
},
|
||||
|
||||
// R2 bucket for the encrypted-at-rest bug-report objects (ADR #5 / issue #9).
|
||||
// The Worker encrypts each scrubbed report with AES-256-GCM before writing, so
|
||||
// only ciphertext is ever stored here. "binding" is the JS var the Worker code
|
||||
// reads (internal/storage.BucketBinding); "bucket_name" matches the bucket
|
||||
// provisioned by infra/ (defaultR2BucketName = "libremail-bug-reports").
|
||||
"r2_buckets": [
|
||||
{
|
||||
"binding": "REPORTS_BUCKET",
|
||||
"bucket_name": "libremail-bug-reports"
|
||||
}
|
||||
],
|
||||
|
||||
// Cloudflare Secrets Store secret holding the versioned encryption keyring
|
||||
// (ADR #5, Key custody): a single JSON secret {active, keys{ver: base64-32B}}.
|
||||
// "binding" is read at runtime via env.BUGREPORT_ENC_KEYRING.get()
|
||||
// (internal/storage.KeyringBinding). Replace "<store-id>" with the account's
|
||||
// Secrets Store id at deploy time; it is not needed for `pnpm run build`
|
||||
// (Wasm compile) or the devserver, so CI does not require it.
|
||||
"secrets_store_secrets": [
|
||||
{
|
||||
"binding": "BUGREPORT_ENC_KEYRING",
|
||||
"store_id": "<store-id>",
|
||||
"secret_name": "bugreport-enc-keyring"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user