diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6453dbd..eb2ade0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,8 +50,20 @@ jobs: - run: node scripts/docs-check.mjs + # Every desktop the release ships to, because the parts of this crate that differ by platform are + # the parts nobody runs by hand: the UserNotifications bridge on macOS, the D-Bus one everywhere + # else, the machine id the refresh tokens are sealed against. A break in one of those used to + # surface at release time on a runner nobody was watching. rust: - runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + include: + # Ubuntu 22.04 is the glibc baseline the release builds on, so it is what CI tests on. + - os: ubuntu-22.04 + - os: macos-26 + - os: windows-latest + runs-on: ${{ matrix.os }} defaults: run: working-directory: rust/margin-mail @@ -61,6 +73,7 @@ jobs: path: rust/margin-mail - name: Install Linux dependencies + if: runner.os == 'Linux' run: | sudo apt-get update sudo apt-get install -y \ @@ -79,14 +92,107 @@ jobs: - uses: swatinem/rust-cache@v2 with: workspaces: rust/margin-mail/src-tauri -> target + key: ${{ matrix.os }} # tauri_build::build() wants a frontendDist that exists, and build.rs wants credentials to # embed. The example file is what a fresh clone compiles against, so that is what CI uses. - name: Stub the build inputs + shell: bash run: | mkdir -p dist && touch dist/index.html cp google-credentials.example.json google-credentials.json + # The gate docs/release.md names is the test suites, and that is all this enforces. `cargo fmt + # --check` and `cargo clippy -D warnings` both fail on the tree as it stands; adopting either + # is a cleanup pass to decide on separately, not something to bolt onto CI first. - name: Test working-directory: rust/margin-mail/src-tauri run: cargo test + + # The flake at whatever release nix/release.json pins. A broken flake, or a deb that no longer + # patches against current nixpkgs, shows up here rather than at the next release. Before the + # first release there is nothing pinned and nothing to build. + nix: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - id: pin + run: echo "version=$(jq -r '.version // ""' nix/release.json)" >> "$GITHUB_OUTPUT" + + - uses: cachix/install-nix-action@v31 + if: steps.pin.outputs.version != '' + + # --impure with the environment variable because the licence is FSL rather than MIT, so + # nixpkgs treats the package as unfree and refuses to build it otherwise. + - if: steps.pin.outputs.version != '' + run: NIXPKGS_ALLOW_UNFREE=1 nix build --impure .#margin-mail --print-build-logs + + - if: steps.pin.outputs.version == '' + run: echo "nix/release.json pins no release yet, so there is nothing to build." + + # The flatpak manifest is hand-written rather than produced by Tauri, so nothing else would catch + # a runtime that stopped carrying webkit2gtk-4.1 or a permission the app needs and does not ask + # for. The release job repackages the published deb; this one builds the same manifest against a + # deb built here, which is the only difference between them. + # + # On main rather than on every pull request: it is a full release-mode build plus a runtime + # download, the manifest changes about once a year, and a break in it is worth finding within the + # day rather than within the minute. + flatpak: + if: github.event_name == 'push' + runs-on: ubuntu-22.04 + defaults: + run: + working-directory: rust/margin-mail + steps: + - uses: actions/checkout@v7 + with: + path: rust/margin-mail + + - uses: actions/checkout@v7 + with: + repository: priyanshujain/margin + path: python/margin + + - name: Install Linux dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + libwebkit2gtk-4.1-dev \ + libgtk-3-dev \ + libayatana-appindicator3-dev \ + librsvg2-dev \ + patchelf \ + libxdo-dev \ + libssl-dev \ + build-essential \ + flatpak \ + flatpak-builder + + - uses: actions/setup-node@v6 + with: + node-version: 26 + + - uses: pnpm/action-setup@v6 + with: + version: 10 + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - uses: swatinem/rust-cache@v2 + with: + workspaces: rust/margin-mail/src-tauri -> target + key: flatpak + + - run: pnpm install --frozen-lockfile + + - run: cp google-credentials.example.json google-credentials.json + + - run: pnpm tauri build --bundles deb + + - name: Build the flatpak + run: | + mv src-tauri/target/release/bundle/deb/*.deb flatpak/margin-mail.deb + flatpak/build.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 457ee06..3f99d19 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,6 +44,12 @@ jobs: jq --arg v "$VERSION" '.version = $v' src-tauri/tauri.conf.json > "$tmp" && mv "$tmp" src-tauri/tauri.conf.json jq --arg v "$VERSION" '.version = $v' package.json > "$tmp" && mv "$tmp" package.json sed -i "0,/^version = \".*\"/s//version = \"$VERSION\"/" src-tauri/Cargo.toml + # Cargo.lock records margin-mail's own version, so bumping only Cargo.toml leaves the lock + # a release behind and the next build rewrites it under whoever checked it out. + awk -v v="$VERSION" ' + /^name = "margin-mail"$/ { print; getline; sub(/^version = ".*"/, "version = \"" v "\""); print; next } + { print } + ' src-tauri/Cargo.lock > "$tmp" && mv "$tmp" src-tauri/Cargo.lock - name: Commit and tag env: @@ -51,7 +57,7 @@ jobs: run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - git add src-tauri/tauri.conf.json package.json src-tauri/Cargo.toml + git add src-tauri/tauri.conf.json package.json src-tauri/Cargo.toml src-tauri/Cargo.lock git commit -m "chore(release): $TAG" for attempt in 1 2 3 4 5; do git fetch origin main @@ -93,6 +99,9 @@ jobs: - os: ubuntu-22.04 args: "--config src-tauri/tauri.release.conf.json" rust-targets: "" + - os: windows-latest + args: "--config src-tauri/tauri.release.conf.json" + rust-targets: "" runs-on: ${{ matrix.os }} defaults: run: @@ -109,7 +118,7 @@ jobs: path: python/margin - name: Install Linux dependencies - if: startsWith(matrix.os, 'ubuntu') + if: runner.os == 'Linux' run: | sudo apt-get update sudo apt-get install -y \ @@ -141,6 +150,7 @@ jobs: - uses: swatinem/rust-cache@v2 with: workspaces: rust/margin-mail/src-tauri -> target + key: ${{ matrix.os }} - name: Install frontend dependencies run: pnpm install --frozen-lockfile @@ -160,6 +170,42 @@ jobs: echo "::warning::GOOGLE_CREDENTIALS secret not set, embedding placeholder credentials; this build cannot connect to Gmail." fi + # macOS shows no notifications from a bundle that is not signed, so tauri.conf.json ad-hoc + # signs at minimum; a Developer ID from the secrets replaces that, and the App Store Connect + # key notarizes on top. Only what is present is exported, because Tauri takes an empty + # APPLE_SIGNING_IDENTITY for an identity and fails the signing step on it. + - name: Provision Apple signing + if: runner.os == 'macOS' + shell: bash + env: + APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }} + APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }} + APPLE_SIGNING_IDENTITY: ${{ secrets.APPLE_SIGNING_IDENTITY }} + APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} + APPLE_API_ISSUER: ${{ secrets.APPLE_API_ISSUER }} + APPLE_API_KEY: ${{ secrets.APPLE_API_KEY }} + APPLE_API_KEY_P8: ${{ secrets.APPLE_API_KEY_P8 }} + run: | + if [ -z "$APPLE_CERTIFICATE" ] || [ -z "$APPLE_SIGNING_IDENTITY" ]; then + echo "::warning::APPLE_CERTIFICATE or APPLE_SIGNING_IDENTITY not set; the macOS bundle will be ad-hoc signed and not notarized." + exit 0 + fi + for name in APPLE_CERTIFICATE APPLE_CERTIFICATE_PASSWORD APPLE_SIGNING_IDENTITY APPLE_TEAM_ID; do + { echo "$name<> "$GITHUB_ENV" + done + echo "Signing as $APPLE_SIGNING_IDENTITY" + if [ -n "$APPLE_API_KEY_P8" ] && [ -n "$APPLE_API_KEY" ] && [ -n "$APPLE_API_ISSUER" ]; then + printf '%s' "$APPLE_API_KEY_P8" > "$RUNNER_TEMP/AuthKey.p8" + { + echo "APPLE_API_KEY=$APPLE_API_KEY" + echo "APPLE_API_ISSUER=$APPLE_API_ISSUER" + echo "APPLE_API_KEY_PATH=$RUNNER_TEMP/AuthKey.p8" + } >> "$GITHUB_ENV" + echo "Notarizing with App Store Connect key $APPLE_API_KEY" + else + echo "::warning::APPLE_API_KEY, APPLE_API_ISSUER or APPLE_API_KEY_P8 not set; the macOS bundle will be signed and not notarized." + fi + - name: Build and upload uses: tauri-apps/tauri-action@v0 with: @@ -171,8 +217,45 @@ jobs: TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY }} TAURI_SIGNING_PRIVATE_KEY_PASSWORD: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY_PASSWORD }} - publish: + - name: Verify the bundle is signed and notarized + if: runner.os == 'macOS' && env.APPLE_API_KEY_PATH != '' + run: | + app="src-tauri/target/universal-apple-darwin/release/bundle/macos/Margin Mail.app" + codesign --verify --deep --strict --verbose=2 "$app" + # Gatekeeper only says "accepted" once the notarization ticket is stapled to the bundle, + # so this is the check that somebody double-clicking the dmg will actually get past. + spctl --assess --type execute --verbose=4 "$app" + xcrun stapler validate "$app" + + # The flatpak is not a Tauri bundle target, so it is built here from the deb the Linux job just + # published and uploaded to the same draft release. Before publish, so a release never goes out + # with the Linux artifacts half there. + flatpak: needs: [prepare, build] + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ needs.prepare.outputs.tag }} + + - run: | + sudo apt-get update + sudo apt-get install -y flatpak flatpak-builder + + - name: Build the flatpak from the published deb + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + TAG: ${{ needs.prepare.outputs.tag }} + VERSION: ${{ needs.prepare.outputs.version }} + run: | + gh release download "$TAG" --repo "$REPO" \ + --pattern "Margin.Mail_${VERSION}_amd64.deb" --output flatpak/margin-mail.deb + flatpak/build.sh "$PWD/Margin.Mail_${VERSION}_amd64.flatpak" + gh release upload "$TAG" --repo "$REPO" "Margin.Mail_${VERSION}_amd64.flatpak" --clobber + + publish: + needs: [prepare, build, flatpak] runs-on: ubuntu-latest steps: - name: Verify manifest is complete, then publish @@ -184,10 +267,71 @@ jobs: gh release download "$TAG" --repo "$REPO" --pattern latest.json --output latest.json --clobber echo "Platforms in latest.json:" jq '.platforms | keys' latest.json - for key in darwin-aarch64 darwin-x86_64 linux-x86_64; do + for key in darwin-aarch64 darwin-x86_64 linux-x86_64 windows-x86_64; do if ! jq -e ".platforms[\"$key\"].url" latest.json > /dev/null; then echo "::error::latest.json is missing platform '$key', refusing to publish a partial update manifest. Re-run the release." exit 1 fi done gh release edit "$TAG" --repo "$REPO" --draft=false --latest + + # Nix is the other Linux package. The AppImage carries Ubuntu's GTK stack, which cannot talk to a + # modern Wayland compositor and silently falls back to Xwayland; the Nix package relinks the + # published deb against nixpkgs' webkit2gtk and runs as a native Wayland client. Runs after + # publish so the flake can only ever point at a release that survived the manifest check. + nix: + needs: [prepare, publish] + runs-on: ubuntu-latest + steps: + # main rather than the tag: the pin lands on main, and the tag was cut before the artifact + # it needs the hash of existed. + - uses: actions/checkout@v7 + with: + ref: main + + - name: Pin the flake to this release + env: + VERSION: ${{ needs.prepare.outputs.version }} + REPO: ${{ github.repository }} + run: | + URL="https://github.com/$REPO/releases/download/v$VERSION/Margin.Mail_${VERSION}_amd64.deb" + # From the published asset, so the hash is of the artifact users will actually fetch. + curl -fsSL --retry 3 -o package.deb "$URL" + HASH="sha256-$(openssl dgst -sha256 -binary package.deb | base64)" + jq -n --arg v "$VERSION" --arg h "$HASH" '{version: $v, hash: $h}' > nix/release.json + cat nix/release.json + + - uses: cachix/install-nix-action@v31 + + # Building it is the check: a wrong hash, a library autoPatchelf cannot find or a broken flake + # stops here rather than on someone's machine. --impure and the variable because FSL is not a + # free licence, so nixpkgs refuses to build the package without being told. + - name: Build the package + run: NIXPKGS_ALLOW_UNFREE=1 nix build --impure .#margin-mail --print-build-logs + + - name: Commit the pin + env: + VERSION: ${{ needs.prepare.outputs.version }} + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add nix/release.json + # A rerun after the pin already landed has nothing to commit, and the release is done. + if git diff --cached --quiet; then + echo "The flake already points at v$VERSION, nothing to push." + exit 0 + fi + git commit -m "point the nix package at v$VERSION" + for attempt in 1 2 3 4 5; do + git fetch origin main + git rebase origin/main + if git push origin HEAD:main; then + break + fi + if [ "$attempt" = "5" ]; then + echo "::error::main kept advancing; could not push the nix pin after 5 attempts. The release is published; rerun this job." + exit 1 + fi + echo "main advanced; rebasing and retrying ($attempt)…" + sleep 3 + done diff --git a/README.md b/README.md index 7216cbf..141a1d8 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,20 @@ # Margin Mail -A calm, keyboard-first mail client for Gmail on macOS and iPhone, with the parts of HEY and +A calm, keyboard-first mail client for Gmail on macOS, Linux, Windows and iPhone, with the parts of HEY and Superhuman worth having and none of the tracking. New senders wait at the door until you let them in; people, newsletters and receipts live in three separate boxes; what you owe and what you need sit in two piles at the foot of the list. Every decision you make is yours, kept beside the mail -rather than inside Gmail, and it comes with you if you leave. - -It is a sibling to [margin](https://github.com/priyanshujain/margin) and +rather than inside Gmail, and it comes with you if you leave. It is a sibling to +[margin](https://github.com/priyanshujain/margin) and [Margin Calendar](https://github.com/priyanshujain/margin-calendar) and shares their stack and visual language. -Nothing is built yet. The product is defined in [docs/design.md](docs/design.md), the features in +The product is defined in [docs/design.md](docs/design.md), the features in [docs/features.md](docs/features.md), the screens in [docs/ui.md](docs/ui.md), the keys in -[docs/keyboard.md](docs/keyboard.md), and how it will be built in -[docs/architecture.md](docs/architecture.md). The research it rests on is in -[docs/research/](docs/research/). +[docs/keyboard.md](docs/keyboard.md) and the settings in [docs/settings.md](docs/settings.md). How +it gets built is [docs/architecture.md](docs/architecture.md) and [docs/plan.md](docs/plan.md), how +it ships is [docs/release.md](docs/release.md), and the research it rests on is in +[docs/research/](docs/research/). To install it, [docs/install.md](docs/install.md). Licensed FSL-1.1-MIT: use it for anything except building a competing product, and every version turns MIT two years after its release. diff --git a/docs/architecture.md b/docs/architecture.md index 03f2695..416fa03 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -2,8 +2,9 @@ Tauri 2, React 19, Vite, TypeScript and zustand on the front, Rust behind. The same stack as margin and Margin Calendar, so OAuth, token sealing, the build and bundle setup, the overlay -title bar and the phone chrome carry over rather than being invented again. The conventions are -the calendar's, in `../margin-caledar/docs/conventions.md`, and apply here unchanged. +title bar and the phone chrome carry over rather than being invented again. The house rules for +writing it are in [conventions.md](conventions.md), which is the calendar's file with its +examples pointed here. The split is strict. Rust owns authentication, every byte to and from a mail provider, the local mirror, the sync loop, MIME parsing, HTML sanitising, the portable state database, the journal @@ -12,29 +13,69 @@ provider, which keeps the content security policy locked to `ipc:` as in the sib The facts about the Gmail API that this design rests on are in [research/gmail-api.md](research/gmail-api.md), checked against Google's pages on 3 September -2026. +2026. The milestones, the libraries and the order they land in are in [plan.md](plan.md). ## Two databases, one boundary There are two SQLite databases per account and the boundary between them is the product. The **mirror** is a copy of the mailbox: messages, threads, labels, bodies, attachment metadata, -an FTS5 index, the provider's sync cursor, and an outbox. It is derived from the provider and can -be thrown away and rebuilt. Its keys are the provider's ids. +an FTS5 index, the provider's sync cursor, and an outbox. It holds a window of the mailbox rather +than all of it, which the next section explains. It is derived from the provider and can be +thrown away and rebuilt. Its keys are the provider's ids. The **state** database is everything the user decided: sender rules, pile membership, snoozes, notes, renames, merges, clips, ignore flags, notification opt-ins, contact notes. It is never -derived, it is never sent to the provider, and its keys are portable: a thread is identified by -the RFC `Message-ID` of its earliest message (the thread key), a message by its own `Message-ID`, -a sender by their address or domain. Provider ids appear in the state database nowhere. When the -user moves to another provider and the same mail arrives through IMAP with the same -`Message-ID`s, every decision reattaches. +derived, it is never sent to the provider, and its keys are portable. A sender is their address +or their domain. A message is its own `Message-ID`. A thread is the **thread key**: the first +entry of the message's `References` header, or its `In-Reply-To` when there is no `References`, +or its own `Message-ID` when it starts the conversation. + +That definition matters more than it looks. A key derived from the earliest message the mirror +happens to hold changes when the window moves, so the same thread would carry two keys on two +devices and lose its pile on the one that had less of it. A key read off the headers of any +message in the thread is the same everywhere: on a phone holding a month, on a laptop holding a +year, and in an IMAP mailbox reached two providers from now. Provider ids appear in the state +database nowhere. A view is the mirror joined to the state. The Inbox is "threads whose sender rule says Inbox, or that carry a reply to a thread we are in, minus piles, minus snoozes, minus archived", grouped by seen state. The join is computed in Rust and served to the frontend as a flat, ordered list of thread summaries; the frontend never sees a label id or a rule. +## The window + +Mail on the device is a window, not the mailbox. Each account carries a `window` setting of 30, +90, 180 or 365 days, or everything, and it is 30 days out of the box. Thirty days is what a +mail client is actually used for, it turns a first sync from an afternoon into a few minutes, +and everything outside it is still one search away on the provider. + +The mirror holds every thread whose latest message falls inside the window. On top of that it +holds, at any age: every thread that carries app state (a pile, a snooze, a note, a rename, a +merge, a clip, ignore, notify), every starred thread, every draft, and the outbox. A thread the +user has touched is a thread the user expects to find, and the state database would otherwise +point at rows that are not there. + +Eviction runs once a day and again whenever the window shrinks. It removes bodies, attachments +and rows for threads that fall outside, leaving the state database untouched; nothing a person +decided is ever evicted. Widening the window is the mirror image: the newly covered range is +backfilled in the background, newest first, through the same hydration path as the first sync, so +there is one code path for filling the mirror and not two. + +The arithmetic of the first sync is the reason for all of it. `messages.list` with an `after:` +term returns the ids for the window cheaply, and metadata hydration runs in batches of 50 at 20 +units a message, which is about 300 messages a minute inside the per-user budget. A month of a +busy mailbox is minutes. The whole of it, at the same rate, is hours for twenty thousand messages +and most of a working day for a hundred thousand, which is what the setting exists to let someone +choose deliberately rather than discover. + +The window is visible in two places and nowhere else. The Everything place ends with one quiet +line, "Showing the last month. Older mail is on Gmail.", with the setting one click away. Search +results end with "Search older mail on Gmail", which runs the provider's search and hydrates the +hits as transient rows; the next eviction pass removes them again unless they gained state in the +meantime. A query carrying a `before:` or `after:` that lands outside the window skips the local +index and goes straight to the provider, because a local answer would be confidently wrong. + ## The provider trait ``` @@ -54,10 +95,44 @@ trait Provider { } ``` -Gmail implements it with the REST API. IMAP and SMTP will implement it next with folders mapped -onto the flag set and labels onto `X-GM-LABELS` or keywords, and JMAP after that with its own -change log. The trait is shaped by what every provider can do, and everything the trait cannot do -is done in the state database instead, which is why the state database exists. +Gmail implements it with the REST API. A mailbox reached over IMAP and SMTP with a password is the +second implementation and is what every account that is not Google uses; JMAP is a third if anyone +ever ships it. The trait is shaped by what every provider can do, and everything the trait cannot +do is done in the state database instead, which is why the state database exists. + +An account carries a `kind` saying which of the two it is. Almost nothing branches on it: the +places, the piles, the Screener, snoozes, notes and search are all above the trait and cannot tell +the difference. The screens that do differ are the ones about the account itself, because an IMAP +account has no scopes to grant and no Google account page to revoke from, and it has a server and +a port to show that a Google account does not. + +IMAP specifics that live only inside that implementation: a per-mailbox cursor of UIDVALIDITY, +UIDNEXT and HIGHESTMODSEQ in place of a change log, CONDSTORE with QRESYNC for the fast path and +a UID scan when the server has neither, mailbox roles read from SPECIAL-USE and XLIST flags before +falling back to names, and folders standing in for labels so a label change is a MOVE. A UIDVALIDITY +change is reported as a full resync rather than remapped, which reuses a recovery path the engine +already had rather than inventing a second one. + +Three consequences of IMAP that are worth knowing before they surprise somebody. There is no +server-side snippet, so the preview line on a list row is derived from the body the first time it +is fetched and is empty until then. There is no thread id, so the portable thread key does all the +work; it is computed by the same rule on both sides, so the two agree by construction. And there is +no equivalent of archiving: a mailbox with no `\Archive` special use and no folder named like one +gets an Archive mailbox created on first sync, because a keybinding that means something different +per account is worse than a folder somebody did not ask for. + +Sign-in is a password, not OAuth. Both Thunderbird and Mailspring ship their own OAuth client +credentials in the binary for Google and Microsoft, and Mailspring's source says outright that +anyone can extract theirs. Doing OAuth over IMAP would mean registering another client for +`AUTH XOAUTH2`, so a provider that publishes OAuth as its first choice, which Gmail and Fastmail +both do, is offered its own second choice instead: an app password. + +Certificates are decided in one place. The loopback is trusted without asking, because a local +bridge listens there with a certificate it generated for itself and nothing sits between this +process and that socket to impersonate anybody. Every other host is verified against the webpki +roots, and a failure becomes a question carrying a fingerprint, remembered per host and port. The +one exception is a certificate naming a different host: that is refused outright and never offered +as a choice, because it is the single failure indistinguishable from an interception. Gmail specifics that live only inside the Gmail implementation: `history.list` as the change log and its 404 recovery, the 6,000 units per minute per user budget, `format=metadata` with a fixed @@ -65,21 +140,74 @@ header list for hydration, `format=raw` for bodies, batch requests of 25 to 50, exponential backoff on 429, threading rules on send, `CATEGORY_*` labels read as a hint for the suggestion function, and the People API for contact autocomplete. -## Authentication and distribution +## One client for the whole suite -Loopback OAuth with PKCE from the calendar, unchanged: the consent page opens in the system -browser, never in a webview the app owns, and the code lands on `127.0.0.1`. Refresh tokens are +Margin, Margin Calendar and Margin Mail share one Google Cloud project, `margin-500217`, and one +OAuth client of type installed. A person who opens the third party access page of their Google +account sees a single entry called "Margin", and revoking it revokes the suite. That is the right +shape: three apps by the same author, on the same machine, holding the same person's data, should +not look like three vendors. + +Mechanically it is the calendar's arrangement unchanged. `build.rs` copies +`google-credentials.json` from the repository root into `OUT_DIR`, falling back to +`google-credentials.example.json` when the real file is absent, so a fresh clone compiles and +fails at runtime with a readable "not set up yet" rather than at build time with a missing file. +Loopback OAuth with PKCE, also from the calendar: the consent page opens in the system browser, +never in a webview the app owns, and the code comes back on `127.0.0.1`. Refresh tokens are sealed with XChaCha20-Poly1305 in the app data directory as the calendar does, for the same reasons (no `keyring` on Android, code-signature churn on macOS, no Secret Service on minimal -Linux). Scopes: `gmail.modify`, `gmail.settings.basic`, `contacts.other.readonly`, -`contacts.readonly`, and `calendar.events` for RSVP. No `mail.google.com` until IMAP is real. +Linux). -Every Gmail scope that reads mail is restricted. The plan is the one in the research: push the -consent screen to production unverified for the friends release (100 lifetime users, no weekly -re-login), file restricted-scope verification at once with the statement that there is no server -and all Google user data stays on the device, ask in writing whether the security assessment -applies, and budget for it anyway. Bring-your-own OAuth client stays as an escape hatch in -settings for the technical. +Connect asks for `openid email`, `gmail.modify`, `gmail.settings.basic`, `contacts.readonly` and +`contacts.other.readonly`. It does not ask for `calendar.events`, and this is the one place where +Google's rules cost the user something: installed apps get no incremental authorization, so a +scope cannot be added to a live token. The first time somebody answers an invite, the app runs +the whole authorization again with `calendar.events` in the list and replaces the stored token. +Backup does the same with `drive.file` when it is turned on. Both say so before they start. + +Granular consent is always on for this client, which means the tick boxes on the consent page are +the user's to clear and the app cannot assume it got what it asked for. The `scope` field of the +token response is stored per account and is the truth. Every feature that needs a scope checks it +first and, when it is missing, renders one sentence and a Grant button that re-runs consent for +the full list. Without `gmail.modify` there is no app at all, so the account is not added and the +screen says which permission was declined and what it was for. + +### What the shared client costs + +`gmail.modify` and `gmail.settings.basic` are restricted scopes. The shared project must pass +restricted scope verification, and until it does, every Margin app shows the unverified screen and +the three of them share one pool of 100 lifetime users. Verification has to be renewed annually, +and a lapse blocks the whole suite rather than one app. That is the price of the single "Margin" +entry, and it is worth stating in the same breath as the benefit. + +The plan is the one in the research: push to production unverified for the friends release, file +restricted scope verification immediately with the statement that there is no server and all +Google user data stays on the device, ask in writing whether the security assessment applies, and +budget for it anyway. Bring your own OAuth client stays in settings as the escape hatch for the +technical. + +### The hedge + +If verification is refused, or priced at a number that is not worth paying, Gmail over IMAP and +SMTP with a Google app password is the way in. It needs no Cloud project, no verification, and it +has no user cap. Gmail's IMAP extensions carry the pieces the REST API was giving us: +`X-GM-LABELS` for labels, `X-GM-THRID` for the provider's thread id, `X-GM-MSGID` for a stable +per-message id, all of which land in the same mirror columns. IDLE and CONDSTORE replace +`history.list` as the change log. RSVP goes out as an iMIP reply by mail instead of through the +Calendar API. Contacts come from the mirror, which is where autocomplete looks first anyway. + +This is not a rewrite, it is the `Provider` trait's second implementation, and knowing that is +half the reason the trait is drawn where it is. + +Removing an account therefore reaches further than this device. It hands the grant back to Google +first, and because the endpoint acts on the authorization rather than on the string it is handed, +that signs the person out of every Margin app on every machine they own; the confirmation says so +before it runs. Then the token and the registry entry go, and by default the account's mirror and +state database with them. The confirmation carries one box, ticked, for that last part: unticked, +the pair is moved from `accounts/` to `kept/`, where nothing that enumerates accounts can +see it, and it is moved back the day the same account is added again, decisions and all. An IMAP +account has nothing at Google to hand back, so removing one forgets its passwords and stops there. +The two actions used to be two buttons, Remove and Revoke, and read as a choice nobody could make. ## Sync @@ -88,17 +216,61 @@ grant no desktop app should hold, and a relay is the thing that might drag the a annual security assessment. `history.list` costs 2 units; polling every 12 seconds in the foreground and 60 seconds in the background is a rounding error against the budget. -Initial sync is the expensive part after the May 2026 quota change: `messages.get` is 20 units, -so hydration runs at about 300 messages a minute per account. A 20,000 message mailbox takes -about an hour of background work; a 100,000 message mailbox most of a working day. The order is -newest first, the app is usable as soon as the first page lands, and a thin bar in the account -chip says how far back the mirror reaches. Bodies are fetched on open and prefetched for the -last 90 days when idle. Attachments are fetched on open and cached with a size cap. The mirror -is the whole mailbox by decision; a setting caps the age for people who want less on disk. +Initial sync fills the window, newest first, and the app is usable as soon as the first page +lands. A thin bar in the account chip says how far the mirror has got. Attachments are fetched on +open and cached under a size cap. An account that fails repeatedly is paused by a circuit breaker +rather than retried into a rate limit, and the account chip says so. + +An account joins the engine the moment it is connected, and its first pass starts then rather +than at the next tick, which can be a minute away while the consent browser has the focus. One +pass at a time per account: a pass somebody asked for while the loop's is running gets the status +the running one is producing, and a first sync is never listed twice. A first sync that a quit or +a tunnel cut short is picked up by the next pass and reported the same way, with the count +carrying on from where it stopped, so an account still arriving never looks idle. + +Bodies are the exception, and the rule is worth stating plainly: **opening a thread never waits on +the network.** `thread_view` is a local read. A message whose body has not arrived comes back with +`bodyPending` and the pane draws a placeholder where the text goes, then `thread_hydrate` fetches +what is missing eight at a time and the bodies appear on a `store-changed` of scope `thread`, which +refreshes the open thread and deliberately does not touch the list. + +Behind that, the cache warms itself. Once the first sync has finished, each foreground pass takes +the forty newest bodies it does not have, which is about two hundred a minute against Gmail's six +thousand units and leaves room for roughly a hundred explicit opens in the same minute. The phase +is `caching` while it runs and the header says so. Two exclusions keep the queue moving: a body the +provider will not give up is remembered for the life of the process, so a handful of unfetchable +messages at the head cannot stall everything behind them, and transient rows pulled in by a +provider search are skipped, because a body fetched for a row the next eviction pass deletes is +twenty units spent on nothing. Both are excluded from the progress count as well as the fetch: work +nobody is going to do is not work outstanding. Every write is optimistic: it lands in the mirror, renders, and is pushed behind. Consecutive flag changes are coalesced into `batchModify`. Offline writes queue in the outbox and drain on -reconnect, with the thread showing "Waiting to send" until a send goes. +reconnect, with the thread showing "Waiting to send" until a send goes. A failure that is about +the connection or the account holds the queue; one the provider raised about the row itself (an +id it no longer has, a label it never had) is retried once and then dropped with its reason said +once, so a dead row never blocks the rows behind it. While a flag or label change is still queued, +the change log's word on those labels is applied under it rather than over it: what this device +decided is the truth about a message until the server has heard it. + +Failures are handled the way Mailspring handles them, which is why nobody using Mailspring has seen +a sync error. A connection that dropped under a request (reset, closed before the answer, cut off in +the body) is tried again at once and then after a second, on a fresh connection, at the call site; +the pool keeps its connections alive with HTTP/2 pings so one the machine slept through is found +dead before a request lands on it. A pass that still fails is written down and otherwise kept +quiet: the chip does not move for one failure, because the next poll is twelve seconds away and is +the retry. It moves on the second in a row ("Offline" for the network, "Sync trouble" for the +rest) and the account pauses for five minutes on the fourth. Only three things are ever toasted: +the pause, because pressing sync is the way out of it; a refused token or a missing scope, because +nothing else mends them; and a write the provider refused for good, once, because the change did +not take. Sending a message and creating a draft are the two calls never repeated on a dropped +connection, since the first attempt may have gone through. + +Every failure is also appended to `margin-mail.log` in the app data directory, capped at 256 KB: +each failed pass with the provider's sentence, each body that would not come, each command the +frontend called that answered with an error, and each uncaught error in the webview. An app +launched from the Finder has no stderr anybody will read, and a report of "sync failed" with +nothing behind it cannot be debugged. Seen, starred, archived, trashed and spam are provider flags and go through the trait. Nothing in the piles, the Screener, snoozes or notes ever touches the provider, so a user who screens out @@ -112,18 +284,25 @@ payload. The tables are a materialised view of the journal. This is what makes r without a server and what makes the backup meaningful. A **backup store** is a trait with three operations: put a blob at a name, get a blob by name, -list names under a prefix. Two implementations ship: Google Drive's app-data folder, which every -Gmail user already has and which margin's backup already uses, and Cloudflare R2 through S3 +list names under a prefix. Two implementations ship: Google Drive and Cloudflare R2 through S3 credentials for people who run their own. Each device uploads its own journal segments under its device id and downloads every other device's. Merging is last-writer-wins per key by timestamp, which is correct for every kind of state here (a pile toggle, a note, a rule), and a device that has been offline for a month simply replays what it missed. +The Drive implementation follows margin's, with one thing worth being accurate about: margin does +not use Drive's app-data space. It holds the `drive.file` scope and writes whole unencrypted files +into a visible folder named `margin`, which is deliberate, because a person should be able to see +their own backup. Margin Mail keeps the same scope, adds no new one, and writes encrypted journal +segments under `margin/mail///`. Files created by the shared client are +visible to every Margin app, which is what makes one folder work for three of them. The HTTP +parts of margin's `gdrive.rs` port across; the encryption, the journal and the merge are new here. + Everything uploaded is encrypted on the device with XChaCha20-Poly1305 under a key that is generated on first backup, stored sealed like the tokens, and shown once as a recovery phrase. Drive and R2 hold ciphertext and names; neither can read a note or a rule. The recovery phrase is -the only way to attach a second device or restore after a lost one, and the settings panel says -so in one sentence. +the only way to attach a second device or restore after a lost one, and the Backup section of +settings says so in one sentence. Version one on macOS alone does not need the merge. The journal shape is there from the first commit so that the iPhone can join without a migration. @@ -134,18 +313,21 @@ Message bodies are parsed from raw RFC 2822 with a real MIME parser (`mail-parse the provider's pre-parsed payload beyond headers, because encoded words, parameter continuations and legacy charsets appear daily. HTML is sanitised in Rust before it reaches the webview: scripts, forms, event handlers, `` refreshes, external stylesheets, `javascript:` -and `data:` navigation are removed; `cid:` references are rewritten to a local resource scheme; -every remote `` is replaced with a placeholder and its source recorded. The body renders in -an iframe with a strict CSP inside the app's webview so the message can never touch the app. +and `data:` navigation are removed; `cid:` references are rewritten to inline data so the body +carries its own images. The body renders in a sandboxed iframe inside the app's webview so the +message can never touch the app, and it renders on the paper surface in both palettes when it +arrived as HTML: mail written for a white page usually sets a text colour and no background, and +inverting it breaks more than it fixes. Plain text is rendered by this app rather than by its +sender, so it follows the theme like everything else. Tracker stripping happens in the same pass: images with a known tracking host (a maintained list, shipped with the app and updated with it), images of one pixel or hidden by style, and images whose URL carries a recipient token are removed and counted, and the vendor is named in the banner. When the user asks to show images, Rust fetches them without cookies or referrer and serves them from cache; the user's IP is exposed to the image host at that moment and only then, -which the privacy setting says plainly. Outgoing mail never contains a tracker and the app never -requests a read receipt. Links are rewritten on click to drop known tracking parameters, with a -setting to turn that off. +which the Privacy section of settings says plainly. Outgoing mail never contains a tracker and the +app never requests a read receipt. Links are rewritten on click to drop known tracking parameters, +with a setting to turn that off. Refresh tokens and the backup key are sealed, never in SQLite. The mirror and the state database are files in the app data directory and inherit the OS's disk encryption; encrypting them again @@ -158,31 +340,18 @@ macOS: overlay title bar with the traffic lights on the header's centre line, cl hides it and Cmd-Q quits, all from the calendar. iOS second: the same code with the phone chrome from the calendar's `data-phone` and `data-touch` scheme, overlays as bottom sheets, the OAuth flow through `ASWebAuthenticationSession`, and background app refresh used only to run the -snooze evaluation and a short sync. Linux afterwards: no traffic lights, closing quits, deb and -AppImage. +snooze evaluation and a short sync. + +Linux and Windows are the same code with the platform branches taken the other way: no traffic +lights, closing quits, and the menu bar built rather than adjusted, because neither is given the +File and View submenus macOS starts with. Linux ships as a deb, an AppImage, a flatpak and a Nix +package, Windows as an msi and an exe; which of the four a Linux reader should pick is in +[install.md](install.md). ## Order of work The sync engine and the reading pane are the two hard things and neither proves the other, so the -first milestone is one account, authentication, the mirror, and a read-only Inbox with a -sanitised, tracker-stripped reading pane. Everything the app is for depends on those being -right. - -Second, triage on the mirror: seen, archive, star, trash, spam, selection, the keyboard, the -palette, local search. At this point it is a fast Gmail client and nothing more. - -Third, the state database and the four places: sender rules, the suggestion function, the -Screener, the first-run pass, Feed and Paper Trail. This is the milestone where it stops being a -Gmail client. - -Fourth, the piles and their friends: Reply later, Set aside, Focus & Reply, snooze with lazy -evaluation, notes, rename, merge, clips, All files, ignore, per-thread notifications, the contact -card. - -Fifth, writing: reply, compose, drafts, the outbox with undo send, attachments, remind me if no -reply, instant intro, calendar RSVP. - -Sixth, more than one account, the unified view, settings, export, and the backup store with the -journal behind it. - -Then the iPhone, then Linux, then the IMAP provider, in that order. +first milestone after the scaffold is one account, authentication, the mirror, and a read-only +Inbox with a sanitised, tracker-stripped reading pane. Everything the app is for depends on those +being right, and everything after them is additive. The milestones and their work packages are in +[plan.md](plan.md). diff --git a/docs/conventions.md b/docs/conventions.md new file mode 100644 index 0000000..c60599c --- /dev/null +++ b/docs/conventions.md @@ -0,0 +1,139 @@ +# Conventions + +This project is a sibling to margin and Margin Calendar and follows their conventions deliberately +rather than inventing new ones. When something here is unclear, the answer is almost always "do +what the calendar does", and the file to look at is named below. + +## Rust + +`Result` everywhere. No `anyhow`, no custom error enum except +`provider::ProviderError`, which exists only because the sync engine has to branch: a revoked +token, a missing scope, a 429 to back off from, and a 404 from `history.list` that means the +change log expired and a full list is the answer rather than a failure. + +DTOs crossing the IPC boundary live in `src-tauri/src/dto.rs` and are marked +`#[serde(rename_all = "camelCase")]`. That file is the contract and is frozen: implementation +modules add bodies, not fields. Its mirror is `src/ipc.ts`. + +Read Google's responses through the ported `read_json`, which takes the body to a `String` first +so the error payload survives into the message rather than becoming "expected value at line 1". +It is the calendar's function and it lives with the Gmail client. + +Heavy synchronous work goes behind `#[tauri::command(async)]` on a synchronous fn, which is +margin's trick in `pdf.rs` for getting off the main thread without hand-writing `spawn_blocking`. +Every command that touches SQLite qualifies. + +Provider-specific behaviour stays inside the provider's module. Nothing above +`src-tauri/src/provider/` may know what a Gmail label id looks like, and nothing outside the +mirror may know that a thread has a provider id at all. The sync engine talks to the trait, which +is what lets `provider::fake` drive the whole engine in `cargo test` without credentials. + +Comments are rare and explain why, never what. Match the density in `lib.rs`. + +## TypeScript + +One zustand store per domain in `src/store/`. No middleware. One selector call per field +(`useThing((s) => s.field)`, never a destructured object), actions as inline arrow properties, and +`set((s) => ...)` returning `{}` to no-op. + +Async actions use a string phase union (`"idle" | "syncing" | "error"`), never boolean loading +flags. Errors stringify with `String(e)` and surface as a toast. + +Side effects that touch disk, the DOM or Tauri live in a sibling module, never inside the store. + +The OAuth connect flow in `src/store/useAccounts.ts` reuses margin's +promise-holding-its-own-resolver pattern from `useBackup.ts`: `connect()` returns a promise whose +`resolve` is stashed in state for a later Tauri event to settle. + +Typed IPC wrappers live in `src/api/`, one module per domain, one thin function per command. They +are written once against the frozen contract; add bodies to Rust, not new wrappers. + +## The design system + +Three layers, and the rule is that each may only reach down. + +Tokens are `src/styles/tokens.css`, which is a seam rather than a list: it imports the set +margin-shared holds for all three apps, then `src/styles/mail.css` for what mail adds on top. +Every colour, radius, size, duration and font stack is in one of those two, and nothing else in +the app may declare a token. + +Primitives are `src/ui/`: the button, the keycap, the row, the avatar, the panel, the popover, the +toast. They read tokens and nothing else, and every one of them appears in every state on the Kit +page at `#/kit`, which is how a restyle gets reviewed. + +Screens are `src/screens/`. A screen composes primitives and may never write a colour, a radius or +a size. If a screen needs a value that is not available to it, the answer is a new token or a new +primitive, not a literal. + +## Places and stages + +A `Place` in the frozen contract is a query over threads, and three of the palette's entries are +not that: Contacts lists people, Clips lists passages, All files lists attachments. Focus & Reply is +a fourth, a page over the Reply later pile rather than somewhere you can be. None of them belongs in +`Place`, and none is an overlay either, because an overlay is something you dismiss to get back to +what you were doing and these are somewhere you go. + +So they are a stage, in `src/store/useStage.ts`, and the rule is that a stage wins over a place: +opening Contacts leaves the Inbox where it was and Escape puts you back on it. The current place +and the current stage are both on the root element as `data-place` and `data-stage`, which is what +a test reads and what a stylesheet keys off, so neither has to ask the app what it thinks it is +showing. + +## CSS + +Flat kebab-case class names, not BEM. State is a `data-*` attribute, never an `is-` class. + +Every colour, radius and size goes through a token. If a value is not in the token layer, add it +to `src/styles/mail.css` rather than writing a literal. + +Dark mode is `data-theme` on ``, with both palettes defining an identical variable set. +Never a media query for theme. + +Transitions name explicit properties and use `var(--ease)`. Never `transition: all`. + +Responsiveness is JS-driven. `usePhone()` and `useTouch()` in `src/useMedia.ts` write `data-phone` +and `data-touch` on the root, and styles read those attributes rather than adding media queries. +They answer different questions. `data-phone` is a window too narrow for the desktop chrome and it governs +layout; `data-touch` is a coarse pointer and it governs interaction. A tablet is touch and not a +phone, a narrow desktop window is a phone and not touch, and treating either as a proxy for the +other is how a hover-only control ends up unreachable. Both are also set by the boot script in +`index.html`, so the first paint is already the right shape. + +A rule that reads "you cannot hover here" belongs on `data-touch`. A rule that reads "there is no +room for this" belongs on `data-phone`. + +There is no container query in this repository. The calendar has exactly one, on the event block, +and it earned it: what decides how many lines of a title fit is the block's own width and not the +window's. Nothing here has met that bar yet, and nothing may reach for one without the same kind +of reason. + +Overlays follow margin's `.overlay` and `.panel` idiom, which is in `src/styles/app.css`, and +every one of them registers with `useEscapeLayer` from `src/escape.ts` so Escape unwinds the +layers in order. + +## Icons + +Feather-style 24x24 stroke `d` strings, named in `src/ui/icons.ts` and passed to +``. The handful of glyphs the whole suite shares are re-exported from +`margin-shared/icons` so a search here and a search in the calendar are the same drawing; the +verbs and the piles are mail's own. There is no icon set and no registry, and there will not be +one. An icon-only button always carries a `title` with its shortcut written in real glyphs. + +## Storage keys + +Anything in `localStorage` is prefixed `marginmail-`, following margin's convention: the theme is +`marginmail-theme` in `src/theme.ts`, and the fonts, the text size and the reading pane follow the +same shape. Keys read before first paint are restored by the blocking IIFE in `index.html`, which +is why they are flat strings rather than one blob. + +## Work packages + +A work package owns a set of files and never edits another package's files. When a package needs +something that lives in another one, the answer is to agree the interface up front (which is what +`dto.rs` and `src/ipc.ts` are for) and stub behind it, not to reach across. A package that has to +edit somebody else's file was cut in the wrong place. + +## Never + +No CSS framework, no component library, no router, no zustand middleware, no directory trees in +any document, and no em dashes anywhere including code comments. diff --git a/docs/design.md b/docs/design.md index 68beb30..b301ed7 100644 --- a/docs/design.md +++ b/docs/design.md @@ -17,10 +17,10 @@ named. We refused HEY's layout (one column, a page per thread, a round trip for insistence that routing is per sender only, and its refusal to let you archive. From Superhuman: speed and the keyboard. One key per verb, the key printed on every button, and a -command palette that is the whole settings and discovery surface, so the app teaches itself. A -list beside a reading pane, so you triage without leaving the list. Remind me if no reply, undo -send, the contact card. We refused the tracking pixels, the AI surface that ships your mail to a -vendor, the inbox-zero streak, the tiny fixed type, and the price. +command palette that reaches every place, every command and every setting, so the app teaches +itself. A list beside a reading pane, so you triage without leaving the list. Remind me if no +reply, undo send, the contact card. We refused the tracking pixels, the AI surface that ships your +mail to a vendor, the inbox-zero streak, the tiny fixed type, and the price. From neither: the app is the only place your state lives, and that state is yours. Your piles, screening decisions, notes, renames and clips are keyed on the mail itself (the RFC Message-ID @@ -34,9 +34,9 @@ There are three boxes and one gate, and every sender has exactly one destination **Inbox** is for people and for the few services you want to hear from as they arrive. It is a stream, not a queue: what you have not looked at sits under New for you at the top, and -everything you have opened or sent sinks to Previously seen beneath it. Nothing counts anything. -A reply pulls a thread back up. There is an archive key, because some people need an empty list -to feel finished, but nothing in the design pushes you towards it. +everything you have opened or sent sinks to Previously seen beneath it. Neither group carries a +count. A reply pulls a thread back up. There is an archive key, because some people need an empty +list to feel finished, but nothing in the design pushes you towards it. **Feed** is for newsletters and long reads. Every item is already open, in one scrolling column, newest first, with a marker where you left off. There is no read state and no obligation. You @@ -53,9 +53,10 @@ decision can be per address or per domain, which is the thing HEY's users ask fo will not give them. Replies to a thread you are already in bypass the whole system and land in the Inbox, because the sender rule is about first contact, not about conversations. -On first run there is no Screener avalanche. Everyone who has ever written to you is screened in, -routed by the same suggestion rules, and movable later from their contact card. Only genuinely -new senders from that point on are held. +On first run there is no Screener avalanche. Everyone the account already knows is screened in, +routed by the same suggestion rules, and movable later from their contact card: your Google +contacts, everyone in the mail that came down with the first sync, and everyone you wrote to in +it. Only genuinely new senders from that point on are held. ## The two piles @@ -76,16 +77,20 @@ Every verb is one unmodified key, the same key Gmail and Superhuman use where th (`j`, `k`, `e`, `r`, `a`, `f`, `c`, `/`, `x`, `u`, `z`) and HEY's letters for HEY's verbs (`l` Reply Later, `s` Set Aside, `b` snooze, `y` note, `m` ignore). Number keys go to places. There are no two-key chords, nothing is modal, and the key is printed on every button so the mouse -teaches the keyboard. Cmd-K opens the palette, which also lists every place, every command and -every setting. The full map and the reasoning for each conflict are in [keyboard.md](keyboard.md). +teaches the keyboard. Cmd-K opens the palette, which reaches every place, every command and +every setting, though settings itself is a screen you can sit in rather than a list you pass +through. The full map and the reasoning for each conflict are in [keyboard.md](keyboard.md). ## Quiet by design -No badge, no unread count, no streak, no photograph when the list is empty. No notification -unless you turned it on for that thread or that person. Remote images do not load until you ask, -tracking pixels are removed before the message renders, and the banner tells you whose pixel it -was. Nothing you send carries a tracker and nothing reports when it was opened. Links open with -their tracking parameters removed. +No unread count, no streak, no photograph when the list is empty. No notification unless you +turned it on for that thread or that person. The one number anywhere is the dock badge, and it +counts New for you rather than unread mail: what is waiting for a decision once the Screener, the +Feed and the Paper Trail have taken everything that is not. That is a fact about your Inbox rather +than a reason to open the app, and it turns off in Notifications. Remote images do not load until +you ask, tracking pixels are removed before the message renders, and the banner tells you whose +pixel it was. Nothing you send carries a tracker and nothing reports when it was opened. Links open +with their tracking parameters removed. There is no AI in this version. Classification is by headers and by your decisions, the way HEY does it, and it is explainable in a sentence on every Screener card. The design leaves a seat for @@ -113,15 +118,17 @@ passage with a link back. All of it roams through the backup store, none of it t Not a team tool: no shared threads, comments, or read statuses. Not a calendar: invites hand off to Margin Calendar. Not a scheduler: no send later in this version. Not an assistant: nothing is -summarised or drafted for you. Not a Gmail skin: the Gmail web UI is not a consideration, since -the whole point is never opening it. +summarised or drafted for you. Not an archive: the device keeps a window of recent mail, a month +unless you ask for more, and the rest stays on Gmail where a search still reaches it. Not a Gmail +skin: the Gmail web UI is not a consideration, since the whole point is never opening it. ## Visual language Lifted from margin and the calendar unchanged: warm paper, ink and two softer inks, hairline borders, a four-step type scale, three radii, one easing curve, light and dark driven by `data-theme`. Hanken Grotesk for the interface, Literata for the subject line and for message -bodies, because mail is reading and reading deserves a text face. The additions are mail-specific: +bodies, because mail is reading and reading deserves a text face. Both are the default rather than +the law: Appearance offers the six faces the suite bundles and whatever else is on the machine. The additions are mail-specific: a row hover and a row selection wash, a warmer band for Previously seen, a note surface, a dot colour for new mail, and the eight muted hues the calendar already uses, here for avatars and account edges. No CSS framework, no component library, hand-written CSS on tokens. diff --git a/docs/features.md b/docs/features.md index 2b2c70f..7a524ad 100644 --- a/docs/features.md +++ b/docs/features.md @@ -21,8 +21,9 @@ reaches all of them. | `5` | Set aside | The Set aside pile | | `6` | Screener | First messages from senders with no decision yet | | `7` | Snoozed | Threads waiting to return, with their return time | -| `0` | Everything | Every thread in the account, including archived, in date order | -| palette | Sent, Drafts, Starred, Screened out, Spam, Trash | The usual folders | +| `0` | Everything | Every thread on the device, including archived, spam and screened out, in date order | +| palette | Sent, Drafts, Starred | The usual folders | +| palette | Screened out, Spam, Trash | Under Other: the places you go looking in rather than read | | palette | All files, Clips, Contacts | The libraries | | palette | Labels | The provider's labels or folders, one place each | @@ -30,6 +31,13 @@ Every place except Feed, Screener and Focus & Reply is a list column beside the Feed and Screener take the whole stage because their content is inline. A place remembers its scroll position and selection while the app is open. +Every place is a view over what is on the device, and what is on the device is a window of the +mailbox: the last 30 days by default, or 90, 180, 365 days or everything, set per account. Threads +you have done something to are kept whatever their age. Only Everything says any of this out loud, +in one quiet line at the foot of the list, "Showing the last month. Older mail is on Gmail.", with +the setting one click away. The mechanism is in [architecture.md](architecture.md) and the setting +is in [settings.md](settings.md). + ## 2. Routing and the Screener ### Destinations @@ -43,8 +51,8 @@ Two overrides apply before the sender rule: - A message whose `In-Reply-To` or `References` points at a thread the account is already in goes where that thread is, or to the Inbox if the thread was screened. A reply is never held. -- A message from an address in the account's contacts, or one the account has ever sent to, is - screened in on first run and routed by the suggestion rules, never held. +- A message from someone the account already knows is screened in on first run and routed by the + suggestion rules, never held. Who counts as known is settled under First run below. ### The Screener @@ -60,9 +68,10 @@ subject, snippet, and a one-line reason with the suggested destination. Keys: sender into the Inbox and opens a reply. - Clear all screens out every sender currently waiting, after a confirmation. -Screened out mail is kept for 90 days in the Screened out place, then trashed. Reversing a -decision is done from the sender's contact card, and re-screening someone in brings back whatever -they sent in the last 90 days. +Screened out mail sits in the Screened out place for as long as the storage window keeps it and +falls off the device with everything else of that age. There is no second retention rule to +remember, and a wider window means a longer memory. Reversing a decision is done from the sender's +contact card, and re-screening someone in brings back whatever they sent that is still here. ### Suggestions @@ -85,10 +94,26 @@ wrong suggestion is a one-line fix. ### First run -When an account is added, every sender in the mirror is screened in with a rule set by the same -suggestion function, silently. The user can move any sender from the contact card, and the move -applies to that sender's existing threads immediately. Only senders whose first message arrives -after the account was added are held in the Screener. +When an account is added, everyone it already knows is screened in with a rule set by the same +suggestion function, silently. A month of mail is not by itself a good answer to who a person +knows, so the seed is drawn from three cheap sources at once: every sender and every recipient +inside the storage window, the People API's `connections` and `otherContacts` lists, and the Sent +mail inside the window. Anyone in any of the three is screened in. All three are already fetched +or already on the way, so this costs a pair of extra calls and no waiting. + +The pass runs at the end of the first sync rather than when the first-run panel is dismissed, so +the Inbox fills as the mail arrives instead of sitting empty for as long as somebody takes to read +a panel. It runs once per account and is guarded, which is the whole of what makes the Screener a +gate: if it ran again on a later sync it would screen in every new sender the moment they wrote. +The guard cuts the other way too: asked before the crawl has finished, as it is the moment an +account is added from Settings, the seed answers "not yet" rather than marking itself done over a +mirror with nothing in it, and the sync's own idle asks again. A seed on record as having +screened in nobody is run once more when the mirror is ready, which repairs an account that was +marked that way before the rule existed without anybody removing it and connecting it again. + +The user can move any sender from the contact card, and the move applies to that sender's existing +threads immediately. Only senders whose first message arrives after the account was added are held +in the Screener. Alongside this, a first-run panel offers "Start fresh": mark everything older than a chosen age (default one week) as seen, so New for you holds only what is recent. This is the only bulk @@ -110,8 +135,12 @@ holds it until it is opened. - A new message in a Previously seen thread moves the thread to New for you. - `e` archives: the thread leaves the Inbox and lives in Everything. A new message in an archived thread brings it back to New for you. Archive is a provider change (Gmail: remove `INBOX`). -- There are no counts on the groups, on the place, or on the app icon. A dock badge for New for - you exists as a setting and is off. +- There are no counts on the groups or on the place. The one count anywhere is the dock badge, + which is the size of New for you across every account, and it is on by default and turns off in + Notifications. It is not the mailbox's unread count: a thread held in the Screener, routed to the + Feed or the Paper Trail, piled, snoozed or ignored is unread and is not waiting for you. Zero + takes the badge off rather than showing a nought. macOS and Linux carry it; Windows would need a + drawn overlay icon and does not have one yet. - A note on a thread shows as a single line under its row. Seen state is the provider's read state (Gmail: `UNREAD`), so it is not app state. Everything @@ -200,12 +229,14 @@ focused message, `Shift+O` expands all. Quoted text is collapsed behind a pill. - Message bodies render in a sandboxed webview with scripts, forms and external styles removed. Remote images are blocked by default; a banner says how many trackers were stripped and names the vendor; Show images loads them for this message, and the contact card can allow them for a - sender always. Attachments are chips; images and PDFs preview inline on demand. + sender always. Attachments are chips; pressing one opens the file with whatever owns its type. - Attachments are fetched when the thread is opened, not during sync, and cached. - A calendar invite (`text/calendar` with `METHOD:REQUEST`) renders as a card: date, title, time, location, organiser, and Accept (`y`), Maybe (`m`), Decline (`n`), plus Open in Margin Calendar. RSVP goes through the Calendar API on the invited calendar; when the event is not - there yet it is imported first. Without the Calendar scope the card still renders read-only. + there yet it is imported first. The Calendar permission is not asked for when the account is + added, so the first RSVP says it needs it and runs the consent page again; until then the card + renders read-only. - Links show their real destination on hover and open with known tracking parameters removed. - Read together: select several threads with `x` and press `Enter`; the pane shows them one after another with a heading each. @@ -290,15 +321,26 @@ focused. Built from the local index; nothing is fetched until you open one. ### Ignore -`m` on a thread. New messages still arrive and append, but the thread never returns to New for -you and never notifies. A banner on the thread says "You are ignoring this thread" with Stop -ignoring. Local. +`m` on a thread. New messages still arrive and append, and the thread rises with them because the +Inbox is in time order, but it never reads as new, never counts on the badge and never notifies. A +banner on the thread says "You are ignoring this thread" with Stop ignoring. Local. ### Notifications Off by default everywhere. `Shift+N` on a thread turns them on for that thread; the contact card -turns them on for a person. A notification shows the sender and subject, and opening it opens the -thread. There is no badge unless the setting is turned on. Local. +turns them on for a person; Settings turns them on for a place, and has one switch over all of it +for the machine, which is what "nothing on this laptop" means without touching a single thread. The +sync pass that brings a message in is what posts the notification, and only for mail that arrived +after the app came up, so a first sync, a rebuild or a week away says nothing about the backlog; one +message is three lines, the app's name, the sender and the subject, and several in one pass are one +notification counting them and naming the senders. On macOS the system asks once whether the app +may notify at all, the first time anything here is turned on or the test button is pressed, and a +refusal is undone in System Settings rather than here. Clicking one brings the app to the front and +opens the thread in the list it shows in; a click on the grouped one opens the account's Inbox, and +a click on the sample from Settings only brings the app to the front. The dock badge is a setting +in the same section rather than a notification, it counts New for you, and it is the one thing here +that is on. Local. + ## 11. Contacts and the contact card @@ -319,8 +361,38 @@ State: notes, delivery, notify are local. Provider: nothing. the app POSTs and confirms; with a `mailto:` header it sends the message; otherwise it opens the link. Either way it offers "and trash everything from them" and "and screen them out". - Screen out from the contact card is the block: future mail goes to Screened out. Nothing is sent. -- `!` marks spam (provider), `#` trashes (provider), both with undo. Trash empties after 30 days - on the provider's schedule; the Trash place has an Empty button. +- `!` marks spam and `#` trashes, both provider changes, both with an undo toast. + +### Screened out, Spam and Trash + +Three places under Other in the palette, below the daily ones and below the labels, with no number +keys of their own. Where a place sits is the honest statement of how often you should be in it, and +these are the ones you go looking in rather than the ones you read. + +None of them is a filter of ours. Gmail's spam filter runs on Gmail's side before the app sees a +message, and the mirror takes what the mailbox holds: every list call sets `includeSpamTrash`, so a +junked message is already on the device with its body indexed whether or not anything shows it. +Screened out is the one we own, and it is a routing destination rather than a folder, so its rules +are in section 2. + +Getting mail back out is the point of all three, and the verb that puts it back is the verb that +put it there, the way the piles already work. `#` in Trash puts a thread back, `!` in Spam takes +the spam mark off. Gmail restores a message's labels when the `TRASH` label comes off, so a thread +put back lands where it was rather than in the Inbox. A thread taken out of Spam is routed like any +other: to its sender's box if that sender has a rule, and to the Screener if they do not, which is +the decision you still owe them. + +There are three ways back, in the order you will want them. A wrong keystroke is the toast that is +already up, "Trashed · Undo" and `z`. A rescue a week later is the place and the verb. After thirty +days Gmail empties its own trash, the message leaves the mirror with it, and nothing local changes +that. + +There is no Empty button. Permanently deleting through the Gmail API needs the +`https://mail.google.com/` scope, which is total access to the mailbox, and asking every account +for that so a button can destroy things thirty days earlier than Gmail will anyway is a bad trade. +Each place states the rule instead, in one line at the foot of the list: "Gmail empties this after +30 days." Screened out carries no such line, because it falls off with the storage window like +everything else of its age and there is no second retention rule to learn. ## 13. Selection and bulk actions @@ -332,16 +404,28 @@ Enter for Read together. Every bulk action is one undo. ## 14. Search `/` focuses search. Results replace the list column and the reading pane works as usual. Search -is local over the full mirror: subject, participants, snippet and body text, with operators +is local over what is on the device: subject, participants, snippet and body text, with operators `from:`, `to:`, `subject:`, `has:attachment`, `filename:`, `in:` (any place), `before:` and -`after:`, `label:`. When the query touches mail that is not yet hydrated, the provider's search -runs as a second pass and its results append with a note. Results open in place and `Esc` -returns to the previous place. +`after:`, `label:`. + +Because the device holds a window, a local result set is a partial answer and says so. Every list +of results ends with "Search older mail on Gmail", which runs the provider's search, appends the +hits and hydrates them as they arrive. Those rows behave like any other row, and the next eviction +pass takes them away again unless they picked up a pile, a note or some other decision in the +meantime. A query whose `before:` or `after:` falls outside the window skips the local index +entirely and goes to the provider, because a local answer to that question would be wrong rather +than merely short. Results open in place and `Esc` returns to the previous place. + +Search reaches Spam, Trash and Screened out, and names the place on the row when it does. The +message you most need to find is the one something else decided you should not see, and a search +that skipped those three would be one you had to already know the answer to use. `in:` narrows to +a single place when that is what you meant. ## 15. Accounts -Add as many Gmail accounts as you like. Each has its own places, sender rules, piles and -Screener. The account chip in the title bar switches (`Ctrl+1` to `Ctrl+9`) and offers All +Add as many Gmail accounts as you like. Each has its own places, sender rules, piles, Screener, +storage window and granted permissions, so a work account can keep a year while a personal one +keeps a month. The account chip in the title bar switches (`Ctrl+1` to `Ctrl+9`) and offers All accounts (`Ctrl+0`), which merges every account's version of the current place into one list with a coloured edge on each row. Compose picks the account from the thread you are replying to, or the account you are looking at, and the From field switches it. Sent mail goes through the @@ -355,13 +439,38 @@ are the provider's, they roam with the mailbox, and they are not how Margin orga ## 17. Settings and export -`Cmd+,` opens settings as a panel: Accounts (add, remove, signature, aliases), Backup (Google -Drive or Cloudflare R2, and the recovery phrase), Appearance (theme, reading pane, row density -for the phone), Sending (undo delay, reply-all default, instant intro text), Privacy (remote -images, link cleaning, per-sender allowances), Notifications (badge, sound), Keyboard (the keymap -file), Data (export mail as mbox per account, export app state as JSON, import app state). +Settings is a place, not a panel. `Cmd+,`, the palette, the account chip and the app menu all lead +to the same full-stage screen, with a rail of sections down the left and one section at a time on +the right. Twelve sections cover the accounts and their permissions, appearance, the storage +window, privacy, the Screener, the piles and snooze, writing, notifications, the keymap, backup and +the recovery phrase, every export the app offers, and the version. Each is specified in +[settings.md](settings.md), including which of them live on the device and which roam. -## 18. Not in version one +## 18. Help + +The app is unlike the mail clients people arrive from, and none of the differences are +discoverable by poking at the interface, so there is a tour, a question mark in the corner, and a +guide behind it. Each is specified in [help.md](help.md). + +The tour is nine slides in a sheet, over the Inbox, run once for every account that is added and +skipped with one key. It follows the first-run panel above and names the Screener, the three +boxes, the two piles, snooze, the keyboard, the palette, the things kept beside the mail, what is +off by default, and undo. + +The corner button opens the tour again, the guide, and the keyboard shortcuts. It is not drawn +while the compose card is open, while an overlay is up, in the guide itself, or on a phone. + +The guide is a panel over the whole window: a search field, a rail of sections, and one article at +a time. How to do each thing the app does, and last, the questions people actually ask. Every +article leads with its answer and shows it in a drawn figure, a screenshot or a table of keys, and +a search nothing answers offers to file the question against the repository. Closing it puts back +the place, the open thread and the scroll. Its pictures come from the dev fixture through +`just guide-shots` and are committed, because they ship in the bundle. + +State: which accounts have had their first run, and therefore their tour, is a device fact in +localStorage beside the first-run panel's own flag. Provider: nothing. + +## 19. Not in version one Send later. Snippets. Any AI. Shared threads, team comments, read statuses. Workflows, collections, cover art. A calendar sidebar. Unified search across accounts (search is per account until the diff --git a/docs/help.md b/docs/help.md new file mode 100644 index 0000000..53fd3b7 --- /dev/null +++ b/docs/help.md @@ -0,0 +1,103 @@ +# Help + +Three things, and they are separate on purpose: a tour that runs once when an account is added, a +question mark in the corner that is there for good, and a guide behind it that answers the +questions this app raises by not working like the last one. Behaviour that is being explained is +specified in [features.md](features.md); the keys are in [keyboard.md](keyboard.md) and every +keycap printed anywhere here is generated from the same binding table, so a remapped key is what +the help says. + +The premise is that this app is unusual and that pretending otherwise is what makes people leave. +Mail from somebody new does not arrive. There are three boxes rather than one. Flags are two piles +with keys on them. None of that is discoverable by poking at it, and none of it is a reason to +turn the first hour into a wizard either. + +## The tour + +Nine slides in a sheet, over the Inbox it is talking about. It follows the first-run panel, which +is the moment the account is set up and the mail is in, and it ends where the app expects you to +carry on: the Screener, the three boxes, the two piles, snooze, the keyboard, the palette, the +things kept beside the mail, what is quiet by default, and undo. + +Skip is the first control on it and Escape is the same answer, so it costs one keystroke to +refuse. The arrows and the return key move through it, the dots at the foot say how far in you are +and can be pressed to jump, and the last slide says where to find all of this again. + +It runs for every account that is added rather than only the first. The panel it follows is per +account too: a second mailbox on a shared machine is somebody else's first look at the app, and +the flag that remembers is on the device rather than in the state that roams. Somebody adding +their own second account has seen it before and skips it, which is one key. + +The slides state facts and nothing else. No animation beyond a fade, nothing bouncing, no +illustration that is not a diagram of the real thing, and no screenshot: the app is behind the +sheet, so a picture of it would be a picture of what is already there. + +## The corner + +A small round question mark fixed in the bottom right, and the only permanent chrome the app has +outside the header. It opens three rows: the tour again, the guide, and the keyboard shortcuts +with its key printed. + +It gets out of the way rather than floating over everything. It is not drawn while the compose +card is open, because they share that corner and compose is the thing you are doing; not while any +overlay is up, because it would be under the scrim; not in the guide, which is where it goes; and +not on a phone, where the tab bar owns that corner and the palette behind the places button is +what reaches all three. + +The same three are in the native Help menu, above Report an Issue, because that is the first place +a Mac user looks and a menu bar item costs nothing. + +## The guide + +A panel over the whole window, with the app dimmed behind it. It is read about the app rather than +instead of it, so nothing behind it can be pressed while it is up and the close control puts back +exactly what was there: the place, the open thread, the scroll position. Escape is the same answer. +Inside, a search field across the whole width, then a rail of sections down the left and one +article at a time on the right at a reading measure. + +Search is the first thing in the panel and it takes the keyboard the moment the guide opens, +because somebody who came here has a question rather than an appetite for a table of contents. It +reads what the articles say and not only what they are called. A query nothing answers is the one +moment the guide has failed, so that is where it offers to open a question against the repository, +labelled `question` and titled with what was typed, and it says out loud that the page it opens is +public. + +Articles are written to be looked at before they are read. The answer is the first paragraph, a +drawn figure or a screenshot carries the idea under it, and a verb that would otherwise be a +sentence in a list of six is a table of the key and what it does, generated from the binding table. +The rule is enforced rather than hoped for: `guide.test.ts` fails on a paragraph over seventy five +words, and on an article over a hundred and twenty words with nothing in it to look at. + +The figures are drawn from the app's own parts rather than exported from a drawing program, in +`src/screens/guide/Figures.tsx`. A picture is the app photographed and a figure is an idea drawn, +and the ideas are the things no camera can be pointed at: where mail goes, what a pile does, how +long a send is held. + +It was a stage first, and that was wrong twice over. A stage wins over a place, so the header sat +above it with three box buttons that changed a place nobody could see; and a library about the app +is not somewhere you go instead of your mail, it is something you hold up in front of it. Making it +a panel found the second half of the same bug: the title bar carries a stacking order of its own so +that a popover can hang off the account chip, and it was above every scrim in the app, which left +the palette, the shortcut sheet and the tour all reachable past their own scrim at the top of the +window. Overlays now sit above it, which is the rule the phone stylesheet had already worked out +for its tab bar. + +The sections run in the order somebody meets the problem: getting started, the Screener, reading, +triage, writing, organising, accounts, and then the questions. An article is prose with steps +where there are steps and the key printed where there is a key. The questions are the last section +and they are the ones people actually ask: where a newsletter went, whether a screened-out sender +is told (they are not), why mail from last year is not here, why there is no Empty Trash button, +whether anything reads the mail with AI (nothing does), and what happens to the mailbox if the app +is deleted. + +The pictures come from the browser suite rather than from anybody's mailbox. `just guide-shots` +drives the dev fixture and writes ten PNGs into `public/guide/`, at twice the display size for a +retina screen, and they are committed because they ship inside the bundle. The recipe is not part +of `just test-ui`: an ordinary run of the suite must not rewrite files that are in the tree. + +## Not here + +No coach marks over the interface, no tooltip that follows you around, no "did you know" after the +third launch, no checklist of things to finish, no progress bar over your own mailbox. The tour is +one sheet you can refuse, and after that help is somewhere you go and find rather than something +that arrives while you are reading your mail. diff --git a/docs/install.md b/docs/install.md new file mode 100644 index 0000000..ed15678 --- /dev/null +++ b/docs/install.md @@ -0,0 +1,157 @@ +# Installing Margin Mail + +Every build comes from the same release. Pick the file for your machine from +[the latest release](https://github.com/priyanshujain/margin-mail/releases/latest) and follow the +section below for it. The version number in the file names changes with every release; `0.1.0` +stands in for it throughout. + +Whatever you install, the app keeps its mirror of your mail, your accounts and every decision +you have ever made about a sender in one directory, and uninstalling never touches it. Where that +directory is, and how to move it, is at the end. + +## macOS + +Apple Silicon and Intel are the same file: the dmg carries a universal binary. + +1. Download `Margin.Mail_0.1.0_universal.dmg`. +2. Open it and drag **Margin Mail** onto the Applications folder in the same window. +3. Eject the disk image and open the app from Applications or Spotlight. + +The bundle is signed with a Developer ID and notarized, so it opens on the first double click with +no right-click-and-Open dance and no trip to System Settings. + +macOS asks about notifications the first time the app has something to tell you rather than at +launch. If you say no and change your mind, it is under Notifications in System Settings, and the +app's Settings has a button that takes you straight there. + +The minimum is macOS 10.15. + +## Linux + +Four packages, and they are not equal. If you are on Wayland, or on NixOS, use Nix. Otherwise the +deb on Debian and Ubuntu, the flatpak on anything else, and the AppImage when you want no install +at all. + +### deb, on Debian and Ubuntu + +``` +sudo apt install ./Margin.Mail_0.1.0_amd64.deb +``` + +`apt` rather than `dpkg -i`, because it pulls `libwebkit2gtk-4.1-0` and `libgtk-3-0` in for you. +The app then appears in your launcher. Upgrading is the same command with the newer file, and +`sudo apt remove margin-mail` reverses it. + +Built on Ubuntu 22.04, so 22.04 is the oldest release it runs on. Anything older has a glibc the +binary was not linked against and will refuse to start. + +### flatpak, on everything else + +``` +flatpak install ./Margin.Mail_0.1.0_amd64.flatpak +flatpak run studio.margin.mail +``` + +A single self-contained file: it brings its own GTK and WebKit, so it does not care what your +distribution ships. It is not on Flathub and will not be, because a Flathub build has to come from +source and this app's Google client is embedded at compile time from a file that is deliberately +not in the repository. + +The sandbox is narrow on purpose. The app gets the network, the notification service and your +downloads directory, and nothing else. That means saved attachments and exports land in +`~/Downloads` and can go nowhere else, and the app's data lives under +`~/.var/app/studio.margin.mail/` rather than in the usual place, so a flatpak install and a deb +install do not see each other's mail. + +`flatpak uninstall studio.margin.mail` removes it, and add `--delete-data` to take the mail with +it. + +### Nix + +The package relinks the published deb against nixpkgs' own GTK and WebKit, which is the only build +here that runs as a native Wayland client rather than falling back to Xwayland. + +``` +NIXPKGS_ALLOW_UNFREE=1 nix run --impure github:priyanshujain/margin-mail#margin-mail +``` + +`NIXPKGS_ALLOW_UNFREE` because the licence is FSL rather than MIT, so nix asks first. To keep it, +add the flake as an input and the overlay to your configuration: + +```nix +{ + inputs.margin-mail.url = "github:priyanshujain/margin-mail"; + + # In your nixpkgs configuration: + nixpkgs.overlays = [ inputs.margin-mail.overlays.default ]; + nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ "margin-mail" ]; + environment.systemPackages = [ pkgs.margin-mail ]; +} +``` + +A Nix install does not update itself. The store is read only, so the app reports the new version +and tells you to update the flake instead of trying to replace its own binary. + +### AppImage, when you want no install + +``` +chmod +x Margin.Mail_0.1.0_amd64.AppImage +./Margin.Mail_0.1.0_amd64.AppImage +``` + +Nothing is written outside your home directory and there is nothing to uninstall. It carries +Ubuntu's GTK stack, which cannot talk to a modern Wayland compositor, so on Wayland it runs through +Xwayland and looks slightly soft on a HiDPI screen. That is the trade for a single portable file. + +To get it into your launcher rather than running it from a terminal, `just install` from a clone +does the whole thing, or by hand: + +``` +install -Dm755 Margin.Mail_0.1.0_amd64.AppImage ~/.local/bin/margin-mail +``` + +and write a `.desktop` file pointing `Exec` at it. + +## Windows + +Two installers, and either is fine. The `.exe` is the friendlier one; the `.msi` is what you want +if you are deploying by policy. + +1. Download `Margin.Mail_0.1.0_x64-setup.exe`. +2. Run it. Windows SmartScreen will say it does not recognise the publisher, because the installer + is not code-signed. Click **More info**, then **Run anyway**. +3. The app appears in the Start menu. + +Uninstalling is through Apps in Settings, the same as anything else. + +Windows 10 1803 or newer, and WebView2, which every supported Windows already has. + +## Updating + +Every install except Nix and the flatpak updates itself. The app checks the release feed on launch, +tells you when something newer is out, and installs it when you say so. **Check for Updates** in +the File menu, or the application menu on macOS, asks immediately. + +Nix updates with the flake. The flatpak updates with `flatpak update studio.margin.mail` once you +have installed the newer bundle file, since a single-file bundle carries no remote to check. + +## Where your mail lives + +One directory per platform, and it survives uninstalling: + +- macOS: `~/Library/Application Support/studio.margin.mail` +- Linux: `~/.local/share/studio.margin.mail`, or `~/.var/app/studio.margin.mail/data/studio.margin.mail` under flatpak +- Windows: `%APPDATA%\studio.margin.mail` + +It holds the mirror of your mail, the state database of every decision you have made, and your +sealed refresh tokens. Copying it to another machine moves everything except the tokens, which are +sealed against the machine that stored them, so the accounts ask to be connected again and nothing +else changes. + +To remove the app and its mail together, uninstall and then delete that directory. + +## Building it yourself + +`just install` from a clone builds for the machine you are sitting at and puts it where that +machine expects to find applications. [release.md](release.md) has what a build needs and how a +release is cut. diff --git a/docs/keyboard.md b/docs/keyboard.md index 55268bb..a88343f 100644 --- a/docs/keyboard.md +++ b/docs/keyboard.md @@ -34,8 +34,8 @@ app. Bindings live in a keymap file the user can edit; the defaults are what fol | `e` | Archive | yes | | `u` | Toggle seen | yes | | `Shift+S` | Toggle star | yes | -| `#` | Trash | yes | -| `!` | Spam | yes | +| `#` | Trash, and put back in Trash (toggle) | yes | +| `!` | Spam, and not spam in Spam (toggle) | yes | | `l` | Reply later (toggle) | no | | `s` | Set aside (toggle) | no | | `b` | Snooze… | no | diff --git a/docs/mockups/connect.png b/docs/mockups/connect.png new file mode 100644 index 0000000..322baf4 Binary files /dev/null and b/docs/mockups/connect.png differ diff --git a/docs/mockups/onboarding.png b/docs/mockups/onboarding.png new file mode 100644 index 0000000..5d90e9a Binary files /dev/null and b/docs/mockups/onboarding.png differ diff --git a/docs/mockups/screener.png b/docs/mockups/screener.png index aac7774..9c2d068 100644 Binary files a/docs/mockups/screener.png and b/docs/mockups/screener.png differ diff --git a/docs/mockups/settings.png b/docs/mockups/settings.png new file mode 100644 index 0000000..f1c2cfa Binary files /dev/null and b/docs/mockups/settings.png differ diff --git a/docs/mockups/src/connect.html b/docs/mockups/src/connect.html new file mode 100644 index 0000000..b4ec86b --- /dev/null +++ b/docs/mockups/src/connect.html @@ -0,0 +1,128 @@ + + + + +Margin Mail: Connect + + + + + +
+
+
+
+
+
+
+ +
+
+
+ + + + + +
+ +

Margin Mail

+

A quiet, keyboard-first client for your mail, where every decision you make stays on your own machine.

+ +
+
+ + +
+ +
+ +

Any mailbox works here: Google, Fastmail, iCloud, Yahoo, Proton through Bridge, a mailbox where you work, or anything else that speaks IMAP. Margin works out the servers from the address, and nothing is stored until the account is added.

+
+
+
+ + diff --git a/docs/mockups/src/onboarding.html b/docs/mockups/src/onboarding.html new file mode 100644 index 0000000..95a1bce --- /dev/null +++ b/docs/mockups/src/onboarding.html @@ -0,0 +1,294 @@ + + + + +Margin Mail: First run + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+ +
+
+ + + +
+
+ + + +
+
+ +
+
+
+

Inbox

+
+
+
New for you
+ +
+
+ MR +
+
Maya Raghunathan311:42
+
Dinner on Thursday?Priya said the place on Church Street takes bookings now, want me to
+
+
+ +
+
+ AK +
+
Arun Kulkarni10:15
+
Re: Lease renewal for the studioAttached the revised draft. The only change is clause 7, which now
+
+
+ +
+
+ Ai +
+
Airbnb09:03
+
Your reservation in Lisbon is confirmedCheck-in Friday 12 September after 15:00. Your host Inês will send
+
+
+ +
+
+ SO +
+
Sam OkaforYesterday
+
Piano lessons, the form you sentGot it, thank you. Wednesdays at five work for us. Is there a
+
+
+ +
+
Previously seen
+ +
+
+ LB +
+
Lena Brandt7Yesterday
+
Kitchen bench quoteSounds good, Julie. Any afternoon next week works for me.
+
+
+ +
+
+ DP +
+
Dev PatelMon
+
Slides from the talkHere they are, plus the reading list I mentioned. The paper on
+
+
+ +
+
+ Ds +
+
DocuSignMon
+
Completed: Studio lease 2026All parties have completed the envelope. You can access the
+
+
+ +
+
+ HW +
+
Hannah Weiss2Sun
+
Photos from the Hawaii tripFinally went through them all. The ones from the ridge walk are
+
+
+ +
+
+ RY +
+
Russell Young30 Aug
+
Pumpkin bread recipeFrom my mother's card, transcribed as best I could. Bake at 175
+
+
+
+
+ +
+ + +
+
+ +
+
+ + + + + + + +
+ +
+
+
+

Lease renewal for the studio

+
+ AKPJ + Arun Kulkarni and you + · + 2 messages +
+
+ + + +
+
+ PJ +
You
Hi Arun, thanks for sending the renewal over. Two things before I sign: the notice period in clause 7 and
+
Mon 14:20
+
+
+ +
+
+ AK +
Arun Kulkarniarun@meridianproperties.in
to you
+
Today 10:15
+
+
+

Hi Priyanshu,

+

Attached the revised draft. The only change is clause 7, which now reads three months either side rather than six. The rent review in clause 12 stays as it was, tied to the index and capped at four percent.

+

If that works, sign when you get a moment and I will countersign the same day. Happy to walk through anything on a call, Thursday afternoon is open.

+

Best,
Arun

+ ··· Show quoted text +
+
PDFStudio-lease-2026-v2.pdf412 KB
+
+
+
+ +
+
+
+
+ +
+
+
+

You are set up

+
+
+

1,284 senders were screened in already, because you have written to them or they are in your contacts: 96 to the Inbox, 71 to the Feed and 1,117 to the Paper Trail.

+

From here on, anyone new waits in the Screener until you say where their mail goes. Nobody is told either way.

+
+ Start fresh +

Optional. Mark older mail as seen, so New for you holds only what is recent. Reversible for seven days.

+
+ Older than + + + + +
+
+
+
+ + +
+
+
+
+ + diff --git a/docs/mockups/src/screener.html b/docs/mockups/src/screener.html index e53d520..0333a9d 100644 --- a/docs/mockups/src/screener.html +++ b/docs/mockups/src/screener.html @@ -105,7 +105,7 @@ -

Screened-out mail is kept for 90 days under Screened out, then deleted. Change your mind from a sender's contact card at any time.

+

Screened-out mail sits under Screened out for as long as this account keeps mail on the device. Change your mind from a sender's contact card at any time.

diff --git a/docs/mockups/src/settings.html b/docs/mockups/src/settings.html new file mode 100644 index 0000000..919845b --- /dev/null +++ b/docs/mockups/src/settings.html @@ -0,0 +1,371 @@ + + + + +Margin Mail: Settings + + + + + + + + + + + + + + + + +
+
+
+
+ +
+
+ + + +
+
+ + + +
+
+ +
+
+ + +
+
+

Accounts

+

Two accounts, each with its own places, rules and piles. The colour is the edge on a row in All accounts.

+ +
+
+ PJ +
+
Priyanshu Jain
+
pj@73ai.org
+
+
+ + + + + + + + +
+
+ +
+
SignaturePriyanshu Jain · 73ai
+
Aliasespriyanshu@73ai.org, hello@73ai.org
+
+ +
+
Permissions
+ +
+ + Read and change your mailGranted +
+
+ + Read your signature and aliasesGranted +
+
+ + Read your contactsGranted +
+
+ + Read the people you have written toGranted +
+
+ + Keep a backup in your DriveGranted +
+
+ + Answer calendar invitations + +
+

Calendar is not connected, so an invitation renders but Accept, Maybe and Decline do nothing. Granting reopens the Google consent page.

+
+
+ +
+
+ PS +
+
Personal
+
priyanshujain@gmail.com
+
+
+ + + + + + + + +
+
+ +
+
SignatureNot set
+
AliasesNone on the provider
+
PermissionsAll granted
+
+
+ +
+ + +
+ +

Margin Mail, Margin and Margin Calendar share one Google client, so your Google account lists them once, as Margin, and revoking it revokes all three.

+
+
+
+
+
+
+ + diff --git a/docs/plan.md b/docs/plan.md new file mode 100644 index 0000000..1ece393 --- /dev/null +++ b/docs/plan.md @@ -0,0 +1,193 @@ +# Plan + +How the app gets built, in the order it gets built, and what it is built out of. The product is +specified in [design.md](design.md) and [features.md](features.md); this document is about the +work rather than the result, and it exists so that somebody picking the repository up in six +months can see why the pieces landed in the order they did. + +Work is cut into milestones, and a milestone into work packages. A package is a unit somebody can +finish: it owns a set of files, it ends with something that runs, and it never edits another +package's files. The package names below are the ones the source comments already use, so a +placeholder that says "the contract lands in F3" means the work package named here. + +## The milestones + +**M0, foundations.** F1 is the scaffold: the Tauri crate, the Vite front end, the icons, the +justfile and CI. F2 is the design system, which is `src/styles/tokens.css` and the primitives that +sit on it, reviewed through the Kit page. F3 is the contracts: `src-tauri/src/dto.rs` and its +mirror `src/ipc.ts`, frozen before anything implements them. F4 is the documentation, this file +among it. None of M0 sends a byte to Google, and that is the point: the shape is settled while it +is still cheap to change. + +**M1, read.** The milestone that proves the two hard things. R1 is authentication and accounts: +the loopback flow, the sealed token, the granted scopes. R2 is the Gmail client behind the +`Provider` trait, with the quota arithmetic and the backoff in it. R3 is the mirror and the sync +loop, including the storage window and eviction. R4 is the message pipeline: the MIME parse, the +sanitiser, the tracker stripper. R5 is the shell, which is the header, the list column, the +reading pane and the keyboard. R6 is connect and onboarding, the first thing a new user meets and +the last thing built in this milestone, because it cannot be designed honestly until the sync it +narrates exists. At the end of M1 the app reads mail and does nothing else. + +**M2, triage.** T1 is flags and undo: seen, star, archive, trash, spam, each optimistic and each +reversible. T2 is selection and bulk actions. T3 is search, local over the window with the +provider as the second pass. T4 is labels. At the end of M2 it is a fast Gmail client and nothing +more, which is worth having in the hands of one user for a week before the next milestone changes +what it is. + +**M3, places.** P1 is the state database and its journal, the schema that everything after this +depends on. P2 is routing: sender rules, the suggestion function, the overrides. P3 is the +Screener and the first-run pass. P4 is the Feed and the Paper Trail. P5 is contacts, the contact +card and autocomplete. This is the milestone where it stops being a Gmail client. + +**M4, piles.** L1 is Reply later, Set aside and Focus & Reply. L2 is snooze with lazy evaluation. +L3 is notes, rename and merge. L4 is the rest of what the state database makes possible: clips, +All files, ignore, per-thread notifications and unsubscribe. + +**M5, writing.** W1 is the editor and drafts. W2 is the send pipeline: the outbox, the undo delay, +attachments, threading headers. W3 is instant intro, remind me if no reply, and calendar invites +including the re-authorization that RSVP needs. + +**M6, accounts, settings and backup.** A1 is more than one account and the unified view. A2 is the +settings screen, specified in [settings.md](settings.md). A3 is the backup store, the encryption +and the recovery phrase, with the journal merge behind it. + +**M7, ship.** S1 is the release pipeline: signing, notarisation, the updater. S2 is the +verification materials Google's restricted scope review wants, which is a privacy policy that is +true, a demo video, and a written justification for each scope. Then the platforms in order: S3 +the iPhone, S4 Linux, S5 the IMAP provider. They are last because each one is a second copy of a +problem already solved once, and solving it twice before it is solved once is how a project stalls. + +## What comes from the siblings + +Very little here is new, and that is deliberate. Margin Mail sits beside margin and Margin +Calendar on disk and takes from both. + +From the calendar: the OAuth loopback flow with PKCE and its Google-specific handling, the sealed +token store, the deep-link path that mobile needs instead of a loopback listener, the overlay +title bar and the macOS window behaviour, the `data-phone` and `data-touch` scheme with the boot +script that sets them before first paint, the escape-layer stack, the palette, the zustand store +idiom, the release workflow's shape, and the `build.rs` trick that embeds +`google-credentials.json` with the example file as a fallback. + +From margin: the HTTP half of `gdrive.rs`, which is folder lookup, upload and download against +Drive's v3 API; the trick of putting heavy synchronous work behind `#[tauri::command(async)]`; and +`margin-shared`, which is a real dependency rather than a copy. The tokens, the icon strings and +the font catalogue come from that package through a relative path in `package.json`, which is why +CI checks out both repositories side by side. + +From neither: the mirror, the state database, the journal, the sanitiser and everything to do with +mail. Those are this repository's own work. + +## The libraries + +Every version below was checked against crates.io and npm on 3 September 2026. + +On the Rust side, `mail-parser` 0.11 with `full_encoding` parses bodies from the raw RFC 2822 +bytes; the feature is not optional, because the charsets it adds are the ones that still turn up +in real mail every day. `mail-builder` 0.5 builds outgoing MIME. `css-inline` 0.21 folds the +editor's stylesheet into the markup before the message is built, so a client that drops ` + + + +
+

It has been 30 days since your last check. +Stop by any store for a free air pressure check, no appointment needed.

+
+
+ + +
+ +

Book +now

+

Find a store near you

+ + + + diff --git a/src-tauri/fixtures/inline-image-attachment-disposition.eml b/src-tauri/fixtures/inline-image-attachment-disposition.eml new file mode 100644 index 0000000..aca6dc0 --- /dev/null +++ b/src-tauri/fixtures/inline-image-attachment-disposition.eml @@ -0,0 +1,29 @@ +Return-Path: +MIME-Version: 1.0 +Date: Mon, 31 Aug 2026 14:22:07 +0100 +Message-ID: +Subject: The signature block, and one photo +From: Clare Dunn +To: Priyanshu Jain +Content-Type: multipart/related; type="text/html"; boundary="_004_AM6PR03MB4419_" + +--_004_AM6PR03MB4419_ +Content-Type: text/html; charset="utf-8" +Content-Transfer-Encoding: 7bit + + +

The photo you asked for is below.

+

The site

+

Clare

+ + +--_004_AM6PR03MB4419_ +Content-Type: image/png; name="site.png" +Content-Description: site.png +Content-Disposition: attachment; filename="site.png"; size=70 +Content-ID: +Content-Transfer-Encoding: base64 + +iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg== + +--_004_AM6PR03MB4419_-- diff --git a/src-tauri/fixtures/inline-image-cid.eml b/src-tauri/fixtures/inline-image-cid.eml new file mode 100644 index 0000000..886cc01 --- /dev/null +++ b/src-tauri/fixtures/inline-image-cid.eml @@ -0,0 +1,75 @@ +Return-Path: +MIME-Version: 1.0 +Date: Sun, 30 Aug 2026 12:19:40 +0200 +Message-ID: +Subject: Photos from the Hawaii trip +From: Hannah Weiss +To: Priyanshu Jain +Content-Type: multipart/related; type="multipart/alternative"; + boundary="rel-hawaii-04" + +--rel-hawaii-04 +Content-Type: multipart/alternative; boundary="alt-hawaii-04" + +--alt-hawaii-04 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 7bit + +Finally went through them all. The ones from the ridge walk are the best +of the lot, two of them are below and the rest are in the folder. + +Hannah + +--alt-hawaii-04 +Content-Type: text/html; charset=UTF-8 +Content-Transfer-Encoding: 7bit + +
+

Finally went through them all. The ones from the ridge walk are the +best of the lot, two of them are below and the rest are in the folder.

+

The ridge at seven

+

Looking back down

+

Hannah

+
+ +--alt-hawaii-04-- + +--rel-hawaii-04 +Content-Type: image/png; name="ridge-walk-1.png" +Content-Transfer-Encoding: base64 +Content-ID: +Content-Disposition: inline; filename="ridge-walk-1.png" + +iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAACJ0lEQVR42g3Loc6GIBQA0P9xvmg0 +EAgEAoFgYMxwA2MEAmOGm5jBwEwkRjA4EsnkA/6efv5+mkyazppRzbkWUkulF9DK6TVoQG127bL2 +RYemt1vj3w/IBHQGRoFzEBKkggVAOVgDAILZwWXwBUKD7YYvWDJZOltGLedWSCuVXcAqZ9dgAa3Z +rcvWFxua3W77BU8mT2fPqOfcC+ml8gt45fwaPKA3u3fZ++JD89vtvxDJFOkcGY2cRyGjVHGBqFxc +QwSMZo8uR19iaHG74xeQTEhnZBQ5RyFRKlwAlcM1ICCaHV1GXzA03G78QiJTonNiNHGehExSpQWS +cmkNCTCZPbmcfEmhpe1OXzjIdND5YPTg/BDykOpY4FDuWMMBeJj9cPnw5Qjt2O7jCyeZTjqfjJ6c +n0KeUp0LnMqdazgBT7OfLp++nKGd231+oZCp0LkwWjgvQhapygJFubKGAljMXlwuvpTQynaXL1Qy +VTpXRivnVcgqVV2gKlfXUAGr2avL1ZcaWt3u+oWLTBedL0Yvzi8hL6muBS7lrjVcgJfZL5cvX67Q +ru2+vtDJ1OncGe2cdyG7VH2BrlxfQwfsZu8ud196aH27+xcGmQadB6OD8yHkkGosMJQbaxiAw+zD +5eHLCG1s9/jCQ6aHzg+jD+ePkI9UzwKPcs8aHsDH7I/Ljy9PaM92P194yfTS+WX05fwV8pXqXeBV +7l3DC/ia/XX59eUN7d3uF/8BbBpFEE55jaYAAAAASUVORK5CYII= + +--rel-hawaii-04 +Content-Type: image/png; name="ridge-walk-2.png" +Content-Transfer-Encoding: base64 +Content-ID: +Content-Disposition: inline; filename="ridge-walk-2.png" + +iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAACJ0lEQVR42g3Loc6GIBQA0P9xvmg0 +EAgEAoFgYMxwA2MEAmOGm5jBwEwkRjA4EsnkA/6efv5+mkyazppRzbkWUkulF9DK6TVoQG127bL2 +RYemt1vj3w/IBHQGRoFzEBKkggVAOVgDAILZwWXwBUKD7YYvWDJZOltGLedWSCuVXcAqZ9dgAa3Z +rcvWFxua3W77BU8mT2fPqOfcC+ml8gt45fwaPKA3u3fZ++JD89vtvxDJFOkcGY2cRyGjVHGBqFxc +QwSMZo8uR19iaHG74xeQTEhnZBQ5RyFRKlwAlcM1ICCaHV1GXzA03G78QiJTonNiNHGehExSpQWS +cmkNCTCZPbmcfEmhpe1OXzjIdND5YPTg/BDykOpY4FDuWMMBeJj9cPnw5Qjt2O7jCyeZTjqfjJ6c +n0KeUp0LnMqdazgBT7OfLp++nKGd231+oZCp0LkwWjgvQhapygJFubKGAljMXlwuvpTQynaXL1Qy +VTpXRivnVcgqVV2gKlfXUAGr2avL1ZcaWt3u+oWLTBedL0Yvzi8hL6muBS7lrjVcgJfZL5cvX67Q +ru2+vtDJ1OncGe2cdyG7VH2BrlxfQwfsZu8ud196aH27+xcGmQadB6OD8yHkkGosMJQbaxiAw+zD +5eHLCG1s9/jCQ6aHzg+jD+ePkI9UzwKPcs8aHsDH7I/Ljy9PaM92P194yfTS+WX05fwV8pXqXeBV +7l3DC/ia/XX59eUN7d3uF/8BbBpFEE55jaYAAAAASUVORK5CYII= + +--rel-hawaii-04-- diff --git a/src-tauri/fixtures/invite-daylight-saving.eml b/src-tauri/fixtures/invite-daylight-saving.eml new file mode 100644 index 0000000..5adbaa1 --- /dev/null +++ b/src-tauri/fixtures/invite-daylight-saving.eml @@ -0,0 +1,65 @@ +Return-Path: +MIME-Version: 1.0 +Date: Wed, 2 Sep 2026 11:04:51 +0100 +Message-ID: <00000000000091f2ab@google.com> +Subject: Invitation: Quarterly review @ Thu 15 Oct 2026 14:00 - 15:00 (BST) +From: Nadia Osei +To: Priyanshu Jain +Content-Type: multipart/alternative; boundary="000000000000c1a4" + +--000000000000c1a4 +Content-Type: text/plain; charset="UTF-8" +Content-Transfer-Encoding: 7bit + +You have been invited to Quarterly review on Thursday 15 October 2026, +14:00 to 15:00 London time. + +--000000000000c1a4 +Content-Type: text/calendar; charset="UTF-8"; method=REQUEST +Content-Transfer-Encoding: 7bit + +BEGIN:VCALENDAR +PRODID:-//Google Inc//Google Calendar 70.9054//EN +VERSION:2.0 +CALSCALE:GREGORIAN +METHOD:REQUEST +BEGIN:VTIMEZONE +TZID:Europe/London +X-LIC-LOCATION:Europe/London +BEGIN:DAYLIGHT +TZOFFSETFROM:+0000 +TZOFFSETTO:+0100 +TZNAME:BST +DTSTART:19700329T010000 +RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=-1SU +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETFROM:+0100 +TZOFFSETTO:+0000 +TZNAME:GMT +DTSTART:19701025T020000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU +END:STANDARD +END:VTIMEZONE +BEGIN:VEVENT +DTSTART;TZID=Europe/London:20261015T140000 +DTEND;TZID=Europe/London:20261015T150000 +DTSTAMP:20260902T100451Z +ORGANIZER;CN=Nadia Osei:mailto:nadia@westferry.example +UID:5m1q9c3k8p7v@google.com +ATTENDEE;CUTYPE=INDIVIDUAL;ROLE=REQ-PARTICIPANT;PARTSTAT=NEEDS-ACTION;RSVP= + TRUE;CN=Priyanshu Jain;X-NUM-GUESTS=0:mailto:pj@73ai.org +ATTENDEE;CUTYPE=INDIVIDUAL;ROLE=REQ-PARTICIPANT;PARTSTAT=ACCEPTED;CN=Nadia + Osei;X-NUM-GUESTS=0:mailto:nadia@westferry.example +CREATED:20260902T100450Z +DESCRIPTION:The quarterly numbers\, then the roadmap. Bring the deck. +LAST-MODIFIED:20260902T100450Z +LOCATION:Westferry\, meeting room 2 +SEQUENCE:0 +STATUS:CONFIRMED +SUMMARY:Quarterly review +TRANSP:OPAQUE +END:VEVENT +END:VCALENDAR + +--000000000000c1a4-- diff --git a/src-tauri/fixtures/nested-multipart.eml b/src-tauri/fixtures/nested-multipart.eml new file mode 100644 index 0000000..a0f23bc --- /dev/null +++ b/src-tauri/fixtures/nested-multipart.eml @@ -0,0 +1,57 @@ +Return-Path: +MIME-Version: 1.0 +Date: Wed, 2 Sep 2026 16:04:51 -0500 +Message-ID: <116400046977232.1756846291@citypower.example> +Subject: Bill payment pending +From: City Power +To: Priyanshu Jain +Auto-Submitted: auto-generated +Content-Type: multipart/mixed; boundary="outer-116400046977232" + +--outer-116400046977232 +Content-Type: multipart/alternative; boundary="inner-116400046977232" + +--inner-116400046977232 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 7bit + +Dear Customer, + +Your one-time electronic payment of 98.26 for your City Power bill has +been received and is pending. Your payment will be posted within two +business days. + +Confirmation number 116400046977232 + +Please keep this message for your records. If you did not make this +payment, call us on the number printed on your statement. + +Thank you, +City Power Customer Care + +--inner-116400046977232 +Content-Type: text/html; charset=UTF-8 +Content-Transfer-Encoding: quoted-printable + + +

Dear Customer,

+

Your one-time electronic payment of 98.26 for your City Power = +bill has been received and is pending. Your payment will be posted with= +in two business days.

+

Confirmation number 116400046977232

+

Thank you,
City Power Customer Care

+ + +--inner-116400046977232-- + +--outer-116400046977232 +Content-Type: application/pdf; name="Statement-August.pdf" +Content-Transfer-Encoding: base64 +Content-Disposition: attachment; filename="Statement-August.pdf" + +JVBERi0xLjQKMSAwIG9iajw8L1R5cGUvQ2F0YWxvZy9QYWdlcyAyIDAgUj4+ZW5kb2JqCjIgMCBv +Ymo8PC9UeXBlL1BhZ2VzL0tpZHNbMyAwIFJdL0NvdW50IDE+PmVuZG9iagozIDAgb2JqPDwvVHlw +ZS9QYWdlL1BhcmVudCAyIDAgUi9NZWRpYUJveFswIDAgNTk1IDg0Ml0+PmVuZG9iagp0cmFpbGVy +PDwvUm9vdCAxIDAgUj4+CiUlRU9GCg== + +--outer-116400046977232-- diff --git a/src-tauri/fixtures/newsletter-list-unsubscribe.eml b/src-tauri/fixtures/newsletter-list-unsubscribe.eml new file mode 100644 index 0000000..4cd1e8b --- /dev/null +++ b/src-tauri/fixtures/newsletter-list-unsubscribe.eml @@ -0,0 +1,52 @@ +Return-Path: +MIME-Version: 1.0 +Date: Thu, 3 Sep 2026 06:00:03 +0100 +Message-ID: <01000198a2f4c1b2-9b0c1d2e-3f4a-5b6c-7d8e-9f0a1b2c3d4e@thebrowser.example> +Subject: Five things worth reading this weekend +From: The Browser +To: pj@73ai.org +List-Id: The Browser +List-Unsubscribe: , + +List-Unsubscribe-Post: List-Unsubscribe=One-Click +List-Post: NO +Precedence: bulk +Feedback-ID: 91827:the-browser:thebrowser +Content-Type: multipart/alternative; boundary="b1-the-browser" + +--b1-the-browser +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: quoted-printable + +Good morning. It is a long weekend in some places and a wet one in most, +so this edition leans toward pieces you can settle into. A long piece on +the history of the pencil, a short one on why bridges hum, and three more. + +THE PENCIL, AT LENGTH. Henry Petroski wrote a whole book about the pencil +and this essay is the argument for why that was a reasonable thing to do. + +WHY BRIDGES HUM. A short explanation of vortex shedding from an engineer +who spent a career listening to cables. + +Unsubscribe: https://thebrowser.example/unsubscribe?u=3D91827&id=3D8f3a1c + +--b1-the-browser +Content-Type: text/html; charset=UTF-8 +Content-Transfer-Encoding: quoted-printable + + +

Good morning. It is a long weekend in some places and a wet one in mo= +st, so this edition leans toward pieces you can settle into.

+

The pencil, at length. Henry Petroski wrote a whole book about = +the pencil and this essay is the argument for why that was a reasonable = +thing to do. Graphite, cedar, the Napoleonic wars and a factory in Nurem= +berg that still runs.

+

Why bridges hum. A short explanation of vortex shedding from an= + engineer who spent a career listening to cables.

+

The last typewriter repairman in Mumbai. A profile that is real= +ly about what it means to keep a trade going after the trade has gone.

+

Unsubscribe

+ + +--b1-the-browser-- diff --git a/src-tauri/fixtures/no-message-id.eml b/src-tauri/fixtures/no-message-id.eml new file mode 100644 index 0000000..794223b --- /dev/null +++ b/src-tauri/fixtures/no-message-id.eml @@ -0,0 +1,15 @@ +Return-Path: +MIME-Version: 1.0 +Date: Wed, 12 Aug 2026 11:02:19 +0530 +Subject: Enrolment received for Cooper +From: Sunny Day Music +To: pj@73ai.org +X-Mailer: PHPMailer 5.2.9 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 7bit + +Enrolment received for Cooper. Term starts the week of 8 September and +your teacher will be in touch to fix a weekly slot. + +Nothing further is needed from you today. The term dates and the studio +address are on the enrolment form you signed. diff --git a/src-tauri/fixtures/outlook-reply.eml b/src-tauri/fixtures/outlook-reply.eml new file mode 100644 index 0000000..a735c10 --- /dev/null +++ b/src-tauri/fixtures/outlook-reply.eml @@ -0,0 +1,46 @@ +Return-Path: +MIME-Version: 1.0 +Date: Tue, 1 Sep 2026 16:41:09 +0100 +Message-ID: +In-Reply-To: <20260901120044.7F1@73ai.org> +References: <20260901120044.7F1@73ai.org> +Subject: RE: The lease, clause 14 +From: "Young, Russell" +To: Priyanshu Jain +Content-Type: multipart/alternative; boundary="_000_AM0PR07MB5218_" + +--_000_AM0PR07MB5218_ +Content-Type: text/plain; charset="utf-8" +Content-Transfer-Encoding: quoted-printable + +Clause 14 is the standard form. I would not sign it as drafted. + +Russell + +-----Original Message----- +From: Priyanshu Jain +Sent: Tuesday, 1 September 2026 12:00 +To: Young, Russell +Subject: The lease, clause 14 + +Russell, is clause 14 negotiable or is it the landlord's standard form? + +--_000_AM0PR07MB5218_ +Content-Type: text/html; charset="utf-8" +Content-Transfer-Encoding: quoted-printable + + +

Clause 14 is the standard form. I would not sign it as drafted.

+

Russell

+
+
+
+

From: Priyanshu Jain <pj@73ai.org>
+Sent: Tuesday, 1 September 2026 12:00
+Subject: The lease, clause 14

+

Russell, is clause 14 negotiable or is it the landlord's standard form?

+
+ + +--_000_AM0PR07MB5218_-- diff --git a/src-tauri/fixtures/plain-text.eml b/src-tauri/fixtures/plain-text.eml new file mode 100644 index 0000000..ed418a8 --- /dev/null +++ b/src-tauri/fixtures/plain-text.eml @@ -0,0 +1,21 @@ +Return-Path: +Received: from mail-pl1-f174.example.com (mail-pl1-f174.example.com [209.85.214.174]) + by mx.73ai.org with ESMTPS id 4c1f9a2b3d + for ; Thu, 3 Sep 2026 11:42:14 +0530 (IST) +MIME-Version: 1.0 +Date: Thu, 3 Sep 2026 11:42:09 +0530 +Message-ID: +Subject: Dinner on Thursday? +From: Maya Raghunathan +To: Priyanshu Jain +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 7bit + +Priya said the place on Church Street takes bookings now, want me to put +us down for four at eight? Everyone can make Thursday except Karthik, who +has the badminton thing and says he will come late. + +If you would rather somewhere quieter there is the Malleswaram place we +went to in June. Say by tomorrow and I will call them. + +Maya diff --git a/src-tauri/fixtures/quoted-printable-soft-breaks.eml b/src-tauri/fixtures/quoted-printable-soft-breaks.eml new file mode 100644 index 0000000..9f57f56 --- /dev/null +++ b/src-tauri/fixtures/quoted-printable-soft-breaks.eml @@ -0,0 +1,24 @@ +Return-Path: +MIME-Version: 1.0 +Date: Mon, 31 Aug 2026 09:30:00 +0000 +Message-ID: +Subject: =?utf-8?Q?September=3A_the_notebook_that_survived_a_washin?= + =?utf-8?Q?g_machine?= +From: Field Notes Dispatch +To: pj@73ai.org +List-Id: +List-Unsubscribe: +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: quoted-printable + +A reader in Troms=C3=B8 sent us a photo of a pocket noteb= +ook that went through a full cycle at forty degrees, in a coat, with the= + coat. Every page is still legible. + +Also this month: the autumn edition ships on the 15th. Three colours, th= +e usual count. The paper weight is unchanged at 100 g/m=C2=B2, and the p= +rice stays at =E2=82=AC12 for the three pack. + +The equals sign is written =3D when it stands alone, which is the part o= +f this encoding that catches parsers out. Trailing whitespace is kept=20 +by encoding it, like the space at the end of the previous line. diff --git a/src-tauri/fixtures/quoted-reply-blockquote.eml b/src-tauri/fixtures/quoted-reply-blockquote.eml new file mode 100644 index 0000000..8df33f3 --- /dev/null +++ b/src-tauri/fixtures/quoted-reply-blockquote.eml @@ -0,0 +1,45 @@ +Return-Path: +MIME-Version: 1.0 +Date: Mon, 31 Aug 2026 21:14:55 +0530 +Message-ID: +In-Reply-To: +References: + +Subject: Re: Slides from the talk +From: Dev Patel +To: Priyanshu Jain +Content-Type: multipart/alternative; boundary="alt-slides-03" + +--alt-slides-03 +Content-Type: text/plain; charset=UTF-8 + +Here they are, plus the reading list I mentioned. The paper on cache +oblivious layouts is the one to start with. + +--alt-slides-03 +Content-Type: text/html; charset=UTF-8 + +
Here they are, plus the reading list I mentioned. The paper +on cache oblivious layouts is the one to start with.
+
+
+
On Mon, 31 Aug 2026 at 18:40, Priyanshu +Jain <pj@73ai.org> wrote:
+
+
Good talk. Could you send the slides, and the reading list +you put up at the end?
+
+
+
On Fri, 28 Aug 2026 at 12:02, Dev Patel +<dev.patel@example.org> +wrote:
+
+
Talk is on Monday at six, room 4.02. Come if you are free.
+
+
+
+
+ +--alt-slides-03-- diff --git a/src-tauri/fixtures/quoted-reply-plain.eml b/src-tauri/fixtures/quoted-reply-plain.eml new file mode 100644 index 0000000..c6c290f --- /dev/null +++ b/src-tauri/fixtures/quoted-reply-plain.eml @@ -0,0 +1,33 @@ +Return-Path: +MIME-Version: 1.0 +Date: Wed, 2 Sep 2026 15:40:02 +0530 +Message-ID: +In-Reply-To: +References: + +Subject: Re: Fw: (no subject) +From: Sam Okafor +To: Priyanshu Jain +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 7bit + +Got it, thank you. Wednesdays at five work for us. + +The first lesson is on the 9th, nothing to prepare. The invitation is +attached, it should land in your calendar. + +Sam + +On Mon, 31 Aug 2026 at 09:12, Priyanshu Jain wrote: +> Hi Sam, form signed and attached. Would a weekday after school suit +> Cooper? He finishes at four and we are ten minutes away on foot. +> +> On Wed, 12 Aug 2026 at 11:02, Sunny Day Music +> wrote: +> > Enrolment received for Cooper. Term starts the week of 8 September +> > and your teacher will be in touch to fix a weekly slot. +> > +> > Nothing further is needed from you today. +> +> Thanks, +> Priyanshu diff --git a/src-tauri/fixtures/receipt-no-reply.eml b/src-tauri/fixtures/receipt-no-reply.eml new file mode 100644 index 0000000..e6a6ef8 --- /dev/null +++ b/src-tauri/fixtures/receipt-no-reply.eml @@ -0,0 +1,18 @@ +Return-Path: <> +MIME-Version: 1.0 +Date: Thu, 3 Sep 2026 06:30:18 +0000 +Message-ID: <20260903063018.4d2a1f9c@mail.spotify.example> +Subject: Your receipt +From: Spotify +To: pj@73ai.org +Auto-Submitted: auto-generated +Precedence: bulk +X-Entity-Ref-ID: 8f21c0aa-receipt +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 7bit + +Thank you for purchasing Spotify Premium. Amount charged 11.99, on the +card ending 4417. Your next payment is due on 3 October 2026. + +This mailbox is not monitored. Manage your subscription from the account +page. diff --git a/src-tauri/fixtures/reply-references-folded.eml b/src-tauri/fixtures/reply-references-folded.eml new file mode 100644 index 0000000..3c82665 --- /dev/null +++ b/src-tauri/fixtures/reply-references-folded.eml @@ -0,0 +1,26 @@ +Return-Path: +MIME-Version: 1.0 +Date: Thu, 3 Sep 2026 10:15:02 +0530 +Message-ID: +In-Reply-To: +References: + + +Subject: Re: Lease renewal for the studio +From: Arun Kulkarni +To: Priyanshu Jain +Cc: Meridian Leasing +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: quoted-printable + +Hi Priyanshu, + +Attached the revised draft. The only change is clause 7, which now reads +three months either side rather than six. The rent review in clause 12 +stays as it was, tied to the index and capped at four percent. + +If that works, sign when you get a moment and I will countersign the same +day. Happy to walk through anything on a call, Thursday afternoon is open. + +Best, +Arun diff --git a/src-tauri/fixtures/rfc2231-filename.eml b/src-tauri/fixtures/rfc2231-filename.eml new file mode 100644 index 0000000..b895328 --- /dev/null +++ b/src-tauri/fixtures/rfc2231-filename.eml @@ -0,0 +1,33 @@ +Return-Path: +MIME-Version: 1.0 +Date: Wed, 2 Sep 2026 09:12:00 +0200 +Message-ID: <20260902071200.2B7C4@brandt-tischlerei.example> +Subject: Kitchen bench quote +From: Lena Brandt +To: Priyanshu Jain +Content-Type: multipart/mixed; boundary="----=_Part_9021_1471903384.1756800720" + +------=_Part_9021_1471903384.1756800720 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 7bit + +The quote is attached. Prices hold for thirty days. + +Lena + +------=_Part_9021_1471903384.1756800720 +Content-Type: application/pdf; + name*0*=utf-8'de'Angebot%20K%C3%BCchenarbeitsplatte%20; + name*1*=Eiche%20massiv%202026.pdf +Content-Transfer-Encoding: base64 +Content-Disposition: attachment; + filename*0*=utf-8'de'Angebot%20K%C3%BCchenarbeitsplatte%20; + filename*1*=Eiche%20massiv%202026.pdf; + size=248 + +JVBERi0xLjQKMSAwIG9iajw8L1R5cGUvQ2F0YWxvZy9QYWdlcyAyIDAgUj4+ZW5kb2JqCjIgMCBv +Ymo8PC9UeXBlL1BhZ2VzL0tpZHNbMyAwIFJdL0NvdW50IDE+PmVuZG9iagozIDAgb2JqPDwvVHlw +ZS9QYWdlL1BhcmVudCAyIDAgUi9NZWRpYUJveFswIDAgNTk1IDg0Ml0+PmVuZG9iagp0cmFpbGVy +PDwvUm9vdCAxIDAgUj4+CiUlRU9GCg== + +------=_Part_9021_1471903384.1756800720-- diff --git a/src-tauri/fixtures/screener-first-contact.eml b/src-tauri/fixtures/screener-first-contact.eml new file mode 100644 index 0000000..9257279 --- /dev/null +++ b/src-tauri/fixtures/screener-first-contact.eml @@ -0,0 +1,18 @@ +Return-Path: +MIME-Version: 1.0 +Date: Mon, 31 Aug 2026 09:48:12 -0500 +Message-ID: +Subject: Re: Life insurance quote +From: Todd Markham +To: Priyanshu Jain +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 7bit + +Hello Ms. Young, I hope all is well with you. I wanted to touch base on +our conversation from a few weeks ago. + +Following up on the quote I sent through last week for the twenty year +term policy. If the premium looks right, I can start the paperwork +whenever you are ready. + +Todd diff --git a/src-tauri/fixtures/service-on-behalf.eml b/src-tauri/fixtures/service-on-behalf.eml new file mode 100644 index 0000000..0e754a6 --- /dev/null +++ b/src-tauri/fixtures/service-on-behalf.eml @@ -0,0 +1,18 @@ +Return-Path: +MIME-Version: 1.0 +Date: Tue, 1 Sep 2026 19:22:41 -0700 +Message-ID: +Subject: You have an invitation from Robyn Madison +From: Evite on behalf of Robyn Madison +Reply-To: Robyn Madison +Sender: +To: pj@73ai.org +Precedence: bulk +List-Unsubscribe: +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 7bit + +Join us for Jack's 8th birthday! Robyn Madison needs your RSVP. + +Saturday 19 September, from two in the afternoon, at the park by the +lake. Bring nothing but a hat. diff --git a/src-tauri/fixtures/surface-dark-inline-text.eml b/src-tauri/fixtures/surface-dark-inline-text.eml new file mode 100644 index 0000000..af8d398 --- /dev/null +++ b/src-tauri/fixtures/surface-dark-inline-text.eml @@ -0,0 +1,16 @@ +Return-Path: +MIME-Version: 1.0 +Date: Mon, 31 Aug 2026 17:05:33 +0200 +Message-ID: <20260831170533.4C1A@brandt-tischlerei.example> +Subject: The worktop, and one more thing +From: Lena Brandt +To: Priyanshu Jain +Content-Type: text/html; charset=UTF-8 +Content-Transfer-Encoding: 7bit + +
+

The worktop is oiled rather than lacquered, as we agreed.

+

The delivery slot moved to the fourteenth.

+

Everything else is unchanged.

+

Lena Brandt, Brandt Tischlerei

+
diff --git a/src-tauri/fixtures/surface-newsletter-background-image.eml b/src-tauri/fixtures/surface-newsletter-background-image.eml new file mode 100644 index 0000000..36bec95 --- /dev/null +++ b/src-tauri/fixtures/surface-newsletter-background-image.eml @@ -0,0 +1,24 @@ +Return-Path: +MIME-Version: 1.0 +Date: Wed, 2 Sep 2026 06:15:00 +0100 +Message-ID: +Subject: The autumn box is back +From: Harbour Market +To: pj@73ai.org +List-Id: Harbour Market +Precedence: bulk +Content-Type: text/html; charset=UTF-8 +Content-Transfer-Encoding: 7bit + + + + +
+ + +
+

The autumn box is back, and it is heavier than last year.

+

See what is in it

+
+
+ diff --git a/src-tauri/fixtures/surface-newsletter-bgcolor.eml b/src-tauri/fixtures/surface-newsletter-bgcolor.eml new file mode 100644 index 0000000..47014ef --- /dev/null +++ b/src-tauri/fixtures/surface-newsletter-bgcolor.eml @@ -0,0 +1,31 @@ +Return-Path: +MIME-Version: 1.0 +Date: Thu, 3 Sep 2026 07:00:11 +0100 +Message-ID: <01000198a3f1-longread@thelongread.example> +Subject: The weekend edition +From: The Long Read +To: pj@73ai.org +List-Id: The Long Read +List-Unsubscribe: +Precedence: bulk +Content-Type: text/html; charset=UTF-8 +Content-Transfer-Encoding: 7bit + + + + +
+ + + +
+

The weekend edition

+

Three pieces to settle into, and one to argue with.

+

The pencil, at length

+

Why bridges hum

+
+You are getting this because you asked for it. +Unsubscribe +
+
+ diff --git a/src-tauri/fixtures/surface-plain-html.eml b/src-tauri/fixtures/surface-plain-html.eml new file mode 100644 index 0000000..ece1fab --- /dev/null +++ b/src-tauri/fixtures/surface-plain-html.eml @@ -0,0 +1,31 @@ +Return-Path: +MIME-Version: 1.0 +Date: Wed, 2 Sep 2026 11:14:02 +0530 +Message-ID: <2026090211140-arun@meridianproperties.example> +Subject: Three things before Thursday +From: Arun Mehta +To: Priyanshu Jain +X-Mailer: Mailspring +Content-Type: text/html; charset=UTF-8 +Content-Transfer-Encoding: 7bit + +
+

Morning Priyanshu,

+

Three things before Thursday, none of them urgent:

+
    +
  • The revised lease is with the landlord's solicitor.
    • +
  • The service charge reconciliation went out on Monday.
    • +
  • Parking bay 12 is yours from the first.
    • +
+

Links, in case you want them now rather than Thursday: +the draft, +the reconciliation, +the bay plan and the +office number.

+

Best,
Arun

+
+

Arun Mehta
+Meridian Properties
++91 22 6100 4400

+
+
diff --git a/src-tauri/fixtures/surface-sender-dark-design.eml b/src-tauri/fixtures/surface-sender-dark-design.eml new file mode 100644 index 0000000..cf68f4b --- /dev/null +++ b/src-tauri/fixtures/surface-sender-dark-design.eml @@ -0,0 +1,24 @@ +Return-Path: +MIME-Version: 1.0 +Date: Fri, 4 Sep 2026 09:30:00 +0100 +Message-ID: +Subject: Tonight at the Station House +From: Station House +To: pj@73ai.org +List-Id: Station House +Content-Type: text/html; charset=UTF-8 +Content-Transfer-Encoding: 7bit + + + +
+

Doors at seven, the band at eight.

+

Tickets on the door only, cash or card.

+

Sold out on Saturday.

+
+ diff --git a/src-tauri/fixtures/surface-wrapper-background.eml b/src-tauri/fixtures/surface-wrapper-background.eml new file mode 100644 index 0000000..37e4126 --- /dev/null +++ b/src-tauri/fixtures/surface-wrapper-background.eml @@ -0,0 +1,19 @@ +Return-Path: +MIME-Version: 1.0 +Date: Tue, 1 Sep 2026 08:22:47 +0100 +Message-ID: +Subject: Your receipt from Bramble Coffee +From: Bramble Coffee +To: pj@73ai.org +Auto-Submitted: auto-generated +Content-Type: text/html; charset=UTF-8 +Content-Transfer-Encoding: 7bit + +
+
+

Thanks, that is paid.

+

Flat white and a cardamom bun

+

£6.40 on the card ending 4417

+

Bramble Coffee, 14 Rivington Street

+
+
diff --git a/src-tauri/fixtures/tracking-pixel.eml b/src-tauri/fixtures/tracking-pixel.eml new file mode 100644 index 0000000..54cce0a --- /dev/null +++ b/src-tauri/fixtures/tracking-pixel.eml @@ -0,0 +1,22 @@ +Return-Path: +MIME-Version: 1.0 +Date: Mon, 31 Aug 2026 12:02:44 +0530 +Message-ID: +Subject: The studio, one more thing +From: Arun Kulkarni +To: Priyanshu Jain +X-HS-Marketing-Email: true +Content-Type: text/html; charset=UTF-8 +Content-Transfer-Encoding: 7bit + + +

Priyanshu, the floor plan you asked for is below. The measurements are +from the survey in June, not the original drawing.

+

Studio floor plan

+

Arun

+ + + diff --git a/src-tauri/fixtures/utf8-raw-headers.eml b/src-tauri/fixtures/utf8-raw-headers.eml new file mode 100644 index 0000000..d0c7ac3 --- /dev/null +++ b/src-tauri/fixtures/utf8-raw-headers.eml @@ -0,0 +1,19 @@ +Return-Path: +MIME-Version: 1.0 +Date: Fri, 4 Sep 2026 08:12:00 +0100 +Message-ID: <20260904071200.aa91@lisbonstays.example> +Subject: Instruções para a chegada, com um café à espera +From: Inês Carvalho +To: Priyanshu Jain +X-Original-Subject-Encoding: 8bit +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Olá Priyanshu, + +O código da porta é 4417 e o elevador está do lado direito. Deixo café e +pão na cozinha para a manhã seguinte. + +Se o comboio atrasar, telefone. Não há problema nenhum. + +Inês diff --git a/src-tauri/src/accounts.rs b/src-tauri/src/accounts.rs new file mode 100644 index 0000000..f98e45b --- /dev/null +++ b/src-tauri/src/accounts.rs @@ -0,0 +1,365 @@ +// Which Google accounts this install has a token for, and the handful of facts about each one +// that are not mail. +// +// A file, `accounts.json` in the app data directory, rather than a table. Every account owns a +// database of its own under `accounts//`, so a table of accounts would have to live in one of +// them and be read before the others could be opened, and the first thing the app does on launch +// is ask which accounts there are. A file answers that before any database is open. +// +// What is not here matters as much as what is. The refresh token is sealed in `google::secrets` +// and never written to this file. The storage window and the signature belong to settings and to +// the state database, which are the places that already own per account decisions. +// +// This is the list of accounts that have a token. `Db::on_disk` is the list that have a database, +// and a removal takes the entry here before it touches anything else, so the two lists differ for +// as long as a removal takes and neither is authoritative on its own. + +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use tauri::Manager; + +use crate::dto::{Account, AccountKind, MailConfig}; +use crate::library::{app_data_dir, atomic_write}; + +const FILE: &str = "accounts.json"; + +/// The stylesheet defines `--hue-1` to `--hue-8`, so those are the only values worth storing. +const HUES: usize = 8; + +/// One row of the registry. `granted_scopes` is what Google said it granted, not what was asked +/// for, because granular consent means those are different lists. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Entry { + pub id: String, + pub email: String, + /// Absent in a registry written before there was a second kind, which is what the default is + /// for: every account that predates IMAP is a Google one, and the file is upgraded in place + /// the next time anything writes it. + #[serde(default)] + pub kind: AccountKind, + pub name: String, + pub color: String, + #[serde(default)] + pub granted_scopes: Vec, + /// An IMAP account's servers. `None` on a Google account, where the mailbox is reached over + /// the API and there is nothing to configure. The passwords are not here: they are sealed + /// alongside the refresh tokens in `google::secrets`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub servers: Option, + pub added_ms: i64, +} + +fn path(app: &tauri::AppHandle) -> Result { + Ok(app_data_dir(app)?.join(FILE)) +} + +pub fn list(app: &tauri::AppHandle) -> Result, String> { + Ok(read(&path(app)?)? + .into_iter() + .map(|entry| account(app, entry)) + .collect()) +} + +pub fn find(app: &tauri::AppHandle, account_id: &str) -> Result, String> { + Ok(read(&path(app)?)? + .into_iter() + .find(|entry| entry.id == account_id)) +} + +/// What Google granted this account, for the features that have to check before they act rather +/// than discover it as a 403 halfway through. +pub fn granted_scopes(app: &tauri::AppHandle, account_id: &str) -> Result, String> { + find(app, account_id)? + .map(|entry| entry.granted_scopes) + .ok_or_else(|| format!("Account {account_id} is not connected.")) +} + +/// A completed consent, arriving from `google::auth::finish`. +/// +/// An account that is already here keeps its name and its colour: consent is re-run to pick up a +/// scope, and having the avatar change colour because somebody agreed to a calendar permission +/// would be a strange thing to watch. The scopes are replaced wholesale, since the new grant is +/// the only one that exists now. +pub fn upsert( + app: &tauri::AppHandle, + id: &str, + email: &str, + name: &str, + granted_scopes: Vec, +) -> Result<(), String> { + let path = path(app)?; + let mut entries = read(&path)?; + match entries.iter_mut().find(|entry| entry.id == id) { + Some(entry) => { + entry.email = email.to_string(); + entry.granted_scopes = granted_scopes; + } + None => { + restore_kept(app, id)?; + let color = next_hue(&entries); + entries.push(Entry { + id: id.to_string(), + email: email.to_string(), + kind: AccountKind::Google, + name: name.to_string(), + color, + granted_scopes, + servers: None, + added_ms: chrono::Utc::now().timestamp_millis(), + }); + } + } + write(&path, &entries) +} + +/// An IMAP account arriving from `imap::connect`, which is the other way into this registry. +/// +/// Deliberately a second function rather than a wider `upsert`: a Google account has scopes and no +/// servers, an IMAP account has servers and no scopes, and one function taking both would be four +/// arguments that are each meaningless in half the calls. +pub fn upsert_imap( + app: &tauri::AppHandle, + id: &str, + email: &str, + name: &str, + servers: MailConfig, +) -> Result<(), String> { + let path = path(app)?; + let mut entries = read(&path)?; + match entries.iter_mut().find(|entry| entry.id == id) { + Some(entry) => { + entry.email = email.to_string(); + entry.servers = Some(servers); + } + None => { + restore_kept(app, id)?; + let color = next_hue(&entries); + entries.push(Entry { + id: id.to_string(), + email: email.to_string(), + kind: AccountKind::Imap, + name: name.to_string(), + color, + granted_scopes: Vec::new(), + servers: Some(servers), + added_ms: chrono::Utc::now().timestamp_millis(), + }); + } + } + write(&path, &entries) +} + +/// An account that was removed with its data kept gets that data back on its way in, before the +/// first sync pass can write a fresh, empty pair over the place it belongs. `Db` is absent in a +/// test and in any build that has not opened one yet, when there is nothing set aside either. +fn restore_kept(app: &tauri::AppHandle, id: &str) -> Result<(), String> { + match app.try_state::() { + Some(db) => db.restore(id), + None => Ok(()), + } +} + +pub fn remove(app: &tauri::AppHandle, account_id: &str) -> Result<(), String> { + let path = path(app)?; + let mut entries = read(&path)?; + let before = entries.len(); + entries.retain(|entry| entry.id != account_id); + if entries.len() == before { + return Ok(()); + } + write(&path, &entries) +} + +#[tauri::command(async)] +pub fn accounts_list(app: tauri::AppHandle) -> Result, String> { + list(&app) +} + +#[tauri::command(async)] +pub fn account_set_color( + app: tauri::AppHandle, + account_id: String, + color: String, +) -> Result<(), String> { + let path = path(&app)?; + let mut entries = read(&path)?; + let entry = entries + .iter_mut() + .find(|entry| entry.id == account_id) + .ok_or_else(|| format!("Account {account_id} is not connected."))?; + entry.color = hue(&color)?; + write(&path, &entries)?; + crate::emit_store_changed(&app, "accounts"); + Ok(()) +} + +#[tauri::command(async)] +pub fn account_set_name( + app: tauri::AppHandle, + account_id: String, + name: String, +) -> Result<(), String> { + let path = path(&app)?; + let mut entries = read(&path)?; + let entry = entries + .iter_mut() + .find(|entry| entry.id == account_id) + .ok_or_else(|| format!("Account {account_id} is not connected."))?; + let name = name.trim(); + if name.is_empty() { + return Err("An account needs a name.".to_string()); + } + entry.name = name.to_string(); + write(&path, &entries)?; + crate::emit_store_changed(&app, "accounts"); + Ok(()) +} + +/// Every account in this file has a token, so `connected` is true without going and looking. A +/// probe would be a file read and a key derivation on every `store-changed`, which sync emits on +/// every pass, for an answer nothing acts on: a secret that has really gone surfaces at the next +/// `valid_access_token` as a sync error, which is the honest place to learn it. +/// +/// `window_days` is not this file's to know. The window is a device setting and `settings.json` +/// owns it; this registry owns the id, the address, the name, the colour and the grant. The DTO +/// carries both, so the value is fetched from the one that owns it on the way past. +fn account(app: &tauri::AppHandle, entry: Entry) -> Account { + Account { + window_days: crate::settings::window_days(app, &entry.id), + id: entry.id, + email: entry.email, + kind: entry.kind, + name: entry.name, + color: entry.color, + connected: true, + granted_scopes: entry.granted_scopes, + } +} + +fn hue(color: &str) -> Result { + let n = color.strip_prefix("hue-").and_then(|n| n.parse::().ok()); + match n { + Some(n) if (1..=HUES).contains(&n) => Ok(color.to_string()), + // The stylesheet owns the value: a hex stored here would reach the avatar as an unknown + // token name and render as nothing at all. + _ => Err(format!("{color} is not one of hue-1 to hue-{HUES}.")), + } +} + +/// The lowest hue nobody is using, rather than the next one along, so removing an account and +/// adding another reuses the colour that went spare instead of leaving two accounts to collide +/// eight connections later. The avatar hue is how the unified inbox says whose mail a row is, so +/// two accounts sharing one while a colour is going spare is the thing to avoid. +fn next_hue(entries: &[Entry]) -> String { + for n in 1..=HUES { + let hue = format!("hue-{n}"); + if !entries.iter().any(|entry| entry.color == hue) { + return hue; + } + } + format!("hue-{}", entries.len() % HUES + 1) +} + +/// A missing file is an install with no accounts, which is every install once. A malformed one is +/// not: it is the only record of which accounts exist, and overwriting it would silently orphan +/// every sealed token and every database on disk. +fn read(path: &Path) -> Result, String> { + let text = match std::fs::read_to_string(path) { + Ok(text) => text, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(e) => return Err(e.to_string()), + }; + serde_json::from_str(&text).map_err(|e| format!("could not read {}: {e}", path.display())) +} + +fn write(path: &Path, entries: &[Entry]) -> Result<(), String> { + let text = serde_json::to_string_pretty(entries).map_err(|e| e.to_string())?; + atomic_write(path, text.as_bytes()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn entry(id: &str, color: &str) -> Entry { + Entry { + id: id.to_string(), + email: format!("{id}@example.test"), + kind: AccountKind::Google, + name: id.to_string(), + color: color.to_string(), + granted_scopes: vec!["https://www.googleapis.com/auth/gmail.modify".to_string()], + servers: None, + added_ms: 1_700_000_000_000, + } + } + + #[test] + fn a_registry_survives_a_write_and_a_read() { + let dir = tempfile::tempdir().expect("a temp dir"); + let path = dir.path().join(FILE); + + let written = vec![entry("11829", "hue-1"), entry("44021", "hue-2")]; + write(&path, &written).expect("write"); + + let read_back = read(&path).expect("read"); + assert_eq!(read_back.len(), 2); + assert_eq!(read_back[1].id, "44021"); + assert_eq!(read_back[1].email, "44021@example.test"); + assert_eq!(read_back[1].color, "hue-2"); + assert_eq!( + read_back[0].granted_scopes, + vec!["https://www.googleapis.com/auth/gmail.modify".to_string()] + ); + assert_eq!(read_back[0].added_ms, 1_700_000_000_000); + } + + #[test] + fn no_file_yet_is_an_install_with_no_accounts() { + let dir = tempfile::tempdir().expect("a temp dir"); + assert!(read(&dir.path().join(FILE)).expect("read").is_empty()); + } + + #[test] + fn a_malformed_registry_is_an_error_rather_than_an_empty_list() { + let dir = tempfile::tempdir().expect("a temp dir"); + let path = dir.path().join(FILE); + std::fs::write(&path, "{ not json").expect("write"); + assert!(read(&path).is_err()); + } + + #[test] + fn hues_are_handed_out_in_order() { + let mut entries: Vec = Vec::new(); + for n in 1..=HUES { + let hue = next_hue(&entries); + assert_eq!(hue, format!("hue-{n}")); + entries.push(entry(&format!("account-{n}"), &hue)); + } + // A ninth account has to share, and starts the cycle again rather than inventing hue-9. + assert_eq!(next_hue(&entries), "hue-1"); + } + + #[test] + fn a_removed_accounts_hue_is_reused_before_a_new_one() { + let mut entries = vec![ + entry("a", "hue-1"), + entry("b", "hue-2"), + entry("c", "hue-3"), + ]; + entries.retain(|entry| entry.id != "b"); + assert_eq!(next_hue(&entries), "hue-2"); + } + + #[test] + fn only_the_eight_token_names_are_a_colour() { + assert_eq!(hue("hue-1").expect("hue-1"), "hue-1"); + assert_eq!(hue("hue-8").expect("hue-8"), "hue-8"); + assert!(hue("hue-9").is_err()); + assert!(hue("hue-0").is_err()); + assert!(hue("#8c6f4a").is_err()); + assert!(hue("").is_err()); + } +} diff --git a/src-tauri/src/attachments.rs b/src-tauri/src/attachments.rs new file mode 100644 index 0000000..c08ca57 --- /dev/null +++ b/src-tauri/src/attachments.rs @@ -0,0 +1,895 @@ +// Bytes fetched because somebody asked for them: the pictures in a message, and the files hanging +// off it. +// +// Nothing here runs during a sync. A message arrives as headers, its body arrives when the thread +// is opened, and everything in this file happens later still, when a reader presses something. The +// two halves sit together because they are the same job under two names: fetch on demand, cap what +// is fetched, and never let the webview make a request of its own. +// +// Remote images are the reason that last clause is written down. The sanitiser has no network by +// design, so the only way to show a picture is for Rust to fetch it, without cookies and without a +// referrer, and hand the bytes back through `RenderOptions::remote_images`. The sender learns that +// somebody asked, once, from an address, and nothing else: no repeat opens, no forwarded reads, +// and no correlation with anything else the app does. + +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::sync::OnceLock; +use std::time::Duration; + +use base64::Engine; +use futures::stream::{self, StreamExt}; +use mail_parser::{MessageParser, MimeHeaders, PartType}; +use rusqlite::Connection; +use tauri::Manager; +use tauri_plugin_opener::OpenerExt; + +use crate::db::Db; +use crate::dto::MessageView; +use crate::mime::{self, RenderOptions, Rendered}; +use crate::mirror::read::{self, AttachmentRow}; +use crate::mirror::write; +use crate::sync::{self, Remote}; + +/// How many remote images one message may have before the answer is a sentence rather than a +/// download. A newsletter has a dozen; a body with hundreds is either broken or a probe, and +/// either way nobody wants to wait for it. +const MAX_IMAGES: usize = 60; +/// Per image, and in total. A picture in a message is a picture, not a payload. +const MAX_IMAGE_BYTES: usize = 4 * 1024 * 1024; +const MAX_IMAGES_TOTAL_BYTES: usize = 16 * 1024 * 1024; +/// How many of one message's pictures are in flight together. A newsletter spreads them over two +/// or three hosts, and one host that will not answer must not hold the others behind it, which is +/// what fetching them one after another did: a dead CDN was a full timeout per picture, in a row, +/// while the reader watched a button that had not changed. +const IMAGES_AT_ONCE: usize = 8; + +/// The ceiling on a `data:` URI. Base64 costs a third on top, the whole string crosses the IPC +/// boundary as JSON and is then held in the webview, so a preview of something enormous is a hang +/// with a progress bar in front of it. Above this the answer is Save. +const MAX_DATA_URL_BYTES: u64 = 8 * 1024 * 1024; + +/// What the cache is allowed to hold when the settings file cannot be read. It matches +/// `settings::defaults`, so an unreadable file behaves like a fresh install rather than like no +/// cache at all. +const DEFAULT_CACHE_MB: u64 = 512; + +const CACHE_DIR: &str = "attachments"; + +// --------------------------------------------------------------------------------------------- +// Remote images +// --------------------------------------------------------------------------------------------- + +/// No cookie store is compiled into this build at all, so there is nothing to send even by +/// accident, and `referer(false)` keeps a redirect from naming the URL it came from. The timeouts +/// are short: this runs while somebody is looking at the message, and a picture that has not +/// arrived in ten seconds is a picture that is not coming. +fn image_client() -> &'static reqwest::Client { + static CLIENT: OnceLock = OnceLock::new(); + CLIENT.get_or_init(|| { + reqwest::Client::builder() + .connect_timeout(Duration::from_secs(5)) + .timeout(Duration::from_secs(10)) + .referer(false) + .redirect(reqwest::redirect::Policy::limited(3)) + .build() + .expect("could not build the image client") + }) +} + +/// The remote images this body would like, or a refusal. Rendering with images off is what names +/// them, so the list is the sanitiser's own and not a second scan of the markup. +fn wanted(raw: &[u8], options: &RenderOptions) -> Result, String> { + let mut urls: Vec = Vec::new(); + for url in mime::render(raw, options)?.blocked_urls { + if !urls.contains(&url) { + urls.push(url); + } + } + if urls.len() > MAX_IMAGES { + return Err(format!( + "This message asks for {} remote images, which is more than Margin Mail will fetch at \ + once ({MAX_IMAGES}). None were fetched.", + urls.len() + )); + } + Ok(urls) +} + +/// The same body again with the fetched bytes in hand. The trackers stay removed: that rule lives +/// in the sanitiser and this passes through it rather than around it, because "show me the +/// pictures" is not "file an open report". +fn shown( + raw: &[u8], + options: &RenderOptions, + fetched: HashMap>, +) -> Result { + let options = RenderOptions { + allow_remote_images: true, + remote_images: fetched, + ..options.clone() + }; + mime::render(raw, &options) +} + +/// One image, capped. Anything that fails, redirects too far or runs over the cap is left blocked +/// rather than reported: a picture that would not come is a picture that is not there. +async fn fetch_image(url: &str) -> Option> { + let mut response = image_client().get(url).send().await.ok()?; + if !response.status().is_success() { + return None; + } + let mut bytes: Vec = Vec::new(); + loop { + match response.chunk().await { + Ok(Some(chunk)) => { + if bytes.len() + chunk.len() > MAX_IMAGE_BYTES { + return None; + } + bytes.extend_from_slice(&chunk); + } + Ok(None) => break, + Err(_) => return None, + } + } + (!bytes.is_empty()).then_some(bytes) +} + +/// All of them, `IMAGES_AT_ONCE` at a time, in whatever order they arrive. Dropping the stream +/// once the total cap is reached is what cancels the fetches still in flight. +async fn fetch_images(urls: &[String]) -> HashMap> { + let mut fetched = HashMap::new(); + let mut total = 0usize; + let mut results = stream::iter(urls.iter().cloned()) + .map(|url| async move { + let bytes = fetch_image(&url).await; + (url, bytes) + }) + .buffer_unordered(IMAGES_AT_ONCE); + while let Some((url, bytes)) = results.next().await { + if total >= MAX_IMAGES_TOTAL_BYTES { + break; + } + let Some(bytes) = bytes else { + continue; + }; + total += bytes.len(); + fetched.insert(url, bytes); + } + fetched +} + +// --------------------------------------------------------------------------------------------- +// The attachment cache +// --------------------------------------------------------------------------------------------- + +fn cache_dir(db: &Db, account_id: &str) -> PathBuf { + db.account_dir(account_id).join(CACHE_DIR) +} + +/// A file name derived from the attachment id, which is already unique within the account. The +/// extension is kept so the OS opens the file with the right thing. +fn cache_name(row: &AttachmentRow) -> String { + let stem: String = row + .id + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '-' }) + .collect(); + match row.filename.rsplit_once('.') { + Some((_, extension)) if !extension.is_empty() && extension.len() <= 8 => { + let extension: String = extension + .chars() + .filter(|c| c.is_ascii_alphanumeric()) + .collect(); + if extension.is_empty() { + stem + } else { + format!("{stem}.{}", extension.to_lowercase()) + } + } + _ => stem, + } +} + +/// The bytes of one part of a message that is already on the device. +/// +/// `mime::render` only carries the bytes of a part small enough to keep beside the row, and the +/// mirror keys an attachment by its MIME part path, so this matches on what the row does hold: the +/// decoded length and the file name. Body parts are skipped the same way the renderer skips them, +/// so a text body the same length as a file cannot be mistaken for it. +fn part_bytes(raw: &[u8], row: &AttachmentRow) -> Option> { + let message = MessageParser::default().parse(raw)?; + let bodies: HashSet = message + .html_body + .iter() + .chain(message.text_body.iter()) + .map(|index| *index as usize) + .collect(); + + let mut fallback = None; + for (index, part) in message.parts.iter().enumerate() { + if bodies.contains(&index) { + continue; + } + let bytes = match &part.body { + PartType::Binary(bytes) | PartType::InlineBinary(bytes) => bytes.to_vec(), + PartType::Text(text) | PartType::Html(text) => text.as_bytes().to_vec(), + PartType::Multipart(_) | PartType::Message(_) => continue, + }; + if bytes.len() as u64 != row.size { + continue; + } + if part.attachment_name() == Some(row.filename.as_str()) { + return Some(bytes); + } + fallback.get_or_insert(bytes); + } + fallback +} + +/// The bytes of one attachment: from the cache, else out of the message already on the device, +/// else from the provider. +/// +/// The provider is last because it is rarely needed and, for Gmail, rarely possible: the mirror +/// keys an attachment by its MIME part path and `attachments.get` wants Gmail's own attachment id, +/// which nothing stores. A message with an attachment row has a body, and a body is the whole +/// message, so the bytes are almost always here already. +async fn bytes_of( + db: &Db, + account_id: &str, + remote: Option<&dyn Remote>, + row: &AttachmentRow, + cap_bytes: u64, +) -> Result, String> { + let dir = cache_dir(db, account_id); + if let Some(path) = &row.cached_path { + if let Ok(bytes) = std::fs::read(path) { + return Ok(bytes); + } + db.with(account_id, |conn| write::attachment_uncached(conn, &row.id))?; + } + + let raw = db.with(account_id, |conn| read::raw_body(conn, &row.message_id))?; + let bytes = match raw.as_deref().and_then(|raw| part_bytes(raw, row)) { + Some(bytes) => bytes, + None => { + let remote = remote.ok_or("that file is not on this device")?; + remote + .fetch_attachment(&row.message_id, &row.part_id) + .await + .map_err(|e| e.to_string())? + } + }; + + put(db, account_id, &dir, row, &bytes, cap_bytes)?; + Ok(bytes) +} + +/// Writes the bytes into the cache and keeps the cache inside its cap. +fn put( + db: &Db, + account_id: &str, + dir: &Path, + row: &AttachmentRow, + bytes: &[u8], + cap_bytes: u64, +) -> Result<(), String> { + std::fs::create_dir_all(dir).map_err(|e| e.to_string())?; + let path = dir.join(cache_name(row)); + std::fs::write(&path, bytes).map_err(|e| e.to_string())?; + db.with(account_id, |conn| { + write::attachment_cached(conn, &row.id, &path.to_string_lossy(), write::now_ms())?; + trim(conn, dir, cap_bytes) + }) +} + +/// Keeps the cache honest in both directions. +/// +/// `mirror::evict` deletes an attachment row without touching the file it points at, because +/// eviction runs inside one SQL transaction and has no business on the filesystem. A file with no +/// row is therefore the expected shape rather than a bug, and this is where it goes. After that, +/// least recently fetched first, until what is left fits the cap. +/// +/// The newest is never given up, whatever the cap says. It is the file somebody is waiting on, and +/// a cache too small for one file should still hand that file over. +fn trim(conn: &Connection, dir: &Path, cap_bytes: u64) -> Result<(), String> { + let held = read::cached_attachments(conn)?; + let known: HashSet = held.iter().map(|(_, path, _)| PathBuf::from(path)).collect(); + + if let Ok(entries) = std::fs::read_dir(dir) { + for entry in entries.flatten() { + let path = entry.path(); + if path.is_file() && !known.contains(&path) { + let _ = std::fs::remove_file(&path); + } + } + } + + let mut total: u64 = held.iter().map(|(_, _, size)| *size).sum(); + let mut index = 0; + while total > cap_bytes && index + 1 < held.len() { + let (id, path, size) = &held[index]; + let _ = std::fs::remove_file(path); + write::attachment_uncached(conn, id)?; + total = total.saturating_sub(*size); + index += 1; + } + Ok(()) +} + +/// The cap the settings screen set, in bytes. A settings file that will not load is not a reason +/// to refuse somebody their attachment, so it falls back rather than failing. +fn cap_bytes(app: &tauri::AppHandle) -> u64 { + crate::settings::load(app) + .map(|settings| settings.attachment_cache_mb as u64) + .ok() + .filter(|megabytes| *megabytes > 0) + .unwrap_or(DEFAULT_CACHE_MB) + * 1024 + * 1024 +} + +// --------------------------------------------------------------------------------------------- +// Finding things +// --------------------------------------------------------------------------------------------- + +fn db_of(app: &tauri::AppHandle) -> Result, String> { + app.try_state::() + .ok_or_else(|| "the mirror is not open yet".to_string()) +} + +fn account_holding(db: &Db, sql: &str, id: &str) -> Result { + for (account_id, _) in sync::accounts(db, None) { + let held = db.with(&account_id, |conn| { + conn.query_row(sql, [id], |row| row.get::<_, i64>(0)) + .map_err(|e| e.to_string()) + })?; + if held > 0 { + return Ok(account_id); + } + } + Err("that is not on this device".to_string()) +} + +fn attachment_of(db: &Db, id: &str) -> Result<(String, AttachmentRow), String> { + let account_id = account_holding(db, "SELECT COUNT(*) FROM attachments WHERE id = ?1", id)?; + let row = db + .with(&account_id, |conn| read::attachment_row(conn, id))? + .ok_or("that file is not on this device")?; + Ok((account_id, row)) +} + +/// Somewhere in the downloads directory that is not already taken. +fn free_path(dir: &Path, filename: &str) -> PathBuf { + let safe: String = filename + .chars() + .map(|c| if std::path::is_separator(c) { '-' } else { c }) + .collect(); + let safe = safe.trim().trim_matches('.').to_string(); + let safe = if safe.is_empty() { + "attachment".to_string() + } else { + safe + }; + let path = dir.join(&safe); + if !path.exists() { + return path; + } + let (stem, extension) = match safe.rsplit_once('.') { + Some((stem, extension)) => (stem.to_string(), format!(".{extension}")), + None => (safe.clone(), String::new()), + }; + for nth in 2..1000 { + let candidate = dir.join(format!("{stem} ({nth}){extension}")); + if !candidate.exists() { + return candidate; + } + } + dir.join(format!("{stem} ({}){extension}", write::now_ms())) +} + +// --------------------------------------------------------------------------------------------- +// Commands +// --------------------------------------------------------------------------------------------- + +/// The reader has asked to see the pictures. +/// +/// The answer is a view and not a stored row on purpose: the `bodies` table has no column saying +/// whether images were loaded, so this is per view rather than remembered, and closing the thread +/// puts the block back. Inventing a column for it would also be inventing a policy, and the policy +/// that roams with a person is the per sender allowance on the contact card, not a flag on a body. +#[tauri::command] +pub async fn message_show_images( + app: tauri::AppHandle, + message_id: String, +) -> Result { + let db = db_of(&app)?; + let account_id = account_holding( + db.inner(), + "SELECT COUNT(*) FROM messages WHERE id = ?1", + &message_id, + )?; + + let (raw, options) = db.with(&account_id, |conn| { + let raw = read::raw_body(conn, &message_id)? + .ok_or("that message has not been fetched yet")?; + Ok((raw, sync::hydrate::render_options(conn)?)) + })?; + + let urls = wanted(&raw, &options)?; + let rendered = shown(&raw, &options, fetch_images(&urls).await)?; + + let mut view = db.with(&account_id, |conn| { + read::message_view(conn, &account_id, &message_id, &options.own_addresses) + })?; + view.html = rendered.html; + view.quoted_html = rendered.quoted_html; + view.trackers = rendered.trackers; + view.blocked_images = rendered.blocked_images; + view.images_loaded = true; + Ok(view) +} + +/// The inline preview. Refuses anything too big to be a data URI before it fetches a byte, so the +/// answer to a hundred megabyte video is a sentence rather than a spinner. +#[tauri::command] +pub async fn attachment_data_url( + app: tauri::AppHandle, + attachment_id: String, +) -> Result { + let db = db_of(&app)?; + let (account_id, row) = attachment_of(db.inner(), &attachment_id)?; + if let Some(refusal) = too_big_to_preview(&row) { + return Err(refusal); + } + let remote = sync::remote_for(&account_id); + let bytes = bytes_of( + db.inner(), + &account_id, + remote.as_deref(), + &row, + cap_bytes(&app), + ) + .await?; + Ok(format!( + "data:{};base64,{}", + row.mime_type, + base64::engine::general_purpose::STANDARD.encode(&bytes) + )) +} + +/// Writes it to the downloads directory and hands back the path, which is what the toast names. +#[tauri::command] +pub async fn attachment_save( + app: tauri::AppHandle, + attachment_id: String, +) -> Result { + let db = db_of(&app)?; + let (account_id, row) = attachment_of(db.inner(), &attachment_id)?; + let remote = sync::remote_for(&account_id); + let bytes = bytes_of( + db.inner(), + &account_id, + remote.as_deref(), + &row, + cap_bytes(&app), + ) + .await?; + + let downloads = app.path().download_dir().map_err(|e| e.to_string())?; + std::fs::create_dir_all(&downloads).map_err(|e| e.to_string())?; + let path = free_path(&downloads, &row.filename); + std::fs::write(&path, &bytes).map_err(|e| e.to_string())?; + Ok(path.to_string_lossy().to_string()) +} + +/// Hands the cached file to the OS. The cache is where it opens from, so opening the same file +/// twice costs nothing the second time. +#[tauri::command] +pub async fn attachment_open(app: tauri::AppHandle, attachment_id: String) -> Result<(), String> { + let db = db_of(&app)?; + let (account_id, row) = attachment_of(db.inner(), &attachment_id)?; + let remote = sync::remote_for(&account_id); + bytes_of( + db.inner(), + &account_id, + remote.as_deref(), + &row, + cap_bytes(&app), + ) + .await?; + + let path = db + .with(&account_id, |conn| read::attachment_row(conn, &row.id))? + .and_then(|row| row.cached_path) + .ok_or("that file could not be put on the disk")?; + app.opener() + .open_path(path, None::<&str>) + .map_err(|e| e.to_string()) +} + +/// Decided from the row rather than from the bytes, so an enormous file is refused before anything +/// is fetched and the answer arrives at once. +fn too_big_to_preview(row: &AttachmentRow) -> Option { + (row.size > MAX_DATA_URL_BYTES).then(|| { + format!( + "{} is {}, which is too big to preview here. Save it instead.", + row.filename, + megabytes(row.size) + ) + }) +} + +fn megabytes(bytes: u64) -> String { + format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)) +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use rusqlite::Connection; + use tauri::async_runtime::block_on; + + use crate::db; + use crate::provider::fake::FakeProvider; + use crate::sync::{hydrate, Store}; + + use super::*; + + struct Memory(Mutex); + + impl Store for Memory { + fn with Result>(&self, f: F) -> Result { + let conn = self.0.lock().map_err(|e| e.to_string())?; + f(&conn) + } + } + + impl Memory { + fn read(&self, f: impl FnOnce(&Connection) -> Result) -> T { + self.with(f).expect("the mirror") + } + } + + fn store() -> Memory { + Memory(Mutex::new(db::memory().expect("a pair of in-memory databases"))) + } + + fn eml(id: &str, headers: &str, body: &str) -> Vec { + format!( + "Message-ID: <{id}@example.test>\r\nFrom: Ana \r\n\ + To: You \r\nSubject: {id}\r\n\ + Date: Wed, 2 Sep 2026 10:00:00 +0000\r\n{headers}\r\n{body}" + ) + .into_bytes() + } + + /// A body with one ordinary remote image and one tracking pixel from a named vendor. + fn with_images() -> Vec { + eml( + "images", + "Content-Type: text/html; charset=utf-8\r\n", + "

Hello

\ + \ + \ + \r\n", + ) + } + + fn options() -> RenderOptions { + RenderOptions { + allow_remote_images: false, + link_cleaning: true, + remote_images: HashMap::new(), + own_addresses: vec!["you@example.test".to_string()], + } + } + + /// Eight bytes of PNG magic, which is all the sanitiser sniffs before it will inline anything. + fn png() -> Vec { + b"\x89PNG\r\n\x1a\nrest".to_vec() + } + + #[test] + fn showing_images_inlines_what_was_fetched_and_leaves_the_trackers_removed() { + let raw = with_images(); + let urls = wanted(&raw, &options()).expect("the blocked images"); + assert_eq!(urls, vec!["https://shop.example/photo.jpg".to_string()]); + + let blocked = mime::render(&raw, &options()).expect("a render"); + assert_eq!(blocked.blocked_images, 1); + assert_eq!(blocked.trackers.len(), 1); + + let mut fetched = HashMap::new(); + fetched.insert(urls[0].clone(), png()); + let rendered = shown(&raw, &options(), fetched).expect("a second render"); + + assert!( + rendered.html.contains("data:image/png;base64,"), + "the fetched bytes are inlined: {}", + rendered.html + ); + assert_eq!(rendered.blocked_images, 0); + assert_eq!( + rendered.trackers.len(), + 1, + "the pixel is still named and still gone" + ); + assert_eq!(rendered.trackers[0].vendor, "HubSpot"); + assert!( + !rendered.html.contains("track.hubspot.com"), + "showing the pictures is not filing an open report: {}", + rendered.html + ); + } + + #[test] + fn a_body_with_more_images_than_the_cap_is_refused_before_anything_is_fetched() { + let mut body = String::from(""); + for index in 0..(MAX_IMAGES + 1) { + body.push_str(&format!( + "" + )); + } + body.push_str("\r\n"); + let raw = eml("many", "Content-Type: text/html; charset=utf-8\r\n", &body); + + let refused = wanted(&raw, &options()).expect_err("a refusal"); + assert!(refused.contains("more than Margin Mail will fetch"), "{refused}"); + assert!(refused.contains("None were fetched."), "{refused}"); + } + + #[test] + fn showing_images_changes_nothing_that_is_stored() { + let store = store(); + let fake = FakeProvider::new(); + let raw = with_images(); + fake.add_eml("m1", "t1", &["INBOX"], &raw); + block_on(hydrate::headers(&store, &fake, &["m1".to_string()], false)).expect("headers"); + block_on(hydrate::body(&store, &fake, "m1")).expect("a body"); + + let before = store.read(|conn| { + conn.query_row( + "SELECT html, blocked_images FROM bodies WHERE message_id = 'm1'", + [], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)), + ) + .map_err(|e| e.to_string()) + }); + assert_eq!(before.1, 1, "the stored render has the image blocked"); + + let held = store.read(|conn| read::raw_body(conn, "m1")).expect("the bytes"); + let urls = wanted(&held, &options()).expect("the blocked images"); + let mut fetched = HashMap::new(); + fetched.insert(urls[0].clone(), png()); + let rendered = shown(&held, &options(), fetched).expect("a render with images"); + assert!(rendered.html.contains("data:image/png;base64,")); + + let after = store.read(|conn| { + conn.query_row( + "SELECT html, blocked_images FROM bodies WHERE message_id = 'm1'", + [], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)), + ) + .map_err(|e| e.to_string()) + }); + assert_eq!(after, before, "the row is untouched, so the next open blocks again"); + } + + // -- the attachment cache ------------------------------------------------------------------- + + /// A message with one attached file, built as multipart so the parts are real parts. + fn with_attachment(id: &str, filename: &str, bytes: &[u8]) -> Vec { + let encoded = base64::engine::general_purpose::STANDARD.encode(bytes); + let mut body = String::from("--sep\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n"); + body.push_str("Here it is.\r\n"); + body.push_str(&format!( + "--sep\r\nContent-Type: application/octet-stream\r\n\ + Content-Disposition: attachment; filename=\"{filename}\"\r\n\ + Content-Transfer-Encoding: base64\r\n\r\n" + )); + for chunk in encoded.as_bytes().chunks(76) { + body.push_str(&String::from_utf8_lossy(chunk)); + body.push_str("\r\n"); + } + body.push_str("--sep--\r\n"); + eml( + id, + "Content-Type: multipart/mixed; boundary=\"sep\"\r\n", + &body, + ) + } + + fn only_attachment(store: &Memory, message_id: &str) -> AttachmentRow { + let rows = store.read(|conn| read::attachments(conn, message_id)); + assert_eq!(rows.len(), 1, "{rows:?}"); + store + .read(|conn| read::attachment_row(conn, &rows[0].id)) + .expect("the row") + } + + /// The cache without the Tauri handle in front of it: the same order of preference, against a + /// directory a test owns. + fn fetch( + store: &Memory, + dir: &Path, + remote: Option<&dyn Remote>, + row: &AttachmentRow, + cap: u64, + ) -> Result, String> { + let row = store + .read(|conn| read::attachment_row(conn, &row.id)) + .expect("the row"); + if let Some(path) = &row.cached_path { + if let Ok(bytes) = std::fs::read(path) { + return Ok(bytes); + } + store.read(|conn| write::attachment_uncached(conn, &row.id)); + } + let raw = store.read(|conn| read::raw_body(conn, &row.message_id)); + let bytes = match raw.as_deref().and_then(|raw| part_bytes(raw, &row)) { + Some(bytes) => bytes, + None => { + let remote = remote.ok_or("that file is not on this device")?; + block_on(remote.fetch_attachment(&row.message_id, &row.part_id)) + .map_err(|e| e.to_string())? + } + }; + std::fs::create_dir_all(dir).map_err(|e| e.to_string())?; + let path = dir.join(cache_name(&row)); + std::fs::write(&path, &bytes).map_err(|e| e.to_string())?; + store.read(|conn| { + write::attachment_cached(conn, &row.id, &path.to_string_lossy(), write::now_ms())?; + trim(conn, dir, cap) + }); + Ok(bytes) + } + + fn a_message_with_a_file( + store: &Memory, + fake: &FakeProvider, + id: &str, + filename: &str, + bytes: &[u8], + ) -> AttachmentRow { + fake.add_eml(id, id, &["INBOX"], &with_attachment(id, filename, bytes)); + block_on(hydrate::headers(store, fake, &[id.to_string()], false)).expect("headers"); + block_on(hydrate::body(store, fake, id)).expect("a body"); + only_attachment(store, id) + } + + #[test] + fn a_file_already_on_the_device_is_served_without_asking_the_provider() { + let store = store(); + let fake = FakeProvider::new(); + let dir = tempfile::tempdir().expect("a cache directory"); + let row = a_message_with_a_file(&store, &fake, "m1", "notes.pdf", b"the file itself"); + + let bytes = fetch(&store, dir.path(), Some(&fake), &row, 1 << 30).expect("the bytes"); + assert_eq!(bytes, b"the file itself"); + assert!( + !fake.calls().iter().any(|call| call.starts_with("fetch_attachment")), + "the whole message was already here: {:?}", + fake.calls() + ); + } + + #[test] + fn an_attachment_is_fetched_once_and_served_from_the_cache_the_second_time() { + let store = store(); + let fake = FakeProvider::new(); + let dir = tempfile::tempdir().expect("a cache directory"); + let row = a_message_with_a_file(&store, &fake, "m1", "notes.pdf", b"the file itself"); + fake.set_attachment(&row.part_id, b"the file itself".to_vec()); + + // The message is gone but its rows are not, which is the one shape that has to reach the + // provider. + store.read(|conn| { + conn.execute("UPDATE bodies SET raw = NULL WHERE message_id = 'm1'", []) + .map(|_| ()) + .map_err(|e| e.to_string()) + }); + + let first = fetch(&store, dir.path(), Some(&fake), &row, 1 << 30).expect("the bytes"); + let second = fetch(&store, dir.path(), Some(&fake), &row, 1 << 30).expect("the bytes again"); + assert_eq!(first, b"the file itself"); + assert_eq!(second, first); + + let fetches = fake + .calls() + .into_iter() + .filter(|call| call.starts_with("fetch_attachment")) + .count(); + assert_eq!(fetches, 1, "the second time came off the disk"); + assert!( + store + .read(|conn| read::attachment_row(conn, &row.id)) + .expect("the row") + .cached_path + .is_some(), + "and the row says where" + ); + } + + #[test] + fn a_file_too_big_for_a_data_uri_is_refused_with_a_reason() { + let store = store(); + let fake = FakeProvider::new(); + let row = a_message_with_a_file(&store, &fake, "m1", "film.mov", b"small in the test"); + assert_eq!(too_big_to_preview(&row), None, "an ordinary file previews"); + + store.read(|conn| { + conn.execute( + "UPDATE attachments SET size = ?1 WHERE id = ?2", + rusqlite::params![(MAX_DATA_URL_BYTES + 1) as i64, row.id], + ) + .map(|_| ()) + .map_err(|e| e.to_string()) + }); + let row = store + .read(|conn| read::attachment_row(conn, &row.id)) + .expect("the row"); + + assert_eq!( + too_big_to_preview(&row).as_deref(), + Some("film.mov is 8.0 MB, which is too big to preview here. Save it instead."), + "and it says which file and how big before it fetches a byte" + ); + } + + #[test] + fn the_cache_gives_up_the_oldest_file_to_stay_inside_its_cap() { + let store = store(); + let fake = FakeProvider::new(); + let dir = tempfile::tempdir().expect("a cache directory"); + let first = a_message_with_a_file(&store, &fake, "m1", "one.bin", &vec![b'a'; 4_000]); + let second = a_message_with_a_file(&store, &fake, "m2", "two.bin", &vec![b'b'; 4_000]); + + // Room for one of them, and the older one is the one that goes. + fetch(&store, dir.path(), Some(&fake), &first, 5_000).expect("the first"); + fetch(&store, dir.path(), Some(&fake), &second, 5_000).expect("the second"); + + let held: Vec = store + .read(read::cached_attachments) + .into_iter() + .map(|(id, _, _)| id) + .collect(); + assert_eq!(held, vec![second.id.clone()]); + + let files: Vec = std::fs::read_dir(dir.path()) + .expect("the cache directory") + .flatten() + .map(|entry| entry.file_name().to_string_lossy().to_string()) + .collect(); + assert_eq!(files.len(), 1, "{files:?}"); + assert!( + std::fs::read(dir.path().join(&files[0])).expect("the file") == vec![b'b'; 4_000], + "the one that is left is the one that was asked for last" + ); + assert!( + store + .read(|conn| read::attachment_row(conn, &first.id)) + .expect("the row") + .cached_path + .is_none(), + "and the row it dropped says so, so storage_used stays honest" + ); + } + + #[test] + fn a_file_whose_row_was_evicted_is_swept_off_the_disk() { + let store = store(); + let fake = FakeProvider::new(); + let dir = tempfile::tempdir().expect("a cache directory"); + let row = a_message_with_a_file(&store, &fake, "m1", "one.bin", b"contents"); + fetch(&store, dir.path(), Some(&fake), &row, 1 << 30).expect("the bytes"); + + let orphan = dir.path().join("left-behind.bin"); + std::fs::write(&orphan, b"whatever").expect("an orphan"); + + store.read(|conn| trim(conn, dir.path(), 1 << 30)); + assert!(!orphan.exists(), "eviction cannot reach the disk, so this does"); + assert!(dir.path().join(cache_name(&row)).exists()); + } +} diff --git a/src-tauri/src/backup/crypto.rs b/src-tauri/src/backup/crypto.rs new file mode 100644 index 0000000..6652923 --- /dev/null +++ b/src-tauri/src/backup/crypto.rs @@ -0,0 +1,144 @@ +// What leaves the device, and what the store can and cannot learn from it. +// +// Everything uploaded is one segment of the state journal, encrypted here before it goes and +// decrypted here when it comes back. The key is 32 bytes, derived once from a BIP39 recovery +// phrase in `phrase.rs`, sealed on disk beside the OAuth tokens, and uploaded nowhere. +// +// So what a Google Drive folder or an S3 bucket holds is this: +// +// It can see how many segments there are, how large each one is, when each was written, and +// the names, which carry a hash of the account's address, a device id and a range +// of sequence numbers. That is enough to say "somebody made about forty decisions +// on Tuesday, on one of their two machines", and it is the whole of it. +// It cannot see a note, a rule, a pile, a rename, a snooze, an address, a subject or a thread +// key. Every one of those is inside a record, and a record only exists on the +// store as ciphertext. +// It cannot lie about what it holds. Each segment is authenticated under its own name, so a +// store that moves a segment between devices, swaps two of them, replays an old +// one into a new name or flips a byte gets a decryption failure here rather than a +// wrong answer in somebody's mailbox. +// +// XChaCha20-Poly1305, one fresh 24 byte nonce per segment from the OS random source. The X is the +// reason it can be random rather than counted: 192 bits is wide enough that a collision is not a +// thing that happens, and there is no counter two devices could have agreed on without a server. + +use base64::engine::general_purpose::STANDARD as BASE64; +use base64::Engine; +use chacha20poly1305::aead::{Aead, Payload}; +use chacha20poly1305::{KeyInit, XChaCha20Poly1305, XNonce}; +use rand::RngCore; + +use crate::google::secrets; + +const NONCE_LEN: usize = 24; + +/// Four bytes in front of every segment, so a file that is not one of ours says so before the tag +/// does, and so a second format later can be told from this one rather than guessed at. +const MAGIC: &[u8; 4] = b"MMB1"; + +/// The sealed key's name in the token store. It is not an account id and cannot become one: a +/// Google `sub` is digits and an email address cannot carry a space. +const KEY_ID: &str = "margin-mail backup key"; + +/// The key, and a `Debug` that will not print it. Everything that carries one of these ends up in +/// an error message eventually. +#[derive(Clone, PartialEq, Eq)] +pub struct Key([u8; 32]); + +impl std::fmt::Debug for Key { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("Key(not shown)") + } +} + +impl Key { + pub fn from_bytes(bytes: [u8; 32]) -> Key { + Key(bytes) + } +} + +/// Seals one segment. `name` is authenticated but not encrypted: it is the file name the caller is +/// about to write to, and binding it here is what stops the store rearranging the backup. +pub fn seal(key: &Key, name: &str, plain: &[u8]) -> Result, String> { + let cipher = XChaCha20Poly1305::new((&key.0).into()); + let mut nonce = [0u8; NONCE_LEN]; + rand::thread_rng().fill_bytes(&mut nonce); + let body = cipher + .encrypt( + (&nonce).into(), + Payload { + msg: plain, + aad: name.as_bytes(), + }, + ) + .map_err(|_| "could not encrypt a backup segment".to_string())?; + + let mut out = Vec::with_capacity(MAGIC.len() + NONCE_LEN + body.len()); + out.extend_from_slice(MAGIC); + out.extend_from_slice(&nonce); + out.extend_from_slice(&body); + Ok(out) +} + +/// The other direction, with the name the segment was actually found under. A wrong key, a wrong +/// name and a changed byte are all the same failure here, which is the point of an AEAD: there is +/// no partial answer to hand back and nothing that half decrypts. +pub fn open(key: &Key, name: &str, sealed: &[u8]) -> Result, String> { + if sealed.len() < MAGIC.len() + NONCE_LEN || &sealed[..MAGIC.len()] != MAGIC { + return Err(format!("{name} is not a Margin Mail backup segment")); + } + let (nonce, body) = sealed[MAGIC.len()..].split_at(NONCE_LEN); + let nonce: &XNonce = nonce + .try_into() + .map_err(|_| format!("{name} is truncated"))?; + XChaCha20Poly1305::new((&key.0).into()) + .decrypt( + nonce, + Payload { + msg: body, + aad: name.as_bytes(), + }, + ) + .map_err(|_| { + format!("{name} could not be decrypted. Either the recovery phrase is not the one this backup was made with, or the file has been changed since it was written.") + }) +} + +// --------------------------------------------------------------------------------------------- +// The key at rest +// --------------------------------------------------------------------------------------------- +// +// `google::secrets` is the OAuth token store: an XChaCha20-Poly1305 blob in the app data +// directory, 0600 from creation, keyed from a per install salt mixed with a machine identifier so +// a copied home directory does not open it. The backup key is sealed by that code rather than +// beside it, through its own front door, because a second implementation of the same idea is a +// second thing to get wrong and it would have to be reviewed on five platforms too. +// +// Losing the sealed key is not losing the backup. The phrase derives it again. + +pub fn stored() -> Result, String> { + let Some(held) = secrets::load(KEY_ID)? else { + return Ok(None); + }; + let raw = BASE64 + .decode(held) + .map_err(|e| format!("the stored backup key is malformed: {e}"))?; + let bytes: [u8; 32] = raw + .try_into() + .map_err(|_| "the stored backup key is not 32 bytes".to_string())?; + Ok(Some(Key(bytes))) +} + +pub fn remember(key: &Key) -> Result<(), String> { + secrets::store(KEY_ID, &BASE64.encode(key.0)) +} + +pub fn forget() -> Result<(), String> { + secrets::delete(KEY_ID) +} + +/// Whether this install has a key at all, which is the same question as whether the phrase has +/// been shown yet. +pub fn have_key() -> bool { + matches!(stored(), Ok(Some(_))) +} diff --git a/src-tauri/src/backup/drive.rs b/src-tauri/src/backup/drive.rs new file mode 100644 index 0000000..559e464 --- /dev/null +++ b/src-tauri/src/backup/drive.rs @@ -0,0 +1,172 @@ +// `impl BackupStore for Drive`: three operations onto the folder half of `google::drive`. +// +// The folder is `margin/mail///`, inside the same visible `margin` folder +// the other two apps in the suite write to. That is the shape the architecture settled on and the +// reason it matters is worth repeating here: margin holds `drive.file` and writes into an ordinary +// folder rather than the hidden app data space, so a person can open their Drive, see their backup +// and delete it without asking anyone. Mail keeps the scope, adds none, and puts encrypted segments +// under it. +// +// `drive.file` also means this app can only see files it created, so listing a folder returns +// Margin's own files and nothing else. That is what makes a bare "everything in this folder" query +// safe here when it would be a privacy problem under a wider scope. +// +// Drive has no paths, only folders with parents, so a name is walked segment by segment. The walk +// creates what is missing, on a read as well as a write, because the first thing a fresh device +// does is list a folder that does not exist yet and the answer to that is an empty folder rather +// than an error. + +use std::future::Future; + +use tauri::Manager; + +use crate::accounts; +use crate::google::auth::{self, AuthState}; +use crate::google::drive as gdrive; + +use super::store::BackupStore; + +pub struct Drive { + app: tauri::AppHandle, + account_id: String, +} + +impl Drive { + pub fn new(app: tauri::AppHandle, account_id: impl Into) -> Drive { + Drive { + app, + account_id: account_id.into(), + } + } + + /// The scope is checked before the token is asked for, so a cleared tick box on the consent + /// page reads as the sentence naming that tick box rather than as a 403 from Drive. + async fn token(&self) -> Result { + let granted = accounts::granted_scopes(&self.app, &self.account_id)?; + if !gdrive::has_scope(&granted) { + return Err(gdrive::REAUTH_MESSAGE.to_string()); + } + let state = self + .app + .try_state::() + .ok_or_else(|| "this app has no Google session store".to_string())?; + auth::valid_access_token(&self.app, &state, &self.account_id).await + } +} + +/// A name is `//`, which is the folders it lives under and the file +/// itself. Anything else is a caller bug rather than a user's problem, so it says so plainly. +fn split(name: &str) -> Result<(&str, &str, &str), String> { + let mut parts = name.split('/'); + match (parts.next(), parts.next(), parts.next(), parts.next()) { + (Some(account), Some(device), Some(file), None) + if !account.is_empty() && !device.is_empty() && !file.is_empty() => + { + Ok((account, device, file)) + } + _ => Err(format!("{name} is not a backup segment name")), + } +} + +impl BackupStore for Drive { + fn put(&self, name: &str, bytes: &[u8]) -> impl Future> + Send { + let name = name.to_string(); + let bytes = bytes.to_vec(); + async move { + let token = self.token().await?; + let (account, device, file) = split(&name)?; + let folder = gdrive::ensure_path( + &token, + &[gdrive::FOLDER_NAME, gdrive::MAIL_FOLDER, account, device], + ) + .await + .map_err(|e| e.to_string())?; + // Overwriting in place rather than adding a second file with the same name, which + // Drive would allow and which would make the segment ambiguous. A segment is written + // once in the ordinary course of things; this is the re-run after an interruption. + let existing = gdrive::find_file(&token, &folder, file) + .await + .map_err(|e| e.to_string())?; + gdrive::upload( + &token, + &folder, + file, + &bytes, + existing.as_ref().map(|held| held.id.as_str()), + ) + .await + .map(|_| ()) + .map_err(|e| e.to_string()) + } + } + + fn get(&self, name: &str) -> impl Future, String>> + Send { + let name = name.to_string(); + async move { + let token = self.token().await?; + let (account, device, file) = split(&name)?; + let folder = gdrive::ensure_path( + &token, + &[gdrive::FOLDER_NAME, gdrive::MAIL_FOLDER, account, device], + ) + .await + .map_err(|e| e.to_string())?; + let held = gdrive::find_file(&token, &folder, file) + .await + .map_err(|e| e.to_string())? + .ok_or_else(|| format!("{name} is not in the backup folder any more"))?; + gdrive::download(&token, &held.id) + .await + .map_err(|e| e.to_string()) + } + } + + /// A prefix is either one account's folder or one device's folder inside it. There is no third + /// depth, so the walk is two loops rather than a recursion, and a listing that came back with + /// something at another depth would be somebody else's file rather than a segment. + fn list(&self, prefix: &str) -> impl Future, String>> + Send { + let prefix = prefix.to_string(); + async move { + let token = self.token().await?; + let parts: Vec<&str> = prefix.split('/').filter(|part| !part.is_empty()).collect(); + let mut names = Vec::new(); + match parts.as_slice() { + [account] => { + let folder = gdrive::ensure_path( + &token, + &[gdrive::FOLDER_NAME, gdrive::MAIL_FOLDER, account], + ) + .await + .map_err(|e| e.to_string())?; + for device in gdrive::list_folder(&token, &folder) + .await + .map_err(|e| e.to_string())? + { + for file in gdrive::list_folder(&token, &device.id) + .await + .map_err(|e| e.to_string())? + { + names.push(format!("{account}/{}/{}", device.name, file.name)); + } + } + } + [account, device] => { + let folder = gdrive::ensure_path( + &token, + &[gdrive::FOLDER_NAME, gdrive::MAIL_FOLDER, account, device], + ) + .await + .map_err(|e| e.to_string())?; + for file in gdrive::list_folder(&token, &folder) + .await + .map_err(|e| e.to_string())? + { + names.push(format!("{account}/{device}/{}", file.name)); + } + } + _ => return Err(format!("{prefix} is not a backup folder")), + } + Ok(names) + } + } +} diff --git a/src-tauri/src/backup/mod.rs b/src-tauri/src/backup/mod.rs new file mode 100644 index 0000000..a7205f2 --- /dev/null +++ b/src-tauri/src/backup/mod.rs @@ -0,0 +1,405 @@ +// The backup, which is the state journal encrypted, put somewhere the person chose, and brought +// back. +// +// store.rs put a blob at a name, get a blob by name, list names under a prefix. Three operations +// drive.rs that trait over the Drive client in `google::drive` +// r2.rs that trait over S3 signature version four, for somebody who runs their own +// crypto.rs what leaves the device and what the store can learn from it +// phrase.rs the twenty-four words and the key they derive +// +// Nothing here decides anything. `state::merge` already knows how two logs meet, `state::journal` +// already guarantees that replaying a log rebuilds the tables, and this package is the transport +// and the encryption around that. What it adds is a rhythm: a device uploads its own segments under +// its own device id and downloads every other device's, so two machines that never met still land +// on the same tables, and a machine that was off for a month replays what it missed in one pass. +// +// The one hazard worth naming, because it is a product rule rather than a bug to fix here. A second +// device attaches with the phrase; it must never turn backup on by itself. Two devices that each +// generate their own phrase for the same account write two backups under two keys into one folder, +// and neither can read the other's. That is what the sentence in the Backup section of settings is +// for, and a segment that will not decrypt says so by name rather than being skipped, because a +// backup that quietly ignores what it cannot read is a backup that quietly loses half of itself. + +pub mod crypto; +pub mod drive; +pub mod phrase; +pub mod r2; +pub mod store; + +#[cfg(test)] +mod tests; + +use std::collections::HashMap; + +use rusqlite::Connection; +use sha2::{Digest, Sha256}; +use tauri::Manager; + +use crate::db::Db; +use crate::dto::BackupSettings; +use crate::state::{device, journal::Record, merge}; + +use crypto::Key; +use store::BackupStore; + +/// Events per segment. Small enough that an upload cut off halfway has lost very little work and +/// large enough that a month of decisions is a handful of files rather than a directory listing +/// nobody can read. +const SEGMENT_RECORDS: usize = 500; + +const SUFFIX: &str = ".seg"; + +/// A hash rather than the address, because a folder listing in somebody's Drive should not be a +/// list of the email addresses they have accounts for. It is the same on every device, which is +/// what lets a second one find the first one's folder from the phrase and the account alone. +/// +/// Truncated to 128 bits: this names one folder among a person's handful and is not a secret, and +/// a name that fits on one line is worth more here than the other half of the digest. +pub fn account_hash(email: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(b"margin-mail backup account v1"); + hasher.update(email.trim().to_lowercase().as_bytes()); + let digest = hasher.finalize(); + digest[..16].iter().fold(String::new(), |mut out, byte| { + out.push_str(&format!("{byte:02x}")); + out + }) +} + +/// `//-.seg`. +/// +/// The name carries a range of sequence numbers and nothing else. It cannot say what is in the +/// segment, because everything a record holds is inside the ciphertext, and it does say what a +/// device needs in order to skip a download: a segment ending at or below what this device already +/// holds for that device is a segment it has already absorbed. That is what makes catching up a +/// month cost one listing and only the files that were missed. +fn segment_name(account_hash: &str, device_id: &str, first: i64, last: i64) -> String { + format!("{account_hash}/{device_id}/{first:012}-{last:012}{SUFFIX}") +} + +/// What a name says. Anything that does not parse belongs to somebody else and is left alone. +fn parse_name(name: &str) -> Option<(String, i64, i64)> { + let mut parts = name.split('/'); + let _account = parts.next()?; + let device = parts.next()?; + let file = parts.next()?; + if parts.next().is_some() { + return None; + } + let (first, last) = file.strip_suffix(SUFFIX)?.split_once('-')?; + Some((device.to_string(), first.parse().ok()?, last.parse().ok()?)) +} + +/// The journal side of a pass, reached through a closure. +/// +/// A `&Connection` may not be held across an await and the transport in the middle of a pass is +/// asynchronous, so the database work happens in these closures and the network work happens +/// between them. Deliberately not `sync::Store`, which is the same shape: that one means the +/// mirror and this one means the journal, and sharing the name would couple two packages through a +/// word that means different things in each. +pub trait Journal: Sync { + fn with Result>(&self, f: F) -> Result; +} + +/// The app's journal: the shared connection pool, narrowed to one account. +pub struct Account<'a> { + pub db: &'a Db, + pub account_id: &'a str, +} + +impl Journal for Account<'_> { + fn with Result>(&self, f: F) -> Result { + self.db.with(self.account_id, f) + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Pass { + pub uploaded: usize, + pub downloaded: usize, + pub absorbed: usize, + pub skipped: usize, +} + +/// One account's whole exchange with the store: what is new here goes up, what is new anywhere else +/// comes down and is absorbed. +/// +/// Both halves are driven by the listing rather than by a local watermark, so there is one source +/// of truth about what the store holds and it is the store. A pass that died halfway through its +/// uploads leaves whole segments behind it, and the next pass sees them and carries on from there. +/// +/// Reading happens before writing, which matters on the pass a restore runs: a key that cannot open +/// what is already in the folder fails before it has added anything of its own, so a phrase typed +/// wrongly cannot leave a second backup's worth of segments that nothing can read. +pub async fn pass( + journal: &J, + store: &S, + key: &Key, + account_hash: &str, +) -> Result { + let held = store.list(&format!("{account_hash}/")).await?; + let device_id = journal.with(device::device_id)?; + let mut pass = Pass::default(); + + let high: HashMap = journal.with(merge::high_water)?.into_iter().collect(); + let mut wanted: Vec<&String> = held + .iter() + .filter(|name| match parse_name(name) { + Some((device, _, last)) => { + device != device_id && last > *high.get(&device).unwrap_or(&0) + } + None => false, + }) + .collect(); + wanted.sort(); + + let mut records: Vec = Vec::new(); + for name in wanted { + let sealed = store.get(name).await?; + let plain = crypto::open(key, name, &sealed)?; + let text = String::from_utf8(plain).map_err(|_| format!("{name} is not a segment"))?; + records.extend(merge::decode(&text)?); + pass.downloaded += 1; + } + + if !records.is_empty() { + let report = journal.with(|conn| merge::absorb(conn, &records))?; + pass.absorbed = report.applied; + pass.skipped = report.skipped; + } + + let uploaded_to = held + .iter() + .filter_map(|name| parse_name(name)) + .filter(|(device, _, _)| device == &device_id) + .map(|(_, _, last)| last) + .max() + .unwrap_or(0); + + // Only this device's own events, which is why two logs meeting is a union rather than a + // conflict: no other device can have written under this sequence. + let mine = journal.with(|conn| merge::export(conn, &device_id, uploaded_to))?; + for chunk in mine.chunks(SEGMENT_RECORDS) { + let (Some(first), Some(last)) = (chunk.first(), chunk.last()) else { + continue; + }; + let name = segment_name(account_hash, &device_id, first.seq, last.seq); + let body = crypto::seal(key, &name, merge::encode(chunk)?.as_bytes())?; + store.put(&name, &body).await?; + pass.uploaded += 1; + } + Ok(pass) +} + +// --------------------------------------------------------------------------------------------- +// The commands +// --------------------------------------------------------------------------------------------- + +/// Which store the settings file names, with whatever it needs to reach it. +enum Chosen { + None, + Drive, + R2(r2::Config), +} + +fn chosen(app: &tauri::AppHandle) -> Result { + match crate::settings::load(app)?.backup.store.as_str() { + "drive" => Ok(Chosen::Drive), + "r2" => match r2::stored()? { + Some(config) => Ok(Chosen::R2(config)), + None => Err( + "The R2 credentials are not on this device any more. Enter them again.".to_string(), + ), + }, + _ => Ok(Chosen::None), + } +} + +/// Writes the non-secret half of the backup settings back into `settings.json`, through the +/// settings module's own patch path so there is one writer of that file. +fn record(app: &tauri::AppHandle, patch: serde_json::Value) -> Result { + let settings = + crate::settings::settings_set(app.clone(), serde_json::json!({ "backup": patch }))?; + Ok(with_key_state(settings.backup)) +} + +/// `has_phrase` is not a setting, it is whether a key is sealed on this device, so it is answered +/// from the key store every time rather than trusted from the file. +fn with_key_state(mut settings: BackupSettings) -> BackupSettings { + settings.has_phrase = crypto::have_key(); + settings +} + +/// One account's pass against whichever store is turned on. The store is built here rather than +/// once for the run because the Drive one holds an account: it signs its requests with that +/// account's token, and `drive.file` means it can only see the files it created itself. +async fn pass_for_account( + app: &tauri::AppHandle, + chosen: &Chosen, + key: &Key, + account_id: &str, + email: &str, +) -> Result { + let db = app + .try_state::() + .ok_or_else(|| "the databases are not open yet".to_string())?; + let journal = Account { + db: db.inner(), + account_id, + }; + let hash = account_hash(email); + match chosen { + Chosen::None => Err("No backup store is turned on.".to_string()), + Chosen::Drive => { + let store = drive::Drive::new(app.clone(), account_id); + pass(&journal, &store, key, &hash).await + } + Chosen::R2(config) => { + let store = r2::R2::new(config.clone()); + pass(&journal, &store, key, &hash).await + } + } +} + +#[tauri::command] +pub async fn backup_status(app: tauri::AppHandle) -> Result { + Ok(with_key_state(crate::settings::load(&app)?.backup)) +} + +/// `store` is "drive", "r2" or "none". R2 takes the four S3 fields and nothing else; Drive takes +/// none, because it uses the account that is already connected. +#[tauri::command] +pub async fn backup_configure( + app: tauri::AppHandle, + store: String, + config: HashMap, +) -> Result { + match store.as_str() { + "drive" => { + let accounts = crate::accounts::list(&app)?; + if accounts.is_empty() { + return Err("Connect an account before turning backup on.".to_string()); + } + // Granular consent means the Drive tick box is the user's to clear, so this is asked + // now rather than discovered as a failure during the first backup. + if !accounts + .iter() + .any(|account| crate::google::drive::has_scope(&account.granted_scopes)) + { + return Err(crate::google::drive::REAUTH_MESSAGE.to_string()); + } + record( + &app, + serde_json::json!({ + "store": "drive", + "configured": true, + "r2Bucket": serde_json::Value::Null, + "r2Endpoint": serde_json::Value::Null + }), + ) + } + "r2" => { + let config = r2::Config::from_fields(&config)?; + r2::remember(&config)?; + record( + &app, + serde_json::json!({ + "store": "r2", + "configured": true, + "r2Bucket": config.bucket, + "r2Endpoint": config.endpoint + }), + ) + } + "none" => { + // The key stays. It is what reads the segments already up there, and turning the store + // off is not a decision to make those unreadable. The credentials for somebody else's + // bucket do not stay, because they are not ours to keep once they are not in use. + r2::forget()?; + record( + &app, + serde_json::json!({ + "store": "none", + "configured": false, + "r2Bucket": serde_json::Value::Null, + "r2Endpoint": serde_json::Value::Null + }), + ) + } + other => Err(format!("{other} is not a backup store")), + } +} + +#[tauri::command] +pub async fn backup_now(app: tauri::AppHandle) -> Result { + let key = crypto::stored()?.ok_or( + "Backup has no recovery phrase yet. Turn backup on and write the phrase down first.", + )?; + let chosen = chosen(&app)?; + if matches!(chosen, Chosen::None) { + return Err("No backup store is turned on.".to_string()); + } + let mut absorbed = 0; + for account in crate::accounts::list(&app)? { + let pass = pass_for_account(&app, &chosen, &key, &account.id, &account.email).await?; + absorbed += pass.absorbed; + } + + let settings = record( + &app, + serde_json::json!({ "lastBackupMs": chrono::Utc::now().timestamp_millis() }), + )?; + if absorbed > 0 { + crate::emit_store_changed(&app, "state threads"); + } + Ok(settings) +} + +/// Shown once, at setup, and never returned again. +/// +/// Literally never: the phrase is generated here, its key is sealed, and the phrase itself is +/// dropped at the end of this function. Argon2 does not run backwards, so there is nothing left on +/// the device that could print it a second time even if a screen asked. That is the property being +/// bought, and it is why the second call is an explanation rather than a repeat. +#[tauri::command] +pub async fn backup_phrase(app: tauri::AppHandle) -> Result { + if crypto::have_key() { + return Err("The recovery phrase is shown once and is not kept anywhere, so it cannot be shown again. Backup on this device carries on working without it; the phrase is only needed to attach another device or to restore onto a new one.".to_string()); + } + let phrase = phrase::generate()?; + crypto::remember(&phrase::derive(&phrase)?)?; + record(&app, serde_json::json!({ "hasPhrase": true }))?; + Ok(phrase) +} + +/// Attaches this device to an existing backup, which is also how a lost device is replaced. +/// +/// The phrase becomes the key, one pass brings down every other device's segments and replays +/// them, and only then is the key sealed: a phrase that turns out not to be this backup's leaves +/// the device exactly as it was rather than half attached to something it cannot read. There is +/// nothing else to a restore. The journal rebuilds the tables and the mirror is derived from the +/// provider, so what comes back is every decision and none of the mail, which fills itself in on +/// the next sync. +#[tauri::command] +pub async fn backup_restore(app: tauri::AppHandle, phrase: String) -> Result<(), String> { + let key = phrase::derive(&phrase)?; + let chosen = chosen(&app)?; + if matches!(chosen, Chosen::None) { + return Err("Choose where the backup is kept before restoring from it.".to_string()); + } + + for account in crate::accounts::list(&app)? { + pass_for_account(&app, &chosen, &key, &account.id, &account.email).await?; + } + crypto::remember(&key)?; + + record( + &app, + serde_json::json!({ + "hasPhrase": true, + "lastBackupMs": chrono::Utc::now().timestamp_millis() + }), + )?; + crate::emit_store_changed(&app, "state threads"); + Ok(()) +} diff --git a/src-tauri/src/backup/phrase.rs b/src-tauri/src/backup/phrase.rs new file mode 100644 index 0000000..a027594 --- /dev/null +++ b/src-tauri/src/backup/phrase.rs @@ -0,0 +1,87 @@ +// The twenty-four words, and the only way back into a backup. +// +// The phrase is generated once, on the device, when backup is first turned on. It is shown once +// and then it is gone: what stays behind is the key it derives, sealed by `crypto`, and Argon2 is +// one way, so this app cannot show the phrase again even if a screen asked it to. That is the +// property being bought. A phrase the app can reprint is a phrase the app is storing, and a phrase +// the app is storing is one more file that has to be as well defended as the backup itself. +// +// BIP39 rather than a random string because the words are transcribable. This gets written on +// paper, read out over a phone call and typed on a machine that is not the one it came from, and +// the wordlist carries a checksum that catches a wrong word at the point of typing rather than as +// a decryption failure ten seconds later. + +use argon2::{Algorithm, Argon2, Params, Version}; +use bip39::Mnemonic; + +use super::crypto::Key; + +/// Twenty-four words, so 256 bits of entropy. Twelve would be past anything anyone can brute force +/// too, and the extra dozen words cost one more line on the piece of paper. +pub const WORDS: usize = 24; + +/// Argon2id, 64 MiB, three passes, one lane, 32 bytes out. +/// +/// Argon2id because it is the hybrid: 2i alone gives up GPU resistance and 2d alone is open to a +/// side channel, and RFC 9106 tells anyone without a specific reason to differ to use the hybrid. +/// +/// The cost is RFC 9106's second recommended configuration, the one written for a machine that +/// cannot spare a gigabyte, which is the right shape for something that has to run on a phone. It +/// is a fraction of a second on a laptop and under a second on a handset, and it runs exactly +/// twice in the life of an installation: once at setup and once at a restore. There is no +/// interactive cost to trade it against, so there was no reason to go lower. +/// +/// One lane rather than the four the RFC pairs with 64 MiB because this implementation walks the +/// lanes in sequence rather than in threads. Four lanes would quarter the memory each one touches +/// and shorten nothing, and the ratio between what this costs us and what it costs an attacker is +/// the same either way. +const M_COST_KIB: u32 = 64 * 1024; +const T_COST: u32 = 3; +const LANES: u32 = 1; + +/// A constant, which for a password would be a bug and here is forced. A second device has the +/// phrase and nothing else, so every input to the derivation has to be reachable from the phrase +/// alone; a random salt would have to be stored somewhere, and the only place to store it is the +/// backup, which cannot be read until the key exists. What a per user salt buys is that one +/// precomputed table cannot answer for many users, and there is nothing to precompute against 256 +/// bits from the wordlist. The entropy is doing the work here and Argon2 is the belt to it. +const SALT: &[u8] = b"margin-mail backup phrase v1"; + +/// A fresh phrase. Never returned twice: the caller shows it, seals the key it derives, and this +/// module keeps nothing. +pub fn generate() -> Result { + Mnemonic::generate(WORDS) + .map(|mnemonic| mnemonic.to_string()) + .map_err(|e| format!("could not make a recovery phrase: {e}")) +} + +/// The phrase as the derivation sees it: the wordlist's own spelling, single spaced. Typing it in +/// a different case, with an extra space or with a line break in the middle is the same phrase, and +/// a mistyped word is refused here by name rather than becoming a wrong key and an unreadable +/// backup twenty seconds later. +/// +/// The case folding is not politeness. This gets typed on the phone that the backup is being +/// restored onto, every soft keyboard capitalises the first word of what looks like a sentence, and +/// the wordlist is lower case, so without this the commonest way of entering a phrase correctly +/// fails as if the phrase were wrong. +pub fn normalise(phrase: &str) -> Result { + let tidied = phrase + .split_whitespace() + .map(|word| word.to_lowercase()) + .collect::>() + .join(" "); + Mnemonic::parse(&tidied) + .map(|mnemonic| mnemonic.to_string()) + .map_err(|e| format!("that is not a recovery phrase: {e}")) +} + +pub fn derive(phrase: &str) -> Result { + let normalised = normalise(phrase)?; + let params = Params::new(M_COST_KIB, T_COST, LANES, Some(32)) + .map_err(|e| format!("the key derivation is misconfigured: {e}"))?; + let mut out = [0u8; 32]; + Argon2::new(Algorithm::Argon2id, Version::V0x13, params) + .hash_password_into(normalised.as_bytes(), SALT, &mut out) + .map_err(|e| format!("could not derive the backup key: {e}"))?; + Ok(Key::from_bytes(out)) +} diff --git a/src-tauri/src/backup/r2.rs b/src-tauri/src/backup/r2.rs new file mode 100644 index 0000000..96593fe --- /dev/null +++ b/src-tauri/src/backup/r2.rs @@ -0,0 +1,426 @@ +// `impl BackupStore for R2`: the same three operations over S3 signature version four, for +// somebody who would rather their backup went to a bucket they own than to Drive. +// +// Cloudflare R2 is what this was written against and what the settings screen names, but nothing +// below is R2 specific beyond the region: it signs the way S3 does, so any endpoint that speaks +// path-style S3 with sigv4 works. The four fields are an endpoint, a bucket, an access key and a +// secret, and the secret never crosses the IPC boundary in the reading direction, which is why +// `dto::BackupSettings` carries the endpoint and the bucket and neither of the other two. +// +// The signing is by hand rather than through an SDK. `aws-sdk-s3` is a hundred crates and a +// runtime of its own for three verbs, and the signing is a page of code with two published test +// vectors to hold it to, which is what `tests.rs` does. Even the HMAC is here: the app does not +// depend on the `hmac` crate, and the construction over a hash it already has is six lines. + +use std::collections::HashMap; +use std::future::Future; +use std::sync::LazyLock; +use std::time::Duration; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::google::secrets; + +use super::store::BackupStore; + +/// The sealed credentials' name in the token store. Not an account id and unable to become one: a +/// Google `sub` is digits and an email address cannot carry a space. +const SECRET_ID: &str = "margin-mail r2 credentials"; + +const ALGORITHM: &str = "AWS4-HMAC-SHA256"; +const SERVICE: &str = "s3"; + +/// R2 has one region and calls it `auto`. It is not optional in a sigv4 signature even so, and +/// Cloudflare's own SDK signs against this string. +const REGION: &str = "auto"; + +/// Its own client rather than the Gmail layer's. They share no host, so one pool would never be +/// reused, and a backup upload's timeout has nothing to do with a metadata batch's. +static HTTP: LazyLock = LazyLock::new(|| { + reqwest::Client::builder() + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(120)) + .build() + .expect("could not build the HTTP client") +}); + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Config { + pub endpoint: String, + pub bucket: String, + pub access_key: String, + pub secret: String, +} + +impl Config { + /// What the settings screen sends. Refused here rather than at the first upload, because a + /// typed-in endpoint that is wrong should say so while the person still has the field open. + pub fn from_fields(fields: &HashMap) -> Result { + let field = |camel: &str, snake: &str, label: &str| -> Result { + let value = fields + .get(camel) + .or_else(|| fields.get(snake)) + .map(|value| value.trim().to_string()) + .unwrap_or_default(); + if value.is_empty() { + return Err(format!("The {label} is missing.")); + } + Ok(value) + }; + + let endpoint = field("endpoint", "endpoint", "endpoint")?; + let endpoint = endpoint.trim_end_matches('/').to_string(); + // Not https means the request that carries the signature, and everything the bucket policy + // rests on, travels in the open. The blob itself is already encrypted; the credentials in + // the header are not. + if !endpoint.starts_with("https://") { + return Err("The endpoint has to start with https://".to_string()); + } + if url::Url::parse(&endpoint) + .ok() + .and_then(|url| url.host_str().map(|host| host.to_string())) + .is_none() + { + return Err(format!("{endpoint} is not an endpoint address.")); + } + + let bucket = field("bucket", "bucket", "bucket name")?; + if bucket.contains('/') { + return Err("A bucket name is one word, with no slashes in it.".to_string()); + } + + Ok(Config { + endpoint, + bucket, + access_key: field("accessKey", "access_key", "access key")?, + secret: field("secret", "secret", "secret access key")?, + }) + } +} + +/// Sealed the way the OAuth tokens are, and for a stronger reason than the backup key: these are +/// credentials to somebody else's paid account, and unlike the key they cannot be derived again +/// from anything the person wrote down. +pub fn remember(config: &Config) -> Result<(), String> { + let json = serde_json::to_string(config).map_err(|e| e.to_string())?; + secrets::store(SECRET_ID, &json) +} + +pub fn stored() -> Result, String> { + match secrets::load(SECRET_ID)? { + Some(json) => serde_json::from_str(&json) + .map(Some) + .map_err(|e| format!("the stored R2 credentials are malformed: {e}")), + None => Ok(None), + } +} + +pub fn forget() -> Result<(), String> { + secrets::delete(SECRET_ID) +} + +pub struct R2 { + config: Config, +} + +impl R2 { + pub fn new(config: Config) -> R2 { + R2 { config } + } + + fn host(&self) -> Result { + let url = url::Url::parse(&self.config.endpoint).map_err(|e| e.to_string())?; + let host = url + .host_str() + .ok_or_else(|| format!("{} is not an endpoint address.", self.config.endpoint))?; + // The signed host has to be the host that goes on the wire, port and all, or the signature + // covers a request nobody sent. + Ok(match url.port() { + Some(port) => format!("{host}:{port}"), + None => host.to_string(), + }) + } + + /// Builds and sends one signed request. Everything that differs between the three operations is + /// a parameter, so there is one place that knows how a request is signed. + async fn send( + &self, + method: reqwest::Method, + path: &str, + query: &[(&str, &str)], + body: Vec, + ) -> Result { + let host = self.host()?; + let payload_sha = hex(&Sha256::digest(&body)); + let amz_date = chrono::Utc::now().format("%Y%m%dT%H%M%SZ").to_string(); + + let canonical_query = canonical_query(query); + let headers = [ + ("host", host.clone()), + ("x-amz-content-sha256", payload_sha.clone()), + ("x-amz-date", amz_date.clone()), + ]; + let authorization = authorization( + &self.config.access_key, + &self.config.secret, + REGION, + SERVICE, + method.as_str(), + path, + &canonical_query, + &headers, + &payload_sha, + &amz_date, + ); + + let mut url = format!("{}{path}", self.config.endpoint); + if !canonical_query.is_empty() { + url.push('?'); + url.push_str(&canonical_query); + } + + HTTP.request(method, url) + .header("x-amz-content-sha256", &payload_sha) + .header("x-amz-date", &amz_date) + .header(reqwest::header::AUTHORIZATION, authorization) + .body(body) + .send() + .await + .map_err(|e| format!("could not reach the bucket: {e}")) + } + + fn path_for(&self, name: &str) -> String { + format!( + "/{}/{}", + encode(&self.config.bucket, false), + encode(name, true) + ) + } +} + +/// What the store said when it said no. The XML body is worth keeping: an S3 error names the code +/// in it, and "SignatureDoesNotMatch" is a different afternoon from "NoSuchBucket". +async fn refused(context: &str, resp: reqwest::Response) -> String { + let status = resp.status().as_u16(); + let body = resp.text().await.unwrap_or_default(); + let detail = tag(&body, "Message").or_else(|| tag(&body, "Code")); + match detail { + Some(detail) => format!("{context} failed: {status}, {detail}"), + None => format!("{context} failed: {status}"), + } +} + +impl BackupStore for R2 { + fn put(&self, name: &str, bytes: &[u8]) -> impl Future> + Send { + let path = self.path_for(name); + let name = name.to_string(); + let bytes = bytes.to_vec(); + async move { + let resp = self.send(reqwest::Method::PUT, &path, &[], bytes).await?; + if !resp.status().is_success() { + return Err(refused(&format!("Uploading {name}"), resp).await); + } + Ok(()) + } + } + + fn get(&self, name: &str) -> impl Future, String>> + Send { + let path = self.path_for(name); + let name = name.to_string(); + async move { + let resp = self + .send(reqwest::Method::GET, &path, &[], Vec::new()) + .await?; + if !resp.status().is_success() { + return Err(refused(&format!("Downloading {name}"), resp).await); + } + resp.bytes() + .await + .map(|bytes| bytes.to_vec()) + .map_err(|e| format!("could not read {name}: {e}")) + } + } + + /// `list-type=2`, paged until the store says it has stopped truncating. The keys come back in + /// an XML document; the two fields that matter are scraped out of it rather than parsed, + /// because a dependency on an XML crate to read `` would be a poor trade. + fn list(&self, prefix: &str) -> impl Future, String>> + Send { + let path = format!("/{}", encode(&self.config.bucket, false)); + let prefix = prefix.to_string(); + async move { + let mut names = Vec::new(); + let mut token: Option = None; + loop { + let mut query = vec![("list-type", "2"), ("prefix", prefix.as_str())]; + if let Some(token) = &token { + query.push(("continuation-token", token.as_str())); + } + let resp = self + .send(reqwest::Method::GET, &path, &query, Vec::new()) + .await?; + if !resp.status().is_success() { + return Err(refused("Listing the bucket", resp).await); + } + let body = resp + .text() + .await + .map_err(|e| format!("could not read the bucket listing: {e}"))?; + names.extend(tags(&body, "Key")); + match tag(&body, "NextContinuationToken") { + Some(next) if tag(&body, "IsTruncated").as_deref() == Some("true") => { + token = Some(next) + } + _ => return Ok(names), + } + } + } + } +} + +// --------------------------------------------------------------------------------------------- +// Signature version four +// --------------------------------------------------------------------------------------------- + +fn hex(bytes: &[u8]) -> String { + bytes.iter().fold(String::new(), |mut out, byte| { + out.push_str(&format!("{byte:02x}")); + out + }) +} + +/// HMAC-SHA256, RFC 2104, over the hash the app already depends on. The block size is 64 bytes for +/// SHA-256 and every key sigv4 uses is shorter than that, but the long-key branch is here because +/// leaving it out would make this a function that is right for one caller rather than an HMAC. +pub(super) fn hmac_sha256(key: &[u8], message: &[u8]) -> [u8; 32] { + let mut block = [0u8; 64]; + if key.len() > 64 { + block[..32].copy_from_slice(&Sha256::digest(key)); + } else { + block[..key.len()].copy_from_slice(key); + } + let mut inner_pad = [0x36u8; 64]; + let mut outer_pad = [0x5cu8; 64]; + for at in 0..64 { + inner_pad[at] ^= block[at]; + outer_pad[at] ^= block[at]; + } + let inner = Sha256::new() + .chain_update(inner_pad) + .chain_update(message) + .finalize(); + Sha256::new() + .chain_update(outer_pad) + .chain_update(inner) + .finalize() + .into() +} + +/// The four nested HMACs that turn a secret into a key good for one day, one region and one +/// service. This is the derivation AWS publishes a test vector for, and `tests.rs` uses it. +pub(super) fn signing_key(secret: &str, date: &str, region: &str, service: &str) -> [u8; 32] { + let start = format!("AWS4{secret}"); + let key = hmac_sha256(start.as_bytes(), date.as_bytes()); + let key = hmac_sha256(&key, region.as_bytes()); + let key = hmac_sha256(&key, service.as_bytes()); + hmac_sha256(&key, b"aws4_request") +} + +/// Percent encoding as sigv4 defines it, which is stricter than a URL's: everything outside the +/// unreserved set is encoded, and the slash survives only where it is separating path segments. +pub(super) fn encode(value: &str, keep_slash: bool) -> String { + let mut out = String::with_capacity(value.len()); + for byte in value.bytes() { + match byte { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'.' | b'_' | b'~' => { + out.push(byte as char) + } + b'/' if keep_slash => out.push('/'), + other => out.push_str(&format!("%{other:02X}")), + } + } + out +} + +/// Sorted by name, encoded, joined. Every query this module sends has distinct names, so there is +/// no tie to break on the value. +fn canonical_query(query: &[(&str, &str)]) -> String { + let mut pairs: Vec = query + .iter() + .map(|(name, value)| format!("{}={}", encode(name, false), encode(value, false))) + .collect(); + pairs.sort(); + pairs.join("&") +} + +/// The whole signature, as the `Authorization` header wants it. +/// +/// Everything it needs is a parameter, including the moment and the header list, so that the +/// published test vectors can be run through this exact function rather than through a +/// reimplementation of it that might differ in the one place that matters. +#[allow(clippy::too_many_arguments)] +pub(super) fn authorization( + access_key: &str, + secret: &str, + region: &str, + service: &str, + method: &str, + uri: &str, + canonical_query: &str, + headers: &[(&str, String)], + payload_sha: &str, + amz_date: &str, +) -> String { + let mut sorted: Vec<(String, String)> = headers + .iter() + .map(|(name, value)| (name.to_lowercase(), value.trim().to_string())) + .collect(); + sorted.sort(); + let canonical_headers: String = sorted + .iter() + .map(|(name, value)| format!("{name}:{value}\n")) + .collect(); + let signed_headers: Vec<&str> = sorted.iter().map(|(name, _)| name.as_str()).collect(); + let signed_headers = signed_headers.join(";"); + + let canonical_request = format!( + "{method}\n{uri}\n{canonical_query}\n{canonical_headers}\n{signed_headers}\n{payload_sha}" + ); + let date = &amz_date[..8.min(amz_date.len())]; + let scope = format!("{date}/{region}/{service}/aws4_request"); + let to_sign = format!( + "{ALGORITHM}\n{amz_date}\n{scope}\n{}", + hex(&Sha256::digest(canonical_request.as_bytes())) + ); + let signature = hex(&hmac_sha256( + &signing_key(secret, date, region, service), + to_sign.as_bytes(), + )); + format!( + "{ALGORITHM} Credential={access_key}/{scope}, SignedHeaders={signed_headers}, Signature={signature}" + ) +} + +// --------------------------------------------------------------------------------------------- +// Just enough XML +// --------------------------------------------------------------------------------------------- + +pub(super) fn tags(xml: &str, name: &str) -> Vec { + let open = format!("<{name}>"); + let close = format!(""); + let mut out = Vec::new(); + let mut rest = xml; + while let Some(start) = rest.find(&open) { + let after = &rest[start + open.len()..]; + let Some(end) = after.find(&close) else { + break; + }; + out.push(after[..end].to_string()); + rest = &after[end + close.len()..]; + } + out +} + +fn tag(xml: &str, name: &str) -> Option { + tags(xml, name).into_iter().next() +} diff --git a/src-tauri/src/backup/store.rs b/src-tauri/src/backup/store.rs new file mode 100644 index 0000000..27919f3 --- /dev/null +++ b/src-tauri/src/backup/store.rs @@ -0,0 +1,30 @@ +// Three operations, and the reason there are only three. +// +// A journal segment is a whole blob written once under a name that never changes meaning, so +// nothing above this trait needs a rename, a delete, a directory, a lock, a range read or a +// conditional write. What is left is put, get and list, which is the intersection of Drive's REST +// API and S3, and small enough that the second implementation was an afternoon and the one in +// `tests.rs` is thirty lines. A backup whose store trait needs a transaction is a backup that +// cannot be pointed at somebody's own bucket. +// +// A name is a path with forward slashes: `//-.seg`. Drive +// has no paths, so it maps them onto folders; S3 has no folders, so it takes the name as the key. +// Neither is allowed to invent a layout of its own, because two implementations that disagree +// about where a segment lives are two backups that cannot be swapped. +// +// Async in the `Provider` trait's shape: `impl Future + Send` on a `Sync` trait, so there is no +// boxing and no `async_trait`. + +use std::future::Future; + +pub trait BackupStore: Sync { + /// Whole, or not at all. Both implementations write a blob in one request, so a name either + /// does not exist or names every byte of a segment; a pass cut off halfway leaves the segments + /// it finished and nothing else. `pass` depends on that and `tests.rs` holds it to it. + fn put(&self, name: &str, bytes: &[u8]) -> impl Future> + Send; + + fn get(&self, name: &str) -> impl Future, String>> + Send; + + /// Every name under a prefix, in no particular order. The caller sorts what it needs sorted. + fn list(&self, prefix: &str) -> impl Future, String>> + Send; +} diff --git a/src-tauri/src/backup/tests.rs b/src-tauri/src/backup/tests.rs new file mode 100644 index 0000000..f94d727 --- /dev/null +++ b/src-tauri/src/backup/tests.rs @@ -0,0 +1,650 @@ +// The backup over a store that is a map in memory, which is the whole reason the store trait is +// three operations. +// +// Nothing here touches a network, a Google account or a bucket. What is being tested is the part +// that would be wrong in a way nobody notices: that two devices which never met land on the same +// tables, that the ciphertext is ciphertext, and that a pass cut off halfway leaves the store in a +// state the next pass can carry on from. + +use std::collections::BTreeMap; +use std::future::Future; +use std::sync::Mutex; + +use rusqlite::Connection; +use tauri::async_runtime::block_on; + +use crate::db; +use crate::dto::{Destination, Person, Pile}; +use crate::state::journal::{self, Payload}; +use crate::state::{device, merge, write}; + +use super::crypto::{self, Key}; +use super::store::BackupStore; +use super::{account_hash, pass, phrase, r2, Journal, Pass}; + +// -- the harness ------------------------------------------------------------------------------- + +/// A data directory: one pair of in-memory databases, with a device id of its own. +struct Device(Mutex); + +impl Journal for Device { + fn with Result>(&self, f: F) -> Result { + let conn = self.0.lock().map_err(|e| e.to_string())?; + f(&conn) + } +} + +impl Device { + fn new() -> Device { + Device(Mutex::new( + db::memory().expect("a pair of in-memory databases"), + )) + } + + fn on(&self, f: impl FnOnce(&Connection) -> T) -> T { + let conn = self.0.lock().expect("the connection"); + f(&conn) + } + + fn id(&self) -> String { + self.on(|conn| device::device_id(conn).expect("a device id")) + } + + /// Every materialised table, row by row, in an order that does not depend on how they were + /// written. Two devices have converged when these are equal. + fn snapshot(&self) -> Vec { + self.on(|conn| { + let mut out = Vec::new(); + for table in TABLES { + let mut stmt = conn + .prepare(&format!("SELECT * FROM state.{table}")) + .expect("prepare"); + let columns = stmt.column_count(); + let rows = stmt + .query_map([], |row| { + let mut cells = Vec::new(); + for at in 0..columns { + cells.push(format!("{:?}", row.get::<_, rusqlite::types::Value>(at)?)); + } + Ok(cells.join("|")) + }) + .expect("query"); + let mut table_rows: Vec = rows + .map(|row| format!("{table}: {}", row.expect("row"))) + .collect(); + table_rows.sort(); + out.extend(table_rows); + } + out + }) + } +} + +const TABLES: [&str; 11] = [ + "sender_rules", + "piles", + "snoozes", + "notes", + "renames", + "merges", + "clips", + "thread_flags", + "contacts", + "prefs", + "markers", +]; + +/// A store that is a map. `put` replaces a whole value under a lock, so a name never exists with +/// half a segment behind it, which is the property both real stores get from writing a blob in one +/// request. +#[derive(Default)] +struct Memory { + blobs: Mutex>>, + /// How many more puts will be accepted before the store starts refusing, for the pass that gets + /// cut off halfway. + allowed: Mutex>, +} + +impl Memory { + fn names(&self) -> Vec { + self.blobs.lock().expect("blobs").keys().cloned().collect() + } + + fn blob(&self, name: &str) -> Vec { + self.blobs + .lock() + .expect("blobs") + .get(name) + .cloned() + .unwrap_or_else(|| panic!("{name} is not in the store")) + } + + fn breaks_after(&self, puts: usize) { + *self.allowed.lock().expect("allowed") = Some(puts); + } + + fn mended(&self) { + *self.allowed.lock().expect("allowed") = None; + } +} + +impl BackupStore for Memory { + fn put(&self, name: &str, bytes: &[u8]) -> impl Future> + Send { + let name = name.to_string(); + let bytes = bytes.to_vec(); + async move { + let mut allowed = self.allowed.lock().map_err(|e| e.to_string())?; + match allowed.as_mut() { + Some(0) => return Err("the network went away".to_string()), + Some(left) => *left -= 1, + None => {} + } + self.blobs + .lock() + .map_err(|e| e.to_string())? + .insert(name, bytes); + Ok(()) + } + } + + fn get(&self, name: &str) -> impl Future, String>> + Send { + let name = name.to_string(); + async move { + self.blobs + .lock() + .map_err(|e| e.to_string())? + .get(&name) + .cloned() + .ok_or_else(|| format!("{name} is not in the store")) + } + } + + fn list(&self, prefix: &str) -> impl Future, String>> + Send { + let prefix = prefix.to_string(); + async move { + Ok(self + .blobs + .lock() + .map_err(|e| e.to_string())? + .keys() + .filter(|name| name.starts_with(&prefix)) + .cloned() + .collect()) + } + } +} + +const ACCOUNT: &str = "ana@example.com"; + +fn hash() -> String { + account_hash(ACCOUNT) +} + +fn key() -> Key { + Key::from_bytes([7u8; 32]) +} + +fn run(device: &Device, store: &Memory, key: &Key) -> Pass { + block_on(pass(device, store, key, &hash())).expect("a pass") +} + +fn ana() -> Person { + Person { + name: Some("Ana".to_string()), + address: ACCOUNT.to_string(), + } +} + +// -- what goes up ------------------------------------------------------------------------------ + +#[test] +fn what_goes_up_comes_back_byte_for_byte_after_decryption() { + let device = Device::new(); + let store = Memory::default(); + device.on(|conn| { + write::set_rule( + conn, + "landlord@example.com", + false, + Destination::Inbox, + None, + ) + .expect("a rule"); + write::add_note(conn, "thread-1", "the roof is leaking", None).expect("a note"); + }); + + let pass = run(&device, &store, &key()); + assert_eq!(pass.uploaded, 1); + assert_eq!(pass.downloaded, 0); + + let names = store.names(); + assert_eq!(names.len(), 1); + let expected = device.on(|conn| { + let id = device::device_id(conn).expect("device"); + merge::encode(&merge::export(conn, &id, 0).expect("export")).expect("encode") + }); + let plain = crypto::open(&key(), &names[0], &store.blob(&names[0])).expect("decrypt"); + assert_eq!(String::from_utf8(plain).expect("utf8"), expected); + + // And a second pass finds nothing new to send, because the store is what says what it holds. + assert_eq!(run(&device, &store, &key()).uploaded, 0); +} + +#[test] +fn the_store_holds_ciphertext_and_names_that_say_nothing() { + let device = Device::new(); + let store = Memory::default(); + device.on(|conn| { + write::set_rule(conn, "landlord@example.com", false, Destination::Feed, None) + .expect("a rule"); + write::add_note(conn, "thread-roof", "the roof is leaking", None).expect("a note"); + write::add_clip( + conn, + "thread-roof", + "message-1", + "the plumber comes on Thursday", + &ana(), + "About the roof", + ) + .expect("a clip"); + }); + run(&device, &store, &key()); + + let secrets = [ + "landlord@example.com", + "the roof is leaking", + "the plumber comes on Thursday", + "About the roof", + "thread-roof", + "note", + "clip", + "sender-rule", + ACCOUNT, + ]; + for name in store.names() { + let blob = store.blob(&name); + let text = String::from_utf8_lossy(&blob).to_string(); + for secret in secrets { + assert!( + !text.contains(secret), + "{secret} is readable in the segment at {name}" + ); + assert!(!name.contains(secret), "{secret} is readable in {name}"); + } + } +} + +#[test] +fn a_segment_moved_to_another_devices_folder_does_not_open() { + let device = Device::new(); + let store = Memory::default(); + device.on(|conn| write::set_pile(conn, "thread-1", Pile::ReplyLater).expect("a pile")); + run(&device, &store, &key()); + + let name = store.names().into_iter().next().expect("a segment"); + let blob = store.blob(&name); + assert!(crypto::open(&key(), &name, &blob).is_ok()); + + let moved = name.replace(&device.id(), "somebody-elses-device"); + assert!( + crypto::open(&key(), &moved, &blob).is_err(), + "a segment is authenticated under the name it was written to" + ); +} + +// -- two devices ------------------------------------------------------------------------------- + +/// The done check for this package. Two data directories, each with its own device id, each holding +/// decisions the other never saw, meeting only through a store, in both orders. +#[test] +fn two_devices_converge_on_identical_tables_in_either_order() { + for reversed in [false, true] { + let one = Device::new(); + let two = Device::new(); + let store = Memory::default(); + + one.on(|conn| { + write::set_rule( + conn, + "landlord@example.com", + false, + Destination::Inbox, + None, + ) + .expect("a rule"); + write::add_note(conn, "thread-roof", "the roof is leaking", None).expect("a note"); + write::set_pile(conn, "thread-roof", Pile::ReplyLater).expect("a pile"); + // The same key on both devices, decided at two different moments, so that convergence + // here means agreeing about a conflict rather than merely taking a union. + journal::append_at( + conn, + 5_000, + "thread-shared", + &Payload::Rename { + name: Some("what one called it".into()), + }, + ) + .expect("a rename"); + }); + two.on(|conn| { + write::set_rule( + conn, + "bank@example.com", + false, + Destination::PaperTrail, + None, + ) + .expect("a rule"); + write::set_snooze( + conn, + "thread-bill", + 9_000, + crate::dto::SnoozeKind::Tomorrow, + 0, + ) + .expect("a snooze"); + journal::append_at( + conn, + 9_000, + "thread-shared", + &Payload::Rename { + name: Some("what two called it".into()), + }, + ) + .expect("a rename"); + }); + + let (first, second) = if reversed { (&two, &one) } else { (&one, &two) }; + run(first, &store, &key()); + run(second, &store, &key()); + // The first device has not seen the second's yet: it uploaded before there was anything to + // fetch. One more pass each way is what "converged" means. + run(first, &store, &key()); + + assert_eq!( + one.snapshot(), + two.snapshot(), + "the two devices disagree, reversed = {reversed}" + ); + assert!(!one.snapshot().is_empty()); + // The later decision owns the shared key on both of them, whichever order they met in. + assert!(one + .snapshot() + .iter() + .any(|row| row.contains("what two called it"))); + } +} + +#[test] +fn a_device_that_missed_a_month_catches_up_in_one_pass() { + let store = Memory::default(); + let busy = Device::new(); + let kept_up = Device::new(); + let away = Device::new(); + + // Four weeks of decisions, with the device that was on backing up after each of them. + for week in 0..4 { + busy.on(|conn| { + write::set_rule( + conn, + &format!("sender-{week}@example.com"), + false, + Destination::Feed, + None, + ) + .expect("a rule"); + write::add_note(conn, &format!("thread-{week}"), "worth remembering", None) + .expect("a note"); + }); + run(&busy, &store, &key()); + run(&kept_up, &store, &key()); + } + + let pass = run(&away, &store, &key()); + assert_eq!(pass.downloaded, 4, "one pass, four segments, nothing else"); + assert_eq!(away.snapshot(), kept_up.snapshot()); + assert_eq!(away.snapshot(), busy.snapshot()); + + // And having caught up, it asks for nothing the next time. + assert_eq!(run(&away, &store, &key()).downloaded, 0); +} + +#[test] +fn a_second_device_attaches_with_the_phrase_and_reads_what_the_first_wrote() { + let written = phrase::generate().expect("a phrase"); + assert_eq!(written.split_whitespace().count(), phrase::WORDS); + let key = phrase::derive(&written).expect("the key"); + + let store = Memory::default(); + let first = Device::new(); + first.on(|conn| { + write::add_note(conn, "thread-roof", "the roof is leaking", None).expect("a note"); + write::set_rule( + conn, + "landlord@example.com", + false, + Destination::Inbox, + None, + ) + .expect("a rule"); + }); + run(&first, &store, &key); + + // The second device has the words off a piece of paper, typed the way somebody types. + let second = Device::new(); + let typed = format!(" {} ", written.to_uppercase()); + let second_key = phrase::derive(&typed).expect("the same key"); + run(&second, &store, &second_key); + + assert_eq!(second.snapshot(), first.snapshot()); + assert!(!second.snapshot().is_empty()); +} + +#[test] +fn a_wrong_phrase_fails_clearly_rather_than_producing_garbage() { + let store = Memory::default(); + let first = Device::new(); + let right = phrase::derive(&phrase::generate().expect("a phrase")).expect("a key"); + first.on(|conn| { + write::add_note(conn, "thread-roof", "the roof is leaking", None).expect("a note"); + }); + run(&first, &store, &right); + + let second = Device::new(); + let wrong = phrase::derive(&phrase::generate().expect("another phrase")).expect("a key"); + let refused = block_on(pass(&second, &store, &wrong, &hash())).expect_err("the wrong key"); + assert!( + refused.contains("recovery phrase"), + "the failure has to say what went wrong: {refused}" + ); + assert!( + second.snapshot().is_empty(), + "nothing half decrypted landed in the tables" + ); + + // A mistyped word is caught by the wordlist before any of that, and names itself. + let mistyped = phrase::normalise( + "abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon zzzz", + ); + assert!(mistyped + .expect_err("not a phrase") + .contains("recovery phrase")); +} + +// -- a pass that does not finish ----------------------------------------------------------------- + +#[test] +fn an_interrupted_upload_leaves_whole_segments_and_the_next_pass_carries_on() { + let store = Memory::default(); + let device = Device::new(); + // Two segments' worth, so there is a second put to refuse. + device.on(|conn| { + for at in 0..600 { + write::set_marker(conn, &format!("place-{at}"), at as i64).expect("a marker"); + } + }); + + store.breaks_after(1); + let cut_off = block_on(pass(&device, &store, &key(), &hash())).expect_err("the network went"); + assert!(cut_off.contains("network")); + + // What is on the store is one whole segment, not one and a half. + let names = store.names(); + assert_eq!(names.len(), 1); + let records = { + let plain = crypto::open(&key(), &names[0], &store.blob(&names[0])).expect("decrypt"); + merge::decode(&String::from_utf8(plain).expect("utf8")).expect("decode") + }; + assert_eq!(records.len(), super::SEGMENT_RECORDS); + assert_eq!(records.first().expect("first").seq, 1); + + store.mended(); + let mended = run(&device, &store, &key()); + assert_eq!( + mended.uploaded, 1, + "the segment that landed is not sent again" + ); + assert_eq!(store.names().len(), 2); + + let other = Device::new(); + run(&other, &store, &key()); + assert_eq!(other.snapshot(), device.snapshot()); +} + +// -- the names ----------------------------------------------------------------------------------- + +#[test] +fn an_account_folder_is_named_by_a_hash_rather_than_by_an_address() { + let hash = account_hash(ACCOUNT); + assert_eq!(hash.len(), 32); + assert!(!hash.contains('@')); + assert_eq!( + hash, + account_hash(" Ana@Example.COM "), + "one address, one folder, whatever the casing" + ); + assert_ne!(hash, account_hash("ana@example.org")); +} + +#[test] +fn a_segment_name_carries_a_range_and_nothing_else() { + let name = super::segment_name("abc", "device-1", 1, 500); + assert_eq!(name, "abc/device-1/000000000001-000000000500.seg"); + assert_eq!( + super::parse_name(&name), + Some(("device-1".to_string(), 1, 500)) + ); + // Sorting names sorts the segments, which is why the numbers are padded. + let mut names = vec![ + super::segment_name("abc", "device-1", 1001, 1500), + super::segment_name("abc", "device-1", 1, 500), + super::segment_name("abc", "device-1", 501, 1000), + ]; + names.sort(); + assert_eq!(names[0], super::segment_name("abc", "device-1", 1, 500)); + assert_eq!(names[2], super::segment_name("abc", "device-1", 1001, 1500)); + + assert_eq!(super::parse_name("abc/device-1/notes.txt"), None); + assert_eq!( + super::parse_name("abc/device-1/deeper/000000000001-000000000002.seg"), + None + ); +} + +// -- signature version four ------------------------------------------------------------------------ + +/// AWS publishes the signing key for these inputs, so this is the vector rather than a value read +/// off this implementation and pasted back in. +#[test] +fn the_signing_key_matches_the_published_derivation() { + let key = r2::signing_key( + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + "20120215", + "us-east-1", + "iam", + ); + let hex = key.iter().fold(String::new(), |mut out, byte| { + out.push_str(&format!("{byte:02x}")); + out + }); + assert_eq!( + hex, + "f4780e2d9f65fa895f9c67b32ce1baf0b0d8a43505a000a1a9e090d414db404d" + ); +} + +/// `get-vanilla` from AWS's own signature version four test suite, end to end through the same +/// function the uploads go through. +#[test] +fn the_authorization_header_matches_the_published_test_suite() { + let header = r2::authorization( + "AKIDEXAMPLE", + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + "us-east-1", + "service", + "GET", + "/", + "", + &[ + ("host", "example.amazonaws.com".to_string()), + ("x-amz-date", "20150830T123600Z".to_string()), + ], + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "20150830T123600Z", + ); + assert_eq!( + header, + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, \ + SignedHeaders=host;x-amz-date, \ + Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31" + ); +} + +#[test] +fn a_bucket_listing_is_read_out_of_the_xml_it_arrives_in() { + let body = "false\ + abc/device-1/000000000001-000000000500.seg10\ + abc/device-2/000000000001-000000000004.seg"; + assert_eq!( + r2::tags(body, "Key"), + vec![ + "abc/device-1/000000000001-000000000500.seg".to_string(), + "abc/device-2/000000000001-000000000004.seg".to_string() + ] + ); + assert!(r2::tags(body, "NextContinuationToken").is_empty()); +} + +#[test] +fn the_four_fields_are_checked_while_the_person_still_has_the_form_open() { + let fields = |pairs: &[(&str, &str)]| { + pairs + .iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect::>() + }; + + let good = r2::Config::from_fields(&fields(&[ + ("endpoint", "https://example.r2.cloudflarestorage.com/"), + ("bucket", "margin"), + ("accessKey", "key"), + ("secret", "shh"), + ])) + .expect("four fields"); + assert_eq!(good.endpoint, "https://example.r2.cloudflarestorage.com"); + + assert!(r2::Config::from_fields(&fields(&[ + ("endpoint", "http://example.com"), + ("bucket", "margin"), + ("accessKey", "key"), + ("secret", "shh") + ])) + .is_err()); + assert!(r2::Config::from_fields(&fields(&[ + ("endpoint", "https://example.com"), + ("bucket", "margin"), + ("accessKey", "key") + ])) + .expect_err("no secret") + .contains("secret")); +} diff --git a/src-tauri/src/badge.rs b/src-tauri/src/badge.rs new file mode 100644 index 0000000..1e05cd6 --- /dev/null +++ b/src-tauri/src/badge.rs @@ -0,0 +1,122 @@ +// The number on the dock icon. +// +// One number, one icon, every account summed, and it is the count of unseen threads in the Inbox +// rather than the mailbox's unread count. Mailspring shows 999+ on a Gmail account because it +// counts every unread message carrying the `INBOX` label, and that number is exactly the noise +// this app was written to remove: what it is supposed to say is how many things are actually +// waiting for a decision. +// +// The count itself is `mirror::read::inbox_unseen`, which is the list's own definition of the +// group counted rather than selected. Nothing in this file knows what the Inbox is, on purpose. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use tauri::Manager; + +use crate::db::Db; +use crate::mirror::read; + +/// Long enough for a sync pass to finish landing what it landed, short enough that nobody sees the +/// lag on a number they read with their eyes. +const SETTLE_MS: u64 = 200; + +/// True while a recount is already on its way, so a burst of changes costs one query rather than +/// one each. +static QUEUED: AtomicBool = AtomicBool::new(false); + +/// macOS and Linux, and nowhere else. +/// +/// `set_badge_count` compiles everywhere, so this is about honesty rather than about the build: on +/// Windows the call does nothing and the taskbar wants `set_overlay_icon` with an image drawn for +/// it, which is not written here, and Android has no badge at all. A platform with no badge to +/// paint has no reason to run the count either, so this gates the whole of it rather than the last +/// line of it. +const CARRIES_A_BADGE: bool = cfg!(any(target_os = "macos", target_os = "linux")); + +/// Which `store-changed` scopes can move the number. +/// +/// `emit_store_changed` fires for everything the app does and most of it cannot touch this count. A +/// body landing is scoped `thread`, a clip is `state`, a drained outbox is `outbox`, and none of +/// the three adds or removes a thread from New for you. Reading the scope here is what keeps the +/// badge a query per real change rather than a query per event. +/// +/// `settings` is in the list because the toggle itself has to be obeyed at once, and `accounts` +/// because an account added or removed changes what there is to sum over. +pub fn moves_the_count(reason: &str) -> bool { + reason + .split_whitespace() + .any(|scope| matches!(scope, "threads" | "accounts" | "settings")) +} + +/// What the dock should show, which is nothing at all rather than a `0`. +/// +/// Separate from the call that puts it there so that the rule can be tested without a running app, +/// and because `Some(0)` and `None` mean the same thing to the platform while only one of them is +/// what is meant here. +pub fn to_show(on: bool, count: i64) -> Option { + (on && count > 0).then_some(count) +} + +/// Every account's New for you, added up. +pub fn count(db: &Db) -> Result { + let mut total = 0; + for account_id in db.on_disk() { + total += db.with(&account_id, read::inbox_unseen)?; + } + Ok(total) +} + +/// The hook on the one notification path there is. `lib.rs` calls this from `emit_store_changed`, +/// so the badge follows the same signal the frontend does rather than needing its own. +pub fn on_store_changed(app: &tauri::AppHandle, reason: &str) { + if moves_the_count(reason) { + refresh(app); + } +} + +/// Recounts and repaints, once, shortly. +/// +/// Deferred rather than done here because a first sync emits as it goes and thirty-nine counts on +/// the way to the fortieth are thirty-nine wasted, and because the emit sits on the return path of +/// whatever command made the change, which has no business waiting on a query nobody asked it for. +pub fn refresh(app: &tauri::AppHandle) { + if !CARRIES_A_BADGE || QUEUED.swap(true, Ordering::SeqCst) { + return; + } + let app = app.clone(); + tauri::async_runtime::spawn(async move { + tokio::time::sleep(Duration::from_millis(SETTLE_MS)).await; + QUEUED.store(false, Ordering::SeqCst); + apply(&app); + }); +} + +fn apply(app: &tauri::AppHandle) { + let on = crate::settings::load(app) + .map(|settings| settings.badge) + .unwrap_or(false); + + // Turning it off is a clear, and it does not need the count to know that. + if !on { + show(app, None); + return; + } + let Some(db) = app.try_state::() else { + return; + }; + // A badge cleared because a query failed reads as "nothing is waiting", which is the one thing + // it must never say wrongly. Leave what is on the dock and try again at the next change. + if let Ok(count) = count(db.inner()) { + show(app, to_show(true, count)); + } +} + +/// The window rather than the app, because that is where tauri hangs the call. On macOS it reaches +/// the dock tile, which is the app's and not the window's, so a window that has been closed to the +/// menu bar still carries the right number. +fn show(app: &tauri::AppHandle, count: Option) { + if let Some(window) = app.get_webview_window("main") { + let _ = window.set_badge_count(count); + } +} diff --git a/src-tauri/src/clips.rs b/src-tauri/src/clips.rs new file mode 100644 index 0000000..6cfc047 --- /dev/null +++ b/src-tauri/src/clips.rs @@ -0,0 +1,378 @@ +// Clips, and the All files place. +// +// Two libraries built from what is already on the device. A clip is a sentence somebody kept, held +// in the state database with the thread and the sender it came from, so it survives the mirror +// being thrown away. All files is the other way round: it is entirely derived, it fetches nothing, +// and opening a card is what puts a byte on the network. +// +// The exclusion in `files_list` is the difference between a library and a wall of logos. Every +// newsletter carries a signature image, every one of them is inline and a few kilobytes, and a +// files place that lists them buries the contract somebody is looking for under two hundred of +// them. + +use rusqlite::{params_from_iter, types::Value, Connection, OptionalExtension}; + +use crate::decisions::{account_holding, db_of}; +use crate::dto::{Attachment, Clip, FileCard, Person, Undo}; +use crate::state::{self, journal::Payload}; +use crate::sync; +use crate::undo::Stack; + +static UNDO: Stack = Stack::new("clip"); + +/// Enough that nobody's library is truncated, and a ceiling rather than none at all. +const MAX_CLIPS: u32 = 5_000; + +/// Signature junk, as `docs/features.md` section 10 defines it: an image under ten kilobytes that +/// the body refers to rather than lists. +const SIGNATURE_BYTES: u64 = 10 * 1024; + +// --------------------------------------------------------------------------------------------- +// Clips +// --------------------------------------------------------------------------------------------- + +fn clip_by_id(conn: &Connection, account_id: &str, id: &str) -> Result, String> { + Ok(state::read::clips(conn, account_id, MAX_CLIPS)? + .into_iter() + .find(|clip| clip.id == id)) +} + +/// Who wrote the message the clip was taken from, and what the thread is called. Kept on the clip +/// rather than looked up later, because a clip outlives the message: the window moves on and the +/// card still has to say where the words came from. +fn provenance(conn: &Connection, message_id: &str) -> Result<(Person, String), String> { + conn.query_row( + "SELECT m.from_name, m.from_address, COALESCE(rn.name, m.subject) + FROM messages m + LEFT JOIN state.renames rn ON rn.thread_key = m.thread_key + WHERE m.id = ?1", + [message_id], + |row| { + Ok(( + Person { + name: row.get(0)?, + address: row.get(1)?, + }, + row.get(2)?, + )) + }, + ) + .optional() + .map_err(|e| e.to_string())? + .ok_or_else(|| "that message is not on this device".to_string()) +} + +pub fn save( + conn: &Connection, + account_id: &str, + thread_key: &str, + message_id: &str, + text: &str, +) -> Result { + let text = text.trim(); + if text.is_empty() { + return Err("a clip needs some words in it".into()); + } + let (sender, subject) = provenance(conn, message_id)?; + let id = state::write::add_clip(conn, thread_key, message_id, text, &sender, &subject)?; + clip_by_id(conn, account_id, &id)?.ok_or_else(|| "that clip did not land".to_string()) +} + +/// Puts a deleted clip back. Deletion is a flag rather than a missing row so that two devices +/// resolve it the same way, and that is what makes this possible. +fn restore(conn: &Connection, clip: &Clip) -> Result<(), String> { + state::journal::append( + conn, + &clip.id, + &Payload::Clip { + thread_key: clip.thread_key.clone(), + message_id: clip.message_id.clone(), + text: clip.text.clone(), + sender_name: clip.sender.name.clone(), + sender_address: clip.sender.address.clone(), + subject: clip.subject.clone(), + created_at: clip.created_at_ms, + deleted: false, + }, + ) + .map(|_| ()) +} + +// --------------------------------------------------------------------------------------------- +// All files +// --------------------------------------------------------------------------------------------- + +fn extension(filename: &str) -> String { + filename + .rsplit_once('.') + .map(|(_, ext)| ext.to_lowercase()) + .unwrap_or_default() +} + +/// The eight buckets from section 10, decided from the MIME type first and the extension second. +/// +/// Both, because neither is reliable on its own: half the world sends a spreadsheet as +/// `application/octet-stream`, and a file called `notes` with `text/calendar` on it is an invite. +/// The order matters in one place, which is that an invite is checked before a document, because +/// `text/calendar` would otherwise be caught as text. +pub fn category_of(mime_type: &str, filename: &str) -> &'static str { + let mime = mime_type.trim().to_lowercase(); + let mime = mime.split(';').next().unwrap_or("").trim(); + let ext = extension(filename); + + if mime == "text/calendar" || mime == "application/ics" || ext == "ics" || ext == "ical" { + return "invites"; + } + if mime == "application/pdf" || ext == "pdf" { + return "pdfs"; + } + if mime.starts_with("image/") + || matches!( + ext.as_str(), + "png" | "jpg" | "jpeg" | "gif" | "webp" | "heic" | "bmp" | "tiff" | "svg" | "avif" + ) + { + return "images"; + } + if SPREADSHEET_TYPES.contains(&mime) + || matches!(ext.as_str(), "xls" | "xlsx" | "xlsm" | "ods" | "csv" | "tsv" | "numbers") + { + return "spreadsheets"; + } + if PRESENTATION_TYPES.contains(&mime) + || matches!(ext.as_str(), "ppt" | "pptx" | "odp" | "key") + { + return "presentations"; + } + if ARCHIVE_TYPES.contains(&mime) + || matches!(ext.as_str(), "zip" | "tar" | "gz" | "tgz" | "bz2" | "xz" | "7z" | "rar") + { + return "archives"; + } + if DOCUMENT_TYPES.contains(&mime) + || mime.starts_with("text/") + || matches!(ext.as_str(), "doc" | "docx" | "odt" | "rtf" | "txt" | "md" | "pages" | "epub") + { + return "documents"; + } + "other" +} + +const SPREADSHEET_TYPES: [&str; 4] = [ + "application/vnd.ms-excel", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.oasis.opendocument.spreadsheet", + "text/csv", +]; + +const PRESENTATION_TYPES: [&str; 3] = [ + "application/vnd.ms-powerpoint", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "application/vnd.oasis.opendocument.presentation", +]; + +const ARCHIVE_TYPES: [&str; 8] = [ + "application/zip", + "application/x-zip-compressed", + "application/x-tar", + "application/gzip", + "application/x-gzip", + "application/x-7z-compressed", + "application/vnd.rar", + "application/x-rar-compressed", +]; + +const DOCUMENT_TYPES: [&str; 4] = [ + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.oasis.opendocument.text", + "application/rtf", +]; + +/// True for the images a signature block hangs off the bottom of a message. +fn signature_junk(attachment: &Attachment, category: &str) -> bool { + category == "images" && attachment.inline && attachment.size < SIGNATURE_BYTES +} + +/// Every file on the device as a card, newest first, filtered by type and by sender. +/// +/// Trash and spam are left out. The place is a library and the mail is still there to be found in +/// Trash; a library that lists the attachments of deleted mail is a library nobody trusts. +pub fn files(conn: &Connection, category: &str, sender: &str) -> Result, String> { + let chain = state::read::CHAIN; + let sql = format!( + "{chain} + SELECT a.id, a.message_id, a.filename, a.mime_type, a.size, a.inline, a.content_id, + a.cached_path IS NOT NULL AS cached, + COALESCE((SELECT key FROM chain + WHERE source = t.thread_key + AND key NOT IN (SELECT thread_key FROM state.merges) LIMIT 1), + t.thread_key) AS thread_key, + COALESCE(rn.name, t.subject) AS subject, + m.from_name AS from_name, m.from_address AS from_address, m.date_ms AS date_ms + FROM attachments a + JOIN messages m ON m.id = a.message_id + JOIN threads t ON t.provider_thread_id = m.provider_thread_id + LEFT JOIN state.renames rn ON rn.thread_key = t.thread_key + WHERE t.trashed = 0 AND t.spam = 0 + AND (? = '' OR lower(m.from_address) = ?) + ORDER BY m.date_ms DESC, a.id ASC" + ); + let sender = sender.trim().to_lowercase(); + let params = vec![Value::Text(sender.clone()), Value::Text(sender)]; + + let mut stmt = conn.prepare(&sql).map_err(|e| e.to_string())?; + let rows = stmt + .query_map(params_from_iter(params), |row| { + Ok(( + Attachment { + id: row.get("id")?, + message_id: row.get("message_id")?, + filename: row.get("filename")?, + mime_type: row.get("mime_type")?, + size: row.get::<_, i64>("size")?.max(0) as u64, + inline: row.get::<_, i64>("inline")? != 0, + content_id: row.get("content_id")?, + cached: row.get::<_, i64>("cached")? != 0, + }, + row.get::<_, String>("thread_key")?, + row.get::<_, String>("subject")?, + Person { + name: row.get("from_name")?, + address: row.get("from_address")?, + }, + row.get::<_, i64>("date_ms")?, + )) + }) + .map_err(|e| e.to_string())? + .collect::, _>>() + .map_err(|e| e.to_string())?; + + let wanted = category.trim(); + let mut out = Vec::new(); + for (attachment, thread_key, subject, sender, date_ms) in rows { + let category = category_of(&attachment.mime_type, &attachment.filename); + if signature_junk(&attachment, category) { + continue; + } + if !wanted.is_empty() && wanted != category { + continue; + } + out.push(FileCard { + attachment, + thread_key, + subject, + sender, + date_ms, + category: category.to_string(), + }); + } + Ok(out) +} + +// --------------------------------------------------------------------------------------------- +// Undo +// --------------------------------------------------------------------------------------------- + +pub struct UndoClip { + pub account_id: String, + pub clip: Clip, +} + +pub fn owns(token: &str) -> bool { + UNDO.owns(token) +} + +pub fn undo_apply(app: &tauri::AppHandle, token: &str) -> Result<(), String> { + let entry = UNDO + .take(token) + .ok_or("that clip can no longer be brought back")?; + let db = db_of(app)?; + db.with(&entry.account_id, |conn| restore(conn, &entry.clip))?; + crate::emit_store_changed(app, "state"); + Ok(()) +} + +// --------------------------------------------------------------------------------------------- +// Commands +// --------------------------------------------------------------------------------------------- + +#[tauri::command(async)] +pub fn clip_save( + app: tauri::AppHandle, + account_id: String, + thread_key: String, + message_id: String, + text: String, +) -> Result { + let db = db_of(&app)?; + let clip = db.with(&account_id, |conn| { + save(conn, &account_id, &thread_key, &message_id, &text) + })?; + crate::emit_store_changed(&app, "state"); + Ok(clip) +} + +#[tauri::command(async)] +pub fn clips_list( + app: tauri::AppHandle, + account_id: Option, +) -> Result, String> { + let db = db_of(&app)?; + let mut all = Vec::new(); + for (id, _) in sync::accounts(db.inner(), account_id.as_deref()) { + all.extend(db.with(&id, |conn| state::read::clips(conn, &id, MAX_CLIPS))?); + } + all.sort_by(|a, b| b.created_at_ms.cmp(&a.created_at_ms).then(b.id.cmp(&a.id))); + Ok(all) +} + +#[tauri::command(async)] +pub fn clip_delete(app: tauri::AppHandle, id: String) -> Result { + let db = db_of(&app)?; + let account_id = account_holding( + db.inner(), + "SELECT COUNT(*) FROM state.clips WHERE id = ?1 AND deleted = 0", + &id, + ) + .map_err(|_| "that clip is not here".to_string())?; + let clip = db.with(&account_id, |conn| { + let held = clip_by_id(conn, &account_id, &id)? + .ok_or_else(|| "that clip is not here".to_string())?; + state::write::delete_clip(conn, &id)?; + Ok(held) + })?; + crate::emit_store_changed(&app, "state"); + + let token = UNDO.push(UndoClip { account_id, clip }, "Clip deleted"); + Ok(Undo { + token, + label: "Clip deleted".to_string(), + undo_ms: 0, + }) +} + +/// The All files place. Built from the local index and fetches nothing: the card knows the name, +/// the type, the size, the sender and the thread without a byte leaving the machine. +#[tauri::command(async)] +pub fn files_list( + app: tauri::AppHandle, + account_id: Option, + category: String, + sender: String, +) -> Result, String> { + let db = db_of(&app)?; + let mut all = Vec::new(); + for (id, _) in sync::accounts(db.inner(), account_id.as_deref()) { + all.extend(db.with(&id, |conn| files(conn, &category, &sender))?); + } + all.sort_by(|a, b| { + b.date_ms + .cmp(&a.date_ms) + .then(a.attachment.id.cmp(&b.attachment.id)) + }); + Ok(all) +} + +#[cfg(test)] +mod tests; diff --git a/src-tauri/src/clips/tests.rs b/src-tauri/src/clips/tests.rs new file mode 100644 index 0000000..5e5d642 --- /dev/null +++ b/src-tauri/src/clips/tests.rs @@ -0,0 +1,250 @@ +// Clips and All files, over a pair of in-memory databases. Nothing here fetches anything, which is +// the point of the place: the cards are built from the index the sync already wrote. + +use rusqlite::Connection; + +use crate::db; +use crate::state; + +fn open() -> Connection { + db::memory().expect("a pair of in-memory databases") +} + +/// One file, and the message and thread it hangs off. A struct rather than nine arguments, which +/// is what `screener::tests` does with a sender for the same reason. +struct File<'a> { + id: &'a str, + address: &'a str, + subject: &'a str, + at: i64, + filename: &'a str, + mime_type: &'a str, + size: i64, + /// Referenced from the body by `cid:` rather than listed as a chip. + inline: bool, +} + +fn with_file(conn: &Connection, file: &File<'_>) -> String { + let File { + id, + address, + subject, + at, + filename, + mime_type, + size, + inline, + } = *file; + let key = format!("<{id}@example>"); + let tid = format!("t-{id}"); + conn.execute( + "INSERT INTO threads (provider_thread_id, thread_key, latest_ms, message_count, unseen, + subject, snippet, from_name, from_address, in_inbox, has_attachment) + VALUES (?1, ?2, ?3, 1, 0, ?4, 'snippet', 'Someone', ?5, 1, 1)", + rusqlite::params![tid, key, at, subject, address], + ) + .expect("a thread"); + conn.execute( + "INSERT INTO messages (id, provider_thread_id, thread_key, message_id, date_ms, + from_name, from_address, subject, snippet, hydrated, + has_attachment, labels) + VALUES (?1, ?2, ?3, ?3, ?4, 'Someone', ?5, ?6, 'snippet', 1, 1, '[\"INBOX\"]')", + rusqlite::params![format!("m-{id}"), tid, key, at, address, subject], + ) + .expect("a message"); + conn.execute( + "INSERT INTO attachments (id, message_id, part_id, filename, mime_type, size, inline) + VALUES (?1, ?2, '2', ?3, ?4, ?5, ?6)", + rusqlite::params![ + format!("a-{id}"), + format!("m-{id}"), + filename, + mime_type, + size, + inline as i64 + ], + ) + .expect("a file"); + key +} + +fn pdf<'a>(id: &'a str, address: &'a str, subject: &'a str, at: i64) -> File<'a> { + File { + id, + address, + subject, + at, + filename: "quote.pdf", + mime_type: "application/pdf", + size: 90_000, + inline: false, + } +} + +fn image<'a>( + id: &'a str, + address: &'a str, + subject: &'a str, + at: i64, + filename: &'a str, + size: i64, + inline: bool, +) -> File<'a> { + File { + id, + address, + subject, + at, + filename, + mime_type: "image/png", + size, + inline, + } +} + +// --------------------------------------------------------------------------------------------- +// Clips +// --------------------------------------------------------------------------------------------- + +#[test] +fn a_clip_keeps_the_words_and_where_they_came_from() { + let conn = open(); + let key = with_file(&conn, &pdf("one", "maya@example.org", "The kitchen quote", 300)); + + let clip = super::save(&conn, "acct", &key, "m-one", " the price holds until March ") + .expect("a clip"); + assert_eq!(clip.text, "the price holds until March"); + assert_eq!(clip.sender.address, "maya@example.org"); + assert_eq!(clip.subject, "The kitchen quote"); + assert_eq!(clip.thread_key, key); + + assert_eq!( + state::read::clips(&conn, "acct", 50).expect("the clips").len(), + 1 + ); + + state::write::delete_clip(&conn, &clip.id).expect("deleted"); + assert!(state::read::clips(&conn, "acct", 50).expect("the clips").is_empty()); + + super::restore(&conn, &clip).expect("put back"); + let held = state::read::clips(&conn, "acct", 50).expect("the clips"); + assert_eq!(held.len(), 1); + assert_eq!(held[0].text, "the price holds until March"); +} + +// --------------------------------------------------------------------------------------------- +// All files +// --------------------------------------------------------------------------------------------- + +#[test] +fn a_file_is_categorised_from_its_type_and_its_extension() { + for (mime_type, filename, expected) in [ + ("application/pdf", "quote.pdf", "pdfs"), + ("image/png", "plan.png", "images"), + ("text/calendar", "invite.ics", "invites"), + ( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "costs.xlsx", + "spreadsheets", + ), + ("application/vnd.ms-powerpoint", "deck.ppt", "presentations"), + ("application/msword", "letter.doc", "documents"), + ("application/zip", "photos.zip", "archives"), + ("application/octet-stream", "notes.md", "documents"), + // The type is nothing and the extension is everything, which is most of what arrives. + ("application/octet-stream", "costs.csv", "spreadsheets"), + ("application/octet-stream", "backup.tar.gz", "archives"), + ("application/octet-stream", "firmware.bin", "other"), + ] { + assert_eq!( + super::category_of(mime_type, filename), + expected, + "{filename} as {mime_type}" + ); + } + // A charset on the header does not make it a different type. + assert_eq!( + super::category_of("text/calendar; charset=utf-8", "invite"), + "invites" + ); +} + +#[test] +fn the_library_lists_every_file_newest_first_and_filters_by_type_and_sender() { + let conn = open(); + with_file(&conn, &pdf("pdf", "maya@example.org", "Quote", 300)); + with_file(&conn, &image("img", "ana@example.org", "Plans", 200, "plan.png", 400_000, false)); + with_file( + &conn, + &File { + id: "csv", + address: "maya@example.org", + subject: "Costs", + at: 100, + filename: "costs.csv", + mime_type: "text/csv", + size: 2_000, + inline: false, + }, + ); + + let all = super::files(&conn, "", "").expect("the library"); + assert_eq!(all.len(), 3); + assert_eq!(all[0].attachment.filename, "quote.pdf"); + assert_eq!(all[0].category, "pdfs"); + assert_eq!(all[0].sender.address, "maya@example.org"); + assert_eq!(all[0].subject, "Quote"); + assert_eq!(all[2].attachment.filename, "costs.csv"); + + let images = super::files(&conn, "images", "").expect("the images"); + assert_eq!(images.len(), 1); + assert_eq!(images[0].attachment.filename, "plan.png"); + + let hers = super::files(&conn, "", "MAYA@example.org").expect("one sender"); + assert_eq!(hers.len(), 2); + assert!(hers.iter().all(|card| card.sender.address == "maya@example.org")); +} + +#[test] +fn a_signature_image_is_not_in_the_library() { + let conn = open(); + with_file(&conn, &image("sig", "news@brand.example", "The week", 300, "logo.png", 6_144, true)); + with_file(&conn, &image("real", "ana@example.org", "Plans", 200, "plan.png", 400_000, false)); + // Inline and an image, but far too big to be a signature: an embedded photograph is a file. + with_file(&conn, &image("big", "ana@example.org", "Photo", 100, "photo.png", 60_000, true)); + + let images = super::files(&conn, "images", "").expect("the images"); + let names: Vec<&str> = images + .iter() + .map(|card| card.attachment.filename.as_str()) + .collect(); + assert_eq!(names, vec!["plan.png", "photo.png"]); +} + +#[test] +fn the_library_leaves_out_trash_and_spam() { + let conn = open(); + let key = with_file(&conn, &pdf("gone", "maya@example.org", "Quote", 300)); + conn.execute("UPDATE threads SET trashed = 1 WHERE thread_key = ?1", [&key]) + .expect("trashed"); + + assert!(super::files(&conn, "", "").expect("the library").is_empty()); +} + +#[test] +fn a_file_on_a_merged_thread_opens_the_thread_it_shows_under() { + let conn = open(); + let merged = with_file(&conn, &pdf("one", "maya@example.org", "Quote", 300)); + let source = with_file( + &conn, + &image("two", "ana@example.org", "Re: quote", 200, "plan.png", 400_000, false), + ); + state::write::merge_threads(&conn, std::slice::from_ref(&source), &merged).expect("a merge"); + + let cards = super::files(&conn, "", "").expect("the library"); + assert_eq!(cards.len(), 2); + assert!( + cards.iter().all(|card| card.thread_key == merged), + "a card opens the thread the list shows, not the one the message arrived in" + ); +} diff --git a/src-tauri/src/contacts.rs b/src-tauri/src/contacts.rs new file mode 100644 index 0000000..1d90e56 --- /dev/null +++ b/src-tauri/src/contacts.rs @@ -0,0 +1,565 @@ +// The person behind an address: where their mail delivers, whether it notifies, the private note, +// and the threads and the files they have sent. +// +// Every field of the card is a journalled event through `state::write`, so a decision made here +// roams with the state database and survives the mirror being thrown away. The destination is the +// one that looks different and is not: it is a sender rule, so it goes through the same +// `screener::decide` the Screener's own keys use, and it therefore applies to the sender's existing +// threads at once. A move that only affected future mail would read as a move that did not work. +// +// The card is also the only way to reverse a screening decision, which is why screening somebody +// out is one of the four destinations rather than a verb of its own. + +use rusqlite::{Connection, OptionalExtension}; +use tauri::Manager; + +use crate::db::Db; +use crate::dto::{ + Attachment, ContactCard, ContactPatch, Destination, Person, Place, SenderRule, ThreadQuery, + ThreadSummary, Unsubscribe, +}; +use crate::mirror; +use crate::routing; +use crate::screener; +use crate::state; +use crate::sync; + +/// As many as the popover has room for without becoming a list of its own. +const RECENT_THREADS: u32 = 5; +const RECENT_FILES: usize = 6; +const SUGGESTIONS: usize = 8; + +// --------------------------------------------------------------------------------------------- +// Reading one person +// --------------------------------------------------------------------------------------------- + +/// The name to print. The state database holds decisions about an address and never a name, because +/// a name is something the mail said rather than something the person chose, so this is the +/// mirror's answer: the address book's spelling first, then the latest message's. +fn name_of(conn: &Connection, address: &str) -> Result, String> { + let found: Option = conn + .query_row( + "SELECT name FROM correspondents WHERE lower(address) = ?1 AND name IS NOT NULL", + [address], + |row| row.get(0), + ) + .optional() + .map_err(|e| e.to_string())?; + if let Some(name) = found.filter(|name: &String| !name.trim().is_empty()) { + return Ok(Some(name)); + } + let latest: Option = conn + .query_row( + "SELECT from_name FROM messages + WHERE lower(from_address) = ?1 AND from_name IS NOT NULL AND from_name <> '' + ORDER BY date_ms DESC LIMIT 1", + [address], + |row| row.get(0), + ) + .optional() + .map_err(|e| e.to_string())?; + Ok(latest.filter(|name| !name.trim().is_empty())) +} + +/// The sender's recent threads, as a list page rather than as a second query. +/// +/// `from:` is the search index's own operator, so this is the query the search bar would run and +/// the row that comes back is the row a list draws. The exact address is checked again afterwards +/// because the index is tokenised: a phrase match over `ana example org` would also accept +/// `ana@example.org.uk`, and a card showing somebody else's mail is worse than a card showing less. +fn recent( + conn: &Connection, + account_id: &str, + account_color: &str, + address: &str, + now: i64, +) -> Result, String> { + let query = ThreadQuery { + account_id: Some(account_id.to_string()), + place: Place::Search, + label_id: None, + query: Some(format!("from:{address}")), + // Room to drop the near misses the index let through and still fill the card. + limit: RECENT_THREADS * 4, + cursor: None, + }; + let page = mirror::read::threads_list(conn, account_id, account_color, &query, now)?; + let wanted = address.trim().to_lowercase(); + Ok(page + .threads + .into_iter() + .filter(|thread| { + thread.from.address.to_lowercase() == wanted + || thread + .participants + .iter() + .any(|person| person.address.to_lowercase() == wanted) + }) + .map(|mut thread| { + // A group head is a list's answer to "where am I", and a card is not a place: these + // threads sit under the card's own Recent threads heading, newest first, with nothing + // between them. The empty group is what a view with no grouping already means, and it + // is what a list renderer already draws no head for. + thread.group = String::new(); + thread + }) + .take(RECENT_THREADS as usize) + .collect()) +} + +/// What this sender has attached, newest first. Inline parts are the body's own images rather than +/// files somebody sent, so they are not files. +fn files(conn: &Connection, address: &str, limit: usize) -> Result, String> { + let mut stmt = conn + .prepare( + "SELECT a.id, a.message_id, a.filename, a.mime_type, a.size, a.inline, a.content_id, + a.cached_path IS NOT NULL + FROM attachments a JOIN messages m ON m.id = a.message_id + WHERE lower(m.from_address) = ?1 AND a.inline = 0 AND a.filename <> '' + ORDER BY m.date_ms DESC, a.filename ASC + LIMIT ?2", + ) + .map_err(|e| e.to_string())?; + let rows = stmt + .query_map(rusqlite::params![address, limit as i64], |row| { + Ok(Attachment { + id: row.get(0)?, + message_id: row.get(1)?, + filename: row.get(2)?, + mime_type: row.get(3)?, + size: row.get::<_, i64>(4)? as u64, + inline: row.get::<_, i64>(5)? != 0, + content_id: row.get(6)?, + cached: row.get::<_, i64>(7)? != 0, + }) + }) + .map_err(|e| e.to_string())?; + rows.collect::, _>>() + .map_err(|e| e.to_string()) +} + +/// `List-Unsubscribe` in the three forms it arrives in, from the latest message that carried one. +/// RFC 8058's one-click is the header pair, and without the `POST` half the URL is a page to visit +/// rather than a button to press. +fn unsubscribe(conn: &Connection, address: &str) -> Result, String> { + let held: Option<(String, Option)> = conn + .query_row( + "SELECT list_unsubscribe, list_unsub_post FROM messages + WHERE lower(from_address) = ?1 AND list_unsubscribe IS NOT NULL + AND list_unsubscribe <> '' + ORDER BY date_ms DESC LIMIT 1", + [address], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional() + .map_err(|e| e.to_string())?; + let Some((header, post)) = held else { + return Ok(None); + }; + let mut mailto = None; + let mut url = None; + for part in header.split(',') { + let value = part + .trim() + .trim_start_matches('<') + .trim_end_matches('>') + .trim(); + if value.starts_with("mailto:") { + mailto = Some(value.to_string()); + } else if value.starts_with("http") { + url = Some(value.to_string()); + } + } + if mailto.is_none() && url.is_none() { + return Ok(None); + } + Ok(Some(Unsubscribe { + one_click: url.is_some() + && post + .map(|value| value.to_ascii_lowercase().contains("one-click")) + .unwrap_or(false), + mailto, + url, + })) +} + +/// Where this sender's mail goes, and when that was decided. +/// +/// Nobody has to have been decided about: a card can be opened on somebody still waiting in the +/// Screener, and it says what would happen to them rather than nothing at all. That is the +/// suggestion function's answer, which is the same sentence the Screener card would print. +fn routed(conn: &Connection, address: &str) -> Result<(Destination, bool, Option), String> { + if let Some(rule) = state::read::rule_for(conn, "", address)? { + return Ok((rule.destination, rule.is_domain, Some(rule.decided_at_ms))); + } + let facts = routing::facts_for_sender(conn, address)?.unwrap_or_default(); + Ok((routing::suggest(&facts).destination, false, None)) +} + +/// The whole card for one person, on one account. +pub fn card( + conn: &Connection, + account_id: &str, + account_color: &str, + address: &str, + now: i64, +) -> Result { + let address = address.trim().to_lowercase(); + if address.is_empty() { + return Err("a contact card needs an address".into()); + } + let held = state::read::contact(conn, &address)? + .unwrap_or_else(|| state::read::Contact::unknown(&address)); + let (destination, domain_rule, screened_at_ms) = routed(conn, &address)?; + Ok(ContactCard { + person: Person { + name: name_of(conn, &address)?, + address: address.clone(), + }, + account_id: account_id.to_string(), + destination, + domain_rule, + domain_rule_allowed: domain_rule_allowed(&address), + notify: held.notify, + screened_at_ms, + note: held.note, + allow_remote_images: held.allow_remote_images, + auto_trash_days: held.auto_trash_days, + bundle: held.bundle, + recent_threads: recent(conn, account_id, account_color, &address, now)?, + files: files(conn, &address, RECENT_FILES)?, + unsubscribe: unsubscribe(conn, &address)?, + }) +} + +/// Whether "everyone at this domain" is a group of any kind. Consumer domains are not, which is why +/// the toggle is not offered rather than offered and then refused. +fn domain_rule_allowed(address: &str) -> bool { + match state::write::domain_of(address) { + Some(domain) => !state::write::is_consumer_domain(&domain), + None => false, + } +} + +// --------------------------------------------------------------------------------------------- +// Writing +// --------------------------------------------------------------------------------------------- + +/// Where a sender's mail delivers, by address or by their whole domain. +/// +/// Turning the domain toggle on has to take the address rule away first, because an address rule +/// beats a domain rule and the toggle would otherwise read as off the moment the card was asked +/// again. Turning it off does the opposite and leaves the domain rule standing: it still decides +/// everyone else at that domain, and quietly returning all of them to the Screener is a much larger +/// change than the one this card offered. +fn set_destination( + conn: &Connection, + address: &str, + destination: Destination, + whole_domain: bool, +) -> Result<(), String> { + if whole_domain { + let domain = state::write::domain_of(address) + .ok_or_else(|| format!("{address} has no domain to set a rule for"))?; + // `set_rule` owns the refusal, so it is asked before anything is undone: a change that is + // going to be refused must not leave the sender with no rule at all on the way there. + if state::write::is_consumer_domain(&domain) { + return state::write::set_rule(conn, &domain, true, destination, None); + } + state::write::clear_rule(conn, address)?; + } + screener::decide(conn, address, destination, whole_domain) +} + +/// True when the patch touches the contact record rather than the sender rule. Written out rather +/// than inferred, because `set_contact` appends the whole record and a patch that only moved +/// somebody would otherwise write a second event saying nothing. +fn touches_contact(patch: &ContactPatch) -> bool { + patch.notify.is_some() + || patch.note.is_some() + || patch.allow_remote_images.is_some() + || patch.auto_trash_days.is_some() + || patch.bundle.is_some() +} + +/// Applies the card's patch. Absent is unchanged, in both halves. +pub fn update(conn: &Connection, address: &str, patch: &ContactPatch) -> Result<(), String> { + let address = address.trim().to_lowercase(); + if address.is_empty() { + return Err("a contact needs an address".into()); + } + if patch.destination.is_some() || patch.domain_rule.is_some() { + let (held, held_domain, _) = routed(conn, &address)?; + set_destination( + conn, + &address, + patch.destination.unwrap_or(held), + patch.domain_rule.unwrap_or(held_domain), + )?; + } + if touches_contact(patch) { + state::write::set_contact(conn, &address, patch)?; + } + Ok(()) +} + +// --------------------------------------------------------------------------------------------- +// The Contacts place +// --------------------------------------------------------------------------------------------- + +/// Everyone with a rule, searchable. +/// +/// A domain rule decides senders rather than being one, so it contributes the addresses at that +/// domain the device has actually heard from rather than a row spelling the domain out: the card is +/// about a person and `contact_card` is keyed on an address. +/// +/// Three statements for the whole page rather than a card's worth per row: the rules, the contact +/// records and the names, joined in Rust. `ContactCard` is the frozen return type and it carries a +/// person's threads, files and unsubscribe header, none of which a row draws, so they are left +/// empty here and filled when a card is opened. Answering them per row would be three more queries +/// per sender for something nothing reads. +pub fn list(conn: &Connection, account_id: &str, query: &str) -> Result, String> { + let rules = state::read::rules(conn, account_id)?; + let mut addresses: Vec = Vec::new(); + let mut domains: Vec = Vec::new(); + for rule in &rules { + if rule.is_domain { + domains.push(rule.subject.clone()); + } else { + addresses.push(rule.subject.clone()); + } + } + for domain in &domains { + let mut stmt = conn + .prepare( + "SELECT DISTINCT lower(address) FROM correspondents + WHERE instr(lower(address), '@') > 1 + AND substr(lower(address), instr(address, '@') + 1) = ?1", + ) + .map_err(|e| e.to_string())?; + let found = stmt + .query_map([domain], |row| row.get::<_, String>(0)) + .map_err(|e| e.to_string())? + .collect::, _>>() + .map_err(|e| e.to_string())?; + addresses.extend(found); + } + addresses.sort(); + addresses.dedup(); + + let records: std::collections::HashMap = + state::read::contacts(conn)? + .into_iter() + .map(|record| (record.address.clone(), record)) + .collect(); + let names = names(conn)?; + let needle = query.trim().to_lowercase(); + + let mut out = Vec::new(); + for address in addresses { + let name = names.get(&address).cloned(); + if !needle.is_empty() { + let matched = address.contains(&needle) + || name + .as_deref() + .map(|name| name.to_lowercase().contains(&needle)) + .unwrap_or(false); + if !matched { + continue; + } + } + let held = records + .get(&address) + .cloned() + .unwrap_or_else(|| state::read::Contact::unknown(&address)); + // Every address here came from a rule, so one decides it. The one that does is found in the + // set already in hand rather than asked for again, because a query per row is what a list + // must not do. + let Some(rule) = deciding(&rules, &address) else { + continue; + }; + out.push(ContactCard { + person: Person { + name, + address: address.clone(), + }, + account_id: account_id.to_string(), + destination: rule.destination, + domain_rule: rule.is_domain, + domain_rule_allowed: domain_rule_allowed(&address), + notify: held.notify, + screened_at_ms: Some(rule.decided_at_ms), + note: held.note, + allow_remote_images: held.allow_remote_images, + auto_trash_days: held.auto_trash_days, + bundle: held.bundle, + recent_threads: Vec::new(), + files: Vec::new(), + unsubscribe: None, + }); + } + Ok(out) +} + +/// The rule deciding an address, out of the set already in hand. An address rule beats a domain +/// rule, which is the order `state::read::rule_for` puts them in and the order this repeats. +fn deciding<'a>(rules: &'a [SenderRule], address: &str) -> Option<&'a SenderRule> { + if let Some(rule) = rules + .iter() + .find(|rule| !rule.is_domain && rule.subject == address) + { + return Some(rule); + } + let domain = state::write::domain_of(address)?; + rules + .iter() + .find(|rule| rule.is_domain && rule.subject == domain) +} + +/// Every name the mirror knows, in one statement, so the list above is not a lookup per row. +fn names(conn: &Connection) -> Result, String> { + let mut stmt = conn + .prepare( + "SELECT lower(address), name FROM correspondents + WHERE name IS NOT NULL AND name <> ''", + ) + .map_err(|e| e.to_string())?; + let rows = stmt + .query_map([], |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))) + .map_err(|e| e.to_string())?; + let mut out = std::collections::HashMap::new(); + for row in rows { + let (address, name) = row.map_err(|e| e.to_string())?; + out.insert(address, name); + } + Ok(out) +} + +// --------------------------------------------------------------------------------------------- +// Autocomplete +// --------------------------------------------------------------------------------------------- + +/// A `LIKE` pattern's own wildcards, so somebody typing a percent sign is typing a percent sign. +fn like_escaped(prefix: &str) -> String { + prefix + .trim() + .to_lowercase() + .replace('\\', "\\\\") + .replace('%', "\\%") + .replace('_', "\\_") +} + +/// Autocomplete, in one local query. +/// +/// Both passes are the same table. `correspondents` is everyone this account has written to or +/// heard from, and the sync engine tops it up from the provider's address book with `source` saying +/// which is which, so the mirror's own people rank above the provider's and nobody's address is +/// ever sent anywhere to be looked up. That last part is what the schema promises and it is the +/// reason this is not a network call. +/// +/// Frequency before recency, with a message you sent counting double: writing to somebody is a +/// stronger statement that you know them than receiving from them, which is why a mailing list you +/// have never answered does not outrank a colleague. Recency breaks the ties. +pub fn suggest(conn: &Connection, prefix: &str, limit: usize) -> Result, String> { + let needle = like_escaped(prefix); + if needle.is_empty() { + return Ok(Vec::new()); + } + let mut stmt = conn + .prepare( + "SELECT address, name FROM correspondents + WHERE address <> '' + AND (lower(address) LIKE ?1 || '%' ESCAPE '\\' + OR lower(name) LIKE ?1 || '%' ESCAPE '\\' + OR lower(name) LIKE '% ' || ?1 || '%' ESCAPE '\\') + ORDER BY CASE WHEN source = 'mirror' THEN 0 ELSE 1 END ASC, + seen_count + 2 * sent_count DESC, + last_ms DESC, + address ASC + LIMIT ?2", + ) + .map_err(|e| e.to_string())?; + let rows = stmt + .query_map(rusqlite::params![needle, limit as i64], |row| { + Ok(Person { + name: row + .get::<_, Option>(1)? + .filter(|name| !name.trim().is_empty()), + address: row.get::<_, String>(0)?.to_lowercase(), + }) + }) + .map_err(|e| e.to_string())?; + rows.collect::, _>>() + .map_err(|e| e.to_string()) +} + +// --------------------------------------------------------------------------------------------- +// Commands +// --------------------------------------------------------------------------------------------- + +fn db_of(app: &tauri::AppHandle) -> Result, String> { + app.try_state::() + .ok_or_else(|| "the mirror is not open yet".to_string()) +} + +#[tauri::command(async)] +pub fn contact_card( + app: tauri::AppHandle, + account_id: String, + address: String, +) -> Result { + let db = db_of(&app)?; + let color = sync::accounts(db.inner(), Some(&account_id)) + .into_iter() + .next() + .map(|(_, color)| color) + .unwrap_or_else(|| "hue-1".to_string()); + let now = mirror::write::now_ms(); + db.with(&account_id, |conn| { + card(conn, &account_id, &color, &address, now) + }) +} + +#[tauri::command(async)] +pub fn contact_update( + app: tauri::AppHandle, + account_id: String, + address: String, + patch: ContactPatch, +) -> Result<(), String> { + let db = db_of(&app)?; + let moved = patch.destination.is_some() || patch.domain_rule.is_some(); + db.with(&account_id, |conn| update(conn, &address, &patch))?; + // A destination is a sender rule, and every routed place is a query over those rules, so the + // lists somebody is looking at are already wrong by the time this returns. + crate::emit_store_changed(&app, if moved { "threads state" } else { "state" }); + Ok(()) +} + +#[tauri::command(async)] +pub fn contacts_list( + app: tauri::AppHandle, + account_id: Option, + query: String, +) -> Result, String> { + let db = db_of(&app)?; + let mut all = Vec::new(); + for (id, _) in sync::accounts(db.inner(), account_id.as_deref()) { + all.extend(db.with(&id, |conn| list(conn, &id, &query))?); + } + all.sort_by(|a, b| a.person.address.cmp(&b.person.address)); + Ok(all) +} + +#[tauri::command(async)] +pub fn contacts_suggest( + app: tauri::AppHandle, + account_id: String, + prefix: String, +) -> Result, String> { + let db = db_of(&app)?; + db.with(&account_id, |conn| suggest(conn, &prefix, SUGGESTIONS)) +} + +#[cfg(test)] +mod tests; diff --git a/src-tauri/src/contacts/tests.rs b/src-tauri/src/contacts/tests.rs new file mode 100644 index 0000000..6c9df51 --- /dev/null +++ b/src-tauri/src/contacts/tests.rs @@ -0,0 +1,556 @@ +// The contact card, over a pair of in-memory databases. +// +// Threads and messages go in with SQL rather than through the sync engine, the way the Screener's +// tests do it: what is being asserted is what a card says given a mailbox and a set of decisions, +// and building the mailbox through a fake provider would test the engine again and hide the answer. +// +// The one thing that is not raw SQL is `fts::index`. The card's recent threads are a search over +// `from:`, which is the same query the search bar runs, so a message that is in the mirror and not +// in the index is a message the card cannot see. The engine indexes on headers arriving; these +// tests do the same thing by hand. + +use rusqlite::Connection; + +use crate::db; +use crate::dto::{ContactPatch, Destination, Place, ThreadQuery}; +use crate::mirror; +use crate::provider::fake::FakeProvider; +use crate::state; + +const NOW: i64 = 2_000_000_000_000; + +fn open() -> Connection { + db::memory().expect("a pair of in-memory databases") +} + +struct Mail<'a> { + address: &'a str, + name: &'a str, + subject: &'a str, + list_unsubscribe: Option<&'a str>, + attachment: Option<&'a str>, +} + +impl<'a> Mail<'a> { + fn from(address: &'a str, name: &'a str) -> Self { + Mail { + address, + name, + subject: "Hello", + list_unsubscribe: None, + attachment: None, + } + } +} + +/// One thread with one message from one sender, indexed the way the engine indexes it. +fn arrive(conn: &Connection, mail: &Mail<'_>, at: i64) -> String { + let key = format!("<{}-{at}@example>", mail.address); + let tid = format!("t-{}-{at}", mail.address); + let mid = format!("m-{tid}"); + conn.execute( + "INSERT INTO threads (provider_thread_id, thread_key, latest_ms, message_count, unseen, + subject, snippet, from_name, from_address, in_inbox, has_attachment) + VALUES (?1, ?2, ?3, 1, 1, ?4, 'snippet', ?5, ?6, 1, ?7)", + rusqlite::params![ + tid, + key, + at, + mail.subject, + mail.name, + mail.address, + mail.attachment.is_some() as i64 + ], + ) + .expect("a thread"); + conn.execute( + "INSERT INTO messages (id, provider_thread_id, thread_key, message_id, date_ms, from_name, + from_address, subject, snippet, hydrated, labels, list_unsubscribe, + has_attachment) + VALUES (?1, ?2, ?3, ?3, ?4, ?5, ?6, ?7, 'snippet', 1, '[\"INBOX\"]', ?8, ?9)", + rusqlite::params![ + mid, + tid, + key, + at, + mail.name, + mail.address, + mail.subject, + mail.list_unsubscribe, + mail.attachment.is_some() as i64, + ], + ) + .expect("a message"); + if let Some(filename) = mail.attachment { + conn.execute( + "INSERT INTO attachments (id, message_id, filename, mime_type, size, inline) + VALUES (?1, ?2, ?3, 'application/pdf', 1024, 0)", + rusqlite::params![format!("a-{tid}"), mid, filename], + ) + .expect("an attachment"); + } + conn.execute( + "INSERT INTO correspondents (address, name, last_ms, seen_count, sent_count, source) + VALUES (?1, ?2, ?3, 1, 0, 'mirror') + ON CONFLICT(address) DO UPDATE SET + last_ms = MAX(correspondents.last_ms, excluded.last_ms), + seen_count = correspondents.seen_count + 1", + rusqlite::params![mail.address, mail.name, at], + ) + .expect("a correspondent"); + mirror::fts::index(conn, &mid).expect("the index"); + key +} + +fn card(conn: &Connection, address: &str) -> crate::dto::ContactCard { + super::card(conn, "acct", "hue-1", address, NOW).expect("a card") +} + +fn keys_in(conn: &Connection, place: Place) -> Vec { + let query = ThreadQuery { + account_id: None, + place, + label_id: None, + query: None, + limit: 50, + cursor: None, + }; + mirror::read::threads_list(conn, "acct", "hue-1", &query, NOW) + .expect("a page") + .threads + .into_iter() + .map(|thread| thread.key) + .collect() +} + +// -- the card ---------------------------------------------------------------------------------- + +#[test] +fn a_card_for_somebody_with_no_rule_says_where_they_would_go() { + let conn = open(); + let mut letter = Mail::from("hello@thelongread.example", "The Long Read"); + letter.subject = "The week in review"; + letter.list_unsubscribe = Some(""); + arrive(&conn, &letter, NOW - 1000); + + let card = card(&conn, "hello@thelongread.example"); + + assert_eq!(card.person.name.as_deref(), Some("The Long Read")); + assert_eq!(card.person.address, "hello@thelongread.example"); + // Nobody has decided about them, so the card says what the Screener would suggest rather than + // nothing at all, and says out loud that it is not a decision by carrying no date. + assert_eq!(card.destination, Destination::Feed); + assert_eq!(card.screened_at_ms, None); + assert!(!card.domain_rule); + assert!(card.domain_rule_allowed); + assert!(card.unsubscribe.is_some()); + assert_eq!(card.note, None); + assert!(!card.notify); +} + +#[test] +fn a_card_for_somebody_with_a_rule_carries_the_decision_and_its_date() { + let conn = open(); + arrive(&conn, &Mail::from("maya@example.org", "Maya"), NOW - 1000); + super::update( + &conn, + "maya@example.org", + &ContactPatch { + destination: Some(Destination::PaperTrail), + ..ContactPatch::default() + }, + ) + .expect("a move"); + + let card = card(&conn, "maya@example.org"); + + assert_eq!(card.destination, Destination::PaperTrail); + let decided = card.screened_at_ms.expect("a screening date"); + assert!(decided > 0, "a decision has a moment"); +} + +#[test] +fn a_card_carries_the_senders_recent_threads_and_files() { + let conn = open(); + let mut with_file = Mail::from("sam@sunnydaymusic.example", "Sam Okafor"); + with_file.subject = "Enrolment form"; + with_file.attachment = Some("Enrolment-form.pdf"); + arrive(&conn, &with_file, NOW - 3000); + let mut later = Mail::from("sam@sunnydaymusic.example", "Sam Okafor"); + later.subject = "Piano on Wednesdays"; + arrive(&conn, &later, NOW - 1000); + // Somebody else at the same domain, whose mail is not Sam's. + arrive( + &conn, + &Mail::from("enrolments@sunnydaymusic.example", "Sunny Day Music"), + NOW - 2000, + ); + + let card = card(&conn, "sam@sunnydaymusic.example"); + + let subjects: Vec = card + .recent_threads + .iter() + .map(|thread| thread.subject.clone()) + .collect(); + assert_eq!(subjects, vec!["Piano on Wednesdays", "Enrolment form"]); + // A card is not a place, so its rows sit under the card's own heading and carry no group. + assert!(card.recent_threads.iter().all(|thread| thread.group.is_empty())); + assert_eq!(card.files.len(), 1); + assert_eq!(card.files[0].filename, "Enrolment-form.pdf"); +} + +#[test] +fn the_domain_toggle_is_not_offered_on_a_consumer_domain() { + let conn = open(); + arrive(&conn, &Mail::from("someone@gmail.com", "Someone"), NOW - 1000); + + assert!(!card(&conn, "someone@gmail.com").domain_rule_allowed); + assert!(card(&conn, "ana@northgate.example").domain_rule_allowed); +} + +// -- moving somebody ----------------------------------------------------------------------------- + +#[test] +fn a_destination_change_moves_the_mail_that_is_already_here() { + // The whole point of the card: a move that only affected future mail reads as a move that did + // not work. + let conn = open(); + let mut keys = Vec::new(); + for at in [NOW - 3000, NOW - 2000, NOW - 1000] { + keys.push(arrive( + &conn, + &Mail::from("news@brand.example", "Brand"), + at, + )); + } + super::update( + &conn, + "news@brand.example", + &ContactPatch { + destination: Some(Destination::Feed), + ..ContactPatch::default() + }, + ) + .expect("a move"); + + let feed = keys_in(&conn, Place::Feed); + assert_eq!(feed.len(), 3); + for key in keys { + assert!(feed.contains(&key), "{key} did not move to the Feed"); + } + assert!(keys_in(&conn, Place::Screener).is_empty()); +} + +#[test] +fn screening_somebody_out_from_the_card_is_the_block() { + let conn = open(); + let key = arrive( + &conn, + &Mail::from("reach@talentpartners.example", "Talent Partners"), + NOW - 1000, + ); + super::update( + &conn, + "reach@talentpartners.example", + &ContactPatch { + destination: Some(Destination::ScreenedOut), + ..ContactPatch::default() + }, + ) + .expect("a block"); + + assert_eq!(keys_in(&conn, Place::ScreenedOut), vec![key]); + assert!(keys_in(&conn, Place::Inbox).is_empty()); +} + +#[test] +fn turning_the_domain_toggle_on_decides_everyone_at_the_domain() { + let conn = open(); + let ana = arrive(&conn, &Mail::from("ana@northgate.example", "Ana"), NOW - 2000); + let ben = arrive(&conn, &Mail::from("ben@northgate.example", "Ben"), NOW - 1000); + + super::update( + &conn, + "ana@northgate.example", + &ContactPatch { + destination: Some(Destination::PaperTrail), + ..ContactPatch::default() + }, + ) + .expect("a move"); + super::update( + &conn, + "ana@northgate.example", + &ContactPatch { + domain_rule: Some(true), + ..ContactPatch::default() + }, + ) + .expect("the toggle"); + + let trail = keys_in(&conn, Place::PaperTrail); + assert!(trail.contains(&ana) && trail.contains(&ben)); + // The address rule would beat the domain rule, so turning the toggle on has to take it away or + // the card would read as off the moment it was asked again. + let card = card(&conn, "ana@northgate.example"); + assert!(card.domain_rule); + assert_eq!(card.destination, Destination::PaperTrail); +} + +#[test] +fn a_domain_rule_on_a_consumer_domain_is_refused_and_changes_nothing() { + let conn = open(); + arrive(&conn, &Mail::from("someone@gmail.com", "Someone"), NOW - 1000); + super::update( + &conn, + "someone@gmail.com", + &ContactPatch { + destination: Some(Destination::Inbox), + ..ContactPatch::default() + }, + ) + .expect("a move"); + + let refused = super::update( + &conn, + "someone@gmail.com", + &ContactPatch { + domain_rule: Some(true), + ..ContactPatch::default() + }, + ); + + let message = refused.expect_err("everyone at gmail.com is not one sender"); + assert!(message.contains("gmail.com"), "the message was {message}"); + // Refused before anything was undone: the address rule they already had is still deciding them. + let card = card(&conn, "someone@gmail.com"); + assert_eq!(card.destination, Destination::Inbox); + assert!(!card.domain_rule); +} + +// -- the rest of the card ------------------------------------------------------------------------ + +#[test] +fn a_patch_of_one_field_leaves_the_others_alone() { + let conn = open(); + arrive(&conn, &Mail::from("sam@example.org", "Sam"), NOW - 1000); + super::update( + &conn, + "sam@example.org", + &ContactPatch { + destination: Some(Destination::Inbox), + notify: Some(true), + note: Some("Cooper's teacher".to_string()), + allow_remote_images: Some(true), + ..ContactPatch::default() + }, + ) + .expect("the first patch"); + + super::update( + &conn, + "sam@example.org", + &ContactPatch { + notify: Some(false), + ..ContactPatch::default() + }, + ) + .expect("one field"); + + let card = card(&conn, "sam@example.org"); + assert!(!card.notify); + assert_eq!(card.note.as_deref(), Some("Cooper's teacher")); + assert!(card.allow_remote_images); + assert_eq!(card.destination, Destination::Inbox); + assert!(card.bundle, "the default a contact starts with survives a patch"); +} + +#[test] +fn the_note_and_the_switches_survive_a_replay_of_the_journal() { + // The card's decisions roam, which means they are the log rather than the tables: the same + // events landing on an empty database have to produce the same card. + let written = open(); + arrive(&written, &Mail::from("sam@example.org", "Sam"), NOW - 1000); + super::update( + &written, + "sam@example.org", + &ContactPatch { + destination: Some(Destination::PaperTrail), + notify: Some(true), + note: Some("Cooper's teacher".to_string()), + bundle: Some(false), + auto_trash_days: Some(Some(30)), + ..ContactPatch::default() + }, + ) + .expect("a patch"); + + let roamed = open(); + arrive(&roamed, &Mail::from("sam@example.org", "Sam"), NOW - 1000); + let log = state::journal::records(&written).expect("the log"); + let report = state::merge::absorb(&roamed, &log).expect("absorb"); + assert_eq!(report.applied, log.len()); + + let here = card(&written, "sam@example.org"); + let there = card(&roamed, "sam@example.org"); + assert_eq!(there.destination, here.destination); + assert_eq!(there.screened_at_ms, here.screened_at_ms); + assert_eq!(there.notify, here.notify); + assert_eq!(there.note, here.note); + assert_eq!(there.bundle, here.bundle); + assert_eq!(there.auto_trash_days, here.auto_trash_days); + assert_eq!(there.auto_trash_days, Some(30)); +} + +// -- the Contacts place -------------------------------------------------------------------------- + +#[test] +fn the_list_holds_everyone_with_a_rule_and_nobody_who_is_waiting() { + let conn = open(); + arrive(&conn, &Mail::from("maya@example.org", "Maya"), NOW - 3000); + arrive(&conn, &Mail::from("dev@example.net", "Dev Patel"), NOW - 2000); + arrive(&conn, &Mail::from("stranger@example.com", "A Stranger"), NOW - 1000); + for address in ["maya@example.org", "dev@example.net"] { + super::update( + &conn, + address, + &ContactPatch { + destination: Some(Destination::Inbox), + ..ContactPatch::default() + }, + ) + .expect("a decision"); + } + + let everyone = super::list(&conn, "acct", "").expect("the list"); + let addresses: Vec = everyone + .iter() + .map(|card| card.person.address.clone()) + .collect(); + assert_eq!(addresses, vec!["dev@example.net", "maya@example.org"]); + assert_eq!(everyone[0].person.name.as_deref(), Some("Dev Patel")); + // A row draws none of these, so the list does not answer three more queries per sender for them. + assert!(everyone.iter().all(|card| card.recent_threads.is_empty())); + assert!(everyone.iter().all(|card| card.files.is_empty())); +} + +#[test] +fn a_domain_rule_puts_the_people_it_decides_in_the_list_rather_than_the_domain() { + let conn = open(); + arrive(&conn, &Mail::from("ana@northgate.example", "Ana"), NOW - 2000); + arrive(&conn, &Mail::from("ben@northgate.example", "Ben"), NOW - 1000); + super::update( + &conn, + "ana@northgate.example", + &ContactPatch { + destination: Some(Destination::Feed), + domain_rule: Some(true), + ..ContactPatch::default() + }, + ) + .expect("a domain rule"); + + let everyone = super::list(&conn, "acct", "").expect("the list"); + let addresses: Vec = everyone + .iter() + .map(|card| card.person.address.clone()) + .collect(); + assert_eq!( + addresses, + vec!["ana@northgate.example", "ben@northgate.example"] + ); + assert!(everyone.iter().all(|card| card.domain_rule)); +} + +#[test] +fn the_search_narrows_the_list_by_name_and_by_address() { + let conn = open(); + arrive(&conn, &Mail::from("maya@example.org", "Maya Raghunathan"), NOW - 3000); + arrive(&conn, &Mail::from("dev@northgate.example", "Dev Patel"), NOW - 2000); + for address in ["maya@example.org", "dev@northgate.example"] { + super::update( + &conn, + address, + &ContactPatch { + destination: Some(Destination::Inbox), + ..ContactPatch::default() + }, + ) + .expect("a decision"); + } + + let by_name = super::list(&conn, "acct", "raghu").expect("by name"); + assert_eq!(by_name.len(), 1); + assert_eq!(by_name[0].person.address, "maya@example.org"); + + let by_domain = super::list(&conn, "acct", "northgate").expect("by address"); + assert_eq!(by_domain.len(), 1); + assert_eq!(by_domain[0].person.address, "dev@northgate.example"); + + assert!(super::list(&conn, "acct", "nobody").expect("no match").is_empty()); +} + +// -- autocomplete --------------------------------------------------------------------------------- + +fn correspondent(conn: &Connection, address: &str, name: &str, source: &str, seen: i64, last: i64) { + conn.execute( + "INSERT INTO correspondents (address, name, last_ms, seen_count, sent_count, source) + VALUES (?1, ?2, ?3, ?4, 0, ?5)", + rusqlite::params![address, name, last, seen, source], + ) + .expect("a correspondent"); +} + +#[test] +fn autocomplete_ranks_the_mirror_above_the_provider_and_asks_nobody() { + // The provider is here to prove it is not asked. Everyone this account has actually written to + // or heard from is already on the device, which is what makes autocomplete local: an address + // typed into the compose field is never sent anywhere to be looked up. + let provider = FakeProvider::new(); + provider.set_contacts(vec![crate::dto::Person { + name: Some("Anita Desai".to_string()), + address: "anita@northwind.example".to_string(), + }]); + + let conn = open(); + correspondent(&conn, "ana@example.org", "Ana Ruiz", "mirror", 12, NOW - 5000); + correspondent(&conn, "andrew@example.net", "Andrew Bell", "mirror", 2, NOW - 1000); + correspondent(&conn, "anita@northwind.example", "Anita Desai", "people-api", 0, NOW); + + let found = super::suggest(&conn, "an", 8).expect("suggestions"); + let addresses: Vec = found.iter().map(|person| person.address.clone()).collect(); + + assert_eq!( + addresses, + vec![ + "ana@example.org", + "andrew@example.net", + "anita@northwind.example" + ], + "the mirror's own people come before the provider's address book" + ); + assert!( + provider.calls().is_empty(), + "the mirror answered, so nothing was asked of anybody: {:?}", + provider.calls() + ); +} + +#[test] +fn autocomplete_matches_a_name_as_well_as_an_address_and_nothing_else() { + let conn = open(); + correspondent(&conn, "sam@sunnydaymusic.example", "Sam Okafor", "mirror", 3, NOW); + correspondent(&conn, "billing@citypower.example", "City Power", "mirror", 9, NOW); + + let by_name = super::suggest(&conn, "okaf", 8).expect("by name"); + assert_eq!(by_name.len(), 1); + assert_eq!(by_name[0].address, "sam@sunnydaymusic.example"); + + let by_address = super::suggest(&conn, "billing", 8).expect("by address"); + assert_eq!(by_address.len(), 1); + + // A wildcard is a character somebody typed, not a pattern. + assert!(super::suggest(&conn, "%", 8).expect("a wildcard").is_empty()); + assert!(super::suggest(&conn, "", 8).expect("nothing typed").is_empty()); +} diff --git a/src-tauri/src/db.rs b/src-tauri/src/db.rs new file mode 100644 index 0000000..da27939 --- /dev/null +++ b/src-tauri/src/db.rs @@ -0,0 +1,252 @@ +// Opening the two databases and handing out the one connection that sees both. +// +// Each account gets its own pair of files under `accounts//`: `mirror.db`, which is derived and +// disposable, and `state.db`, which is everything the person decided. They are separate files +// because they have separate lifetimes: clearing the mirror is deleting a file, exporting the +// state is copying one, and the backup uploads segments of the second and never a byte of the +// first. +// +// They are opened on one connection, with the state file attached as `state`, so a view can be one +// SQL statement across both rather than two queries joined in Rust. Every join in `mirror::read` +// depends on that, and it is the reason this module exists rather than each side opening its own. +// +// One connection per account, behind a mutex, reached from `#[tauri::command(async)]` handlers as +// the calendar does. WAL so a long hydration does not block a read, and a busy timeout because +// several accounts in one process on one disk will collide eventually and the alternative is a +// spurious "database is locked" in front of somebody's mail. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; + +use rusqlite::Connection; + +use crate::library::app_data_dir; +use crate::{mirror, state}; + +/// Long enough to outlast a hydration batch's write, short enough that a real deadlock is still a +/// bug that shows up rather than a hang. +const BUSY_TIMEOUT_MS: u32 = 5_000; + +pub struct Db { + root: PathBuf, + /// Where a removed account's pair goes when the person chose to keep it: `kept/`, beside + /// `accounts/` rather than inside it, so nothing that reads `on_disk` can take it for a live + /// mailbox. + kept: PathBuf, + connections: Mutex>, +} + +impl Db { + pub fn open(app: &tauri::AppHandle) -> Result { + let data = app_data_dir(app)?; + // The sync log lives beside the account directories, and opening the databases is the + // one moment the engine's own code is handed the directory: the engine itself never sees + // an app handle, which is what lets it run in a test with nothing behind it. + crate::log::init(&data); + let root = data.join("accounts"); + std::fs::create_dir_all(&root).map_err(|e| e.to_string())?; + Ok(Db { + root, + kept: data.join("kept"), + connections: Mutex::new(HashMap::new()), + }) + } + + pub fn account_dir(&self, account_id: &str) -> PathBuf { + self.root.join(account_id) + } + + /// Runs a closure against one account's connection. Every read and every write goes through + /// here, so there is one place that knows a connection might need opening first. + pub fn with( + &self, + account_id: &str, + f: impl FnOnce(&Connection) -> Result, + ) -> Result { + let mut open = self.connections.lock().map_err(|e| e.to_string())?; + if !open.contains_key(account_id) { + let dir = self.account_dir(account_id); + std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; + open.insert(account_id.to_string(), connect(&dir)?); + } + let conn = open.get(account_id).expect("just inserted"); + f(conn) + } + + /// Closes the connection so the files can be moved or deleted. Opening again is automatic. + pub fn close(&self, account_id: &str) -> Result<(), String> { + let mut open = self.connections.lock().map_err(|e| e.to_string())?; + open.remove(account_id); + Ok(()) + } + + /// The accounts that have a database on disk, which is not always the same list as the accounts + /// that have a token: a removal takes one away before the other. + pub fn on_disk(&self) -> Vec { + std::fs::read_dir(&self.root) + .map(|entries| { + entries + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.path().is_dir()) + .map(|entry| entry.file_name().to_string_lossy().to_string()) + .collect() + }) + .unwrap_or_default() + } + + /// Moves an account's pair out of the live set without deleting it. `on_disk` stops listing + /// it, so no list, badge or sync pass sees a mailbox nobody is signed in to, and `restore` + /// brings it back the day the account is added again. An older kept copy of the same account + /// is replaced: the pair just removed is the one the person was looking at. Call `close` + /// first, because a connection open on the old path would keep writing there. + pub fn set_aside(&self, account_id: &str) -> Result<(), String> { + let live = self.account_dir(account_id); + if !live.exists() { + return Ok(()); + } + std::fs::create_dir_all(&self.kept).map_err(|e| e.to_string())?; + let kept = self.kept.join(account_id); + if kept.exists() { + std::fs::remove_dir_all(&kept).map_err(|e| e.to_string())?; + } + std::fs::rename(&live, &kept).map_err(|e| e.to_string()) + } + + /// The other half: an account added again gets the pair that was set aside when it was + /// removed, decisions and all. Nothing happens when a live pair already exists, because a pair + /// that has been written to since is the newer truth and a rename over it would lose it. + pub fn restore(&self, account_id: &str) -> Result<(), String> { + let kept = self.kept.join(account_id); + let live = self.account_dir(account_id); + if !kept.exists() || live.exists() { + return Ok(()); + } + std::fs::rename(&kept, &live).map_err(|e| e.to_string()) + } +} + +fn connect(dir: &Path) -> Result { + let conn = Connection::open(dir.join("mirror.db")).map_err(|e| e.to_string())?; + prepare(&conn, &dir.join("state.db").to_string_lossy())?; + Ok(conn) +} + +fn prepare(conn: &Connection, state_path: &str) -> Result<(), String> { + // Not a prepared statement: ATTACH takes a literal, and the path is ours rather than anyone's + // input. The escaping is still done, because a home directory with an apostrophe in it exists. + conn.execute_batch(&format!( + "PRAGMA journal_mode = WAL; + PRAGMA synchronous = NORMAL; + PRAGMA foreign_keys = OFF; + ATTACH DATABASE '{}' AS state;", + state_path.replace('\'', "''") + )) + .map_err(|e| e.to_string())?; + conn.busy_timeout(std::time::Duration::from_millis(BUSY_TIMEOUT_MS as u64)) + .map_err(|e| e.to_string())?; + + mirror::schema::migrate(conn)?; + state::schema::migrate(conn) +} + +/// A pair of in-memory databases with both schemas, for tests. Attaching `:memory:` gives a second, +/// separate in-memory database, so the boundary the real thing has is the boundary a test has. +#[cfg(test)] +pub fn memory() -> Result { + let conn = Connection::open_in_memory().map_err(|e| e.to_string())?; + conn.execute_batch("ATTACH DATABASE ':memory:' AS state;") + .map_err(|e| e.to_string())?; + mirror::schema::migrate(&conn)?; + state::schema::migrate(&conn)?; + Ok(conn) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn on_disk(data: &Path) -> Db { + Db { + root: data.join("accounts"), + kept: data.join("kept"), + connections: Mutex::new(HashMap::new()), + } + } + + #[test] + fn a_kept_pair_leaves_the_live_set_and_comes_back_on_restore() { + let tmp = tempfile::tempdir().expect("tempdir"); + let db = on_disk(tmp.path()); + let dir = db.account_dir("a1"); + std::fs::create_dir_all(&dir).expect("live dir"); + std::fs::write(dir.join("state.db"), b"decisions").expect("a file to keep"); + + db.set_aside("a1").expect("set aside"); + assert!(db.on_disk().is_empty(), "a kept account is not a live one"); + assert!(!dir.exists()); + + db.restore("a1").expect("restore"); + assert_eq!(db.on_disk(), vec!["a1".to_string()]); + assert_eq!(std::fs::read(dir.join("state.db")).expect("the file"), b"decisions"); + } + + #[test] + fn keeping_again_replaces_the_older_copy_and_restore_never_overwrites_a_live_pair() { + let tmp = tempfile::tempdir().expect("tempdir"); + let db = on_disk(tmp.path()); + let dir = db.account_dir("a1"); + + std::fs::create_dir_all(&dir).expect("live dir"); + std::fs::write(dir.join("state.db"), b"first").expect("write"); + db.set_aside("a1").expect("first keep"); + + std::fs::create_dir_all(&dir).expect("live dir again"); + std::fs::write(dir.join("state.db"), b"second").expect("write"); + db.set_aside("a1").expect("second keep"); + assert_eq!(std::fs::read(tmp.path().join("kept/a1/state.db")).expect("kept"), b"second"); + + std::fs::create_dir_all(&dir).expect("a live pair in the way"); + std::fs::write(dir.join("state.db"), b"live").expect("write"); + db.restore("a1").expect("restore is a no-op here"); + assert_eq!(std::fs::read(dir.join("state.db")).expect("live"), b"live"); + assert!(tmp.path().join("kept/a1").exists(), "the kept copy is not thrown away either"); + + // Nothing to do when nothing was kept. + db.set_aside("nobody").expect("no live dir is fine"); + db.restore("nobody").expect("no kept dir is fine"); + } + + #[test] + fn a_fresh_pair_has_both_schemas_and_one_connection_sees_both() { + let conn = memory().expect("open"); + + conn.execute( + "INSERT INTO threads (provider_thread_id, thread_key, latest_ms) VALUES ('t1', 'k1', 10)", + [], + ) + .expect("mirror write"); + conn.execute( + "INSERT INTO state.piles (thread_key, pile) VALUES ('k1', 'reply-later')", + [], + ) + .expect("state write"); + + let pile: String = conn + .query_row( + "SELECT p.pile FROM threads t JOIN state.piles p ON p.thread_key = t.thread_key", + [], + |row| row.get(0), + ) + .expect("the join across both databases"); + assert_eq!(pile, "reply-later"); + } + + #[test] + fn migrating_twice_changes_nothing() { + let conn = memory().expect("open"); + mirror::schema::migrate(&conn).expect("mirror again"); + state::schema::migrate(&conn).expect("state again"); + assert_eq!(mirror::schema::version(&conn).expect("version"), mirror::schema::VERSION); + } +} diff --git a/src-tauri/src/decisions.rs b/src-tauri/src/decisions.rs new file mode 100644 index 0000000..a08b18a --- /dev/null +++ b/src-tauri/src/decisions.rs @@ -0,0 +1,584 @@ +// The decisions that are about a thread rather than about a sender: the note, the name, the merge, +// the ignore and the notify switch. +// +// Every one of them is a thin command over a function in `state::write` that already exists and +// already journals, which is what makes them roam. What is here and not there is the part that +// needs an app: which account's state file the decision belongs in, what the toast says, and how to +// put it back. +// +// The two lookups at the top are shared with `piles`, `snooze` and `clips`. They are here because a +// pile and a snooze are decisions about a thread too, and every one of the five command modules +// asks the same two questions before it can write anything. + +use rusqlite::{Connection, OptionalExtension}; +use tauri::Manager; + +use crate::db::Db; +use crate::dto::{Note, Undo}; +use crate::state::{self, journal::Payload, read::ThreadFlags}; +use crate::sync; +use crate::undo::Stack; + +static UNDO: Stack = Stack::new("decision"); + +// --------------------------------------------------------------------------------------------- +// Which account a decision belongs in +// --------------------------------------------------------------------------------------------- + +pub(crate) fn db_of(app: &tauri::AppHandle) -> Result, String> { + app.try_state::() + .ok_or_else(|| "the mirror is not open yet".to_string()) +} + +/// Which of these keys one account holds. A merged thread answers to its own key, so a source key +/// finds its account through the merge as well as through the mirror. +fn held_keys(conn: &Connection, keys: &[String]) -> Result, String> { + let mut out = Vec::new(); + for key in keys { + let held: i64 = conn + .query_row( + "SELECT COUNT(*) FROM threads + WHERE thread_key = ?1 + OR thread_key IN (SELECT thread_key FROM state.merges WHERE merged_key = ?1)", + [key], + |row| row.get(0), + ) + .map_err(|e| e.to_string())?; + if held > 0 { + out.push(key.clone()); + } + } + Ok(out) +} + +/// The keys grouped by the account that holds them, so a decision is written into the state file +/// beside the mail it is about and into no other. A key claimed by one account is not offered to +/// the next, because a thread lives in one mailbox and a second row for it would roam as a second +/// decision. +pub(crate) fn holders(db: &Db, keys: &[String]) -> Result)>, String> { + let mut out: Vec<(String, Vec)> = Vec::new(); + let mut claimed: Vec = Vec::new(); + for (account_id, _) in sync::accounts(db, None) { + let wanted: Vec = keys + .iter() + .filter(|key| !claimed.contains(key)) + .cloned() + .collect(); + if wanted.is_empty() { + break; + } + let mine = db.with(&account_id, |conn| held_keys(conn, &wanted))?; + if mine.is_empty() { + continue; + } + claimed.extend(mine.iter().cloned()); + out.push((account_id, mine)); + } + Ok(out) +} + +/// The account whose state file holds a row, for the commands the frontend names by id alone. +/// `attachments` asks the mirror the same question the same way. +pub(crate) fn account_holding(db: &Db, sql: &str, id: &str) -> Result { + for (account_id, _) in sync::accounts(db, None) { + let held = db.with(&account_id, |conn| { + conn.query_row(sql, [id], |row| row.get::<_, i64>(0)) + .map_err(|e| e.to_string()) + })?; + if held > 0 { + return Ok(account_id); + } + } + Err("that is not on this device".to_string()) +} + +pub(crate) fn plural(count: usize, one: &str) -> String { + if count == 1 { + one.to_string() + } else { + format!("{one} {count} threads") + } +} + +// --------------------------------------------------------------------------------------------- +// Notes +// --------------------------------------------------------------------------------------------- + +/// The message that is latest in the thread right now, which is where the pane puts a new note. +fn latest_message(conn: &Connection, thread_key: &str) -> Result, String> { + conn.query_row( + "SELECT m.id FROM messages m + WHERE m.provider_thread_id IN ( + SELECT provider_thread_id FROM threads + WHERE thread_key = ?1 + OR thread_key IN (SELECT thread_key FROM state.merges WHERE merged_key = ?1)) + ORDER BY m.date_ms DESC, m.id DESC LIMIT 1", + [thread_key], + |row| row.get(0), + ) + .optional() + .map_err(|e| e.to_string()) +} + +/// One note by its id, deleted or not, which is what a reversal needs and what `state::read` does +/// not answer: a note is read by its thread everywhere else. +fn note_row(conn: &Connection, id: &str) -> Result, String> { + conn.query_row( + "SELECT id, thread_key, body, created_at, after_message_id, deleted FROM state.notes + WHERE id = ?1", + [id], + |row| { + Ok(( + Note { + id: row.get(0)?, + thread_key: row.get(1)?, + body: row.get(2)?, + created_at_ms: row.get(3)?, + after_message_id: row.get(4)?, + }, + row.get::<_, i64>(5)? != 0, + )) + }, + ) + .optional() + .map_err(|e| e.to_string()) +} + +pub fn add_note(conn: &Connection, thread_key: &str, body: &str) -> Result { + let body = body.trim(); + if body.is_empty() { + return Err("a note needs something in it".into()); + } + let after = latest_message(conn, thread_key)?; + let id = state::write::add_note(conn, thread_key, body, after.as_deref())?; + note_row(conn, &id)? + .map(|(note, _)| note) + .ok_or_else(|| "that note did not land".to_string()) +} + +/// Puts a deleted note back. There is no un-delete in `state::write` because nothing but a reversal +/// wants one, and deletion is a flag rather than a missing row precisely so this is possible. +fn restore_note(conn: &Connection, note: &Note) -> Result<(), String> { + state::journal::append( + conn, + ¬e.id, + &Payload::Note { + thread_key: note.thread_key.clone(), + body: note.body.clone(), + after_message_id: note.after_message_id.clone(), + created_at: note.created_at_ms, + deleted: false, + }, + ) + .map(|_| ()) +} + +// --------------------------------------------------------------------------------------------- +// Renames and merges +// --------------------------------------------------------------------------------------------- + +fn subject_of(conn: &Connection, thread_key: &str) -> Result, String> { + conn.query_row( + "SELECT subject FROM threads WHERE thread_key = ?1 ORDER BY latest_ms DESC LIMIT 1", + [thread_key], + |row| row.get(0), + ) + .optional() + .map_err(|e| e.to_string()) +} + +/// A merge with no name typed takes the longest of the subjects it swallowed, which is section 10's +/// rule and is usually the one that says the most about the conversation. +fn longest_subject(conn: &Connection, keys: &[String]) -> Result, String> { + let mut best: Option = None; + for key in keys { + let Some(subject) = subject_of(conn, key)? else { + continue; + }; + let longer = best + .as_ref() + .map(|held| subject.chars().count() > held.chars().count()) + .unwrap_or(true); + if longer { + best = Some(subject); + } + } + Ok(best) +} + +/// The threads pointing directly at this one. `state::read::merge_sources` walks the whole chain, +/// which is the right answer for a view and the wrong one for a reversal: putting a chain back +/// means putting each of its links back, not flattening it. +fn direct_sources(conn: &Connection, merged_key: &str) -> Result, String> { + let mut stmt = conn + .prepare("SELECT thread_key FROM state.merges WHERE merged_key = ?1 ORDER BY thread_key") + .map_err(|e| e.to_string())?; + let rows = stmt + .query_map([merged_key], |row| row.get::<_, String>(0)) + .map_err(|e| e.to_string())?; + rows.collect::, _>>().map_err(|e| e.to_string()) +} + +/// Takes one source back out of a merge. `state::write::unmerge` breaks the whole thread apart, +/// which is what the banner offers; taking back a merge of three into a thread that already held +/// two is not that. +fn unmerge_one(conn: &Connection, source: &str) -> Result<(), String> { + state::journal::append(conn, source, &Payload::Merge { merged_key: None }).map(|_| ()) +} + +fn set_name(conn: &Connection, thread_key: &str, name: Option<&str>) -> Result<(), String> { + match name.map(str::trim).filter(|name| !name.is_empty()) { + Some(name) => state::write::set_rename(conn, thread_key, name), + None => state::write::clear_rename(conn, thread_key), + } +} + +/// Makes several threads show as one. The first key is the thread the rest join, which is the key +/// the banner's Unmerge undoes from and the key every list shows the merged thread under. +pub fn merge( + conn: &Connection, + keys: &[String], + name: Option<&str>, +) -> Result<(String, Vec, Option), String> { + let merged_key = keys.first().ok_or("a merge needs at least two threads")?.clone(); + let sources: Vec = keys + .iter() + .skip(1) + .filter(|key| **key != merged_key) + .cloned() + .collect(); + if sources.is_empty() { + return Err("a merge needs at least two threads".into()); + } + let rename_before = state::read::rename_of(conn, &merged_key)?; + state::write::merge_threads(conn, &sources, &merged_key)?; + + let typed = name.map(str::trim).filter(|name| !name.is_empty()); + let chosen = match typed { + Some(name) => Some(name.to_string()), + None => longest_subject(conn, keys)?, + }; + // A name that says exactly what the subject already says would make the pane print + // "renamed, was ..." about nothing. + if chosen.is_some() && chosen != subject_of(conn, &merged_key)? { + set_name(conn, &merged_key, chosen.as_deref())?; + } + Ok((merged_key, sources, rename_before)) +} + +// --------------------------------------------------------------------------------------------- +// Undo +// --------------------------------------------------------------------------------------------- + +/// What each of these reverses is a row in the state database rather than a set of provider labels, +/// so they share a stack with each other and not with `mirror`. +/// +/// Every variant but the last names one account, because a note, a name and a merge are all about a +/// single thread. Ignore and notify take a selection, and a selection can span two mailboxes. +pub enum What { + Note { + account_id: String, + note: Note, + }, + Rename { + account_id: String, + before: Vec<(String, Option)>, + }, + Merge { + account_id: String, + merged_key: String, + sources: Vec, + rename_before: Option, + }, + Unmerge { + account_id: String, + merged_key: String, + sources: Vec, + }, + Flags { + before: Vec<(String, Vec<(String, ThreadFlags)>)>, + }, +} + +pub struct UndoDecision { + pub label: String, + pub what: What, +} + +pub fn owns(token: &str) -> bool { + UNDO.owns(token) +} + +pub fn undo_apply(app: &tauri::AppHandle, token: &str) -> Result<(), String> { + let entry = UNDO + .take(token) + .ok_or("that change can no longer be taken back")?; + let db = db_of(app)?; + match &entry.what { + What::Note { account_id, note } => db.with(account_id, |conn| restore_note(conn, note))?, + What::Rename { account_id, before } => db.with(account_id, |conn| { + for (key, name) in before { + set_name(conn, key, name.as_deref())?; + } + Ok(()) + })?, + What::Merge { + account_id, + merged_key, + sources, + rename_before, + } => db.with(account_id, |conn| { + for source in sources { + unmerge_one(conn, source)?; + } + set_name(conn, merged_key, rename_before.as_deref()) + })?, + What::Unmerge { + account_id, + merged_key, + sources, + } => db.with(account_id, |conn| { + state::write::merge_threads(conn, sources, merged_key) + })?, + What::Flags { before } => { + for (account_id, held) in before { + db.with(account_id, |conn| { + for (key, flags) in held { + state::write::set_thread_flags( + conn, + key, + Some(flags.ignored), + Some(flags.notify), + )?; + } + Ok(()) + })?; + } + } + } + crate::emit_store_changed(app, "threads state"); + Ok(()) +} + +fn handed_back(label: String, what: What) -> Undo { + let token = UNDO.push( + UndoDecision { + label: label.clone(), + what, + }, + &label, + ); + Undo { + token, + label, + undo_ms: 0, + } +} + +// --------------------------------------------------------------------------------------------- +// Commands +// --------------------------------------------------------------------------------------------- + +#[tauri::command(async)] +pub fn note_add(app: tauri::AppHandle, thread_key: String, body: String) -> Result { + let db = db_of(&app)?; + let account_id = holders(db.inner(), std::slice::from_ref(&thread_key))? + .into_iter() + .next() + .map(|(account_id, _)| account_id) + .ok_or("that thread is not on this device")?; + let note = db.with(&account_id, |conn| add_note(conn, &thread_key, &body))?; + crate::emit_store_changed(&app, &format!("threads thread:{thread_key} state")); + Ok(note) +} + +#[tauri::command(async)] +pub fn note_update(app: tauri::AppHandle, id: String, body: String) -> Result<(), String> { + let db = db_of(&app)?; + let account_id = account_holding( + db.inner(), + "SELECT COUNT(*) FROM state.notes WHERE id = ?1", + &id, + ) + .map_err(|_| "that note is not here".to_string())?; + let thread_key = db.with(&account_id, |conn| { + state::write::edit_note(conn, &id, &body)?; + Ok(note_row(conn, &id)?.map(|(note, _)| note.thread_key)) + })?; + let scope = thread_key + .map(|key| format!("threads thread:{key} state")) + .unwrap_or_else(|| "threads state".to_string()); + crate::emit_store_changed(&app, &scope); + Ok(()) +} + +#[tauri::command(async)] +pub fn note_delete(app: tauri::AppHandle, id: String) -> Result { + let db = db_of(&app)?; + let account_id = account_holding( + db.inner(), + "SELECT COUNT(*) FROM state.notes WHERE id = ?1", + &id, + ) + .map_err(|_| "that note is not here".to_string())?; + let note = db.with(&account_id, |conn| { + let held = note_row(conn, &id)? + .filter(|(_, deleted)| !deleted) + .map(|(note, _)| note) + .ok_or_else(|| "that note is not here".to_string())?; + state::write::delete_note(conn, &id)?; + Ok(held) + })?; + let scope = format!("threads thread:{} state", note.thread_key); + crate::emit_store_changed(&app, &scope); + Ok(handed_back( + "Note deleted".to_string(), + What::Note { account_id, note }, + )) +} + +/// A null name is back to the real subject, which is what the pane's "renamed · was …" is offering. +#[tauri::command(async)] +pub fn thread_rename( + app: tauri::AppHandle, + key: String, + name: Option, +) -> Result { + let db = db_of(&app)?; + let account_id = holders(db.inner(), std::slice::from_ref(&key))? + .into_iter() + .next() + .map(|(account_id, _)| account_id) + .ok_or("that thread is not on this device")?; + let before = db.with(&account_id, |conn| { + let before = state::read::rename_of(conn, &key)?; + set_name(conn, &key, name.as_deref())?; + Ok(before) + })?; + let label = match name.as_deref().map(str::trim).filter(|n| !n.is_empty()) { + Some(_) => "Renamed".to_string(), + None => "Name put back".to_string(), + }; + crate::emit_store_changed(&app, &format!("threads thread:{key} state")); + Ok(handed_back( + label, + What::Rename { + account_id, + before: vec![(key, before)], + }, + )) +} + +#[tauri::command(async)] +pub fn thread_merge( + app: tauri::AppHandle, + keys: Vec, + name: Option, +) -> Result { + let db = db_of(&app)?; + // One account, and it is the first key's: a merge across two mailboxes would be a thread whose + // messages no single provider holds, and neither list could show it. + let first = keys + .first() + .ok_or("a merge needs at least two threads")? + .clone(); + let (account_id, held) = holders(db.inner(), &keys)? + .into_iter() + .find(|(_, held)| held.contains(&first)) + .ok_or("those threads are not on this device")?; + let ordered: Vec = keys.iter().filter(|key| held.contains(key)).cloned().collect(); + let (merged_key, sources, rename_before) = + db.with(&account_id, |conn| merge(conn, &ordered, name.as_deref()))?; + + let label = format!("Merged {} threads", sources.len() + 1); + crate::emit_store_changed(&app, &format!("threads thread:{merged_key} state")); + Ok(handed_back( + label, + What::Merge { + account_id, + merged_key, + sources, + rename_before, + }, + )) +} + +#[tauri::command(async)] +pub fn thread_unmerge(app: tauri::AppHandle, key: String) -> Result { + let db = db_of(&app)?; + let account_id = holders(db.inner(), std::slice::from_ref(&key))? + .into_iter() + .next() + .map(|(account_id, _)| account_id) + .ok_or("that thread is not on this device")?; + let sources = db.with(&account_id, |conn| { + let sources = direct_sources(conn, &key)?; + if sources.is_empty() { + return Err("that thread is not a merge".to_string()); + } + state::write::unmerge(conn, &key)?; + Ok(sources) + })?; + crate::emit_store_changed(&app, "threads state"); + Ok(handed_back( + "Unmerged".to_string(), + What::Unmerge { + account_id, + merged_key: key, + sources, + }, + )) +} + +/// An ignored thread still receives its messages and still appends them. What changes is that it +/// stops coming back to New, which `mirror::read` decides when it decides the group. +#[tauri::command(async)] +pub fn thread_ignore(app: tauri::AppHandle, keys: Vec, on: bool) -> Result { + flags(&app, &keys, Some(on), None, |count| { + plural(count, if on { "Ignoring" } else { "No longer ignoring" }) + }) +} + +#[tauri::command(async)] +pub fn thread_notify(app: tauri::AppHandle, keys: Vec, on: bool) -> Result { + flags(&app, &keys, None, Some(on), |count| { + plural( + count, + if on { + "Notifications on" + } else { + "Notifications off" + }, + ) + }) +} + +fn flags( + app: &tauri::AppHandle, + keys: &[String], + ignored: Option, + notify: Option, + label: impl Fn(usize) -> String, +) -> Result { + let db = db_of(app)?; + let mut before: Vec<(String, Vec<(String, ThreadFlags)>)> = Vec::new(); + let mut touched = 0usize; + for (account_id, held) in holders(db.inner(), keys)? { + let held = db.with(&account_id, |conn| { + let mut before = Vec::new(); + for key in &held { + before.push((key.clone(), state::read::flags_of(conn, key)?)); + state::write::set_thread_flags(conn, key, ignored, notify)?; + } + Ok(before) + })?; + touched += held.len(); + before.push((account_id, held)); + } + let label = label(touched); + crate::emit_store_changed(app, "threads state"); + Ok(handed_back(label, What::Flags { before })) +} + +#[cfg(test)] +mod tests; diff --git a/src-tauri/src/decisions/tests.rs b/src-tauri/src/decisions/tests.rs new file mode 100644 index 0000000..2a3f80f --- /dev/null +++ b/src-tauri/src/decisions/tests.rs @@ -0,0 +1,270 @@ +// Notes, names, merges and the two switches, over a pair of in-memory databases. + +use rusqlite::Connection; + +use crate::db; +use crate::dto::{Destination, Place, ThreadQuery, ThreadSummary}; +use crate::mirror; +use crate::state; + +const NOW: i64 = 2_000_000_000_000; + +fn open() -> Connection { + db::memory().expect("a pair of in-memory databases") +} + +fn arrive(conn: &Connection, address: &str, subject: &str, at: i64) -> String { + let key = format!("<{address}-{at}@example>"); + let tid = format!("t-{address}-{at}"); + conn.execute( + "INSERT INTO threads (provider_thread_id, thread_key, latest_ms, message_count, unseen, + subject, snippet, from_name, from_address, in_inbox) + VALUES (?1, ?2, ?3, 1, 1, ?4, 'snippet', 'Someone', ?5, 1)", + rusqlite::params![tid, key, at, subject, address], + ) + .expect("a thread"); + conn.execute( + "INSERT INTO messages (id, provider_thread_id, thread_key, message_id, date_ms, + from_address, subject, snippet, hydrated, labels) + VALUES (?1, ?2, ?3, ?3, ?4, ?5, ?6, 'snippet', 1, '[\"INBOX\"]')", + rusqlite::params![format!("m-{tid}"), tid, key, at, address, subject], + ) + .expect("a message"); + state::write::set_rule(conn, address, false, Destination::Inbox, None).expect("a rule"); + key +} + +/// A new message in an existing thread, appended the way a sync would append it. +fn append(conn: &Connection, key: &str, at: i64) { + let tid: String = conn + .query_row( + "SELECT provider_thread_id FROM threads WHERE thread_key = ?1", + [key], + |row| row.get(0), + ) + .expect("the thread"); + conn.execute( + "INSERT INTO messages (id, provider_thread_id, thread_key, message_id, date_ms, + from_address, subject, snippet, hydrated, labels) + VALUES (?1, ?2, ?3, ?1, ?4, 'them@example.org', 'Hello', 'snippet', 1, '[\"INBOX\"]')", + rusqlite::params![format!("m-{tid}-{at}"), tid, key, at], + ) + .expect("another message"); + conn.execute( + "UPDATE threads SET latest_ms = ?2, message_count = message_count + 1, unseen = 1 + WHERE thread_key = ?1", + rusqlite::params![key, at], + ) + .expect("the thread moved on"); +} + +fn page(conn: &Connection, place: Place) -> Vec { + let query = ThreadQuery { + account_id: None, + place, + label_id: None, + query: None, + limit: 50, + cursor: None, + }; + mirror::read::threads_list(conn, "acct", "hue-1", &query, NOW) + .expect("a page") + .threads +} + +fn keys_in(conn: &Connection, place: Place) -> Vec { + page(conn, place).into_iter().map(|t| t.key).collect() +} + +// --------------------------------------------------------------------------------------------- +// Notes +// --------------------------------------------------------------------------------------------- + +#[test] +fn a_note_round_trips_and_a_second_one_is_a_second_note() { + let conn = open(); + let key = arrive(&conn, "maya@example.org", "The kitchen", 100); + + let first = super::add_note(&conn, &key, "Chase this on Friday").expect("a note"); + assert_eq!(first.body, "Chase this on Friday"); + assert_eq!(first.thread_key, key); + assert_eq!( + first.after_message_id.as_deref(), + Some("m-t-maya@example.org-100"), + "a note sits after the message that was latest when it was written" + ); + + let second = super::add_note(&conn, &key, "Rang, no answer").expect("a second note"); + assert_ne!(second.id, first.id); + let held = state::read::notes_on(&conn, &key).expect("the notes"); + assert_eq!(held.len(), 2); + assert_eq!(held[0].body, "Chase this on Friday"); + + // The list carries the latest one as its single line under the row. + assert_eq!(page(&conn, Place::Inbox)[0].note.as_deref(), Some("Rang, no answer")); +} + +#[test] +fn deleting_a_note_is_reversible() { + let conn = open(); + let key = arrive(&conn, "maya@example.org", "The kitchen", 100); + let note = super::add_note(&conn, &key, "Chase this on Friday").expect("a note"); + + state::write::delete_note(&conn, ¬e.id).expect("deleted"); + assert!(state::read::notes_on(&conn, &key).expect("the notes").is_empty()); + + super::restore_note(&conn, ¬e).expect("put back"); + let held = state::read::notes_on(&conn, &key).expect("the notes"); + assert_eq!(held.len(), 1); + assert_eq!(held[0].id, note.id); + assert_eq!(held[0].body, "Chase this on Friday"); +} + +// --------------------------------------------------------------------------------------------- +// Renames +// --------------------------------------------------------------------------------------------- + +#[test] +fn a_rename_shows_in_a_list_page_with_the_real_subject_beside_it() { + let conn = open(); + let key = arrive(&conn, "maya@example.org", "Re: Fwd: quote v4 FINAL", 100); + + super::set_name(&conn, &key, Some("Kitchen quote")).expect("renamed"); + + let row = &page(&conn, Place::Inbox)[0]; + assert_eq!(row.subject, "Kitchen quote"); + assert_eq!(row.original_subject.as_deref(), Some("Re: Fwd: quote v4 FINAL")); + + super::set_name(&conn, &key, None).expect("put back"); + let row = &page(&conn, Place::Inbox)[0]; + assert_eq!(row.subject, "Re: Fwd: quote v4 FINAL"); + assert!(row.original_subject.is_none()); +} + +// --------------------------------------------------------------------------------------------- +// Merges +// --------------------------------------------------------------------------------------------- + +#[test] +fn a_merge_of_three_shows_as_one_thread_named_after_the_longest() { + let conn = open(); + let first = arrive(&conn, "one@example.org", "Quote", 300); + let second = arrive(&conn, "two@example.org", "The kitchen quote, revised", 200); + let third = arrive(&conn, "three@example.org", "Re: quote", 100); + + let keys = vec![first.clone(), second.clone(), third.clone()]; + let (merged_key, sources, rename_before) = super::merge(&conn, &keys, None).expect("a merge"); + assert_eq!(merged_key, first); + assert_eq!(sources, vec![second.clone(), third.clone()]); + assert!(rename_before.is_none()); + + let inbox = page(&conn, Place::Inbox); + assert_eq!(inbox.len(), 1, "three threads merged show as one"); + assert_eq!(inbox[0].key, first); + assert_eq!(inbox[0].subject, "The kitchen quote, revised"); + assert!(inbox[0].merged); +} + +#[test] +fn a_chain_of_merges_shows_once() { + // Two devices can each merge without either meaning to make a chain: c into b, then b into a. + let conn = open(); + let first = arrive(&conn, "one@example.org", "One", 300); + let second = arrive(&conn, "two@example.org", "Two", 200); + let third = arrive(&conn, "three@example.org", "Three", 100); + + super::merge(&conn, &[second.clone(), third.clone()], Some("Two")).expect("c into b"); + super::merge(&conn, &[first.clone(), second.clone()], Some("One")).expect("b into a"); + + assert_eq!(keys_in(&conn, Place::Inbox), vec![first]); +} + +#[test] +fn an_unmerge_puts_them_all_back() { + let conn = open(); + let first = arrive(&conn, "one@example.org", "One", 300); + let second = arrive(&conn, "two@example.org", "Two", 200); + let third = arrive(&conn, "three@example.org", "Three", 100); + let keys = vec![first.clone(), second.clone(), third.clone()]; + super::merge(&conn, &keys, None).expect("a merge"); + + assert_eq!( + super::direct_sources(&conn, &first).expect("the sources"), + vec![third.clone(), second.clone()], + "the sources are what putting the merge back is made of" + ); + state::write::unmerge(&conn, &first).expect("unmerged"); + + let inbox = keys_in(&conn, Place::Inbox); + assert_eq!(inbox.len(), 3); + for key in keys { + assert!(inbox.contains(&key), "{key} did not come back"); + } +} + +#[test] +fn taking_back_one_merge_leaves_the_merge_it_joined() { + let conn = open(); + let first = arrive(&conn, "one@example.org", "One", 400); + let second = arrive(&conn, "two@example.org", "Two", 300); + let third = arrive(&conn, "three@example.org", "Three", 200); + super::merge(&conn, &[first.clone(), second.clone()], Some("One")).expect("the first merge"); + + let (_, sources, rename_before) = + super::merge(&conn, &[first.clone(), third.clone()], Some("One and three")).expect("more"); + for source in &sources { + super::unmerge_one(&conn, source).expect("back out"); + } + super::set_name(&conn, &first, rename_before.as_deref()).expect("the name back"); + + let inbox = page(&conn, Place::Inbox); + assert_eq!(inbox.len(), 2, "the earlier merge is untouched"); + assert_eq!(inbox[0].key, first); + assert_eq!(inbox[0].subject, "One"); + assert_eq!(inbox[1].key, third); +} + +// --------------------------------------------------------------------------------------------- +// Ignore and notify +// --------------------------------------------------------------------------------------------- + +#[test] +fn an_ignored_thread_receives_its_messages_and_does_not_return_to_new() { + let conn = open(); + let ignored = arrive(&conn, "loud@example.org", "The long thread", 100); + let ordinary = arrive(&conn, "maya@example.org", "Hello", 90); + state::write::set_thread_flags(&conn, &ignored, Some(true), None).expect("ignored"); + + append(&conn, &ignored, 300); + append(&conn, &ordinary, 200); + + let inbox = page(&conn, Place::Inbox); + let held = |key: &str| { + inbox + .iter() + .find(|row| row.key == key) + .expect("the row") + .clone() + }; + + let loud = held(&ignored); + assert_eq!(loud.message_count, 2, "the message still arrived and still appended"); + assert!(loud.ignored); + assert_eq!(loud.group, "seen", "an ignored thread never returns to New for you"); + assert_eq!(held(&ordinary).group, "new"); +} + +#[test] +fn the_two_switches_are_one_row_and_neither_clears_the_other() { + let conn = open(); + let key = arrive(&conn, "maya@example.org", "Hello", 100); + + state::write::set_thread_flags(&conn, &key, None, Some(true)).expect("notify"); + state::write::set_thread_flags(&conn, &key, Some(true), None).expect("ignore"); + + let flags = state::read::flags_of(&conn, &key).expect("the flags"); + assert!(flags.ignored && flags.notify); + + let row = &page(&conn, Place::Inbox)[0]; + assert!(row.ignored && row.notify); +} diff --git a/src-tauri/src/drafts.rs b/src-tauri/src/drafts.rs new file mode 100644 index 0000000..425650f --- /dev/null +++ b/src-tauri/src/drafts.rs @@ -0,0 +1,469 @@ +// Drafts: the copy on this machine and the copy that roams. +// +// Two rhythms, and the reason there are two is the whole of this module. The local write is +// synchronous, costs a row, and happens on every save the composer sends; the provider upload is +// rate limited and coalesces, because a draft saved on a keystroke would be a request on a +// keystroke and Gmail's per-user quota is not a rounding error at that rate. What makes the +// coalescing free is that the `drafts` row is itself the queue: the payload carries the version the +// composer last wrote and the version the provider last had, and ten keystrokes between two uploads +// are one upload rather than ten. +// +// The size check lives here rather than in the composer because the limit is about encoded bytes +// and the frontend has no idea what a base64 part costs. It is an estimate on purpose: building the +// message means reading every attachment off the disk, which is not something to do on a keystroke, +// and the send builds the real thing and checks again. +// +// The upload does not go through `sync::outbox`. That queue is drained against `sync::Remote`, +// which is `Provider` narrowed to what the engine needs and carries `send` but neither `draft_put` +// nor `draft_delete`, so a draft row in it would be a row the drain cannot push and every flag +// change behind it would wait. Widening that trait is the sync package's file to widen; until then +// the queue is the `drafts` table, which already has the two columns it needs. + +use std::collections::HashSet; +use std::sync::{Mutex, OnceLock}; + +use rusqlite::{Connection, OptionalExtension}; +use serde::{Deserialize, Serialize}; + +use crate::decisions::db_of; +use crate::dto::{Draft, DraftAttachment, DraftSaved, Person}; +use crate::mime::build::{self, Outgoing, OutgoingAttachment}; +use crate::mirror::{read, write}; +use crate::sync::{self, Store}; + +/// Gmail's ceiling on one message, encoded. Checked on every save so an attachment is refused while +/// it can still be taken off again, rather than at the send. +pub const MAX_ENCODED_BYTES: u64 = 35 * 1024 * 1024; + +/// How often a draft may reach the provider. The composer's own debounce decides how often a save +/// arrives; this decides how often one of them leaves the machine. +pub const UPLOAD_EVERY_MS: i64 = 5_000; + +/// What sits in the `drafts` row's payload. +/// +/// The version is bumped on every save and is what says whether the provider is behind. A +/// millisecond is not fine enough to tell two keystrokes apart, and a clock is the wrong thing to +/// ask anyway: what matters is whether this is the copy that went, not when it went. `uploaded_at` +/// is only the rate limit. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct Stored { + draft: Draft, + #[serde(default)] + version: i64, + #[serde(default)] + uploaded_version: i64, + #[serde(default)] + uploaded_at: i64, +} + +// --------------------------------------------------------------------------------------------- +// The local copy +// --------------------------------------------------------------------------------------------- + +fn stored(conn: &Connection, id: &str) -> Result, i64)>, String> { + let held: Option<(String, Option, i64)> = conn + .query_row( + "SELECT payload, provider_draft_id, updated_at FROM drafts WHERE id = ?1", + [id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .optional() + .map_err(|e| e.to_string())?; + let Some((payload, provider_draft_id, updated_at)) = held else { + return Ok(None); + }; + let stored: Stored = serde_json::from_str(&payload).map_err(|e| e.to_string())?; + Ok(Some((stored, provider_draft_id, updated_at))) +} + +/// Writes the draft and answers with what the composer needs to know about its size. +/// +/// The second save of a draft updates the row rather than adding one: the id is the app's own and +/// the composer sends it back with every save. +pub fn save(conn: &Connection, draft: &Draft) -> Result { + let id = draft + .id + .clone() + .filter(|id| !id.is_empty()) + .unwrap_or_else(|| write::fresh_id("draft")); + let mut draft = draft.clone(); + draft.id = Some(id.clone()); + + let held = stored(conn, &id)?.map(|(held, _, _)| held); + let payload = serde_json::to_string(&Stored { + draft: draft.clone(), + version: held.as_ref().map(|held| held.version).unwrap_or(0) + 1, + uploaded_version: held.as_ref().map(|held| held.uploaded_version).unwrap_or(0), + uploaded_at: held.map(|held| held.uploaded_at).unwrap_or(0), + }) + .map_err(|e| e.to_string())?; + let now = write::now_ms(); + + conn.execute( + "INSERT INTO drafts (id, thread_key, in_reply_to, payload, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT(id) DO UPDATE SET + thread_key = excluded.thread_key, + in_reply_to = excluded.in_reply_to, + payload = excluded.payload, + updated_at = excluded.updated_at", + rusqlite::params![id, draft.thread_key, draft.in_reply_to, payload, now], + ) + .map_err(|e| e.to_string())?; + + let encoded_size = encoded_size(&draft); + Ok(DraftSaved { + id, + updated_at_ms: now, + encoded_size, + over_limit: encoded_size > MAX_ENCODED_BYTES, + }) +} + +pub fn get(conn: &Connection, id: &str) -> Result { + stored(conn, id)? + .map(|(held, _, _)| held.draft) + .ok_or_else(|| "that draft is not on this device".to_string()) +} + +/// Removes the local row and hands back the provider's draft id, when the draft had reached it, so +/// the caller can take that copy away too. +pub fn delete(conn: &Connection, id: &str) -> Result, String> { + let provider_draft_id = stored(conn, id)?.and_then(|(_, provider, _)| provider); + conn.execute("DELETE FROM drafts WHERE id = ?1", [id]) + .map_err(|e| e.to_string())?; + Ok(provider_draft_id) +} + +// --------------------------------------------------------------------------------------------- +// What it weighs +// --------------------------------------------------------------------------------------------- + +/// Base64 is four characters for every three bytes, with a line ending every 76 of them. +fn base64_len(bytes: u64) -> u64 { + let encoded = bytes.div_ceil(3) * 4; + encoded + encoded.div_ceil(76) * 2 +} + +/// What this draft will weigh on the wire, near enough to refuse an attachment with. +/// +/// Estimated rather than built. The body is counted twice because every message carries a plain +/// text alternative built from the HTML, and each part carries its own headers and boundary. +pub fn encoded_size(draft: &Draft) -> u64 { + const PART_OVERHEAD: u64 = 512; + + let mut total = PART_OVERHEAD * 2 + draft.body_html.len() as u64 * 2; + total += draft.subject.len() as u64; + for person in draft.to.iter().chain(&draft.cc).chain(&draft.bcc) { + total += person.address.len() as u64 + 32; + } + for attachment in &draft.attachments { + total += base64_len(attachment.size) + PART_OVERHEAD + attachment.filename.len() as u64; + } + total +} + +// --------------------------------------------------------------------------------------------- +// The bytes +// --------------------------------------------------------------------------------------------- + +/// One attachment's bytes: a file the composer named, else a part of a message already here, which +/// is what a forward carries. +pub fn attachment_bytes( + conn: &Connection, + attachment: &DraftAttachment, +) -> Result, String> { + if let Some(path) = attachment.path.as_deref().filter(|path| !path.is_empty()) { + return std::fs::read(path) + .map_err(|e| format!("{} could not be read: {e}", attachment.filename)); + } + let id = attachment + .attachment_id + .as_deref() + .filter(|id| !id.is_empty()) + .ok_or_else(|| format!("{} has neither a file nor an attachment", attachment.filename))?; + + let row = read::attachment_row(conn, id)? + .ok_or_else(|| format!("{} is not on this device", attachment.filename))?; + if let Some(path) = &row.cached_path { + if let Ok(bytes) = std::fs::read(path) { + return Ok(bytes); + } + } + let raw = read::raw_body(conn, &row.message_id)? + .ok_or_else(|| format!("{} is not on this device", attachment.filename))?; + part_bytes(&raw, &row.filename, row.size) + .ok_or_else(|| format!("{} could not be read out of its message", attachment.filename)) +} + +/// The bytes of one part of a message on the device, matched on what the attachment row holds: the +/// decoded length and the file name. Body parts are skipped, so a text body the same length as a +/// file cannot be mistaken for it. +fn part_bytes(raw: &[u8], filename: &str, size: u64) -> Option> { + use mail_parser::{MessageParser, MimeHeaders, PartType}; + + let message = MessageParser::default().parse(raw)?; + let bodies: HashSet = message + .html_body + .iter() + .chain(message.text_body.iter()) + .map(|index| *index as usize) + .collect(); + + let mut fallback = None; + for (index, part) in message.parts.iter().enumerate() { + if bodies.contains(&index) { + continue; + } + let bytes = match &part.body { + PartType::Binary(bytes) | PartType::InlineBinary(bytes) => bytes.to_vec(), + PartType::Text(text) | PartType::Html(text) => text.as_bytes().to_vec(), + PartType::Multipart(_) | PartType::Message(_) => continue, + }; + if bytes.len() as u64 != size { + continue; + } + if part.attachment_name() == Some(filename) { + return Some(bytes); + } + fallback.get_or_insert(bytes); + } + fallback +} + +/// The draft as RFC 2822 bytes. The threading headers are the caller's to supply, because what a +/// reply belongs to is a fact about the thread and not about the draft. +pub fn built( + conn: &Connection, + draft: &Draft, + from: &Person, + message_id: Option<&str>, + in_reply_to: Option<&str>, + references: &[String], + subject: &str, +) -> Result, String> { + let mut attachments = Vec::new(); + for attachment in &draft.attachments { + attachments.push(OutgoingAttachment { + filename: attachment.filename.clone(), + mime_type: attachment.mime_type.clone(), + bytes: attachment_bytes(conn, attachment)?, + }); + } + + build::build(&Outgoing { + from: from.clone(), + to: draft.to.clone(), + cc: draft.cc.clone(), + bcc: draft.bcc.clone(), + reply_to: Vec::new(), + subject: subject.to_string(), + html: draft.body_html.clone(), + stylesheet: None, + message_id: message_id.map(str::to_string), + in_reply_to: in_reply_to.map(str::to_string), + references: references.to_vec(), + date_ms: Some(write::now_ms()), + attachments, + inline: Vec::new(), + }) +} + +// --------------------------------------------------------------------------------------------- +// The copy that roams +// --------------------------------------------------------------------------------------------- + +/// The drafts whose local copy is ahead of the provider's and whose moment has come. +fn due(conn: &Connection, now: i64) -> Result, Stored)>, String> { + let mut stmt = conn + .prepare("SELECT id, provider_draft_id, payload FROM drafts") + .map_err(|e| e.to_string())?; + let rows = stmt + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, String>(2)?, + )) + }) + .map_err(|e| e.to_string())? + .collect::, _>>() + .map_err(|e| e.to_string())?; + + let mut out = Vec::new(); + for (id, provider_draft_id, payload) in rows { + let Ok(held) = serde_json::from_str::(&payload) else { + continue; + }; + if held.version > held.uploaded_version && now - held.uploaded_at >= UPLOAD_EVERY_MS { + out.push((id, provider_draft_id, held)); + } + } + Ok(out) +} + +/// Records which version of the draft the provider now has. The payload is read back rather than +/// kept, because the composer may have written again while the upload was in the air and that write +/// must not be lost to this one. +fn uploaded( + conn: &Connection, + id: &str, + provider_draft_id: &str, + version: i64, + now: i64, +) -> Result<(), String> { + let Some((mut held, _, _)) = stored(conn, id)? else { + return Ok(()); + }; + held.uploaded_version = version; + held.uploaded_at = now; + let payload = serde_json::to_string(&held).map_err(|e| e.to_string())?; + conn.execute( + "UPDATE drafts SET provider_draft_id = ?2, payload = ?3 WHERE id = ?1", + rusqlite::params![id, provider_draft_id, payload], + ) + .map(|_| ()) + .map_err(|e| e.to_string()) +} + +/// Uploads every draft that is due, and answers with how many went. +/// +/// Generic over the provider rather than taking `sync::Remote`, because the two calls a draft makes +/// are the two that trait does not carry. +pub async fn upload( + store: &S, + provider: &P, + from: &Person, + now: i64, +) -> Result { + let mut sent = 0u32; + for (id, provider_draft_id, held) in store.with(|conn| due(conn, now))? { + let subject = held.draft.subject.clone(); + let from = crate::send::sender(&held.draft, from); + let raw = store.with(|conn| built(conn, &held.draft, &from, None, None, &[], &subject))?; + let thread_hint = store.with(|conn| thread_hint(conn, held.draft.thread_key.as_deref()))?; + + let put = provider + .draft_put(provider_draft_id.as_deref(), &raw, thread_hint.as_deref()) + .await + .map_err(|e| e.to_string())?; + store.with(|conn| uploaded(conn, &id, &put, held.version, now))?; + sent += 1; + } + Ok(sent) +} + +/// The provider's own id for a thread, so a draft or a reply lands in it on their side too. +pub fn thread_hint(conn: &Connection, thread_key: Option<&str>) -> Result, String> { + let Some(key) = thread_key.filter(|key| !key.is_empty()) else { + return Ok(None); + }; + conn.query_row( + "SELECT provider_thread_id FROM threads WHERE thread_key = ?1 ORDER BY latest_ms DESC + LIMIT 1", + [key], + |row| row.get(0), + ) + .optional() + .map_err(|e| e.to_string()) +} + +/// One account at a time, so two saves in the same second cannot both upload the same draft. +fn uploading() -> &'static Mutex> { + static UPLOADING: OnceLock>> = OnceLock::new(); + UPLOADING.get_or_init(|| Mutex::new(HashSet::new())) +} + +/// Puts every draft of one account that is due in front of the provider. +/// +/// Public because the poll loop is the other honest place to call it from: a draft written just +/// before the app was quit is uploaded when the app comes back, rather than waiting for somebody to +/// open the composer again. +pub async fn upload_pending(app: &tauri::AppHandle, account_id: &str) -> Result { + if !uploading() + .lock() + .map(|mut held| held.insert(account_id.to_string())) + .unwrap_or(false) + { + return Ok(0); + } + let done = upload_now(app, account_id).await; + if let Ok(mut held) = uploading().lock() { + held.remove(account_id); + } + done +} + +async fn upload_now(app: &tauri::AppHandle, account_id: &str) -> Result { + let db = db_of(app)?; + let from = crate::send::own_person(app, db.inner(), account_id)?; + let store = sync::Scoped { + db: db.inner(), + account_id, + }; + // The handle the engine registered for this account, which is the only thing in this module + // that knows the mailbox behind it is Gmail rather than an IMAP server. + let Some(provider) = sync::remote_for(account_id) else { + return Ok(0); + }; + upload(&store, provider.as_ref(), &from, write::now_ms()).await +} + +// --------------------------------------------------------------------------------------------- +// Commands +// --------------------------------------------------------------------------------------------- + +#[tauri::command(async)] +pub fn draft_save(app: tauri::AppHandle, draft: Draft) -> Result { + let db = db_of(&app)?; + let account_id = draft.account_id.clone(); + let saved = db.with(&account_id, |conn| save(conn, &draft))?; + + // The local write is the save. The upload is behind it on its own clock, so a composer that + // saves on every debounce is not a client that uploads on every debounce. + let handle = app.clone(); + tauri::async_runtime::spawn(async move { + tokio::time::sleep(std::time::Duration::from_millis(UPLOAD_EVERY_MS as u64)).await; + let _ = upload_pending(&handle, &account_id).await; + }); + Ok(saved) +} + +#[tauri::command(async)] +pub fn draft_get(app: tauri::AppHandle, id: String) -> Result { + let db = db_of(&app)?; + for (account_id, _) in sync::accounts(db.inner(), None) { + if let Ok(draft) = db.with(&account_id, |conn| get(conn, &id)) { + return Ok(draft); + } + } + Err("that draft is not on this device".to_string()) +} + +#[tauri::command] +pub async fn draft_delete(app: tauri::AppHandle, id: String) -> Result<(), String> { + let db = db_of(&app)?; + for (account_id, _) in sync::accounts(db.inner(), None) { + let held = db.with(&account_id, |conn| stored(conn, &id))?; + if held.is_none() { + continue; + } + let provider_draft_id = db.with(&account_id, |conn| delete(conn, &id))?; + if let Some(provider_draft_id) = provider_draft_id { + // The local row has gone either way. A provider that will not take the delete leaves a + // draft in the mailbox's own Drafts, which is visible and fixable, and refusing the + // command over it would leave the one on this machine that the person asked to be rid + // of. + if let Some(provider) = sync::remote_for(&account_id) { + let _ = provider.draft_delete(&provider_draft_id).await; + } + } + crate::emit_store_changed(&app, "threads"); + return Ok(()); + } + Err("that draft is not on this device".to_string()) +} + +#[cfg(test)] +mod tests; diff --git a/src-tauri/src/drafts/tests.rs b/src-tauri/src/drafts/tests.rs new file mode 100644 index 0000000..6823d53 --- /dev/null +++ b/src-tauri/src/drafts/tests.rs @@ -0,0 +1,193 @@ +// Drafts over a pair of in-memory databases and the fake mailbox. +// +// The two rhythms are what these hold to: a save is a row and nothing else, and the upload is rate +// limited, so the assertions are about how many rows there are and how many calls the provider saw. + +use std::sync::Mutex; + +use rusqlite::Connection; +use tauri::async_runtime::block_on; + +use crate::db; +use crate::dto::{Draft, DraftAttachment, Person}; +use crate::mirror::write; +use crate::provider::fake::FakeProvider; +use crate::sync::Store; + +use super::{save, upload, MAX_ENCODED_BYTES, UPLOAD_EVERY_MS}; + +struct Memory(Mutex); + +impl Store for Memory { + fn with Result>(&self, f: F) -> Result { + let conn = self.0.lock().map_err(|e| e.to_string())?; + f(&conn) + } +} + +fn store() -> Memory { + Memory(Mutex::new(db::memory().expect("a pair of in-memory databases"))) +} + +fn me() -> Person { + Person { + name: Some("You".to_string()), + address: "you@example.com".to_string(), + } +} + +fn draft(body: &str) -> Draft { + Draft { + id: None, + account_id: "acct".to_string(), + thread_key: None, + in_reply_to: None, + from_alias: None, + to: vec![Person { + name: Some("Ana".to_string()), + address: "ana@example.test".to_string(), + }], + cc: Vec::new(), + bcc: Vec::new(), + subject: "About the lease".to_string(), + body_html: format!("

{body}

"), + attachments: Vec::new(), + remind_at_ms: None, + } +} + +fn rows(store: &Memory) -> i64 { + store + .with(|conn| { + conn.query_row("SELECT COUNT(*) FROM drafts", [], |row| row.get(0)) + .map_err(|e| e.to_string()) + }) + .expect("a count") +} + +fn puts(fake: &FakeProvider) -> Vec { + fake.calls() + .into_iter() + .filter(|call| call.starts_with("draft_put")) + .collect() +} + +#[test] +fn a_second_save_updates_the_draft_rather_than_adding_one() { + let store = store(); + let first = store + .with(|conn| save(conn, &draft("Half a sentence"))) + .expect("the first save"); + let mut again = draft("Half a sentence, and the rest of it"); + again.id = Some(first.id.clone()); + let second = store.with(|conn| save(conn, &again)).expect("the second save"); + + assert_eq!(second.id, first.id, "the composer's id is the draft's id"); + assert_eq!(rows(&store), 1, "one draft, not two"); + assert_eq!( + store.with(|conn| super::get(conn, &first.id)).expect("the draft").body_html, + "

Half a sentence, and the rest of it

" + ); +} + +#[test] +fn the_upload_coalesces_rather_than_going_once_per_keystroke() { + let store = store(); + let fake = FakeProvider::new(); + + let mut id = None; + for keystroke in 0..5 { + let mut typing = draft(&format!("Typing {keystroke}")); + typing.id = id.clone(); + id = Some( + store + .with(|conn| save(conn, &typing)) + .expect("a save") + .id, + ); + } + assert_eq!(rows(&store), 1); + assert!(puts(&fake).is_empty(), "saving is not uploading"); + + let now = write::now_ms(); + assert_eq!( + block_on(upload(&store, &fake, &me(), now)).expect("an upload"), + 1 + ); + assert_eq!(puts(&fake).len(), 1, "five saves, one request"); + + // Nothing has changed since, so there is nothing to send. + assert_eq!(block_on(upload(&store, &fake, &me(), now)).expect("nothing to do"), 0); + + // And a keystroke inside the interval waits for it rather than going straight out. + let mut typing = draft("Typing again"); + typing.id = id.clone(); + store.with(|conn| save(conn, &typing)).expect("another save"); + assert_eq!(block_on(upload(&store, &fake, &me(), now)).expect("too soon"), 0); + assert_eq!(puts(&fake).len(), 1); + + assert_eq!( + block_on(upload(&store, &fake, &me(), now + UPLOAD_EVERY_MS)).expect("the interval passed"), + 1 + ); + let puts = puts(&fake); + assert_eq!(puts.len(), 2); + assert!( + puts[1].contains("draft-1"), + "the second upload updates the draft the first one created: {puts:?}" + ); +} + +#[test] +fn a_draft_over_the_size_limit_says_so_before_a_send_is_attempted() { + let store = store(); + let fake = FakeProvider::new(); + + let mut heavy = draft("The plans are attached"); + heavy.attachments = vec![DraftAttachment { + path: Some("/tmp/plans.pdf".to_string()), + attachment_id: None, + filename: "plans.pdf".to_string(), + mime_type: "application/pdf".to_string(), + size: 30 * 1024 * 1024, + }]; + + let saved = store.with(|conn| save(conn, &heavy)).expect("a save"); + assert!( + saved.encoded_size > 30 * 1024 * 1024, + "base64 costs a third on top: {}", + saved.encoded_size + ); + assert!( + saved.over_limit, + "40 MB encoded is over the {} MB one message may be", + MAX_ENCODED_BYTES / (1024 * 1024) + ); + assert!( + fake.calls().is_empty(), + "the refusal is a fact about the draft and costs no request" + ); + + // The same draft without the attachment is not over anything. + let mut light = heavy.clone(); + light.id = saved.id.clone().into(); + light.attachments.clear(); + let saved = store.with(|conn| save(conn, &light)).expect("a save"); + assert!(!saved.over_limit); +} + +#[test] +fn deleting_a_draft_hands_back_the_copy_the_provider_is_holding() { + let store = store(); + let fake = FakeProvider::new(); + let id = store + .with(|conn| save(conn, &draft("A line"))) + .expect("a save") + .id; + block_on(upload(&store, &fake, &me(), write::now_ms())).expect("an upload"); + + let provider_draft_id = store.with(|conn| super::delete(conn, &id)).expect("the delete"); + assert_eq!(provider_draft_id.as_deref(), Some("draft-1")); + assert_eq!(rows(&store), 0); + assert!(store.with(|conn| super::get(conn, &id)).is_err()); +} diff --git a/src-tauri/src/dto.rs b/src-tauri/src/dto.rs index 88ef610..f7a3e9d 100644 --- a/src-tauri/src/dto.rs +++ b/src-tauri/src/dto.rs @@ -1 +1,927 @@ -// The IPC contract. Every type here has a matching declaration in src/ipc.ts. +// The IPC contract. Every type here has a matching declaration in src/ipc.ts, and both sides are +// frozen once written: implementation modules add bodies, not fields. +// +// Two rules run through the whole file. Nothing that crosses this boundary carries a provider +// identifier the frontend could act on, because a view is the mirror joined to the state and the +// frontend is not entitled to know which of the two a value came from. And every list the frontend +// renders arrives already ordered and already grouped, because the grouping is part of the view +// and computing it twice in two languages is how the two drift. + +use serde::{Deserialize, Serialize}; + +// ------------------------------------------------------------------------------------------- +// People, accounts and places +// ------------------------------------------------------------------------------------------- + +/// One correspondent. `name` is whatever the header carried, which is often nothing. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Person { + pub name: Option, + pub address: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Account { + pub id: String, + pub email: String, + /// Which mailbox is behind this account, and therefore which `Provider` drives it. Read by the + /// frontend to know whether a scope list means anything here: an IMAP account has none. + pub kind: AccountKind, + /// The display name from the provider's profile, editable in settings. + pub name: String, + /// One of the eight hues, as a token name (`hue-1`), not a hex. The stylesheet owns the value. + pub color: String, + pub connected: bool, + /// What Google actually granted. Granular consent means a user can untick a scope on the + /// consent screen, so every feature that needs one checks this rather than assuming. + pub granted_scopes: Vec, + /// How far back the mirror keeps this account's mail, in days. Zero means everything. + pub window_days: u32, +} + +/// The two kinds of mailbox, which is the one thing above the `Provider` trait that has to know. +/// +/// Everything else in the app is written against the trait. This exists because a few screens +/// genuinely differ: an IMAP account has no scopes to grant, no Google account page to revoke +/// from, and a server and port to show in settings that a Google account does not have. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum AccountKind { + Google, + Imap, +} + +impl Default for AccountKind { + /// Every account that existed before there was a second kind is a Google one. + fn default() -> Self { + AccountKind::Google + } +} + +// ------------------------------------------------------------------------------------------- +// IMAP and SMTP configuration +// ------------------------------------------------------------------------------------------- + +/// How a socket is protected. +/// +/// The three names Thunderbird's autoconfig format uses, because every published configuration in +/// the world is written in that vocabulary and translating it twice is how the meanings drift. +/// `Tls` is what the format calls SSL: TLS from the first byte, on 993 or 465. `StartTls` is a +/// plaintext connection upgraded by a command, on 143 or 587. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Security { + Plain, + StartTls, + Tls, +} + +/// Which family of SASL mechanism to authenticate with. The exact mechanism is chosen from what +/// the server advertises; this only says which list to choose from. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum AuthKind { + Password, + /// Reachable from a discovered configuration, which is why it can be represented. Nothing + /// builds one yet: OAuth over IMAP needs a client registered with that provider. + OAuth2, +} + +/// One end of a mail account: a host, a port, how the socket is protected and who to log in as. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ServerConfig { + pub host: String, + pub port: u16, + pub security: Security, + pub auth: AuthKind, + /// Already expanded. `%EMAILADDRESS%` and `%EMAILLOCALPART%` are substituted inside discovery, + /// so nothing downstream has to know that the format has placeholders in it. + pub username: String, +} + +/// Both ends, and where they came from. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MailConfig { + pub imap: ServerConfig, + pub smtp: ServerConfig, + /// Which rung of the ladder answered: `autoconfig`, `ispdb`, `mx`, `probe` or `manual`. The + /// connect screen says where the settings came from, because "we found these" and "we guessed + /// these" are different promises and a person about to type a password should be told which. + pub source: String, + /// The provider's own name for itself, when the configuration carried one. + pub display_name: Option, +} + +/// A certificate the user has to decide about before a connection can be made. +/// +/// Only ever raised for a host that is not the loopback. Proton Bridge listens on 127.0.0.1 with a +/// certificate it generated itself, and there is nothing between this process and that socket to +/// impersonate anybody, so loopback is trusted without asking. Every other host is a question. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CertQuestion { + pub host: String, + pub port: u16, + /// SHA-256 of the DER, uppercase hex in colon-separated pairs, which is the form every other + /// tool prints so it can be compared against one. + pub fingerprint: String, + pub subject: String, + pub issuer: String, + pub expires_ms: i64, + /// `self-signed`, `expired` or `unknown-issuer`. A hostname mismatch is never a question: it + /// is the one failure that looks exactly like an interception, so it is refused outright. + pub reason: String, +} + +/// What a connection test found. Not a `Result`, because "the password was wrong" and "the +/// certificate needs a decision" are answers the screen renders rather than errors it reports. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ConnectReport { + pub ok: bool, + /// What kind of refusal it was: `unreachable`, `auth`, `certificate`, `wrong-host` or `other`. + /// + /// A separate field rather than something the screen reads back out of `message`, because the + /// screen has real decisions hanging off it (a loopback that is unreachable means the bridge + /// is not running, and that is a different sentence from a wrong password) and reading them + /// out of prose means a regex over whatever the server happened to say that day. + pub kind: Option, + /// `imap` or `smtp`, when one leg failed and the other did not. + pub failed: Option, + /// One sentence, already fit to read. The server's own words when they are usable. + pub message: Option, + pub cert: Option, + /// Set when the server refused the password but named a reason a person can act on, such as + /// an app password being required. + pub advice: Option, +} + +/// Every view the app can show. `Label` and `Search` carry an argument in `ThreadQuery`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Place { + Inbox, + Feed, + PaperTrail, + ReplyLater, + SetAside, + Screener, + Snoozed, + Everything, + Sent, + Drafts, + Starred, + ScreenedOut, + Spam, + Trash, + Label, + Search, +} + +/// Where a sender's mail goes. Exactly one per sender, which is the whole of the routing model. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Destination { + Inbox, + Feed, + PaperTrail, + ScreenedOut, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Pile { + ReplyLater, + SetAside, +} + +/// The surface a message body renders on, decided from what the sender painted rather than from +/// how the bytes arrived. `sanitize::surface` is where it is worked out and why. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Surface { + /// The sender painted nothing, so the body takes the app's own paper and follows the theme. + #[default] + Theme, + /// The sender painted an opaque light page across the layout, so the body keeps it in both + /// palettes and the pane's chrome goes dark around it. + Paper, +} + +/// What a list asks for. `account_id` of None means every account, which is the All accounts view. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ThreadQuery { + #[serde(default)] + pub account_id: Option, + pub place: Place, + /// The provider's label id, for `Place::Label`. + #[serde(default)] + pub label_id: Option, + /// The query text, for `Place::Search`. + #[serde(default)] + pub query: Option, + pub limit: u32, + /// Opaque, from the previous page's `next_cursor`. None starts at the top. + #[serde(default)] + pub cursor: Option, +} + +// ------------------------------------------------------------------------------------------- +// Threads +// ------------------------------------------------------------------------------------------- + +/// One row of a list, already grouped and already ordered. +/// +/// `key` is the portable thread key: the first entry of the message's `References` header, else its +/// `In-Reply-To`, else its own `Message-ID`. It does not depend on which messages happen to be +/// mirrored, so it survives a storage window changing, a second device, and a move to another +/// provider. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ThreadSummary { + pub key: String, + pub account_id: String, + /// The account's hue token, for the coloured edge in All accounts. + pub account_color: String, + /// What to print. The rename when there is one, the real subject otherwise. + pub subject: String, + /// Set only when the thread was renamed, so the row can say "renamed · was …". + pub original_subject: Option, + /// Whose name goes on the row: the latest correspondent who is not the account. + pub from: Person, + /// Everyone in the thread, for the stacked avatars in the pane. + pub participants: Vec, + pub snippet: String, + /// Epoch milliseconds of the latest message. + pub date_ms: i64, + pub message_count: u32, + /// At least one message the account has not seen. There is no count anywhere in this app. + pub unseen: bool, + pub starred: bool, + /// In the provider's trash, and in its spam. Flags rather than places: a search result carries + /// them into a list that is not Trash or Spam, and neither the row nor the pane can work them + /// out from the place it is being shown in. + pub trashed: bool, + pub spam: bool, + pub has_attachment: bool, + pub has_draft: bool, + pub pile: Option, + /// Epoch milliseconds: the moment this thread was due back. + /// + /// In the future while it is still waiting, in the past once it has come back and is sitting in + /// the Back group, which is how a thread that returned late can say "Due yesterday" rather than + /// pretending. The group tells the two apart, so a row never has to guess. + pub snoozed_until: Option, + pub ignored: bool, + pub notify: bool, + pub merged: bool, + /// The one line under the row. The whole note is in the pane. + pub note: Option, + /// Which group head this row sits under: "back", "new", "seen", "this-week", "earlier" or + /// "sent-to". The view decides both the grouping and the order the groups come in, and the + /// frontend renders a head whenever this changes and never sorts again. Sorting twice is how + /// Back ends up in the middle of the Inbox. + pub group: String, + /// Waiting in the outbox, so the row can say "Waiting to send". + pub sending: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ThreadPage { + pub threads: Vec, + /// Feed the next call. None when this is the end of the list. + pub next_cursor: Option, + /// The quiet line at the foot: "Showing the last month. Older mail is on Gmail." or the + /// search's "Search older mail on Gmail". None when there is nothing to say. + pub footer: Option, +} + +/// A thread this one was merged from, for the banner that offers Unmerge. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MergedSource { + pub key: String, + pub subject: String, +} + +/// The whole thread, for the reading pane. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ThreadView { + pub key: String, + pub account_id: String, + pub subject: String, + pub original_subject: Option, + pub participants: Vec, + pub messages: Vec, + pub notes: Vec, + pub merged_from: Vec, + pub pile: Option, + pub snoozed_until: Option, + pub ignored: bool, + pub notify: bool, + pub starred: bool, + /// The same two flags the row carries, so the pane can offer the way back out. + pub trashed: bool, + pub spam: bool, + /// The provider's labels on this thread, for the label list. Never rendered in a row. + pub labels: Vec, +} + +// ------------------------------------------------------------------------------------------- +// Messages +// ------------------------------------------------------------------------------------------- + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Attachment { + pub id: String, + pub message_id: String, + pub filename: String, + pub mime_type: String, + pub size: u64, + /// Referenced from the body by `cid:` rather than listed as a chip. + pub inline: bool, + pub content_id: Option, + /// Already in the local cache, so opening it costs nothing. + pub cached: bool, +} + +/// A remote image that was removed before the body was rendered, and who it belonged to. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Tracker { + /// The vendor from the shipped list, or the bare host when it is not a known one. + pub vendor: String, + pub url: String, +} + +/// `List-Unsubscribe`, in the three forms it arrives in. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Unsubscribe { + /// RFC 8058: the app POSTs and the sender is required to honour it without a round trip. + pub one_click: bool, + pub mailto: Option, + pub url: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum InviteResponse { + Accepted, + Tentative, + Declined, + NeedsAction, +} + +/// A `text/calendar` part with `METHOD:REQUEST`, rendered as a card. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Invite { + pub uid: String, + pub summary: String, + pub start_ms: i64, + pub end_ms: i64, + pub all_day: bool, + pub location: Option, + pub organizer: Option, + pub description: Option, + /// What this account has already answered, when it has. + pub my_response: InviteResponse, + /// A deep link into Margin Calendar, when the event can be addressed there. + pub calendar_link: Option, +} + +/// One message, sanitised and ready to render. +/// +/// `html` is what goes into the iframe's `srcdoc`. It has been through the sanitiser, so `cid:` +/// images are already `data:` URIs and every remote image is either removed or, once the user has +/// asked for them, fetched by Rust and inlined the same way. The frontend never fetches anything. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MessageView { + /// The provider's message id. Every command argument named `message_id` is this one, never the + /// header below it: the RFC `Message-ID` is what the state database keys on and it never + /// crosses this boundary as an argument. + pub id: String, + /// The RFC `Message-ID`, which is what the state database keys on. + pub message_id: String, + pub thread_key: String, + pub from: Person, + pub to: Vec, + pub cc: Vec, + pub bcc: Vec, + pub reply_to: Vec, + pub date_ms: i64, + pub subject: String, + pub html: String, + /// True when this message's body has not been fetched yet, so `html` is empty because there is + /// nothing to show rather than because the message was. + /// + /// Opening a thread never waits on the network. The rows come back from the mirror at once and + /// anything still missing a body arrives on a later `store-changed`, which is the difference + /// between a thread that opens and a thread that loads. + pub body_pending: bool, + /// The trailing quoted conversation, split off so it can sit behind a pill. + pub quoted_html: Option, + /// True when the source was HTML. A plain text message has been converted, and the pane sets + /// it on a 46em measure in the text face rather than leaving it to the sender's markup. + pub is_html: bool, + /// Which surface the body reads on. Not the same question as `is_html`: what decides it is + /// whether the sender painted a page, and most HTML mail paints nothing. + pub surface: Surface, + pub attachments: Vec, + pub trackers: Vec, + /// Ordinary remote images that were blocked, which is a different count from the trackers. + pub blocked_images: u32, + /// Whether the body currently rendered has remote images loaded. + pub images_loaded: bool, + pub seen: bool, + pub draft: bool, + pub sent_by_me: bool, + pub invite: Option, + pub unsubscribe: Option, + pub list_id: Option, +} + +// ------------------------------------------------------------------------------------------- +// The decisions: rules, piles, snoozes, notes, clips +// ------------------------------------------------------------------------------------------- + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SenderRule { + pub account_id: String, + /// An address, or a domain when `is_domain`. + pub subject: String, + pub is_domain: bool, + pub destination: Destination, + pub decided_at_ms: i64, + /// The suggestion rule that fired when this was set automatically, for the contact card. + pub reason: Option, +} + +/// One waiting sender in the Screener. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScreenerCard { + pub account_id: String, + pub sender: Person, + pub thread_key: String, + pub subject: String, + pub snippet: String, + pub date_ms: i64, + pub suggestion: Destination, + /// The sentence the card prints, which is the row of the rules table that matched. + pub reason: String, + /// How many messages this sender has waiting, when it is more than one. + pub waiting: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum SnoozeKind { + LaterToday, + Tomorrow, + Weekend, + NextWeek, + Date, + /// Comes back only if nobody but the account has written since. + IfNoReply, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Snooze { + pub thread_key: String, + pub return_at_ms: i64, + pub kind: SnoozeKind, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Note { + pub id: String, + pub thread_key: String, + pub body: String, + pub created_at_ms: i64, + /// The message that was latest when the note was written, so the pane can place it. + pub after_message_id: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Clip { + pub id: String, + pub account_id: String, + pub thread_key: String, + pub message_id: String, + pub text: String, + pub sender: Person, + pub subject: String, + pub created_at_ms: i64, +} + +/// Everything the app knows about one correspondent, for the popover and the Contacts place. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ContactCard { + pub person: Person, + pub account_id: String, + pub destination: Destination, + /// The rule that decides them is on their whole domain rather than their address. + pub domain_rule: bool, + /// A consumer domain cannot carry a domain rule, so the toggle is not offered. + pub domain_rule_allowed: bool, + pub notify: bool, + pub screened_at_ms: Option, + pub note: Option, + pub allow_remote_images: bool, + /// Trash this sender's Feed mail after so many days. None is never. + pub auto_trash_days: Option, + /// Bundle this sender's Paper Trail rows into one. + pub bundle: bool, + pub recent_threads: Vec, + pub files: Vec, + pub unsubscribe: Option, +} + +/// The patch the contact card writes back. An absent field means unchanged. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ContactPatch { + #[serde(default)] + pub destination: Option, + #[serde(default)] + pub domain_rule: Option, + #[serde(default)] + pub notify: Option, + #[serde(default)] + pub note: Option, + #[serde(default)] + pub allow_remote_images: Option, + #[serde(default)] + pub auto_trash_days: Option>, + #[serde(default)] + pub bundle: Option, +} + +// ------------------------------------------------------------------------------------------- +// Writing +// ------------------------------------------------------------------------------------------- + +/// A file on the way out, before it has been encoded into a message. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DraftAttachment { + /// Absent for one that is already in the mirror, which is what a forward carries. + #[serde(default)] + pub path: Option, + /// The mirror's attachment id, for a forward. + #[serde(default)] + pub attachment_id: Option, + pub filename: String, + pub mime_type: String, + pub size: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Draft { + /// The app's own draft id. Absent on the first save. + #[serde(default)] + pub id: Option, + pub account_id: String, + /// The thread being replied to, which is what sets the threading headers. + #[serde(default)] + pub thread_key: Option, + /// The message being replied to or forwarded. + #[serde(default)] + pub in_reply_to: Option, + /// A verified alias to send as, when it is not the account's own address. + #[serde(default)] + pub from_alias: Option, + pub to: Vec, + #[serde(default)] + pub cc: Vec, + #[serde(default)] + pub bcc: Vec, + pub subject: String, + /// The editor's HTML. Rust inlines the stylesheet and builds the plain text alternative. + pub body_html: String, + #[serde(default)] + pub attachments: Vec, + /// Set a reminder on the thread if nobody replies by then. + #[serde(default)] + pub remind_at_ms: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DraftSaved { + pub id: String, + pub updated_at_ms: i64, + /// The encoded size, so the composer can refuse an attachment before the send fails. + pub encoded_size: u64, + pub over_limit: bool, +} + +/// A send that is waiting out its undo delay, or waiting for the network. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Outgoing { + pub id: String, + pub account_id: String, + pub thread_key: Option, + /// Who the toast names. + pub to: Vec, + pub subject: String, + /// Epoch milliseconds. Until then the send can still be taken back. + pub hold_until_ms: i64, + pub attempts: u32, + pub last_error: Option, +} + +// ------------------------------------------------------------------------------------------- +// Undo +// ------------------------------------------------------------------------------------------- + +/// What a mutating command hands back so `z` can take it back. +/// +/// The token is a handle on the state before the change, held in a bounded stack in Rust. It is +/// there rather than in the frontend because a bulk archive of forty threads with mixed prior +/// state cannot be reversed from what the frontend knew, and a reversal that guesses is worse than +/// no reversal at all. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Undo { + pub token: String, + /// What the toast says: "Archived 3 threads", "Sent to Ana". + pub label: String, + /// Zero for anything but a send. A send's toast counts down. + pub undo_ms: u32, +} + +// ------------------------------------------------------------------------------------------- +// Search, labels, files +// ------------------------------------------------------------------------------------------- + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SearchResult { + pub page: ThreadPage, + /// The query reached past the storage window, so the provider was asked as a second pass. + pub provider_searched: bool, + /// A note appended under the results when the two sources differ. + pub note: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LabelInfo { + pub id: String, + pub account_id: String, + pub name: String, + /// system or user. The system ones are places already and are not offered for applying. + pub kind: String, +} + +/// One card in the All files place. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FileCard { + pub attachment: Attachment, + pub thread_key: String, + pub subject: String, + pub sender: Person, + pub date_ms: i64, + /// images, pdfs, documents, spreadsheets, presentations, invites, archives, other + pub category: String, +} + +// ------------------------------------------------------------------------------------------- +// Settings +// ------------------------------------------------------------------------------------------- + +/// margin-shared's `FontRef`, which is either one of the six bundled faces or a family off the +/// machine. Stored rather than a bare family name, because a system font can be called "Literata" +/// and must not come back as the bundled one. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "camelCase")] +pub enum FontRef { + #[serde(rename_all = "camelCase")] + Bundled { id: String }, + #[serde(rename_all = "camelCase")] + System { family: String }, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AccountSettings { + pub account_id: String, + pub name: String, + pub color: String, + /// 30, 90, 180, 365, or 0 for everything. + pub window_days: u32, + pub signature: String, + pub aliases: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SnoozeTimes { + /// Hours from now for Later today. + pub later_today_hours: u32, + /// Local time of day, in minutes from midnight. + pub tomorrow_at: u32, + pub weekend_at: u32, + pub next_week_at: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BackupSettings { + /// none, drive or r2 + pub store: String, + pub configured: bool, + pub last_backup_ms: Option, + /// Shown once, at setup. Never returned again. + pub has_phrase: bool, + /// R2 only. The secret never crosses this boundary in the reading direction. + pub r2_bucket: Option, + pub r2_endpoint: Option, +} + +/// Everything the settings screen edits. Device settings live in `settings.json` in the app data +/// directory, written atomically; the per account decisions that should roam live in the state +/// database and its journal instead, and are edited through the contact card and the account +/// sections rather than through this blob. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Settings { + /// light, dark or system + pub theme: String, + pub font_ui: FontRef, + pub font_text: FontRef, + pub text_size: u32, + pub reading_pane: bool, + /// comfortable or compact, on the phone + pub density: String, + + pub accounts: Vec, + pub attachment_cache_mb: u32, + pub prefetch_bodies: bool, + + /// never, ask or always. The per sender allowances are not here: they live on the contact, + /// because they are a decision about a person and they roam with the rest of those. + pub remote_images: String, + pub link_cleaning: bool, + + pub screener_enabled: bool, + /// A reply to a thread you are in is never held. Turning this off holds it anyway. + pub hold_replies: bool, + pub suggestions: bool, + + pub snooze_times: SnoozeTimes, + /// The pile a swipe reaches on the phone: reply-later, set-aside, archive, trash or none. + pub swipe_right: String, + pub swipe_left: String, + pub feed_auto_trash_days: u32, + + pub undo_delay_secs: u32, + pub reply_all_default: bool, + pub instant_intro: String, + + pub badge: bool, + /// The switch over everything below: off, and nothing is posted whatever a thread, a person or + /// a place says. On by default, and absent from files written before it existed. + #[serde(default = "on")] + pub notifications: bool, + /// The places that notify without being asked thread by thread. Empty is the default, which is + /// the product's position: nothing tells you anything until you say so. + pub notify_places: Vec, + + pub backup: BackupSettings, +} + +fn on() -> bool { + true +} + +/// Whether the system will show this app's notifications: what System Settings says on macOS, and +/// Prompt until the app has asked once. Everywhere else the answer is Granted. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum NotifyPermission { + Granted, + Denied, + Prompt, +} + +/// Where a click on a notification goes: the account it was about, the place its thread shows in, +/// and the thread itself when the notification was about one. Several messages in one pass are +/// one notification, and a click on that goes to the place alone. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NotifyTarget { + pub account_id: String, + pub place: Place, + pub thread_key: Option, +} + + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StorageUsed { + pub account_id: String, + pub messages: u64, + pub threads: u64, + pub mirror_bytes: u64, + pub bodies_bytes: u64, + pub attachments_bytes: u64, + pub state_bytes: u64, + /// The date of the oldest message on the device, which is what the window setting shows. + pub oldest_ms: Option, +} + +// ------------------------------------------------------------------------------------------- +// Sync, auth and events +// ------------------------------------------------------------------------------------------- + +/// Per account, because one account being signed out must not be reported as the app being broken. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SyncStatus { + pub account_id: String, + /// idle | syncing | hydrating | caching | backfilling | offline | error | paused + pub phase: String, + pub last_sync_ms: Option, + pub error: Option, + pub pending_writes: u32, + /// The line the progress screen and the account chip show, when there is one. + pub message: Option, + /// Whatever fill is running: the first sync's metadata crawl, and then the body cache behind + /// it. Both zero when there is nothing left to bring in. + pub hydrated: u32, + pub total: u32, + /// How far back the mirror actually reaches, which is not the same as the window setting until + /// the first sync has finished. + pub oldest_ms: Option, +} + +impl SyncStatus { + pub fn idle(account_id: &str) -> Self { + SyncStatus { + account_id: account_id.to_string(), + phase: "idle".to_string(), + last_sync_ms: None, + error: None, + pending_writes: 0, + message: None, + hydrated: 0, + total: 0, + oldest_ms: None, + } + } +} + +/// Payload of the `auth` event. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AuthEvent { + pub ok: bool, + pub error: Option, + pub account_id: Option, + pub email: Option, + /// The user closed the consent browser rather than anything going wrong. Still not `ok`, + /// because no account arrived, but changing your mind is not a failure. + pub cancelled: bool, + /// What was granted, which may be less than what was asked for. + pub granted_scopes: Vec, + /// Set when the account was refused because a scope the app cannot work without was withheld. + pub missing_required: Vec, +} + +/// The flags a triage action sets, and the only thing in this file the provider ever hears about. +/// An absent field means unchanged. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FlagPatch { + #[serde(default)] + pub seen: Option, + #[serde(default)] + pub starred: Option, + #[serde(default)] + pub archived: Option, + #[serde(default)] + pub trashed: Option, + #[serde(default)] + pub spam: Option, +} diff --git a/src-tauri/src/exports.rs b/src-tauri/src/exports.rs new file mode 100644 index 0000000..41d1aab --- /dev/null +++ b/src-tauri/src/exports.rs @@ -0,0 +1,249 @@ +// Taking your mail and your decisions out, and putting the decisions back. +// +// This is the module that makes "ownership is a feature, not a slogan" true. Mail leaves as mbox, +// which every mail client on earth can read, and the decisions leave as the journal itself, which +// is the same thing the backup store carries and the same thing another device absorbs. There is no +// export format invented here: the point of exporting is that somebody else can read it, and a +// shape nobody else knows is not an export. +// +// Nothing here asks the provider for anything. An export is of what is on the device, which is what +// the storage window decided, and saying so is more honest than quietly downloading a mailbox. + +use std::fs; +use std::io::Write; +use std::path::PathBuf; + +use rusqlite::Connection; +use tauri::Manager; + +use crate::db::Db; +use crate::state; + +/// Where an export lands. The downloads directory, because that is where a person looks for a file +/// they asked an app to make, and because writing into the app's own data directory would put the +/// export somewhere uninstalling deletes. +fn downloads(app: &tauri::AppHandle) -> Result { + app.path() + .download_dir() + .or_else(|_| app.path().home_dir()) + .map_err(|e| e.to_string()) +} + +fn stamp() -> String { + chrono::Local::now().format("%Y-%m-%d").to_string() +} + +// --------------------------------------------------------------------------------------------- +// Mail, as mbox +// --------------------------------------------------------------------------------------------- + +/// The mboxo escape, which is the one every reader agrees on: a body line that begins with `From ` +/// gains a `>`, because that sequence at the start of a line is what separates one message from the +/// next and a message quoting an email header would otherwise split in two. +fn escape_from_lines(raw: &[u8], out: &mut Vec) { + let mut at_line_start = true; + let mut i = 0; + while i < raw.len() { + if at_line_start && raw[i..].starts_with(b"From ") { + out.push(b'>'); + } + out.push(raw[i]); + at_line_start = raw[i] == b'\n'; + i += 1; + } + if !out.ends_with(b"\n") { + out.push(b'\n'); + } +} + +/// One message as an mbox entry: the separator line, then the message, then a blank line. +/// +/// The separator carries the envelope sender and a date in asctime form, which is what the format +/// asks for and what readers parse. A message with no raw bytes is skipped rather than written as +/// an empty entry, because the mirror keeps headers for messages whose bodies it never fetched and +/// an entry with no message in it is worse than an absence. +fn mbox_entry(from: &str, date_ms: i64, raw: &[u8], out: &mut Vec) { + let when = chrono::DateTime::from_timestamp_millis(date_ms) + .unwrap_or_default() + .format("%a %b %e %H:%M:%S %Y"); + let sender = if from.is_empty() { "MAILER-DAEMON" } else { from }; + out.extend_from_slice(format!("From {sender} {when}\n").as_bytes()); + escape_from_lines(raw, out); + out.push(b'\n'); +} + +fn write_mbox(conn: &Connection, path: &PathBuf) -> Result { + let mut stmt = conn + .prepare( + "SELECT m.from_address, m.date_ms, b.raw FROM messages m + JOIN bodies b ON b.message_id = m.id + WHERE b.raw IS NOT NULL + ORDER BY m.date_ms ASC", + ) + .map_err(|e| e.to_string())?; + let rows = stmt + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, Vec>(2)?, + )) + }) + .map_err(|e| e.to_string())?; + + let mut file = fs::File::create(path).map_err(|e| e.to_string())?; + let mut written = 0u32; + for row in rows { + let (from, date_ms, raw) = row.map_err(|e| e.to_string())?; + let mut entry = Vec::with_capacity(raw.len() + 96); + mbox_entry(&from, date_ms, &raw, &mut entry); + file.write_all(&entry).map_err(|e| e.to_string())?; + written += 1; + } + file.sync_all().map_err(|e| e.to_string())?; + Ok(written) +} + +/// Every message on the device for one account, as mbox, in the downloads directory. +/// +/// What leaves is what is here: a body the mirror never fetched is not in the file, because the +/// export is of the device rather than of the mailbox. The Data section says so beside the button. +#[tauri::command(async)] +pub fn export_mbox(app: tauri::AppHandle, account_id: String) -> Result { + let db = app.try_state::().ok_or("the mirror is not open yet")?; + let safe: String = account_id + .chars() + .map(|c| if c.is_alphanumeric() { c } else { '-' }) + .collect(); + let path = downloads(&app)?.join(format!("margin-mail-{safe}-{}.mbox", stamp())); + db.with(&account_id, |conn| write_mbox(conn, &path))?; + Ok(path.to_string_lossy().to_string()) +} + +// --------------------------------------------------------------------------------------------- +// The decisions, as the journal +// --------------------------------------------------------------------------------------------- + +/// Every decision on this device, as the journal, one account per key. +/// +/// The journal rather than the tables, because the tables are a view of it and the journal is what +/// another device can absorb without losing the order things happened in. It is the same shape the +/// backup store carries, which is deliberate: one export format, one import path, one thing to get +/// right. +#[tauri::command(async)] +pub fn export_state(app: tauri::AppHandle) -> Result { + let db = app.try_state::().ok_or("the mirror is not open yet")?; + let mut accounts = serde_json::Map::new(); + + for account_id in db.on_disk() { + let records = db.with(&account_id, |conn| { + let mut all = Vec::new(); + for device in state::merge::devices(conn)? { + all.extend(state::merge::export(conn, &device, 0)?); + } + Ok(all) + })?; + accounts.insert( + account_id, + serde_json::to_value(&records).map_err(|e| e.to_string())?, + ); + } + + let document = serde_json::json!({ + "kind": "margin-mail-state", + "version": 1, + "exportedAt": chrono::Utc::now().to_rfc3339(), + "accounts": accounts, + }); + let path = downloads(&app)?.join(format!("margin-mail-decisions-{}.json", stamp())); + crate::library::atomic_write( + &path, + serde_json::to_string_pretty(&document) + .map_err(|e| e.to_string())? + .as_bytes(), + )?; + Ok(path.to_string_lossy().to_string()) +} + +/// Reads an export back in. +/// +/// Absorbing rather than replacing, so importing into an account that has been used since is a +/// merge and not a loss: the journal's own last-writer-wins settles every key, exactly as it does +/// when two devices meet. Importing the same file twice changes nothing, which is the same property +/// that lets a device catch up after a month offline. +#[tauri::command(async)] +pub fn import_state(app: tauri::AppHandle, path: String) -> Result<(), String> { + let db = app.try_state::().ok_or("the mirror is not open yet")?; + let raw = fs::read_to_string(&path).map_err(|e| format!("could not read {path}: {e}"))?; + let document: serde_json::Value = serde_json::from_str(&raw).map_err(|e| e.to_string())?; + + if document.get("kind").and_then(|k| k.as_str()) != Some("margin-mail-state") { + return Err("that file is not a Margin Mail export".to_string()); + } + let accounts = document + .get("accounts") + .and_then(|a| a.as_object()) + .ok_or("that export has no accounts in it")?; + + let here = db.on_disk(); + for (account_id, records) in accounts { + // An account that is not connected here has nowhere for its decisions to go, and creating a + // database for it would be creating an account nobody added. + if !here.contains(account_id) { + continue; + } + let records: Vec = + serde_json::from_value(records.clone()).map_err(|e| e.to_string())?; + db.with(account_id, |conn| { + state::merge::absorb(conn, &records).map(|_| ()) + })?; + } + crate::emit_store_changed(&app, "threads state screener"); + Ok(()) +} + +// --------------------------------------------------------------------------------------------- +// The keymap +// --------------------------------------------------------------------------------------------- + +/// The keymap is a file rather than a table of pickers, which is a decision `docs/keyboard.md` +/// makes: a person who wants to remap a key wants to see the whole map at once. +pub fn keymap_file(app: &tauri::AppHandle) -> Result { + Ok(crate::library::app_data_dir(app)?.join("keymap.json")) +} + +const KEYMAP_TEMPLATE: &str = r#"{ + "$comment": [ + "Margin Mail's keymap. Every command and its default key is in docs/keyboard.md, and the", + "shortcuts sheet behind ? is generated from whatever is in force, so a remapped key is what", + "the buttons print.", + "", + "One entry per command you want to change: the command's id, and the keys it should answer to.", + "A combo is modifiers in cmd+ctrl+alt+shift order and then the key, so cmd+shift+a. Delete this", + "file, or press Reset in Settings, to go back to the defaults." + ], + "bindings": {} +} +"#; + +/// The path, creating the file with its explanation the first time somebody asks for it. A settings +/// screen that offers to open a file has to be sure there is one to open. +#[tauri::command(async)] +pub fn keymap_path(app: tauri::AppHandle) -> Result { + let path = keymap_file(&app)?; + if !path.exists() { + crate::library::atomic_write(&path, KEYMAP_TEMPLATE.as_bytes())?; + } + Ok(path.to_string_lossy().to_string()) +} + +/// Back to the defaults, which is the template rather than an absence: a file that is there and +/// empty is easier to understand than one that has gone. +#[tauri::command(async)] +pub fn keymap_reset(app: tauri::AppHandle) -> Result<(), String> { + let path = keymap_file(&app)?; + crate::library::atomic_write(&path, KEYMAP_TEMPLATE.as_bytes()) +} + +#[cfg(test)] +mod tests; diff --git a/src-tauri/src/exports/tests.rs b/src-tauri/src/exports/tests.rs new file mode 100644 index 0000000..ce5cc3d --- /dev/null +++ b/src-tauri/src/exports/tests.rs @@ -0,0 +1,152 @@ +// The export format, which is the part somebody else's software has to be able to read. + +use rusqlite::Connection; + +use crate::db; +use crate::dto::Destination; +use crate::state; + +use super::{escape_from_lines, mbox_entry, write_mbox}; + +fn open() -> Connection { + db::memory().expect("a pair of in-memory databases") +} + +fn message(conn: &Connection, id: &str, from: &str, date_ms: i64, raw: &str) { + conn.execute( + "INSERT INTO messages (id, provider_thread_id, thread_key, message_id, date_ms, + from_address, subject, hydrated) + VALUES (?1, ?1, ?1, ?1, ?2, ?3, 'Subject', 1)", + rusqlite::params![id, date_ms, from], + ) + .expect("a message"); + conn.execute( + "INSERT INTO bodies (message_id, raw, fetched_at) VALUES (?1, ?2, 1)", + rusqlite::params![id, raw.as_bytes()], + ) + .expect("a body"); +} + +#[test] +fn a_body_line_that_looks_like_a_separator_is_escaped() { + // The one thing mbox can get wrong. A message quoting an email header would otherwise split + // into two messages in whatever reads the file. + let mut out = Vec::new(); + escape_from_lines(b"Hello\nFrom Ana, quoted\nBye\n", &mut out); + assert_eq!( + String::from_utf8(out).unwrap(), + "Hello\n>From Ana, quoted\nBye\n" + ); +} + +#[test] +fn a_from_in_the_middle_of_a_line_is_left_alone() { + let mut out = Vec::new(); + escape_from_lines(b"a note From Ana\n", &mut out); + assert_eq!(String::from_utf8(out).unwrap(), "a note From Ana\n"); +} + +#[test] +fn an_entry_carries_the_separator_the_format_asks_for() { + let mut out = Vec::new(); + mbox_entry("ana@example.org", 1_760_000_000_000, b"Subject: Hi\n\nHello\n", &mut out); + let text = String::from_utf8(out).unwrap(); + assert!(text.starts_with("From ana@example.org "), "got {text}"); + assert!(text.contains("\nSubject: Hi\n")); + assert!(text.ends_with("\n\n"), "an entry ends with a blank line"); +} + +#[test] +fn a_message_with_no_body_on_the_device_is_not_an_empty_entry() { + let conn = open(); + message(&conn, "m1", "ana@example.org", 1_760_000_000_000, "Subject: One\n\nOne\n"); + // Headers with no body, which is what the mirror holds until a thread is opened. + conn.execute( + "INSERT INTO messages (id, provider_thread_id, thread_key, message_id, date_ms, + from_address, subject, hydrated) + VALUES ('m2', 'm2', 'm2', 'm2', 1760000001000, 'bo@example.org', 'Two', 1)", + [], + ) + .expect("a message with no body"); + + let dir = tempfile::tempdir().expect("a temp dir"); + let path = dir.path().join("out.mbox"); + let written = write_mbox(&conn, &path).expect("an mbox"); + + assert_eq!(written, 1, "the export is of what is on the device"); + let text = std::fs::read_to_string(&path).expect("the file"); + assert!(text.contains("From ana@example.org ")); + assert!(!text.contains("bo@example.org")); +} + +#[test] +fn the_decisions_round_trip_through_the_journal() { + let from = open(); + state::write::set_rule(&from, "maya@example.org", false, Destination::Inbox, None) + .expect("a rule"); + state::write::add_note(&from, "", "Ask about the oak finish", None) + .expect("a note"); + + let records: Vec<_> = state::merge::devices(&from) + .expect("devices") + .into_iter() + .flat_map(|device| state::merge::export(&from, &device, 0).expect("export")) + .collect(); + let encoded = state::merge::encode(&records).expect("encode"); + + // A second device with nothing on it, which is what an import into a fresh install is. + let to = open(); + state::merge::absorb(&to, &state::merge::decode(&encoded).expect("decode")).expect("absorb"); + + assert_eq!( + state::read::destination_for(&to, "maya@example.org").expect("rule"), + Some(Destination::Inbox) + ); + assert_eq!(state::read::notes_on(&to, "").expect("notes").len(), 1); +} + +#[test] +fn importing_the_same_export_twice_changes_nothing() { + let from = open(); + state::write::set_rule(&from, "maya@example.org", false, Destination::Feed, None) + .expect("a rule"); + let records: Vec<_> = state::merge::devices(&from) + .expect("devices") + .into_iter() + .flat_map(|device| state::merge::export(&from, &device, 0).expect("export")) + .collect(); + + let to = open(); + state::merge::absorb(&to, &records).expect("first"); + state::merge::absorb(&to, &records).expect("second"); + + let rules = state::read::rules(&to, "acct").expect("rules"); + assert_eq!(rules.len(), 1, "an import is a merge, not an append"); +} + +#[test] +fn importing_into_an_account_that_has_moved_on_is_a_merge_rather_than_a_loss() { + let from = open(); + state::write::set_rule(&from, "maya@example.org", false, Destination::Feed, None) + .expect("the exported rule"); + let records: Vec<_> = state::merge::devices(&from) + .expect("devices") + .into_iter() + .flat_map(|device| state::merge::export(&from, &device, 0).expect("export")) + .collect(); + + let to = open(); + state::write::set_rule(&to, "ben@example.org", false, Destination::Inbox, None) + .expect("a decision made since"); + state::merge::absorb(&to, &records).expect("absorb"); + + assert_eq!( + state::read::destination_for(&to, "ben@example.org").expect("kept"), + Some(Destination::Inbox), + "a decision made here is not lost by importing one made there" + ); + assert_eq!( + state::read::destination_for(&to, "maya@example.org").expect("arrived"), + Some(Destination::Feed) + ); +} diff --git a/src-tauri/src/fixtures.rs b/src-tauri/src/fixtures.rs new file mode 100644 index 0000000..6732bda --- /dev/null +++ b/src-tauri/src/fixtures.rs @@ -0,0 +1,103 @@ +// The `.eml` corpus, compiled into the test binary. +// +// Every file in `fixtures/` is a real message as it comes off the wire: CRLF throughout, headers +// folded the way the sending client folded them, bodies in whatever encoding the sender used. They +// are read as bytes rather than as `&str` because half of them are not UTF-8, which is the point: +// ISO-8859-1 and ISO-2022-JP still arrive daily and a parser that has only seen UTF-8 has not been +// tested. Nothing here is generated at test time, so a fixture cannot quietly change shape between +// the parser's tests and the engine's. + +macro_rules! corpus { + ($($name:ident => $file:literal,)+) => { + $(pub const $name: &[u8] = include_bytes!(concat!("../fixtures/", $file));)+ + + /// The whole corpus, for a test that has to hold every message to the same rule. + pub fn all() -> Vec<(&'static str, &'static [u8])> { + vec![$(($file, $name),)+] + } + }; +} + +corpus! { + ATTACHMENT_FILENAME_ENCODED_WORD => "attachment-filename-encoded-word.eml", + CALENDAR_INVITE => "calendar-invite.eml", + CHARSET_ISO_2022_JP => "charset-iso-2022-jp.eml", + CHARSET_ISO_8859_1 => "charset-iso-8859-1.eml", + CID_MISSING_PART => "cid-missing-part.eml", + DISPLAY_NAME_COMMA_QUOTES => "display-name-comma-quotes.eml", + ENCODED_WORDS_SPLIT_UTF8 => "encoded-words-split-utf8.eml", + ENCODED_WORDS_SUBJECT_B => "encoded-words-subject-b.eml", + ENCODED_WORDS_SUBJECT_Q => "encoded-words-subject-q.eml", + GROUP_ADDRESS_LIST => "group-address-list.eml", + HEADER_ODDITIES => "header-oddities.eml", + HTML_HOSTILE => "html-hostile.eml", + INLINE_IMAGE_ATTACHMENT_DISPOSITION => "inline-image-attachment-disposition.eml", + INLINE_IMAGE_CID => "inline-image-cid.eml", + INVITE_DAYLIGHT_SAVING => "invite-daylight-saving.eml", + NESTED_MULTIPART => "nested-multipart.eml", + NEWSLETTER_LIST_UNSUBSCRIBE => "newsletter-list-unsubscribe.eml", + NO_MESSAGE_ID => "no-message-id.eml", + OUTLOOK_REPLY => "outlook-reply.eml", + PLAIN_TEXT => "plain-text.eml", + QUOTED_PRINTABLE_SOFT_BREAKS => "quoted-printable-soft-breaks.eml", + QUOTED_REPLY_BLOCKQUOTE => "quoted-reply-blockquote.eml", + QUOTED_REPLY_PLAIN => "quoted-reply-plain.eml", + RECEIPT_NO_REPLY => "receipt-no-reply.eml", + REPLY_REFERENCES_FOLDED => "reply-references-folded.eml", + RFC2231_FILENAME => "rfc2231-filename.eml", + SCREENER_FIRST_CONTACT => "screener-first-contact.eml", + SERVICE_ON_BEHALF => "service-on-behalf.eml", + TRACKING_PIXEL => "tracking-pixel.eml", + UTF8_RAW_HEADERS => "utf8-raw-headers.eml", +} + +/// The one fixture with no `Message-ID`, kept named here because the thread key rule falls all the +/// way through on it and more than one test wants that case. +pub const WITHOUT_MESSAGE_ID: &[u8] = NO_MESSAGE_ID; + +mod tests { + use super::all; + use crate::provider::fake::FakeMessage; + use mail_parser::MessageParser; + + /// A date outside this range is a parser that gave up and returned zero rather than a message + /// from an unusual year. + const EARLIEST_MS: i64 = 946_684_800_000; // 2000-01-01 + const LATEST_MS: i64 = 4_102_444_800_000; // 2100-01-01 + + #[test] + fn every_fixture_is_a_message() { + for (name, raw) in all() { + assert!( + raw.windows(4).any(|window| window == b"\r\n\r\n"), + "{name} has no blank line between the headers and the body" + ); + assert!( + !raw.split(|byte| *byte == b'\n') + .any(|line| !line.is_empty() && !line.ends_with(b"\r")), + "{name} has a bare LF, so it is not what the wire delivers" + ); + + let message = FakeMessage::from_eml("id-1", "thread-1", &["INBOX"], raw); + assert_eq!(message.raw, raw, "{name} did not survive from_eml intact"); + assert!( + message.date_ms > EARLIEST_MS && message.date_ms < LATEST_MS, + "{name} came out with an implausible date: {}", + message.date_ms + ); + + let parsed = MessageParser::default() + .parse(&message.raw) + .unwrap_or_else(|| panic!("{name} did not parse at all")); + let from = parsed + .from() + .and_then(|address| address.first()) + .and_then(|address| address.address()) + .unwrap_or_default(); + assert!( + from.contains('@'), + "{name} came out with no From address, got {from:?}" + ); + } + } +} diff --git a/src-tauri/src/google/api.rs b/src-tauri/src/google/api.rs new file mode 100644 index 0000000..338c1f5 --- /dev/null +++ b/src-tauri/src/google/api.rs @@ -0,0 +1,1613 @@ +// Typed wrapper over the Gmail REST API, in the calendar's shape: one function per call the app +// makes, an access token in and a typed response out, and `read_json` taking the body to a String +// before anything tries to deserialise it so a Google error payload survives into the message +// rather than being swallowed by a parse failure on a shape that was never the response type. +// +// Two things live here that are not calls, because every call has to respect them: +// +// `Call` the unit cost and the required scope of each method, in one table +// `Quota` the rolling minute, which is the only reason a first sync finishes at all +// +// The arithmetic, so nobody has to redo it: the budget is 6,000 units per minute per user, +// `messages.get` is 20 units, and hydration is batched 50 at a time. A full batch is 50 * 20 = +// 1,000 units, six batches fill the minute exactly, and the seventh waits. That is 300 messages a +// minute, which is 67 minutes for a twenty thousand message mailbox and most of a working day for +// a hundred thousand. Everything else the app does is noise against it: polling `history.list` +// every 12 seconds is 10 units a minute. + +use std::collections::{HashMap, VecDeque}; +use std::future::Future; +use std::sync::{LazyLock, Mutex}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use base64::alphabet; +use base64::engine::{DecodePaddingMode, GeneralPurpose, GeneralPurposeConfig}; +use base64::Engine; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; + +pub const BASE: &str = "https://gmail.googleapis.com/gmail/v1"; + +/// The batch endpoint is not on `gmail.googleapis.com`. The discovery document puts it at +/// `batch/gmail/v1` on `www.googleapis.com`, and the other host answers 404. +pub const BATCH_URL: &str = "https://www.googleapis.com/batch/gmail/v1"; + +/// Gmail caps a list page at 500 and the ids are cheap, so there is no reason to ask for fewer. +pub const MAX_LIST_RESULTS: &str = "500"; + +/// One `batchModify` takes up to 1,000 ids for its flat 50 units. +pub const MAX_MODIFY_IDS: usize = 1000; + +pub const SCOPE_MODIFY: &str = "https://www.googleapis.com/auth/gmail.modify"; +pub const SCOPE_SETTINGS: &str = "https://www.googleapis.com/auth/gmail.settings.basic"; + +/// The headers hydration asks for, and no others. `format=metadata` without `metadataHeaders` +/// returns every header a message carries, which on a mailing list message is a page of `Received` +/// and DKIM signatures nobody reads: same 20 units, ten times the bytes. +/// +/// These are the ones the reader, the threader and the screener actually parse. `List-Id`, +/// `List-Unsubscribe`, `List-Unsubscribe-Post`, `Precedence` and `Auto-Submitted` together are the +/// "not a human" signal the Feed is sorted by. +pub const METADATA_HEADERS: [&str; 17] = [ + "From", + "To", + "Cc", + "Bcc", + "Reply-To", + "Subject", + "Date", + "Message-ID", + "In-Reply-To", + "References", + "List-Id", + "List-Unsubscribe", + "List-Unsubscribe-Post", + "Precedence", + "Auto-Submitted", + "Content-Type", + // Not read by anything yet. It is the only place SPF, DKIM and DMARC results appear, asking + // for one more header costs nothing at all, and a Screener card that cannot say whether a + // first message was actually from who it claims is a Screener card missing the point. + "Authentication-Results", +]; + +/// Gmail's `raw` is base64url, not standard base64: getting `+/` and `-_` the wrong way round +/// produces MIME that Gmail rejects with a 400 saying nothing useful. Decoding is padding +/// indifferent because both padded and unpadded bodies arrive. +pub const B64: GeneralPurpose = GeneralPurpose::new( + &alphabet::URL_SAFE, + GeneralPurposeConfig::new().with_decode_padding_mode(DecodePaddingMode::Indifferent), +); + +/// A client of this layer's own rather than the OAuth stack's. They share no host: auth talks to +/// `oauth2.googleapis.com`, this talks to `gmail.googleapis.com`, `people.googleapis.com` and +/// `www.googleapis.com`, so one pool would not be reused anyway. The timeout is generous because a +/// batch of 50 metadata responses is around a megabyte before gzip. +pub static HTTP: LazyLock = LazyLock::new(|| { + reqwest::Client::builder() + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(60)) + // Shorter than reqwest's ninety seconds. A connection that sat in the pool through a + // background poll interval, a sleep or a network change is the one that fails with + // "error sending request" on the first call that reuses it, and that first call is the + // outbox push right after somebody archived something. A fresh TLS handshake once a + // minute costs nothing against that. + .pool_idle_timeout(Duration::from_secs(30)) + // And the connection is checked while it sits there. An HTTP/2 ping every twenty seconds + // is how a connection the machine slept through, or the network changed under, is found + // dead in the pool rather than by the request that was about to use it. The pool timeout + // above catches the idle case; this catches the one where the socket is still open on + // this side and closed on the other, which a sleep or a Wi-Fi change leaves behind and + // which no timeout would ever notice. + .http2_keep_alive_interval(Duration::from_secs(20)) + .http2_keep_alive_timeout(Duration::from_secs(10)) + .http2_keep_alive_while_idle(true) + .tcp_keepalive(Duration::from_secs(30)) + .build() + .expect("could not build the HTTP client") +}); + +// -- the call table --------------------------------------------------------------------------- + +/// Every Gmail method the app calls, with the two facts that belong to the call rather than to the +/// response: what it costs against the minute, and which scope a 403 is complaining about. Google +/// never names the scope in the error body, only the service and the method, so the caller is the +/// only one who knows what to put on the Grant button. +/// +/// Costs are Google's table as read on 2026-09-03, after the 1 May 2026 change. Anything quoting +/// `messages.get` at 5 units predates it and is wrong by a factor of four. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Call { + MessagesList, + HistoryList, + MessagesGet, + AttachmentsGet, + LabelsList, + BatchModify, + MessagesModify, + MessagesSend, + DraftsCreate, + DraftsUpdate, + DraftsDelete, + SendAsList, + GetProfile, +} + +impl Call { + pub const fn units(self) -> u32 { + match self { + Call::MessagesSend => 100, + Call::BatchModify => 50, + // `metadata`, `full`, `raw` and `minimal` all cost the same: there is no cheaper way to + // read a message, which is why hydration is the whole budget. + Call::MessagesGet | Call::AttachmentsGet => 20, + Call::DraftsUpdate => 15, + Call::DraftsCreate | Call::DraftsDelete => 10, + Call::MessagesList | Call::MessagesModify => 5, + Call::HistoryList => 2, + Call::LabelsList | Call::SendAsList | Call::GetProfile => 1, + } + } + + pub const fn scope(self) -> &'static str { + match self { + Call::SendAsList => SCOPE_SETTINGS, + _ => SCOPE_MODIFY, + } + } + + pub const fn name(self) -> &'static str { + match self { + Call::MessagesList => "Gmail message list", + Call::HistoryList => "Gmail history", + Call::MessagesGet => "Gmail message fetch", + Call::AttachmentsGet => "Gmail attachment fetch", + Call::LabelsList => "Gmail label list", + Call::BatchModify => "Gmail label change", + Call::MessagesModify => "Gmail label change", + Call::MessagesSend => "Gmail send", + Call::DraftsCreate => "Gmail draft create", + Call::DraftsUpdate => "Gmail draft update", + Call::DraftsDelete => "Gmail draft delete", + Call::SendAsList => "Gmail send-as list", + Call::GetProfile => "Gmail profile", + } + } +} + +// -- errors ----------------------------------------------------------------------------------- + +/// What Google said, at the granularity anything above this file branches on. The `Provider` trait +/// has its own error type and this maps onto it; the split exists so `people`, `calendar` and +/// `drive`, which are not behind the trait, can use the same wrapper. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ApiError { + /// 401. The token was revoked, or the password changed, and refreshing will not help. + Unauthorized(String), + /// 403 whose reason names an insufficient scope. Carries the scope the call needed, because + /// Google's body does not. + InsufficientScope(String), + /// 429, and the 403 rate limit reasons. Carries how long to wait. + RateLimited { retry_after_ms: u64 }, + /// 404. On `history.list` this is the expired cursor and not a failure at all. + NotFound(String), + /// Could not reach Google, as opposed to being turned away by it: a DNS failure, a refused + /// connection, a request that timed out. + Offline(String), + /// A connection that was there and then was not: reset, closed before the answer, cut off in + /// the body. Its own kind because it is the one failure worth trying again at once, on a + /// fresh connection, and the one that a client which never does so shows the person every + /// time the machine wakes up. + Dropped(String), + Other(String), +} + +impl std::fmt::Display for ApiError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ApiError::Unauthorized(m) => write!(f, "signed out: {m}"), + ApiError::InsufficientScope(s) => write!(f, "missing permission: {s}"), + ApiError::RateLimited { retry_after_ms } => { + write!(f, "rate limited, retry in {retry_after_ms}ms") + } + ApiError::NotFound(m) => write!(f, "not found: {m}"), + ApiError::Offline(m) => write!(f, "offline: {m}"), + ApiError::Dropped(m) => write!(f, "connection lost: {m}"), + ApiError::Other(m) => write!(f, "{m}"), + } + } +} + +impl From for String { + fn from(e: ApiError) -> String { + e.to_string() + } +} + +/// A failure to reach Google at all is Offline. A connection that failed under a request that +/// had already started, or under the body of an answer, is Dropped. Anything reqwest raises after +/// a response has arrived whole is a real error and keeps its text. +/// +/// A timeout is Offline rather than Dropped on purpose: sixty seconds have already gone, and +/// trying again on the spot is a minute more of the same. +impl From for ApiError { + fn from(e: reqwest::Error) -> ApiError { + let dropped = !e.is_timeout() && (e.is_request() || e.is_body()); + let offline = e.is_connect() || e.is_timeout(); + let text = transport_text(e); + if dropped { + ApiError::Dropped(text) + } else if offline { + ApiError::Offline(text) + } else { + ApiError::Other(text) + } + } +} + +/// What reqwest has to say, without the URL and with the cause. +/// +/// reqwest's own text is "error sending request for url (https://gmail.googleapis.com/gmail/v1/ +/// users/me/messages/18f9.../modify)" and nothing else: the URL is the whole sentence, and the part +/// worth knowing ("connection closed before message completed", "dns error") is down the source +/// chain where `Display` never looks. So the URL goes and the chain comes. +fn transport_text(e: reqwest::Error) -> String { + let e = e.without_url(); + let mut text = e.to_string(); + let mut source = std::error::Error::source(&e); + while let Some(cause) = source { + let said = cause.to_string(); + if !said.is_empty() && !text.contains(&said) { + text.push_str(": "); + text.push_str(&said); + } + source = cause.source(); + } + strip_urls(&text) +} + +/// The text with every sentence that carried a link taken out. +/// +/// Google's messages end in "See https://developers.google.com/..." and "Enable it by visiting +/// https://console.developers.google.com/... then retry", and a link is the one thing a sentence +/// on screen must not be: nothing in this app can follow it and a person reading a toast cannot +/// either. Whole sentences go rather than the bare URL, because "Enable it by visiting then retry" +/// is not English. +pub fn strip_urls(text: &str) -> String { + let has_link = |s: &str| s.contains("http://") || s.contains("https://") || s.contains("www."); + if !has_link(text) { + return text.trim().to_string(); + } + let mut kept: Vec<&str> = Vec::new(); + let mut start = 0; + let bytes = text.as_bytes(); + for (at, byte) in bytes.iter().enumerate() { + let ends = *byte == b'.' && bytes.get(at + 1).is_none_or(|next| next.is_ascii_whitespace()); + if ends { + kept.push(&text[start..=at]); + start = at + 1; + } + } + if start < text.len() { + kept.push(&text[start..]); + } + let out = kept + .into_iter() + .map(str::trim) + .filter(|s| !s.is_empty() && !has_link(s)) + .collect::>() + .join(" "); + if out.is_empty() { + "Google answered with a link and no reason".to_string() + } else { + out + } +} + +impl From for crate::provider::ProviderError { + fn from(e: ApiError) -> crate::provider::ProviderError { + use crate::provider::ProviderError as P; + match e { + ApiError::Unauthorized(m) => P::Auth(m), + ApiError::InsufficientScope(s) => P::Scope(s), + ApiError::RateLimited { retry_after_ms } => P::RateLimited { retry_after_ms }, + ApiError::NotFound(_) => P::NotFound, + ApiError::Offline(m) | ApiError::Dropped(m) => P::Network(m), + ApiError::Other(m) => P::Other(m), + } + } +} + +/// Google's error body is a wall of JSON carrying the same sentence three times over. The one +/// useful line is `error.message`; a body that will not parse is truncated rather than pasted. +fn explain(body: &str) -> String { + if let Ok(value) = serde_json::from_str::(body) { + if let Some(message) = value + .get("error") + .and_then(|e| e.get("message")) + .and_then(|m| m.as_str()) + { + return strip_urls(message); + } + } + let trimmed = body.trim(); + let cut = match trimmed.char_indices().nth(200) { + Some((end, _)) => format!("{}…", &trimmed[..end]), + None => trimmed.to_string(), + }; + strip_urls(&cut) +} + +/// Google puts the machine-readable reason in three places depending on which decade the API was +/// written in: `error.status`, `error.errors[].reason` and `error.details[].reason`. A 403 for an +/// insufficient scope only says so in `details`, so all three are collected. +fn reasons(body: &str) -> Vec { + let mut out = Vec::new(); + let Ok(value) = serde_json::from_str::(body) else { + return out; + }; + let Some(error) = value.get("error") else { + return out; + }; + if let Some(status) = error.get("status").and_then(|s| s.as_str()) { + out.push(status.to_string()); + } + for key in ["errors", "details"] { + let Some(items) = error.get(key).and_then(|e| e.as_array()) else { + continue; + }; + for item in items { + if let Some(reason) = item.get("reason").and_then(|r| r.as_str()) { + out.push(reason.to_string()); + } + } + } + out +} + +fn mentions(found: &[String], wanted: &[&str]) -> bool { + found + .iter() + .any(|reason| wanted.iter().any(|w| reason.eq_ignore_ascii_case(w))) +} + +/// `dailyLimitExceeded` is deliberately absent: it is the project's day gone, and retrying it in +/// thirty seconds only spends the next day's. +fn is_rate_limit(found: &[String]) -> bool { + mentions( + found, + &[ + "rateLimitExceeded", + "userRateLimitExceeded", + "RESOURCE_EXHAUSTED", + ], + ) +} + +fn is_missing_scope(found: &[String], message: &str) -> bool { + mentions( + found, + &[ + "insufficientPermissions", + "ACCESS_TOKEN_SCOPE_INSUFFICIENT", + "insufficientScopes", + ], + ) || message.contains("insufficient authentication scopes") +} + +/// The Gmail API switched off in the Google Cloud project behind the credentials file. Google's +/// sentence for it is a console link, which is worth saying in words instead. +fn is_api_disabled(found: &[String]) -> bool { + mentions(found, &["accessNotConfigured", "SERVICE_DISABLED"]) +} + +/// When a 429 arrives without a `Retry-After`, this is what the caller waits. Google's guidance is +/// to start retry periods at least one second after the error. +pub const DEFAULT_RETRY_MS: u64 = 1_000; + +/// The whole status-to-meaning decision, pure so the bodies Google actually returns can be pinned +/// in a test. `scope` is what the call needed, since the body never says. +pub fn error_for( + status: u16, + context: &str, + scope: &str, + retry_after_ms: Option, + body: &str, +) -> ApiError { + let message = explain(body); + let found = reasons(body); + match status { + 401 => ApiError::Unauthorized(message), + 403 if is_rate_limit(&found) => ApiError::RateLimited { + retry_after_ms: retry_after_ms.unwrap_or(DEFAULT_RETRY_MS), + }, + 403 if is_missing_scope(&found, &message) => ApiError::InsufficientScope(scope.to_string()), + 403 if is_api_disabled(&found) => ApiError::Other(format!( + "{context} failed ({status}): the Gmail API is switched off in the Google Cloud \ + project this app's credentials belong to" + )), + 404 => ApiError::NotFound(message), + 429 => ApiError::RateLimited { + retry_after_ms: retry_after_ms.unwrap_or(DEFAULT_RETRY_MS), + }, + // 500, 502, 503 and 504 are Google's own bad day, and 408 is a request that took too long + // to arrive. All are retried on the same schedule as a 429. + 408 | 500 | 502 | 503 | 504 => ApiError::RateLimited { + retry_after_ms: retry_after_ms.unwrap_or(DEFAULT_RETRY_MS), + }, + _ => ApiError::Other(format!("{context} failed ({status}): {message}")), + } +} + +/// Seconds, per RFC 9110. Google sends the date form rarely enough that it is not worth parsing. +pub fn retry_after(headers: &reqwest::header::HeaderMap) -> Option { + headers + .get(reqwest::header::RETRY_AFTER)? + .to_str() + .ok()? + .trim() + .parse::() + .ok() + .map(|seconds| seconds.saturating_mul(1000)) +} + +/// The body becomes a String first, so Google's error payload reaches the message instead of being +/// lost to a deserialisation failure. Ported from the calendar's `auth::read_json`. +pub async fn read_json( + resp: reqwest::Response, + context: &str, + scope: &str, +) -> Result { + let status = resp.status().as_u16(); + let retry = retry_after(resp.headers()); + let text = resp.text().await?; + if !(200..300).contains(&status) { + return Err(error_for(status, context, scope, retry, &text)); + } + serde_json::from_str(&text) + .map_err(|e| ApiError::Other(format!("{context}: could not parse response: {e}"))) +} + +/// For the calls whose success is an empty body: `batchModify` and `drafts.delete`. +pub async fn read_empty(resp: reqwest::Response, context: &str, scope: &str) -> Result<(), ApiError> { + let status = resp.status().as_u16(); + let retry = retry_after(resp.headers()); + if (200..300).contains(&status) { + return Ok(()); + } + let text = resp.text().await.unwrap_or_default(); + Err(error_for(status, context, scope, retry, &text)) +} + +pub async fn read_bytes( + resp: reqwest::Response, + context: &str, + scope: &str, +) -> Result, ApiError> { + let status = resp.status().as_u16(); + let retry = retry_after(resp.headers()); + if !(200..300).contains(&status) { + let text = resp.text().await.unwrap_or_default(); + return Err(error_for(status, context, scope, retry, &text)); + } + Ok(resp.bytes().await?.to_vec()) +} + +// -- quota ------------------------------------------------------------------------------------ + +/// Per user, per project, per minute. Google says it cannot be raised for any reason. +pub const BUDGET_PER_MINUTE: u32 = 6_000; + +/// The window the budget is measured over. +pub const WINDOW_MS: u64 = 60_000; + +/// How many messages one batch hydrates. Google allows 100 per batch and recommends no more than +/// 50, and 50 * 20 units is exactly a sixth of the minute, which makes the pacing legible. +pub const BATCH_SIZE: usize = 50; + +/// A rolling minute of spend, so the caller can be told to wait rather than be told off. +/// +/// Not a token bucket: the limit is genuinely "units in the last 60 seconds", so what is stored is +/// the spend and when, and units leave the window on their own. +#[derive(Debug, Default)] +pub struct Quota { + spent: VecDeque<(u64, u32)>, +} + +impl Quota { + pub fn new() -> Self { + Quota::default() + } + + fn expire(&mut self, now_ms: u64) { + while let Some((at, _)) = self.spent.front() { + if now_ms.saturating_sub(*at) >= WINDOW_MS { + self.spent.pop_front(); + } else { + break; + } + } + } + + /// Units spent inside the window ending now. + pub fn spent(&mut self, now_ms: u64) -> u32 { + self.expire(now_ms); + self.spent.iter().map(|(_, units)| units).sum() + } + + /// How long to wait before `units` more would fit. Zero when they fit now. + /// + /// A single call larger than the whole budget would never fit, which cannot happen with this + /// call table (the largest is a batch, and the caller sizes batches), so it waits for the + /// window to clear and then goes anyway rather than deadlocking. + pub fn wait_for(&mut self, now_ms: u64, units: u32) -> u64 { + let spent = self.spent(now_ms); + if spent + units <= BUDGET_PER_MINUTE { + return 0; + } + let mut freed = 0u32; + for (at, entry) in self.spent.iter() { + freed += entry; + if spent.saturating_sub(freed) + units <= BUDGET_PER_MINUTE { + return (at + WINDOW_MS).saturating_sub(now_ms).max(1); + } + } + // Everything in the window would have to go and it still would not fit. + self.spent + .back() + .map(|(at, _)| (at + WINDOW_MS).saturating_sub(now_ms).max(1)) + .unwrap_or(0) + } + + pub fn charge(&mut self, now_ms: u64, units: u32) { + self.expire(now_ms); + self.spent.push_back((now_ms, units)); + } +} + +pub fn now_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64 +} + +/// One rolling minute per account, because the limit is per user and every part of the app that +/// touches one mailbox shares it: the poll loop, hydration and a triage keystroke all spend from +/// the same 6,000. +static LEDGER: LazyLock>> = LazyLock::new(Mutex::default); + +/// Reserves `units` against the account's minute, sleeping until they fit. Charged before the call +/// rather than after, so a burst of concurrent callers cannot all read the same low number and +/// then all spend. +pub async fn spend(account_id: &str, units: u32) { + loop { + let wait = { + let mut ledger = LEDGER.lock().expect("the quota ledger's lock"); + let quota = ledger.entry(account_id.to_string()).or_default(); + let now = now_ms(); + let wait = quota.wait_for(now, units); + if wait == 0 { + quota.charge(now, units); + } + wait + }; + if wait == 0 { + return; + } + tokio::time::sleep(Duration::from_millis(wait)).await; + } +} + +/// What is left of this account's minute, for the account chip. +pub fn remaining(account_id: &str) -> u32 { + let mut ledger = LEDGER.lock().expect("the quota ledger's lock"); + let quota = ledger.entry(account_id.to_string()).or_default(); + BUDGET_PER_MINUTE.saturating_sub(quota.spent(now_ms())) +} + +// -- backoff ---------------------------------------------------------------------------------- + +/// Truncated at a minute and a bit, which is Google's own upper bound. +pub const MAX_BACKOFF_MS: u64 = 64_000; + +/// How many times a rate limited call is retried before the error reaches the engine, which pauses +/// the account instead. Five attempts is roughly half a minute of waiting. +pub const MAX_ATTEMPTS: u32 = 5; + +/// `min(2^n seconds + jitter, 64s)`, Google's truncated exponential backoff, starting at one +/// second because the error guide says to start retry periods at least a second after the error. +/// Jitter is a parameter rather than drawn inside, so the schedule is a pure function. +pub fn backoff_ms(attempt: u32, jitter_ms: u64) -> u64 { + let exponential = 1_000u64.saturating_mul(1u64 << attempt.min(16)); + exponential.saturating_add(jitter_ms).min(MAX_BACKOFF_MS) +} + +/// Up to a second, which is what keeps a thousand clients that all hit the same limit at the same +/// moment from coming back in step. +pub fn jitter_ms() -> u64 { + use rand::Rng; + rand::thread_rng().gen_range(0..1_000) +} + +/// The delay for one attempt, jitter included. +pub fn backoff(attempt: u32) -> u64 { + backoff_ms(attempt, jitter_ms()) +} + +/// How a dropped connection is tried again: at once, then after a beat, on a connection the pool +/// has opened fresh because the old one has just been thrown away. Two more goes and under two +/// seconds, which is what it takes to get past a socket that died while the machine was asleep +/// without turning a real outage into a long wait. +pub const DROPPED_WAITS_MS: [u64; 2] = [250, 1_250]; + +/// Runs a call, retrying the rate limited answers on the schedule above and a dropped connection +/// on the shorter one. When the rate limit attempts run out the error carries the delay the next +/// one would have used, so the engine can pause the account for that long rather than spin. +/// +/// This is for calls that can be made twice without harm: every read, and the label writes, +/// which say what the labels should be rather than what to do to them. `with_retry_no_replay` is +/// for the two that cannot. +pub async fn with_retry(call: F) -> Result +where + F: FnMut() -> Fut, + Fut: Future>, +{ + retrying(call, true).await +} + +/// `with_retry` for a call that must not be made twice. Sending a message on a connection that +/// dropped before the answer came back may have sent it, and a draft created twice is two drafts; +/// for these the dropped connection is reported and whoever asked decides. +pub async fn with_retry_no_replay(call: F) -> Result +where + F: FnMut() -> Fut, + Fut: Future>, +{ + retrying(call, false).await +} + +async fn retrying(mut call: F, replay_dropped: bool) -> Result +where + F: FnMut() -> Fut, + Fut: Future>, +{ + let mut attempt = 0; + let mut dropped = 0; + loop { + match call().await { + Err(ApiError::RateLimited { retry_after_ms }) if attempt + 1 < MAX_ATTEMPTS => { + let wait = retry_after_ms.max(backoff(attempt)); + tokio::time::sleep(Duration::from_millis(wait)).await; + attempt += 1; + } + Err(ApiError::RateLimited { retry_after_ms }) => { + return Err(ApiError::RateLimited { + retry_after_ms: retry_after_ms.max(backoff(attempt)), + }) + } + Err(ApiError::Dropped(_)) if replay_dropped && dropped < DROPPED_WAITS_MS.len() => { + tokio::time::sleep(Duration::from_millis(DROPPED_WAITS_MS[dropped])).await; + dropped += 1; + } + other => return other, + } + } +} + +// -- queries ---------------------------------------------------------------------------------- + +/// The storage window as Gmail sees it. `after:` takes unix seconds, and a bare number is the only +/// form that is unambiguous: the date forms are interpreted in the user's timezone. +pub fn window_query(after_ms: i64) -> String { + format!("after:{}", after_ms.max(0) / 1000) +} + +/// The app's operators (`from:`, `to:`, `subject:`, `has:attachment`, `filename:`, `in:`, +/// `before:`, `after:`, `label:`) are Gmail's own, so a user's query crosses unchanged and the only +/// work is joining it to a window term when there is one. Nothing is quoted or escaped here on +/// purpose: rewriting a search query is how a search silently stops matching what the user typed. +pub fn search_query(user_query: &str, after_ms: Option) -> String { + let user_query = user_query.trim(); + match (user_query.is_empty(), after_ms) { + (true, None) => String::new(), + (true, Some(after)) => window_query(after), + (false, None) => user_query.to_string(), + (false, Some(after)) => format!("{user_query} {}", window_query(after)), + } +} + +/// A query string, built here because reqwest's own builder sits behind a feature this crate does +/// not carry. Keys are literals throughout this file; values are percent-encoded, which matters for +/// a search `q` full of colons and quotes. +pub fn query_string(params: &[(&str, &str)]) -> String { + let mut serializer = url::form_urlencoded::Serializer::new(String::new()); + for (key, value) in params { + serializer.append_pair(key, value); + } + serializer.finish() +} + +/// A URL with its query attached, or the bare URL when there is nothing to attach. +pub fn url_with(base: &str, params: &[(&str, &str)]) -> String { + if params.is_empty() { + base.to_string() + } else { + format!("{base}?{}", query_string(params)) + } +} + +/// Ids can only be hex in practice, but a path segment built by formatting is a path segment that +/// will one day carry something else. +pub fn path_segment(value: &str) -> String { + let mut out = String::with_capacity(value.len()); + for byte in value.as_bytes() { + match byte { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + out.push(*byte as char) + } + _ => out.push_str(&format!("%{byte:02X}")), + } + } + out +} + +/// The path a metadata fetch uses, as a batch part needs it: absolute, with the query string +/// attached. The one definition of the hydration request, shared by the batched and the single +/// form so they cannot drift apart. +pub fn metadata_path(message_id: &str) -> String { + let mut path = format!( + "/gmail/v1/users/me/messages/{}?format=metadata", + path_segment(message_id) + ); + for header in METADATA_HEADERS { + path.push_str("&metadataHeaders="); + path.push_str(header); + } + path +} + +// -- response shapes ---------------------------------------------------------------------------- + +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct MessageId { + pub id: String, + #[serde(default)] + pub thread_id: String, +} + +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct MessageIdsPage { + #[serde(default)] + pub messages: Vec, + #[serde(default)] + pub next_page_token: Option, + /// Gmail's own word for it is an estimate, and it can be wildly wrong. A hint for the progress + /// bar, never a denominator. + #[serde(default)] + pub result_size_estimate: Option, +} + +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +pub struct Header { + #[serde(default)] + pub name: String, + #[serde(default)] + pub value: String, +} + +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Payload { + #[serde(default)] + pub headers: Vec
, +} + +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Message { + #[serde(default)] + pub id: String, + #[serde(default)] + pub thread_id: String, + #[serde(default)] + pub label_ids: Vec, + #[serde(default)] + pub snippet: String, + /// Epoch milliseconds, as a string, because JSON has no int64. The time Gmail received the + /// message, not the `Date` header, which is the sender's claim and can be hours out. + #[serde(default)] + pub internal_date: String, + #[serde(default)] + pub size_estimate: u32, + #[serde(default)] + pub history_id: String, + #[serde(default)] + pub payload: Option, + /// Only with `format=raw`. + #[serde(default)] + pub raw: Option, +} + +impl Message { + pub fn internal_date_ms(&self) -> i64 { + self.internal_date.parse().unwrap_or(0) + } + + pub fn header_pairs(&self) -> Vec<(String, String)> { + self.payload + .as_ref() + .map(|p| { + p.headers + .iter() + .map(|h| (h.name.clone(), h.value.clone())) + .collect() + }) + .unwrap_or_default() + } +} + +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct HistoryMessage { + #[serde(default)] + pub id: String, + #[serde(default)] + pub thread_id: String, + /// The message's whole label set as it stands after the change, not the delta. + #[serde(default)] + pub label_ids: Vec, +} + +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct HistoryMessageRef { + #[serde(default)] + pub message: HistoryMessage, +} + +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct HistoryLabelChange { + #[serde(default)] + pub message: HistoryMessage, + #[serde(default)] + pub label_ids: Vec, +} + +/// Google recommends the specific change-type fields over the `messages` bag, so `messages` is not +/// deserialised at all. +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct HistoryRecord { + #[serde(default)] + pub id: String, + #[serde(default)] + pub messages_added: Vec, + /// Permanently deleted, not trashed. Trashing arrives as a `TRASH` label add. + #[serde(default)] + pub messages_deleted: Vec, + #[serde(default)] + pub labels_added: Vec, + #[serde(default)] + pub labels_removed: Vec, +} + +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct HistoryPage { + #[serde(default)] + pub history: Vec, + #[serde(default)] + pub next_page_token: Option, + /// The cursor to store, once the last page has been applied. + #[serde(default)] + pub history_id: String, +} + +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Label { + #[serde(default)] + pub id: String, + #[serde(default)] + pub name: String, + /// `system` or `user`. Names change and system labels are their own ids, so the id is the key. + #[serde(default, rename = "type")] + pub kind: String, +} + +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct LabelsPage { + #[serde(default)] + pub labels: Vec