commit f773217d21b6dbd265fa30b4c9247a8e888b3e31
parent ad9d7978a7ebd76c5175bff1b57f70540c78cd69
Author: alltheseas <64376233+alltheseas@users.noreply.github.com>
Date: Mon, 16 Feb 2026 17:33:22 -0600
feat: add sitemap.xml and robots.txt for SEO
* ci: add apt-get update before installing deps
Fixes 404 errors when Ubuntu package versions change on mirrors.
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
* upgrade nostrdb to 0.9.0
Update to nostrdb 0.9.0 and fix breaking API changes:
- FilterBuilder.tags() now takes &str instead of String
- Handle new FilterField variants (Search, Relays, Custom)
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
* feat: add NIP-84 highlights and NIP-18 embedded quotes
NIP-84 Highlights (kind:9802):
- Extract and render highlight metadata (context, comment)
- Source attribution for web URLs, notes, and articles
- Blockquote styling with left border accent
NIP-18 Embedded Quotes:
- Parse q tags and inline nevent/note/naddr mentions
- Rich quote cards with avatar, name, @handle, relative time
- Reply detection using nostrdb's NoteReply
- Type indicators for articles/highlights/drafts
Other improvements:
- Draft badge for unpublished articles (kind:30024)
- @username handles displayed under profile names
- Human-readable @mentions (resolve npub to display names)
Closes: #51
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
Co-Authored-By: alltheseas <alltheseas@users.noreply.github.com>
* feat: iOS-style article card for embedded longform quotes
Embedded article quotes now display as cards matching iOS Damus:
- Hero image (if available)
- Bold article title
- Summary text (if available)
- Word count
- DRAFT badge via CSS for kind 30024
Co-Authored-By: alltheseas <alltheseas@users.noreply.github.com>
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
* feat: fetch quoted events from relays with relay provenance
Add an UnknownIds pattern (adapted from notedeck) to fetch quoted events
referenced in q tags and inline mentions. Events are fetched using relay
hints from nevent/naddr bech32 and q tag relay fields.
- Add src/unknowns.rs with UnknownId enum and UnknownIds collection
- Update QuoteRef to include relay hints (Vec<RelayUrl>)
- Extract relay hints from nevent/naddr bech32 and q tag third element
- Add collect_quote_unknowns() and fetch_unknowns() to render.rs
- Fetch quote unknowns in main.rs after primary note is loaded
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
* refactor: generalize unknowns to fetch all missing data
Expand the unknowns pattern beyond just quoted events to collect:
- Author profiles
- Reply chain (root/reply) using nostrdb's NoteReply (NIP-10)
- Mentioned profiles (npub/nprofile with relay hints)
- Mentioned events (nevent/note1 with relay hints)
- Quoted events (q tags, inline mentions)
Move unknowns collection to main.rs for consistent handling
regardless of whether primary note was cached.
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
* chore: fix unused argument warnings
Signed-off-by: William Casarin <jb55@jb55.com>
* fix: refresh profile metadata during background updates
Previously, background profile refreshes only fetched kind 1 (notes),
never updating kind 0 (profile metadata). This caused profiles to remain
stale indefinitely after initial cache.
Now fetch_profile_feed also fetches the latest profile metadata from
relays, allowing nostrdb to update cached profiles with newer versions.
Fixes: https://github.com/damus-io/notecrumbs/issues/52
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
* rustfmt
Signed-off-by: William Casarin <jb55@jb55.com>
* refactor: replace tuple with QuoteProfileInfo struct in build_embedded_quotes_html
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
* feat: integrate rust-nostr PR #1172 relay provenance tracking
Integrate relay provenance tracking from rust-nostr PR #1172 to enable
proper NIP-19 bech32 links with relay hints for better content discoverability.
Changes:
- Update nostr-sdk to alltheseas/rust-nostr relay-provenance-tracking branch
- RelayPool::stream_events returns BoxedStream<RelayEvent> with source relay URL
- NoteAndProfileRenderData stores source_relays captured during fetch
- Generate bech32 links with relay hints for all event types (notes, articles, highlights)
- Filter profile (kind 0) relays from note hints
- Prioritize default relays in source_relays for reliability
- Preserve author/kind fields when rebuilding nevent bech32
- Graceful fallback to original nip19 on encoding failure with metric
- Add bech32_with_relays() helper with 8 unit tests
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
* feat: store relay provenance in nostrdb via IngestMetadata
Use process_event_with instead of process_event to pass the source
relay URL to nostrdb when ingesting events from relay streams.
Fixes: fffaa9e2af82 ("feat: integrate rust-nostr PR #1172 relay provenance tracking")
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
* feat: add sitemap.xml and robots.txt for SEO
Add dynamic sitemap generation from nostrdb cache to improve search
engine discoverability of Nostr content.
New routes:
- GET /robots.txt - crawler directives with sitemap reference
- GET /sitemap.xml - dynamic sitemap from cached notes/profiles/articles
The sitemap queries local nostrdb for:
- Notes (kind:1) → note1xxx URLs
- Long-form articles (kind:30023) → naddr1xxx URLs
- Profiles (kind:0) → npub1xxx URLs
Ref: https://github.com/damus-io/notecrumbs/issues/26
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
* feat: add prometheus metrics for sitemap generation
Track aggregate stats (privacy-preserving, no user tracking):
- sitemap_generations_total: counter for generation requests
- sitemap_generation_duration_seconds: time to generate
- sitemap_urls_total: total URLs in sitemap
- sitemap_notes_count: notes included
- sitemap_articles_count: articles included
- sitemap_profiles_count: profiles included
Metrics available at /metrics endpoint.
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
* fix: address sitemap code review findings
- Skip kind:30023 entries with missing/empty d-tag to avoid ambiguous
URLs and potential collisions across authors
- Add since filter (90 days) to notes and articles queries to prioritize
recent content for SEO freshness
- Log warning when NOTECRUMBS_BASE_URL is not set, to surface potential
misconfiguration in production
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
* fix: cache base URL and use longer lookback for articles
- Cache base URL with OnceLock to avoid logging warning on every request
- Use separate lookback periods: 90 days for notes, 365 days for
evergreen article content (kind:30023)
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
* fix: normalize base url and allow nostr.json
* refactor: flatten nested conditionals with guard clauses
Use early returns and let-else patterns to reduce nesting depth in
generate_sitemap loops. Improves readability by making the happy path
linear instead of deeply indented.
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
---------
Signed-off-by: William Casarin <jb55@jb55.com>
Co-authored-by: William Casarin <jb55@jb55.com>
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
Co-authored-by: alltheseas <alltheseas@users.noreply.github.com>
Diffstat:
| M | .github/workflows/rust.yml | | | 4 | +++- |
| M | Cargo.lock | | | 399 | ++++++++++++++++++++++++++++++++++++++++++++++--------------------------------- |
| M | Cargo.toml | | | 9 | ++++++--- |
| M | assets/damus.css | | | 217 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| M | src/html.rs | | | 1058 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---- |
| M | src/main.rs | | | 38 | +++++++++++++++++++++++++++++++++++++- |
| M | src/nip19.rs | | | 208 | ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--- |
| M | src/relay_pool.rs | | | 26 | ++++++++++++-------------- |
| M | src/render.rs | | | 333 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---------------- |
| A | src/sitemap.rs | | | 356 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | src/unknowns.rs | | | 274 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
11 files changed, 2620 insertions(+), 302 deletions(-)
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -16,7 +16,9 @@ jobs:
steps:
- name: Deps
- run: sudo apt-get install libfontconfig1-dev libfreetype6-dev libssl-dev
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y libfontconfig1-dev libfreetype6-dev libssl-dev
- uses: actions/checkout@v3
- name: Build
run: cargo build --verbose
diff --git a/Cargo.lock b/Cargo.lock
@@ -35,6 +35,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
[[package]]
+name = "adler32"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234"
+
+[[package]]
name = "aead"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -98,21 +104,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
-name = "async-trait"
-version = "0.1.89"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.111",
-]
-
-[[package]]
name = "async-utility"
-version = "0.2.0"
+version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a349201d80b4aa18d17a34a182bdd7f8ddf845e9e57d2ea130a12e10ef1e3a47"
+checksum = "a34a3b57207a7a1007832416c3e4862378c8451b4e8e093e436f48c2d3d2c151"
dependencies = [
"futures-util",
"gloo-timers",
@@ -122,9 +117,9 @@ dependencies = [
[[package]]
name = "async-wsocket"
-version = "0.10.1"
+version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d50cb541e6d09e119e717c64c46ed33f49be7fa592fa805d56c11d6a7ff093c"
+checksum = "9a7d8c7d34a225ba919dd9ba44d4b9106d20142da545e086be8ae21d1897e043"
dependencies = [
"async-utility",
"futures",
@@ -141,12 +136,9 @@ dependencies = [
[[package]]
name = "atomic-destructor"
-version = "0.2.0"
+version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d919cb60ba95c87ba42777e9e246c4e8d658057299b437b7512531ce0a09a23"
-dependencies = [
- "tracing",
-]
+checksum = "ef49f5882e4b6afaac09ad239a4f8c70a24b8f2b0897edb1f706008efd109cf4"
[[package]]
name = "atomic-waker"
@@ -161,16 +153,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
[[package]]
-name = "base58ck"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c8d66485a3a2ea485c1913c4572ce0256067a5377ac8c75c4960e1cda98605f"
-dependencies = [
- "bitcoin-internals",
- "bitcoin_hashes",
-]
-
-[[package]]
name = "base64"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -263,49 +245,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e4b40c7323adcfc0a41c4b88143ed58346ff65a288fc144329c5c45e05d70c6"
[[package]]
-name = "bitcoin"
-version = "0.32.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e499f9fc0407f50fe98af744ab44fa67d409f76b6772e1689ec8485eb0c0f66"
-dependencies = [
- "base58ck",
- "bech32",
- "bitcoin-internals",
- "bitcoin-io",
- "bitcoin-units",
- "bitcoin_hashes",
- "hex-conservative",
- "hex_lit",
- "secp256k1",
- "serde",
-]
-
-[[package]]
-name = "bitcoin-internals"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30bdbe14aa07b06e6cfeffc529a1f099e5fbe249524f8125358604df99a4bed2"
-dependencies = [
- "serde",
-]
-
-[[package]]
name = "bitcoin-io"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dee39a0ee5b4095224a0cfc6bf4cc1baf0f9624b96b367e53b66d974e51d953"
[[package]]
-name = "bitcoin-units"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5285c8bcaa25876d07f37e3d30c303f2609179716e11d688f51e8f1fe70063e2"
-dependencies = [
- "bitcoin-internals",
- "serde",
-]
-
-[[package]]
name = "bitcoin_hashes"
version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -347,6 +292,12 @@ dependencies = [
]
[[package]]
+name = "btreecap"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6160c957d8aa33d0a8ba1dbab98e3cb57023ad9374c501441e88559f99e6c4c9"
+
+[[package]]
name = "bumpalo"
version = "3.19.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -381,9 +332,9 @@ dependencies = [
[[package]]
name = "cc"
-version = "1.2.49"
+version = "1.2.55"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90583009037521a116abf44494efecd645ba48b6622457080f080b85544e2215"
+checksum = "47b26a0954ae34af09b50f0de26458fa95369a0d478d8236d3f93082b219bd29"
dependencies = [
"find-msvc-tools",
"shlex",
@@ -473,6 +424,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
[[package]]
+name = "core2"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
name = "cpufeatures"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -528,7 +488,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
dependencies = [
"generic-array",
- "rand_core",
"typenum",
]
@@ -556,6 +515,12 @@ dependencies = [
]
[[package]]
+name = "dary_heap"
+version = "0.3.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06d2e3287df1c007e74221c49ca10a95d557349e54b3a75dc2fb14712c751f04"
+
+[[package]]
name = "dashmap"
version = "6.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -788,9 +753,9 @@ dependencies = [
[[package]]
name = "find-msvc-tools"
-version = "0.1.5"
+version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844"
+checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
[[package]]
name = "flatbuffers"
@@ -809,6 +774,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb"
dependencies = [
"crc32fast",
+ "libz-rs-sys",
"miniz_oxide",
]
@@ -826,9 +792,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "foldhash"
-version = "0.1.5"
+version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
+checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
[[package]]
name = "foreign-types"
@@ -979,10 +945,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592"
dependencies = [
"cfg-if",
- "js-sys",
"libc",
"wasi",
- "wasm-bindgen",
]
[[package]]
@@ -1015,9 +979,9 @@ checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
[[package]]
name = "gloo-timers"
-version = "0.2.6"
+version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c"
+checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994"
dependencies = [
"futures-channel",
"futures-core",
@@ -1066,9 +1030,9 @@ dependencies = [
[[package]]
name = "hashbrown"
-version = "0.15.5"
+version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
+checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
dependencies = [
"allocator-api2",
"equivalent",
@@ -1076,12 +1040,6 @@ dependencies = [
]
[[package]]
-name = "hashbrown"
-version = "0.16.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
-
-[[package]]
name = "heck"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1109,12 +1067,6 @@ dependencies = [
]
[[package]]
-name = "hex_lit"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd"
-
-[[package]]
name = "hmac"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1458,9 +1410,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
dependencies = [
"cfg-if",
- "js-sys",
- "wasm-bindgen",
- "web-sys",
]
[[package]]
@@ -1537,6 +1486,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091"
[[package]]
+name = "libflate"
+version = "2.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e3248b8d211bd23a104a42d81b4fa8bb8ac4a3b75e7a43d85d2c9ccb6179cd74"
+dependencies = [
+ "adler32",
+ "core2",
+ "crc32fast",
+ "dary_heap",
+ "libflate_lz77",
+]
+
+[[package]]
+name = "libflate_lz77"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a599cb10a9cd92b1300debcef28da8f70b935ec937f44fcd1b70a7c986a11c5c"
+dependencies = [
+ "core2",
+ "hashbrown 0.16.1",
+ "rle-decode-fast",
+]
+
+[[package]]
name = "libloading"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1558,6 +1531,32 @@ dependencies = [
]
[[package]]
+name = "libsodium-sys-stable"
+version = "1.23.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e5d23f4a051a13cf1085b2c5a050d4d890d80c754534cc4247eff525fa5283d"
+dependencies = [
+ "cc",
+ "libc",
+ "libflate",
+ "minisign-verify",
+ "pkg-config",
+ "tar",
+ "ureq 3.2.0",
+ "vcpkg",
+ "zip",
+]
+
+[[package]]
+name = "libz-rs-sys"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c10501e7805cee23da17c7790e59df2870c0d4043ec6d03f67d31e2b53e77415"
+dependencies = [
+ "zlib-rs",
+]
+
+[[package]]
name = "linux-raw-sys"
version = "0.4.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1592,12 +1591,9 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
[[package]]
name = "lru"
-version = "0.12.5"
+version = "0.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38"
-dependencies = [
- "hashbrown 0.15.5",
-]
+checksum = "96051b46fc183dc9cd4a223960ef37b9af631b55191852a8274bfef064cda20f"
[[package]]
name = "mac"
@@ -1728,6 +1724,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]]
+name = "minisign-verify"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e856fdd13623a2f5f2f54676a4ee49502a96a80ef4a62bcedd23d52427c44d43"
+
+[[package]]
name = "miniz_oxide"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1767,15 +1769,9 @@ dependencies = [
[[package]]
name = "negentropy"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e664971378a3987224f7a0e10059782035e89899ae403718ee07de85bec42afe"
-
-[[package]]
-name = "negentropy"
-version = "0.4.3"
+version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43a88da9dd148bbcdce323dd6ac47d369b4769d4a3b78c6c52389b9269f77932"
+checksum = "f0efe882e02d206d8d279c20eb40e03baf7cb5136a1476dc084a324fbc3ec42d"
[[package]]
name = "new_debug_unreachable"
@@ -1801,24 +1797,22 @@ dependencies = [
[[package]]
name = "nostr"
-version = "0.37.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8aad4b767bbed24ac5eb4465bfb83bc1210522eb99d67cf4e547ec2ec7e47786"
+version = "0.44.1"
+source = "git+https://github.com/alltheseas/rust-nostr.git?branch=relay-provenance-tracking#2ea3dbea4d89697e1c2f2bd4dea1bb68432df6f5"
dependencies = [
- "async-trait",
"base64 0.22.1",
"bech32",
"bip39",
- "bitcoin",
+ "bitcoin_hashes",
"cbc",
"chacha20",
"chacha20poly1305",
- "getrandom 0.2.16",
+ "hex",
"instant",
- "negentropy 0.3.1",
- "negentropy 0.4.3",
"once_cell",
+ "rand 0.9.2",
"scrypt",
+ "secp256k1",
"serde",
"serde_json",
"unicode-normalization",
@@ -1827,67 +1821,69 @@ dependencies = [
[[package]]
name = "nostr-database"
-version = "0.37.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23696338d51e45cd44e061823847f4b0d1d362eca80d5033facf9c184149f72f"
+version = "0.44.0"
+source = "git+https://github.com/alltheseas/rust-nostr.git?branch=relay-provenance-tracking#2ea3dbea4d89697e1c2f2bd4dea1bb68432df6f5"
dependencies = [
- "async-trait",
+ "btreecap",
"lru",
"nostr",
- "thiserror 1.0.69",
"tokio",
- "tracing",
+]
+
+[[package]]
+name = "nostr-gossip"
+version = "0.44.0"
+source = "git+https://github.com/alltheseas/rust-nostr.git?branch=relay-provenance-tracking#2ea3dbea4d89697e1c2f2bd4dea1bb68432df6f5"
+dependencies = [
+ "nostr",
]
[[package]]
name = "nostr-relay-pool"
-version = "0.37.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "15fcc6e3f0ca54d0fc779009bc5f2684cea9147be3b6aa68a7d301ea590f95f5"
+version = "0.44.0"
+source = "git+https://github.com/alltheseas/rust-nostr.git?branch=relay-provenance-tracking#2ea3dbea4d89697e1c2f2bd4dea1bb68432df6f5"
dependencies = [
"async-utility",
"async-wsocket",
"atomic-destructor",
- "negentropy 0.3.1",
- "negentropy 0.4.3",
+ "hex",
+ "lru",
+ "negentropy",
"nostr",
"nostr-database",
- "thiserror 1.0.69",
"tokio",
- "tokio-stream",
"tracing",
]
[[package]]
name = "nostr-sdk"
-version = "0.37.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "491221fc89b1aa189a0de640127127d68b4e7c5c1d44371b04d9a6d10694b5af"
+version = "0.44.1"
+source = "git+https://github.com/alltheseas/rust-nostr.git?branch=relay-provenance-tracking#2ea3dbea4d89697e1c2f2bd4dea1bb68432df6f5"
dependencies = [
"async-utility",
- "atomic-destructor",
"nostr",
"nostr-database",
+ "nostr-gossip",
"nostr-relay-pool",
- "thiserror 1.0.69",
"tokio",
"tracing",
]
[[package]]
name = "nostrdb"
-version = "0.5.1"
-source = "git+https://github.com/damus-io/nostrdb-rs?rev=77fbc99a55f6a5e939176085f9a95cf2a4e7eeb5#77fbc99a55f6a5e939176085f9a95cf2a4e7eeb5"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05b2685d093dca579807150b6fafab4dcb974fc6e31017d273ae32d42795d41b"
dependencies = [
"bindgen 0.69.5",
"cc",
"flatbuffers",
"futures",
"libc",
+ "libsodium-sys-stable",
"thiserror 2.0.17",
"tokio",
"tracing",
- "tracing-subscriber",
]
[[package]]
@@ -2043,7 +2039,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166"
dependencies = [
"base64ct",
- "rand_core",
+ "rand_core 0.6.4",
"subtle",
]
@@ -2096,7 +2092,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
dependencies = [
"phf_shared",
- "rand",
+ "rand 0.8.5",
]
[[package]]
@@ -2275,19 +2271,27 @@ version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
dependencies = [
- "libc",
+ "rand_core 0.6.4",
+]
+
+[[package]]
+name = "rand"
+version = "0.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
+dependencies = [
"rand_chacha",
- "rand_core",
+ "rand_core 0.9.3",
]
[[package]]
name = "rand_chacha"
-version = "0.3.1"
+version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
+checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
dependencies = [
"ppv-lite86",
- "rand_core",
+ "rand_core 0.9.3",
]
[[package]]
@@ -2300,6 +2304,15 @@ dependencies = [
]
[[package]]
+name = "rand_core"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"
+dependencies = [
+ "getrandom 0.3.4",
+]
+
+[[package]]
name = "raw-cpuid"
version = "11.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2419,6 +2432,12 @@ dependencies = [
]
[[package]]
+name = "rle-decode-fast"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422"
+
+[[package]]
name = "roxmltree"
version = "0.15.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2557,8 +2576,6 @@ version = "0.29.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9465315bc9d4566e1724f0fffcbcc446268cb522e60f9a27bcded6b19c108113"
dependencies = [
- "bitcoin_hashes",
- "rand",
"secp256k1-sys",
"serde",
]
@@ -2637,7 +2654,6 @@ version = "1.0.145"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c"
dependencies = [
- "indexmap",
"itoa",
"memchr",
"ryu",
@@ -2739,7 +2755,7 @@ dependencies = [
"serde_json",
"tar",
"toml",
- "ureq",
+ "ureq 2.12.1",
]
[[package]]
@@ -3100,21 +3116,10 @@ dependencies = [
]
[[package]]
-name = "tokio-stream"
-version = "0.1.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047"
-dependencies = [
- "futures-core",
- "pin-project-lite",
- "tokio",
-]
-
-[[package]]
name = "tokio-tungstenite"
-version = "0.24.0"
+version = "0.26.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9"
+checksum = "7a9daff607c6d2bf6c16fd681ccb7eecc83e4e2cdc1ca067ffaadfca5de7f084"
dependencies = [
"futures-util",
"log",
@@ -3231,25 +3236,30 @@ checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31"
[[package]]
name = "tungstenite"
-version = "0.24.0"
+version = "0.26.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a"
+checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13"
dependencies = [
- "byteorder",
"bytes",
"data-encoding",
"http 1.4.0",
"httparse",
"log",
- "rand",
+ "rand 0.9.2",
"rustls",
"rustls-pki-types",
"sha1",
- "thiserror 1.0.69",
+ "thiserror 2.0.17",
"utf-8",
]
[[package]]
+name = "typed-path"
+version = "0.12.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3015e6ce46d5ad8751e4a772543a30c7511468070e98e64e20165f8f81155b64"
+
+[[package]]
name = "typenum"
version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3315,6 +3325,31 @@ dependencies = [
]
[[package]]
+name = "ureq"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fdc97a28575b85cfedf2a7e7d3cc64b3e11bd8ac766666318003abbacc7a21fc"
+dependencies = [
+ "base64 0.22.1",
+ "log",
+ "percent-encoding",
+ "ureq-proto",
+ "utf-8",
+]
+
+[[package]]
+name = "ureq-proto"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d81f9efa9df032be5934a46a068815a10a042b494b6a58cb0a1a97bb5467ed6f"
+dependencies = [
+ "base64 0.22.1",
+ "http 1.4.0",
+ "httparse",
+ "log",
+]
+
+[[package]]
name = "url"
version = "2.5.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3876,6 +3911,38 @@ dependencies = [
]
[[package]]
+name = "zip"
+version = "7.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "268bf6f9ceb991e07155234071501490bb41fd1e39c6a588106dad10ae2a5804"
+dependencies = [
+ "crc32fast",
+ "flate2",
+ "indexmap",
+ "memchr",
+ "typed-path",
+ "zopfli",
+]
+
+[[package]]
+name = "zlib-rs"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3"
+
+[[package]]
+name = "zopfli"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249"
+dependencies = [
+ "bumpalo",
+ "crc32fast",
+ "log",
+ "simd-adler32",
+]
+
+[[package]]
name = "zune-inflate"
version = "0.2.54"
source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/Cargo.toml b/Cargo.toml
@@ -13,12 +13,15 @@ hyper-util = { version = "0.1.1", features = ["full"] }
http-body-util = "0.1"
tracing = "0.1.41"
tracing-subscriber = "0.3.19"
-nostrdb = { git = "https://github.com/damus-io/nostrdb-rs", rev = "77fbc99a55f6a5e939176085f9a95cf2a4e7eeb5" }
+nostrdb = "0.9.0"
#nostrdb = { path = "/home/jb55/src/rust/nostrdb-rs" }
#nostrdb = "0.1.6"
#nostr-sdk = { git = "https://github.com/damus-io/nostr-sdk.git", rev = "fc0dc7b38f5060f171228b976b9700c0135245d3" }
-nostr-sdk = "0.37.0"
-nostr = "0.37.0"
+#nostr-sdk = "0.37.0"
+#nostr = "0.37.0"
+# PR #1172: relay provenance tracking - https://github.com/rust-nostr/nostr/pull/1172
+nostr-sdk = { git = "https://github.com/alltheseas/rust-nostr.git", branch = "relay-provenance-tracking" }
+nostr = { git = "https://github.com/alltheseas/rust-nostr.git", branch = "relay-provenance-tracking" }
hex = "0.4.3"
egui = "0.23.0"
egui_extras = { version = "0.23.0", features = ["image", "svg"] }
diff --git a/assets/damus.css b/assets/damus.css
@@ -258,6 +258,13 @@ a:hover {
color: #ffffff;
}
+.damus-note-handle {
+ display: block;
+ font-size: 0.95rem;
+ font-weight: 400;
+ color: var(--damus-muted);
+}
+
.damus-note-time {
font-size: 0.9rem;
color: var(--damus-muted);
@@ -348,6 +355,58 @@ a:hover {
letter-spacing: 0.02em;
}
+/* Draft badge for unpublished articles (kind:30024) */
+.damus-article-draft {
+ background: linear-gradient(135deg, #ff6b35, #f7931a);
+ color: #ffffff;
+ padding: 0.25em 0.65em;
+ border-radius: 6px;
+ font-family: var(--damus-font);
+ font-size: 0.4em;
+ font-weight: 700;
+ letter-spacing: 0.08em;
+ text-transform: uppercase;
+ margin-left: 0.75em;
+ vertical-align: middle;
+ box-shadow: 0 2px 8px rgba(255, 107, 53, 0.3);
+}
+
+/* NIP-84 Highlight styles (kind:9802) */
+.damus-highlight-text {
+ border-left: 3px solid var(--damus-accent);
+ padding-left: 1.25rem;
+ margin: 0;
+ font-style: italic;
+ font-size: 1.15rem;
+ line-height: 1.7;
+ color: var(--damus-text);
+}
+
+.damus-highlight-context {
+ color: var(--damus-muted);
+ font-size: 0.9rem;
+ line-height: 1.6;
+ margin-top: 1rem;
+}
+
+.damus-highlight-source {
+ font-size: 0.9rem;
+ margin-top: 1.25rem;
+ padding-top: 1rem;
+ border-top: 1px solid var(--damus-card-border);
+}
+
+.damus-highlight-source-label {
+ color: var(--damus-muted);
+ margin-right: 0.35em;
+}
+
+.damus-highlight-comment {
+ font-size: 1.1rem;
+ line-height: 1.6;
+ margin-bottom: 1rem;
+}
+
.damus-profile-card {
gap: 1.5rem;
}
@@ -441,6 +500,164 @@ a:hover {
color: var(--damus-muted);
}
+/* NIP-18 Embedded Quotes (q tags) */
+.damus-embedded-quotes {
+ display: flex;
+ flex-direction: column;
+ gap: 0.5rem;
+ margin-top: 0.35rem;
+}
+
+.damus-embedded-quote {
+ background: rgba(0, 0, 0, 0.25);
+ border: 1px solid var(--damus-card-border);
+ border-radius: 16px;
+ padding: 0.875rem 1rem;
+ transition: border-color 160ms ease;
+ text-decoration: none;
+ display: block;
+}
+
+.damus-embedded-quote:hover {
+ border-color: rgba(189, 102, 255, 0.3);
+ text-decoration: none;
+}
+
+.damus-embedded-quote-header {
+ display: flex;
+ align-items: center;
+ gap: 0.35rem;
+ margin-bottom: 0.25rem;
+ flex-wrap: wrap;
+}
+
+.damus-embedded-quote-avatar {
+ width: 20px;
+ height: 20px;
+ border-radius: 50%;
+ object-fit: cover;
+ flex-shrink: 0;
+}
+
+.damus-embedded-quote-author {
+ font-weight: 600;
+ color: #ffffff;
+ font-size: 0.9rem;
+}
+
+.damus-embedded-quote-username {
+ color: var(--damus-muted);
+ font-size: 0.85rem;
+}
+
+.damus-embedded-quote-time {
+ color: var(--damus-muted);
+ font-size: 0.85rem;
+}
+
+.damus-embedded-quote-type {
+ background: rgba(189, 102, 255, 0.2);
+ color: var(--damus-accent);
+ padding: 0.15em 0.5em;
+ border-radius: 4px;
+ font-size: 0.75rem;
+ font-weight: 600;
+ text-transform: uppercase;
+ letter-spacing: 0.03em;
+ margin-left: 0.5rem;
+}
+
+.damus-embedded-quote-type-draft {
+ background: rgba(255, 107, 53, 0.2);
+ color: #ff6b35;
+}
+
+.damus-embedded-quote-reply {
+ color: var(--damus-muted);
+ font-size: 0.8rem;
+ margin-bottom: 0.5rem;
+}
+
+.damus-embedded-quote-content {
+ color: var(--damus-text);
+ font-size: 0.9rem;
+ line-height: 1.5;
+}
+
+.damus-embedded-quote-highlight {
+ border-left: 3px solid var(--damus-accent);
+ padding-left: 0.75rem;
+ font-style: italic;
+}
+
+/* Article card styling for embedded quotes */
+.damus-embedded-quote-article {
+ display: flex;
+ flex-direction: column;
+}
+
+.damus-embedded-article-image {
+ width: 100%;
+ max-height: 180px;
+ object-fit: cover;
+ border-radius: 12px;
+ margin-bottom: 0.75rem;
+}
+
+.damus-embedded-article-title {
+ font-weight: 700;
+ font-size: 1rem;
+ color: #ffffff;
+ line-height: 1.3;
+ margin-bottom: 0.35rem;
+}
+
+.damus-embedded-article-title.damus-embedded-article-draft::after {
+ content: "DRAFT";
+ background: linear-gradient(135deg, #ff6b35, #f7931a);
+ color: #ffffff;
+ padding: 0.15em 0.4em;
+ border-radius: 4px;
+ font-size: 0.6em;
+ font-weight: 700;
+ letter-spacing: 0.05em;
+ margin-left: 0.5em;
+ vertical-align: middle;
+}
+
+.damus-embedded-article-summary {
+ font-size: 0.85rem;
+ color: var(--damus-muted);
+ line-height: 1.4;
+ margin-bottom: 0.5rem;
+}
+
+.damus-embedded-article-wordcount {
+ font-size: 0.8rem;
+ color: var(--damus-muted);
+ font-weight: 500;
+}
+
+.damus-embedded-quote-showmore {
+ color: var(--damus-accent);
+ font-weight: 500;
+}
+
+.damus-embedded-quote-urls {
+ display: flex;
+ flex-wrap: wrap;
+ gap: 0.5rem;
+ margin-top: 0.75rem;
+}
+
+.damus-embedded-quote-url {
+ background: rgba(255, 255, 255, 0.08);
+ color: var(--damus-muted);
+ padding: 0.4rem 0.75rem;
+ border-radius: 8px;
+ font-size: 0.8rem;
+}
+
@media (max-width: 640px) {
.damus-header {
flex-direction: column;
diff --git a/src/html.rs b/src/html.rs
@@ -8,7 +8,7 @@ use ammonia::Builder as HtmlSanitizer;
use http_body_util::Full;
use hyper::{body::Bytes, header, Request, Response, StatusCode};
use nostr::nips::nip19::Nip19Event;
-use nostr_sdk::prelude::{EventId, Nip19, PublicKey, RelayUrl, ToBech32};
+use nostr_sdk::prelude::{EventId, FromBech32, Nip19, PublicKey, RelayUrl, ToBech32};
use nostrdb::{
BlockType, Blocks, Filter, Mention, Ndb, NdbProfile, Note, NoteKey, ProfileRecord, Transaction,
};
@@ -18,6 +18,12 @@ use std::io::Write;
use std::str::FromStr;
use tracing::warn;
+struct QuoteProfileInfo {
+ display_name: Option<String>,
+ username: Option<String>,
+ pfp_url: Option<String>,
+}
+
#[derive(Debug, Clone, PartialEq, Eq)]
struct RelayEntry {
url: String,
@@ -77,6 +83,33 @@ struct ArticleMetadata {
topics: Vec<String>,
}
+/// Metadata extracted from NIP-84 highlight events (kind:9802).
+///
+/// Highlights capture a passage from source content with optional context.
+/// Sources can be: web URLs (r tag), nostr notes (e tag), or articles (a tag).
+#[derive(Default)]
+struct HighlightMetadata {
+ /// Surrounding text providing context for the highlight (from "context" tag)
+ context: Option<String>,
+ /// User's comment/annotation on the highlight (from "comment" tag)
+ comment: Option<String>,
+ /// Web URL source - external article or page (from "r" tag)
+ source_url: Option<String>,
+ /// Nostr note ID - reference to a kind:1 shortform note (from "e" tag)
+ source_event_id: Option<[u8; 32]>,
+ /// Nostr article address - reference to kind:30023/30024 (from "a" tag)
+ /// Format: "30023:{pubkey_hex}:{d-identifier}"
+ source_article_addr: Option<String>,
+}
+
+/// Normalizes text for comparison by trimming whitespace and trailing punctuation.
+/// Used to detect when context and content are essentially the same text.
+fn normalize_for_comparison(s: &str) -> String {
+ s.trim()
+ .trim_end_matches(|c: char| c.is_ascii_punctuation())
+ .to_lowercase()
+}
+
fn collapse_whitespace<S: AsRef<str>>(input: S) -> String {
let mut result = String::with_capacity(input.as_ref().len());
let mut last_space = false;
@@ -151,6 +184,101 @@ fn extract_article_metadata(note: &Note) -> ArticleMetadata {
meta
}
+/// Extracts NIP-84 highlight metadata from a kind:9802 note.
+///
+/// Parses tags to identify the highlight source:
+/// - "context" tag: surrounding text for context
+/// - "comment" tag: user's annotation
+/// - "r" tag: web URL source (external article/page)
+/// - "e" tag: nostr note ID (kind:1 shortform note)
+/// - "a" tag: nostr article address (kind:30023/30024 longform)
+fn extract_highlight_metadata(note: &Note) -> HighlightMetadata {
+ let mut meta = HighlightMetadata::default();
+
+ for tag in note.tags() {
+ let Some(tag_name) = tag.get_str(0) else {
+ continue;
+ };
+
+ match tag_name {
+ "context" => {
+ if let Some(value) = tag.get_str(1) {
+ if !value.trim().is_empty() {
+ meta.context = Some(value.to_owned());
+ }
+ }
+ }
+
+ "comment" => {
+ if let Some(value) = tag.get_str(1) {
+ if !value.trim().is_empty() {
+ meta.comment = Some(value.to_owned());
+ }
+ }
+ }
+
+ "r" => {
+ if let Some(value) = tag.get_str(1) {
+ let trimmed = value.trim();
+ if trimmed.starts_with("http://") || trimmed.starts_with("https://") {
+ meta.source_url = Some(trimmed.to_owned());
+ }
+ }
+ }
+
+ "e" => {
+ // The e tag value is guaranteed to be an ID
+ if let Some(event_id) = tag.get_id(1) {
+ meta.source_event_id = Some(*event_id);
+ }
+ }
+
+ "a" => {
+ if let Some(value) = tag.get_str(1) {
+ let trimmed = value.trim();
+ if trimmed.starts_with("30023:") || trimmed.starts_with("30024:") {
+ meta.source_article_addr = Some(trimmed.to_owned());
+ }
+ }
+ }
+
+ _ => {}
+ }
+ }
+
+ meta
+}
+
+/// Formats a unix timestamp as a relative time string (e.g., "6h", "2d", "3w").
+fn format_relative_time(timestamp: u64) -> String {
+ let now = std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .map(|d| d.as_secs())
+ .unwrap_or(0);
+
+ if timestamp > now {
+ return "now".to_string();
+ }
+
+ let diff = now - timestamp;
+ let minutes = diff / 60;
+ let hours = diff / 3600;
+ let days = diff / 86400;
+ let weeks = diff / 604800;
+
+ if minutes < 1 {
+ "now".to_string()
+ } else if minutes < 60 {
+ format!("{}m", minutes)
+ } else if hours < 24 {
+ format!("{}h", hours)
+ } else if days < 7 {
+ format!("{}d", days)
+ } else {
+ format!("{}w", weeks)
+ }
+}
+
fn render_markdown(markdown: &str) -> String {
let mut options = Options::empty();
options.insert(Options::ENABLE_TABLES);
@@ -256,7 +384,21 @@ fn is_image(url: &str) -> bool {
.any(|ext| ends_with(strip_querystring(url), ext))
}
-pub fn render_note_content(body: &mut Vec<u8>, note: &Note, blocks: &Blocks) {
+/// Gets the display name for a profile, preferring display_name, falling back to name.
+fn get_profile_display_name<'a>(record: Option<&ProfileRecord<'a>>) -> Option<&'a str> {
+ let profile = record?.record().profile()?;
+ let display_name = profile.display_name().filter(|n| !n.trim().is_empty());
+ let username = profile.name().filter(|n| !n.trim().is_empty());
+ display_name.or(username)
+}
+
+pub fn render_note_content(
+ body: &mut Vec<u8>,
+ note: &Note,
+ blocks: &Blocks,
+ ndb: &Ndb,
+ txn: &Transaction,
+) {
for block in blocks.iter(note) {
match block.blocktype() {
BlockType::Url => {
@@ -293,33 +435,596 @@ pub fn render_note_content(body: &mut Vec<u8>, note: &Note, blocks: &Blocks) {
}
BlockType::MentionBech32 => {
- match block.as_mention().unwrap() {
- Mention::Event(_)
- | Mention::Note(_)
- | Mention::Profile(_)
- | Mention::Pubkey(_)
- | Mention::Secret(_)
- | Mention::Addr(_) => {
- let _ = write!(
- body,
- r#"<a href="/{}">@{}</a>"#,
- block.as_str(),
- &abbrev_str(block.as_str())
- );
+ let mention = block.as_mention().unwrap();
+ let pubkey = match mention {
+ Mention::Profile(p) => Some(p.pubkey()),
+ Mention::Pubkey(p) => Some(p.pubkey()),
+ _ => None,
+ };
+
+ if let Some(pk) = pubkey {
+ // Profile/pubkey mentions: show the human-readable name
+ let record = ndb.get_profile_by_pubkey(txn, pk).ok();
+ let display = get_profile_display_name(record.as_ref())
+ .map(|s| s.to_string())
+ .unwrap_or_else(|| abbrev_str(block.as_str()));
+ let display_html = html_escape::encode_text(&display);
+ let _ = write!(
+ body,
+ r#"<a href="/{bech32}">@{display}</a>"#,
+ bech32 = block.as_str(),
+ display = display_html
+ );
+ } else {
+ match mention {
+ // Event/note mentions: skip inline rendering (shown as embedded quotes)
+ Mention::Event(_) | Mention::Note(_) => {}
+
+ // Other mentions: link with abbreviated bech32
+ _ => {
+ let _ = write!(
+ body,
+ r#"<a href="/{bech32}">{abbrev}</a>"#,
+ bech32 = block.as_str(),
+ abbrev = abbrev_str(block.as_str())
+ );
+ }
}
+ }
+ }
+ };
+ }
+}
+
+/// Represents a quoted event reference from a q tag (NIP-18) or inline mention.
+#[derive(Clone, PartialEq)]
+pub enum QuoteRef {
+ Event {
+ id: [u8; 32],
+ bech32: Option<String>,
+ relays: Vec<RelayUrl>,
+ },
+ Article {
+ addr: String,
+ bech32: Option<String>,
+ relays: Vec<RelayUrl>,
+ },
+}
+
+/// Extracts quote references from inline nevent/note mentions in content.
+fn extract_quote_refs_from_content(note: &Note, blocks: &Blocks) -> Vec<QuoteRef> {
+ use nostr_sdk::prelude::Nip19;
- Mention::Relay(relay) => {
- let _ = write!(
- body,
- r#"<a href="/{}">{}</a>"#,
- block.as_str(),
- &abbrev_str(relay.as_str())
+ let mut quotes = Vec::new();
+
+ for block in blocks.iter(note) {
+ if block.blocktype() != BlockType::MentionBech32 {
+ continue;
+ }
+
+ let Some(mention) = block.as_mention() else {
+ continue;
+ };
+
+ match mention {
+ Mention::Event(_ev) => {
+ let bech32_str = block.as_str();
+ // Parse to get relay hints from nevent
+ if let Ok(Nip19::Event(ev)) = Nip19::from_bech32(bech32_str) {
+ let relays: Vec<RelayUrl> = ev.relays.to_vec();
+ quotes.push(QuoteRef::Event {
+ id: *ev.event_id.as_bytes(),
+ bech32: Some(bech32_str.to_string()),
+ relays,
+ });
+ } else if let Ok(Nip19::EventId(id)) = Nip19::from_bech32(bech32_str) {
+ // note1 format has no relay hints
+ quotes.push(QuoteRef::Event {
+ id: *id.as_bytes(),
+ bech32: Some(bech32_str.to_string()),
+ relays: vec![],
+ });
+ }
+ }
+ Mention::Note(_note_ref) => {
+ let bech32_str = block.as_str();
+ // note1 format has no relay hints
+ if let Ok(Nip19::EventId(id)) = Nip19::from_bech32(bech32_str) {
+ quotes.push(QuoteRef::Event {
+ id: *id.as_bytes(),
+ bech32: Some(bech32_str.to_string()),
+ relays: vec![],
+ });
+ }
+ }
+ // naddr mentions - articles (30023/30024) and highlights (9802)
+ Mention::Addr(_) => {
+ let bech32_str = block.as_str();
+ if let Ok(Nip19::Coordinate(coord)) = Nip19::from_bech32(bech32_str) {
+ let kind = coord.kind.as_u16();
+ if kind == 30023 || kind == 30024 || kind == 9802 {
+ let addr = format!(
+ "{}:{}:{}",
+ kind,
+ coord.public_key.to_hex(),
+ coord.identifier
);
+ quotes.push(QuoteRef::Article {
+ addr,
+ bech32: Some(bech32_str.to_string()),
+ relays: coord.relays,
+ });
}
- };
+ }
}
+ _ => {}
+ }
+ }
+
+ quotes
+}
+
+/// Extracts quote references from q tags (NIP-18 quote reposts).
+fn extract_quote_refs_from_tags(note: &Note) -> Vec<QuoteRef> {
+ use nostr_sdk::prelude::Nip19;
+
+ let mut quotes = Vec::new();
+
+ for tag in note.tags() {
+ if tag.get_str(0) != Some("q") {
+ continue;
+ }
+
+ let Some(value) = tag.get_str(1) else {
+ continue;
};
+ let trimmed = value.trim();
+
+ // Optional relay hint in third element of q tag
+ let tag_relay_hint: Option<RelayUrl> = tag
+ .get_str(2)
+ .filter(|s| !s.is_empty())
+ .and_then(|s| RelayUrl::parse(s).ok());
+
+ // Try nevent/note bech32
+ if trimmed.starts_with("nevent1") || trimmed.starts_with("note1") {
+ if let Ok(nip19) = Nip19::from_bech32(trimmed) {
+ match nip19 {
+ Nip19::Event(ev) => {
+ // Combine relays from nevent with q tag relay hint
+ let mut relays: Vec<RelayUrl> = ev.relays.to_vec();
+ if let Some(hint) = &tag_relay_hint {
+ if !relays.contains(hint) {
+ relays.push(hint.clone());
+ }
+ }
+ quotes.push(QuoteRef::Event {
+ id: *ev.event_id.as_bytes(),
+ bech32: Some(trimmed.to_owned()),
+ relays,
+ });
+ continue;
+ }
+ Nip19::EventId(id) => {
+ quotes.push(QuoteRef::Event {
+ id: *id.as_bytes(),
+ bech32: Some(trimmed.to_owned()),
+ relays: tag_relay_hint.clone().into_iter().collect(),
+ });
+ continue;
+ }
+ _ => {}
+ }
+ }
+ }
+
+ // Try naddr bech32
+ if trimmed.starts_with("naddr1") {
+ if let Ok(Nip19::Coordinate(coord)) = Nip19::from_bech32(trimmed) {
+ let addr = format!(
+ "{}:{}:{}",
+ coord.kind.as_u16(),
+ coord.public_key.to_hex(),
+ coord.identifier
+ );
+ // Combine relays from naddr with q tag relay hint
+ let mut relays = coord.relays;
+ if let Some(hint) = &tag_relay_hint {
+ if !relays.contains(hint) {
+ relays.push(hint.clone());
+ }
+ }
+ quotes.push(QuoteRef::Article {
+ addr,
+ bech32: Some(trimmed.to_owned()),
+ relays,
+ });
+ continue;
+ }
+ }
+
+ // Try article address format
+ if trimmed.starts_with("30023:") || trimmed.starts_with("30024:") {
+ quotes.push(QuoteRef::Article {
+ addr: trimmed.to_owned(),
+ bech32: None,
+ relays: tag_relay_hint.into_iter().collect(),
+ });
+ continue;
+ }
+
+ // Try hex event ID
+ if let Ok(bytes) = hex::decode(trimmed) {
+ if let Ok(id) = bytes.try_into() {
+ quotes.push(QuoteRef::Event {
+ id,
+ bech32: None,
+ relays: tag_relay_hint.into_iter().collect(),
+ });
+ }
+ }
+ }
+
+ quotes
+}
+
+/// Collects all quote refs from a note (q tags + inline mentions).
+pub fn collect_all_quote_refs(ndb: &Ndb, txn: &Transaction, note: &Note) -> Vec<QuoteRef> {
+ let mut refs = extract_quote_refs_from_tags(note);
+
+ if let Some(blocks) = note.key().and_then(|k| ndb.get_blocks_by_key(txn, k).ok()) {
+ let inline = extract_quote_refs_from_content(note, &blocks);
+ // Deduplicate - only add inline refs not already in q tags
+ for r in inline {
+ if !refs.contains(&r) {
+ refs.push(r);
+ }
+ }
}
+
+ refs
+}
+
+/// Looks up an article by address (kind:pubkey:d-tag) and returns the note key + optional title.
+fn lookup_article_by_addr(
+ ndb: &Ndb,
+ txn: &Transaction,
+ addr: &str,
+) -> Option<(NoteKey, Option<String>)> {
+ let parts: Vec<&str> = addr.splitn(3, ':').collect();
+ if parts.len() < 3 {
+ return None;
+ }
+
+ let kind: u64 = parts[0].parse().ok()?;
+ let pubkey_bytes = hex::decode(parts[1]).ok()?;
+ let pubkey: [u8; 32] = pubkey_bytes.try_into().ok()?;
+ let d_identifier = parts[2];
+
+ let filter = Filter::new().authors([&pubkey]).kinds([kind]).build();
+ let results = ndb.query(txn, &[filter], 10).ok()?;
+
+ for result in results {
+ let mut found_d_match = false;
+ let mut title = None;
+
+ for tag in result.note.tags() {
+ let tag_name = tag.get_str(0)?;
+ match tag_name {
+ "d" => {
+ if tag.get_str(1) == Some(d_identifier) {
+ found_d_match = true;
+ }
+ }
+ "title" => {
+ if let Some(t) = tag.get_str(1) {
+ if !t.trim().is_empty() {
+ title = Some(t.to_owned());
+ }
+ }
+ }
+ _ => {}
+ }
+ }
+
+ if found_d_match {
+ return Some((result.note_key, title));
+ }
+ }
+
+ None
+}
+
+/// Builds a link URL for a quote reference.
+fn build_quote_link(quote_ref: &QuoteRef) -> String {
+ use nostr_sdk::prelude::{Coordinate, EventId, Kind};
+
+ match quote_ref {
+ QuoteRef::Event { id, bech32, .. } => {
+ if let Some(b) = bech32 {
+ return format!("/{}", b);
+ }
+ if let Ok(b) =
+ EventId::from_slice(id).map(|eid| eid.to_bech32().expect("infallible apparently"))
+ {
+ return format!("/{}", b);
+ }
+ }
+ QuoteRef::Article { addr, bech32, .. } => {
+ if let Some(b) = bech32 {
+ return format!("/{}", b);
+ }
+ let parts: Vec<&str> = addr.splitn(3, ':').collect();
+ if parts.len() >= 3 {
+ if let Ok(kind) = parts[0].parse::<u16>() {
+ if let Ok(pubkey) = PublicKey::from_hex(parts[1]) {
+ let coordinate =
+ Coordinate::new(Kind::from(kind), pubkey).identifier(parts[2]);
+ if let Ok(naddr) = coordinate.to_bech32() {
+ return format!("/{}", naddr);
+ }
+ }
+ }
+ }
+ }
+ }
+ "#".to_string()
+}
+
+/// Builds embedded quote HTML for referenced events.
+fn build_embedded_quotes_html(ndb: &Ndb, txn: &Transaction, quote_refs: &[QuoteRef]) -> String {
+ use nostrdb::NoteReply;
+
+ if quote_refs.is_empty() {
+ return String::new();
+ }
+
+ let mut quotes_html = String::new();
+
+ for quote_ref in quote_refs {
+ let quoted_note = match quote_ref {
+ QuoteRef::Event { id, .. } => match ndb.get_note_by_id(txn, id) {
+ Ok(note) => note,
+ Err(_) => continue,
+ },
+ QuoteRef::Article { addr, .. } => match lookup_article_by_addr(ndb, txn, addr) {
+ Some((note_key, _title)) => match ndb.get_note_by_key(txn, note_key) {
+ Ok(note) => note,
+ Err(_) => continue,
+ },
+ None => continue,
+ },
+ };
+
+ // Get author profile (filter empty strings for proper fallback)
+ let profile_info = ndb
+ .get_profile_by_pubkey(txn, quoted_note.pubkey())
+ .ok()
+ .and_then(|rec| {
+ rec.record().profile().map(|p| {
+ let display_name = p
+ .display_name()
+ .filter(|s| !s.is_empty())
+ .or_else(|| p.name().filter(|s| !s.is_empty()))
+ .map(|n| n.to_owned());
+ let username = p
+ .name()
+ .filter(|s| !s.is_empty())
+ .map(|n| format!("@{}", n));
+ let pfp_url = p.picture().filter(|s| !s.is_empty()).map(|s| s.to_owned());
+ QuoteProfileInfo {
+ display_name,
+ username,
+ pfp_url,
+ }
+ })
+ })
+ .unwrap_or(QuoteProfileInfo {
+ display_name: None,
+ username: None,
+ pfp_url: None,
+ });
+
+ let display_name = profile_info
+ .display_name
+ .unwrap_or_else(|| "nostrich".to_string());
+ let display_name_html = html_escape::encode_text(&display_name);
+ let username_html = profile_info
+ .username
+ .map(|u| {
+ format!(
+ r#" <span class="damus-embedded-quote-username">{}</span>"#,
+ html_escape::encode_text(&u)
+ )
+ })
+ .unwrap_or_default();
+
+ let pfp_html = profile_info
+ .pfp_url
+ .filter(|url| !url.trim().is_empty())
+ .map(|url| {
+ let pfp_attr = html_escape::encode_double_quoted_attribute(&url);
+ format!(
+ r#"<img src="{}" class="damus-embedded-quote-avatar" alt="" />"#,
+ pfp_attr
+ )
+ })
+ .unwrap_or_else(|| {
+ r#"<img src="/img/no-profile.svg" class="damus-embedded-quote-avatar" alt="" />"#
+ .to_string()
+ });
+
+ let relative_time = format_relative_time(quoted_note.created_at());
+ let time_html = html_escape::encode_text(&relative_time);
+
+ // Detect reply using nostrdb's NoteReply
+ let reply_html = NoteReply::new(quoted_note.tags())
+ .reply()
+ .and_then(|reply_ref| ndb.get_note_by_id(txn, reply_ref.id).ok())
+ .and_then(|parent| {
+ get_profile_display_name(
+ ndb.get_profile_by_pubkey(txn, parent.pubkey())
+ .ok()
+ .as_ref(),
+ )
+ .map(|name| format!("@{}", name))
+ })
+ .map(|name| {
+ format!(
+ r#"<div class="damus-embedded-quote-reply">Replying to {}</div>"#,
+ html_escape::encode_text(&name)
+ )
+ })
+ .unwrap_or_default();
+
+ // For articles, we use a special card layout with image, title, summary, word count
+ let (content_preview, is_truncated, type_indicator, content_class, article_card) =
+ match quoted_note.kind() {
+ // For articles, extract metadata and build card layout
+ 30023 | 30024 => {
+ let mut title: Option<&str> = None;
+ let mut image: Option<&str> = None;
+ let mut summary: Option<&str> = None;
+
+ for tag in quoted_note.tags() {
+ let mut iter = tag.into_iter();
+ let Some(tag_name) = iter.next().and_then(|n| n.variant().str()) else {
+ continue;
+ };
+ let tag_value = iter.next().and_then(|n| n.variant().str());
+ match tag_name {
+ "title" => title = tag_value,
+ "image" => image = tag_value.filter(|s| !s.is_empty()),
+ "summary" => summary = tag_value.filter(|s| !s.is_empty()),
+ _ => {}
+ }
+ }
+
+ // Calculate word count
+ let word_count = quoted_note.content().split_whitespace().count();
+ let word_count_text = format!("{} Words", word_count);
+
+ // Build article card HTML
+ let title_text = title.unwrap_or("Untitled article");
+ let title_html = html_escape::encode_text(title_text);
+
+ let image_html = image
+ .map(|url| {
+ let url_attr = html_escape::encode_double_quoted_attribute(url);
+ format!(
+ r#"<img src="{}" class="damus-embedded-article-image" alt="" />"#,
+ url_attr
+ )
+ })
+ .unwrap_or_default();
+
+ let summary_html = summary
+ .map(|s| {
+ let text = html_escape::encode_text(abbreviate(s, 150));
+ format!(
+ r#"<div class="damus-embedded-article-summary">{}</div>"#,
+ text
+ )
+ })
+ .unwrap_or_default();
+
+ let draft_class = if quoted_note.kind() == 30024 {
+ " damus-embedded-article-draft"
+ } else {
+ ""
+ };
+
+ let card_html = format!(
+ r#"{image}<div class="damus-embedded-article-title{draft}">{title}</div>{summary}<div class="damus-embedded-article-wordcount">{words}</div>"#,
+ image = image_html,
+ draft = draft_class,
+ title = title_html,
+ summary = summary_html,
+ words = word_count_text
+ );
+
+ (
+ String::new(),
+ false,
+ "",
+ " damus-embedded-quote-article",
+ Some(card_html),
+ )
+ }
+ // For highlights, use left border styling (no tag needed)
+ 9802 => {
+ let full_content = quoted_note.content();
+ let content = abbreviate(full_content, 200);
+ let truncated = content.len() < full_content.len();
+ (
+ content.to_string(),
+ truncated,
+ "",
+ " damus-embedded-quote-highlight",
+ None,
+ )
+ }
+ _ => {
+ let full_content = quoted_note.content();
+ let content = abbreviate(full_content, 280);
+ let truncated = content.len() < full_content.len();
+ (content.to_string(), truncated, "", "", None)
+ }
+ };
+ let content_html = html_escape::encode_text(&content_preview).replace("\n", " ");
+
+ // Build link to quoted note
+ let link = build_quote_link(quote_ref);
+
+ // For articles, use card layout; for other types, use regular content layout
+ let body_html = if let Some(card) = article_card {
+ card
+ } else {
+ let show_more = if is_truncated {
+ r#" <span class="damus-embedded-quote-showmore">Show more</span>"#
+ } else {
+ ""
+ };
+ format!(
+ r#"<div class="damus-embedded-quote-content{class}">{content}{showmore}</div>"#,
+ class = content_class,
+ content = content_html,
+ showmore = show_more
+ )
+ };
+
+ let _ = write!(
+ quotes_html,
+ r#"<a href="{link}" class="damus-embedded-quote{content_class}">
+ <div class="damus-embedded-quote-header">
+ {pfp}
+ <span class="damus-embedded-quote-author">{name}</span>{username}
+ <span class="damus-embedded-quote-time">· {time}</span>
+ {type_indicator}
+ </div>
+ {reply}
+ {body}
+ </a>"#,
+ link = link,
+ content_class = content_class,
+ pfp = pfp_html,
+ name = display_name_html,
+ username = username_html,
+ time = time_html,
+ type_indicator = type_indicator,
+ reply = reply_html,
+ body = body_html
+ );
+ }
+
+ if quotes_html.is_empty() {
+ return String::new();
+ }
+
+ format!(
+ r#"<div class="damus-embedded-quotes">{}</div>"#,
+ quotes_html
+ )
}
struct Profile<'a> {
@@ -334,11 +1039,22 @@ impl<'a> Profile<'a> {
}
fn author_display_html(profile: Option<&ProfileRecord<'_>>) -> String {
- let profile_name_raw = profile
+ let profile_name_raw = get_profile_display_name(profile).unwrap_or("nostrich");
+ html_escape::encode_text(profile_name_raw).into_owned()
+}
+
+/// Returns the @username handle markup if available, empty string otherwise.
+/// Uses profile.name() (the NIP-01 "name" field) as the handle.
+fn author_handle_html(profile: Option<&ProfileRecord<'_>>) -> String {
+ profile
.and_then(|p| p.record().profile())
.and_then(|p| p.name())
- .unwrap_or("nostrich");
- html_escape::encode_text(profile_name_raw).into_owned()
+ .filter(|name| !name.is_empty())
+ .map(|name| {
+ let escaped = html_escape::encode_text(name);
+ format!(r#"<span class="damus-note-handle">@{}</span>"#, escaped)
+ })
+ .unwrap_or_default()
}
fn build_note_content_html(
@@ -350,16 +1066,18 @@ fn build_note_content_html(
relays: &[RelayUrl],
) -> String {
let mut body_buf = Vec::new();
- if let Some(blocks) = note
+ let blocks = note
.key()
- .and_then(|nk| app.ndb.get_blocks_by_key(txn, nk).ok())
- {
- render_note_content(&mut body_buf, note, &blocks);
+ .and_then(|nk| app.ndb.get_blocks_by_key(txn, nk).ok());
+
+ if let Some(ref blocks) = blocks {
+ render_note_content(&mut body_buf, note, blocks, &app.ndb, txn);
} else {
let _ = write!(body_buf, "{}", html_escape::encode_text(note.content()));
}
let author_display = author_display_html(profile.record.as_ref());
+ let author_handle = author_handle_html(profile.record.as_ref());
let npub = profile.key.to_bech32().unwrap();
let note_body = String::from_utf8(body_buf).unwrap_or_default();
let pfp_attr = pfp_url_attr(
@@ -367,12 +1085,31 @@ fn build_note_content_html(
base_url,
);
let timestamp_attr = note.created_at().to_string();
- let nevent = Nip19Event::new(
- EventId::from_byte_array(note.id().to_owned()),
- relays.iter().map(|r| r.to_string()),
- );
+ let nevent = Nip19Event::new(EventId::from_byte_array(note.id().to_owned()))
+ .relays(relays.iter().cloned());
let note_id = nevent.to_bech32().unwrap();
+ // Extract quote refs from q tags and inline mentions
+ let mut quote_refs = extract_quote_refs_from_tags(note);
+ if let Some(ref blocks) = blocks {
+ for content_ref in extract_quote_refs_from_content(note, blocks) {
+ // Deduplicate by event_id or article_addr
+ let is_dup = quote_refs
+ .iter()
+ .any(|existing| match (existing, &content_ref) {
+ (QuoteRef::Event { id: a, .. }, QuoteRef::Event { id: b, .. }) => a == b,
+ (QuoteRef::Article { addr: a, .. }, QuoteRef::Article { addr: b, .. }) => {
+ a == b
+ }
+ _ => false,
+ });
+ if !is_dup {
+ quote_refs.push(content_ref);
+ }
+ }
+ }
+ let quotes_html = build_embedded_quotes_html(&app.ndb, txn, "e_refs);
+
format!(
r#"<article class="damus-card damus-note">
<header class="damus-note-header">
@@ -382,6 +1119,7 @@ fn build_note_content_html(
<div>
<a href="{base}/{npub}">
<div class="damus-note-author">{author}</div>
+ {handle}
</a>
<a href="{base}/{note_id}">
<time class="damus-note-time" data-timestamp="{ts}" datetime="{ts}" title="{ts}">{ts}</time>
@@ -389,12 +1127,15 @@ fn build_note_content_html(
</div>
</header>
<div class="damus-note-body">{body}</div>
+ {quotes}
</article>"#,
base = base_url,
pfp = pfp_attr,
author = author_display,
+ handle = author_handle,
ts = timestamp_attr,
- body = note_body
+ body = note_body,
+ quotes = quotes_html
)
}
@@ -407,6 +1148,7 @@ fn build_article_content_html(
summary_html: Option<&str>,
article_body_html: &str,
topics: &[String],
+ is_draft: bool,
base_url: &str,
) -> String {
let pfp_attr = pfp_url_attr(
@@ -415,6 +1157,7 @@ fn build_article_content_html(
);
let timestamp_attr = timestamp_value.to_string();
let author_display = author_display_html(profile.record.as_ref());
+ let author_handle = author_handle_html(profile.record.as_ref());
let hero_markup = hero_image
.filter(|url| !url.is_empty())
@@ -448,16 +1191,24 @@ fn build_article_content_html(
topics_markup.push_str("</div>");
}
+ // Draft badge for unpublished articles (kind:30024)
+ let draft_markup = if is_draft {
+ r#"<span class="damus-article-draft">DRAFT</span>"#
+ } else {
+ ""
+ };
+
format!(
r#"<article class="damus-card damus-note">
<header class="damus-note-header">
<img src="{pfp}" class="damus-note-avatar" alt="{author} profile picture" />
<div>
<div class="damus-note-author">{author}</div>
+ {handle}
<time class="damus-note-time" data-timestamp="{ts}" datetime="{ts}" title="{ts}">{ts}</time>
</div>
</header>
- <h1 class="damus-article-title">{title}</h1>
+ <h1 class="damus-article-title">{title}{draft}</h1>
{hero}
{summary}
{topics}
@@ -465,8 +1216,10 @@ fn build_article_content_html(
</article>"#,
pfp = pfp_attr,
author = author_display,
+ handle = author_handle,
ts = timestamp_attr,
title = article_title_html,
+ draft = draft_markup,
hero = hero_markup,
summary = summary_markup,
topics = topics_markup,
@@ -474,6 +1227,169 @@ fn build_article_content_html(
)
}
+/// Builds HTML for a NIP-84 highlight (kind:9802).
+fn build_highlight_content_html(
+ profile: &Profile<'_>,
+ base_url: &str,
+ timestamp_value: u64,
+ highlight_text_html: &str,
+ context_html: Option<&str>,
+ comment_html: Option<&str>,
+ source_markup: &str,
+) -> String {
+ let author_display = author_display_html(profile.record.as_ref());
+ let author_handle = author_handle_html(profile.record.as_ref());
+ let pfp_attr = pfp_url_attr(
+ profile.record.as_ref().and_then(|r| r.record().profile()),
+ base_url,
+ );
+ let timestamp_attr = timestamp_value.to_string();
+
+ let context_markup = context_html
+ .filter(|ctx| !ctx.is_empty())
+ .map(|ctx| format!(r#"<div class="damus-highlight-context">…{ctx}…</div>"#))
+ .unwrap_or_default();
+
+ let comment_markup = comment_html
+ .filter(|c| !c.is_empty())
+ .map(|c| format!(r#"<div class="damus-highlight-comment">{c}</div>"#))
+ .unwrap_or_default();
+
+ format!(
+ r#"<article class="damus-card damus-highlight">
+ <header class="damus-note-header">
+ <img src="{pfp}" class="damus-note-avatar" alt="{author} profile picture" />
+ <div>
+ <div class="damus-note-author">{author}</div>
+ {handle}
+ <time class="damus-note-time" data-timestamp="{ts}" datetime="{ts}" title="{ts}">{ts}</time>
+ </div>
+ </header>
+ {comment}
+ <blockquote class="damus-highlight-text">{highlight}</blockquote>
+ {context}
+ {source}
+ </article>"#,
+ pfp = pfp_attr,
+ author = author_display,
+ handle = author_handle,
+ ts = timestamp_attr,
+ comment = comment_markup,
+ highlight = highlight_text_html,
+ context = context_markup,
+ source = source_markup
+ )
+}
+
+/// Builds source attribution markup for a highlight.
+fn build_highlight_source_markup(ndb: &Ndb, txn: &Transaction, meta: &HighlightMetadata) -> String {
+ // Priority: article > note > URL
+
+ // Case 1: Source is a nostr article (a tag)
+ if let Some(addr) = &meta.source_article_addr {
+ if let Some((note_key, title)) = lookup_article_by_addr(ndb, txn, addr) {
+ let author_name = ndb.get_note_by_key(txn, note_key).ok().and_then(|note| {
+ get_profile_display_name(
+ ndb.get_profile_by_pubkey(txn, note.pubkey()).ok().as_ref(),
+ )
+ .map(|s| s.to_owned())
+ });
+
+ return build_article_source_link(addr, title.as_deref(), author_name.as_deref());
+ }
+ }
+
+ // Case 2: Source is a nostr note (e tag)
+ if let Some(event_id) = &meta.source_event_id {
+ return build_note_source_link(event_id);
+ }
+
+ // Case 3: Source is a web URL (r tag)
+ if let Some(url) = &meta.source_url {
+ return build_url_source_link(url);
+ }
+
+ String::new()
+}
+
+/// Builds source link for an article reference.
+fn build_article_source_link(addr: &str, title: Option<&str>, author: Option<&str>) -> String {
+ use nostr_sdk::prelude::{Coordinate, Kind};
+
+ let parts: Vec<&str> = addr.splitn(3, ':').collect();
+ if parts.len() < 3 {
+ return String::new();
+ }
+
+ let Ok(kind) = parts[0].parse::<u16>() else {
+ return String::new();
+ };
+ let Ok(pubkey) = PublicKey::from_hex(parts[1]) else {
+ return String::new();
+ };
+
+ let coordinate = Coordinate::new(Kind::from(kind), pubkey).identifier(parts[2]);
+ let Ok(naddr) = coordinate.to_bech32() else {
+ return String::new();
+ };
+
+ let display_text = match (title, author) {
+ (Some(t), Some(a)) => format!(
+ "{} by {}",
+ html_escape::encode_text(t),
+ html_escape::encode_text(a)
+ ),
+ (Some(t), None) => html_escape::encode_text(t).into_owned(),
+ (None, Some(a)) => format!("Article by {}", html_escape::encode_text(a)),
+ (None, None) => abbrev_str(&naddr).to_string(),
+ };
+
+ let href_raw = format!("/{naddr}");
+ let href = html_escape::encode_double_quoted_attribute(&href_raw);
+ format!(
+ r#"<div class="damus-highlight-source"><span class="damus-highlight-source-label">From article:</span> <a href="{href}">{display}</a></div>"#,
+ href = href,
+ display = display_text
+ )
+}
+
+/// Builds source link for a note reference.
+fn build_note_source_link(event_id: &[u8; 32]) -> String {
+ use nostr_sdk::prelude::EventId;
+
+ let Ok(id) = EventId::from_slice(event_id) else {
+ return String::new();
+ };
+ let nevent = id.to_bech32().expect("infallible");
+
+ let href_raw = format!("/{nevent}");
+ let href = html_escape::encode_double_quoted_attribute(&href_raw);
+ format!(
+ r#"<div class="damus-highlight-source"><span class="damus-highlight-source-label">From note:</span> <a href="{href}">{abbrev}</a></div>"#,
+ href = href,
+ abbrev = abbrev_str(&nevent)
+ )
+}
+
+/// Builds source link for a web URL.
+fn build_url_source_link(url: &str) -> String {
+ let domain = url
+ .trim_start_matches("https://")
+ .trim_start_matches("http://")
+ .split('/')
+ .next()
+ .unwrap_or(url);
+
+ let href = html_escape::encode_double_quoted_attribute(url);
+ let domain_html = html_escape::encode_text(domain);
+
+ format!(
+ r#"<div class="damus-highlight-source"><span class="damus-highlight-source-label">From:</span> <a href="{href}" target="_blank" rel="noopener noreferrer">{domain}</a></div>"#,
+ href = href,
+ domain = domain_html
+ )
+}
+
const LOCAL_TIME_SCRIPT: &str = r#"
<script>
(function() {
@@ -639,7 +1555,7 @@ pub fn serve_profile_html(
app: &Notecrumbs,
nip: &Nip19,
profile_rd: Option<&ProfileRenderData>,
- r: Request<hyper::body::Incoming>,
+ _r: Request<hyper::body::Incoming>,
) -> Result<Response<Full<Bytes>>, Error> {
let profile_key = match profile_rd {
None | Some(ProfileRenderData::Missing(_)) => {
@@ -991,7 +1907,7 @@ pub fn serve_profile_html(
.body(Full::new(Bytes::from(data)))?)
}
-pub fn serve_homepage(r: Request<hyper::body::Incoming>) -> Result<Response<Full<Bytes>>, Error> {
+pub fn serve_homepage(_r: Request<hyper::body::Incoming>) -> Result<Response<Full<Bytes>>, Error> {
let base_url = get_base_url();
let page_title = "Damus — notecrumbs frontend";
@@ -1107,7 +2023,7 @@ pub fn serve_note_html(
app: &Notecrumbs,
nip19: &Nip19,
note_rd: &NoteAndProfileRenderData,
- r: Request<hyper::body::Incoming>,
+ _r: Request<hyper::body::Incoming>,
) -> Result<Response<Full<Bytes>>, Error> {
let mut data = Vec::new();
@@ -1139,7 +2055,22 @@ pub fn serve_note_html(
profile_record,
);
- let note_bech32 = nip19.to_bech32().unwrap();
+ // Generate bech32 with source relay hints for better discoverability.
+ // This applies to all event types (notes, articles, highlights).
+ // Falls back to original nip19 encoding if relay-enhanced encoding fails.
+ let note_bech32 = match crate::nip19::bech32_with_relays(nip19, ¬e_rd.source_relays) {
+ Some(bech32) => bech32,
+ None => {
+ warn!(
+ "failed to encode bech32 with relays for nip19: {:?}, falling back to original",
+ nip19
+ );
+ metrics::counter!("bech32_encode_fallback_total", 1);
+ nip19
+ .to_bech32()
+ .map_err(|e| Error::Generic(format!("failed to encode nip19: {}", e)))?
+ }
+ };
let base_url = get_base_url();
let canonical_url = format!("{}/{}", base_url, note_bech32);
let fallback_image_url = format!("{}/{}.png", base_url, note_bech32);
@@ -1204,17 +2135,52 @@ pub fn serve_note_html(
summary_display_html.as_deref(),
&article_body_html,
&topics,
+ note.kind() == 30024, // is_draft
&base_url,
)
- } else {
- build_note_content_html(
- app,
- ¬e,
- &txn,
- &base_url,
+ } else if note.kind() == 9802 {
+ // NIP-84: Highlights
+ let highlight_meta = extract_highlight_metadata(¬e);
+
+ display_title_raw = format!("Highlight by {}", profile_name_raw);
+ og_description_raw = collapse_whitespace(abbreviate(note.content(), 200));
+
+ let highlight_text_html = html_escape::encode_text(note.content()).replace("\n", "<br/>");
+
+ // Only show context if it meaningfully differs from the highlight text.
+ // Some clients add/remove trailing punctuation, so we normalize before comparing.
+ let content_normalized = normalize_for_comparison(note.content());
+ let context_html = highlight_meta
+ .context
+ .as_deref()
+ .filter(|ctx| normalize_for_comparison(ctx) != content_normalized)
+ .map(|ctx| html_escape::encode_text(ctx).into_owned());
+
+ let comment_html = highlight_meta
+ .comment
+ .as_deref()
+ .map(|c| html_escape::encode_text(c).into_owned());
+
+ let source_markup = build_highlight_source_markup(&app.ndb, &txn, &highlight_meta);
+
+ build_highlight_content_html(
&profile,
- &crate::nip19::nip19_relays(nip19),
+ &base_url,
+ timestamp_value,
+ &highlight_text_html,
+ context_html.as_deref(),
+ comment_html.as_deref(),
+ &source_markup,
)
+ } else {
+ // Regular notes (kind 1, etc.)
+ // Use source relays from fetch if available, otherwise fall back to nip19 relay hints
+ let relays = if note_rd.source_relays.is_empty() {
+ crate::nip19::nip19_relays(nip19)
+ } else {
+ note_rd.source_relays.clone()
+ };
+ build_note_content_html(app, ¬e, &txn, &base_url, &profile, &relays)
};
if og_description_raw.is_empty() {
diff --git a/src/main.rs b/src/main.rs
@@ -33,6 +33,8 @@ mod nip19;
mod pfp;
mod relay_pool;
mod render;
+mod sitemap;
+mod unknowns;
use relay_pool::RelayPool;
@@ -139,6 +141,31 @@ async fn serve(
"/" => {
return html::serve_homepage(r);
}
+ "/robots.txt" => {
+ let body = sitemap::generate_robots_txt();
+ return Ok(Response::builder()
+ .status(StatusCode::OK)
+ .header(header::CONTENT_TYPE, "text/plain; charset=utf-8")
+ .header(header::CACHE_CONTROL, "public, max-age=86400")
+ .body(Full::new(Bytes::from(body)))?);
+ }
+ "/sitemap.xml" => {
+ match sitemap::generate_sitemap(&app.ndb) {
+ Ok(xml) => {
+ return Ok(Response::builder()
+ .status(StatusCode::OK)
+ .header(header::CONTENT_TYPE, "application/xml; charset=utf-8")
+ .header(header::CACHE_CONTROL, "public, max-age=3600")
+ .body(Full::new(Bytes::from(xml)))?);
+ }
+ Err(err) => {
+ error!("Failed to generate sitemap: {err}");
+ return Ok(Response::builder()
+ .status(StatusCode::INTERNAL_SERVER_ERROR)
+ .body(Full::new(Bytes::from("Failed to generate sitemap\n")))?);
+ }
+ }
+ }
_ => {}
}
@@ -187,6 +214,16 @@ async fn serve(
}
}
+ // Collect and fetch all unknowns from the note (author, mentions, quotes, replies)
+ if let RenderData::Note(note_rd) = &render_data {
+ if let Some(unknowns) = render::collect_note_unknowns(&app.ndb, ¬e_rd.note_rd) {
+ tracing::debug!("fetching {} unknowns", unknowns.ids_len());
+ if let Err(err) = render::fetch_unknowns(&app.relay_pool, &app.ndb, unknowns).await {
+ tracing::warn!("failed to fetch unknowns: {err}");
+ }
+ }
+ }
+
if let RenderData::Profile(profile_opt) = &render_data {
let maybe_pubkey = {
let txn = Transaction::new(&app.ndb)?;
@@ -412,7 +449,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
RelayPool::new(
keys.clone(),
&["wss://relay.damus.io", "wss://nostr.wine", "wss://nos.lol"],
- timeout,
)
.await?,
);
diff --git a/src/nip19.rs b/src/nip19.rs
@@ -1,17 +1,213 @@
-use nostr::nips::nip19::Nip19;
+use nostr::nips::nip19::{Nip19, Nip19Coordinate, Nip19Event};
use nostr_sdk::prelude::*;
/// Do we have relays for this request? If so we can use these when
/// looking for missing data
pub fn nip19_relays(nip19: &Nip19) -> Vec<RelayUrl> {
match nip19 {
- Nip19::Event(ev) => ev
- .relays
- .iter()
- .filter_map(|r| RelayUrl::parse(r).ok())
- .collect(),
+ Nip19::Event(ev) => ev.relays.clone(),
Nip19::Coordinate(coord) => coord.relays.clone(),
Nip19::Profile(p) => p.relays.clone(),
_ => vec![],
}
}
+
+/// Generate a bech32 string with source relay hints.
+/// If source_relays is empty, uses the original nip19 relays.
+/// Otherwise, replaces the relays with source_relays.
+/// Preserves author/kind fields when present.
+pub fn bech32_with_relays(nip19: &Nip19, source_relays: &[RelayUrl]) -> Option<String> {
+ // If no source relays, use original
+ if source_relays.is_empty() {
+ return nip19.to_bech32().ok();
+ }
+
+ match nip19 {
+ Nip19::Event(ev) => {
+ // Preserve author and kind from original nevent
+ let mut new_event = Nip19Event::new(ev.event_id).relays(source_relays.iter().cloned());
+ if let Some(author) = ev.author {
+ new_event = new_event.author(author);
+ }
+ if let Some(kind) = ev.kind {
+ new_event = new_event.kind(kind);
+ }
+ new_event
+ .to_bech32()
+ .ok()
+ .or_else(|| nip19.to_bech32().ok())
+ }
+ Nip19::Coordinate(coord) => {
+ let new_coord =
+ Nip19Coordinate::new(coord.coordinate.clone(), source_relays.iter().cloned());
+ new_coord
+ .to_bech32()
+ .ok()
+ .or_else(|| nip19.to_bech32().ok())
+ }
+ // For other types (note, pubkey), just use original - they don't support relays
+ _ => nip19.to_bech32().ok(),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use nostr::nips::nip01::Coordinate;
+ use nostr::prelude::Keys;
+
+ #[test]
+ fn bech32_with_relays_adds_relay_to_nevent() {
+ let event_id = EventId::from_slice(&[1u8; 32]).unwrap();
+ let nip19_event = Nip19Event::new(event_id);
+ let nip19 = Nip19::Event(nip19_event);
+
+ let source_relays = vec![RelayUrl::parse("wss://relay.damus.io").unwrap()];
+ let result = bech32_with_relays(&nip19, &source_relays).expect("should encode");
+
+ // Result should be longer than original (includes relay hint)
+ let original = nip19.to_bech32().unwrap();
+ assert!(
+ result.len() > original.len(),
+ "bech32 with relay should be longer"
+ );
+
+ // Decode and verify relay is included
+ let decoded = Nip19::from_bech32(&result).unwrap();
+ match decoded {
+ Nip19::Event(ev) => {
+ assert!(!ev.relays.is_empty(), "should have relay hints");
+ assert!(ev.relays[0].to_string().contains("relay.damus.io"));
+ }
+ _ => panic!("expected Nip19::Event"),
+ }
+ }
+
+ #[test]
+ fn bech32_with_relays_adds_relay_to_naddr() {
+ let keys = Keys::generate();
+ let coordinate =
+ Coordinate::new(Kind::LongFormTextNote, keys.public_key()).identifier("test-article");
+ let nip19_coord = Nip19Coordinate::new(coordinate.clone(), Vec::<RelayUrl>::new());
+ let nip19 = Nip19::Coordinate(nip19_coord);
+
+ let source_relays = vec![RelayUrl::parse("wss://nostr.wine").unwrap()];
+ let result = bech32_with_relays(&nip19, &source_relays).expect("should encode");
+
+ // Result should be longer than original (includes relay hint)
+ let original = nip19.to_bech32().unwrap();
+ assert!(
+ result.len() > original.len(),
+ "bech32 with relay should be longer"
+ );
+
+ // Decode and verify relay is included
+ let decoded = Nip19::from_bech32(&result).unwrap();
+ match decoded {
+ Nip19::Coordinate(coord) => {
+ assert!(!coord.relays.is_empty(), "should have relay hints");
+ assert!(coord.relays[0].to_string().contains("nostr.wine"));
+ }
+ _ => panic!("expected Nip19::Coordinate"),
+ }
+ }
+
+ #[test]
+ fn bech32_with_relays_empty_returns_original() {
+ let event_id = EventId::from_slice(&[2u8; 32]).unwrap();
+ let relay = RelayUrl::parse("wss://original.relay").unwrap();
+ let nip19_event = Nip19Event::new(event_id).relays([relay.clone()]);
+ let nip19 = Nip19::Event(nip19_event);
+
+ // Empty source_relays should preserve original
+ let result = bech32_with_relays(&nip19, &[]).expect("should encode");
+ let original = nip19.to_bech32().unwrap();
+
+ assert_eq!(
+ result, original,
+ "empty source_relays should return original bech32"
+ );
+ }
+
+ #[test]
+ fn bech32_with_relays_replaces_existing_relays() {
+ let event_id = EventId::from_slice(&[3u8; 32]).unwrap();
+ let original_relay = RelayUrl::parse("wss://original.relay").unwrap();
+ let nip19_event = Nip19Event::new(event_id).relays([original_relay]);
+ let nip19 = Nip19::Event(nip19_event);
+
+ let new_relay = RelayUrl::parse("wss://new.relay").unwrap();
+ let result = bech32_with_relays(&nip19, &[new_relay.clone()]).expect("should encode");
+
+ // Decode and verify new relay replaced original
+ let decoded = Nip19::from_bech32(&result).unwrap();
+ match decoded {
+ Nip19::Event(ev) => {
+ assert_eq!(ev.relays.len(), 1, "should have exactly one relay");
+ assert!(ev.relays[0].to_string().contains("new.relay"));
+ }
+ _ => panic!("expected Nip19::Event"),
+ }
+ }
+
+ #[test]
+ fn bech32_with_relays_preserves_author_and_kind() {
+ let event_id = EventId::from_slice(&[5u8; 32]).unwrap();
+ let keys = Keys::generate();
+ let nip19_event = Nip19Event::new(event_id)
+ .author(keys.public_key())
+ .kind(Kind::TextNote);
+ let nip19 = Nip19::Event(nip19_event);
+
+ let source_relays = vec![RelayUrl::parse("wss://test.relay").unwrap()];
+ let result = bech32_with_relays(&nip19, &source_relays).expect("should encode");
+
+ // Decode and verify author/kind are preserved
+ let decoded = Nip19::from_bech32(&result).unwrap();
+ match decoded {
+ Nip19::Event(ev) => {
+ assert!(ev.author.is_some(), "author should be preserved");
+ assert_eq!(ev.author.unwrap(), keys.public_key());
+ assert!(ev.kind.is_some(), "kind should be preserved");
+ assert_eq!(ev.kind.unwrap(), Kind::TextNote);
+ assert!(!ev.relays.is_empty(), "should have relay");
+ }
+ _ => panic!("expected Nip19::Event"),
+ }
+ }
+
+ #[test]
+ fn nip19_relays_extracts_from_event() {
+ let event_id = EventId::from_slice(&[4u8; 32]).unwrap();
+ let relay = RelayUrl::parse("wss://test.relay").unwrap();
+ let nip19_event = Nip19Event::new(event_id).relays([relay.clone()]);
+ let nip19 = Nip19::Event(nip19_event);
+
+ let relays = nip19_relays(&nip19);
+ assert_eq!(relays.len(), 1);
+ assert!(relays[0].to_string().contains("test.relay"));
+ }
+
+ #[test]
+ fn nip19_relays_extracts_from_coordinate() {
+ let keys = Keys::generate();
+ let coordinate =
+ Coordinate::new(Kind::LongFormTextNote, keys.public_key()).identifier("article");
+ let relay = RelayUrl::parse("wss://coord.relay").unwrap();
+ let nip19_coord = Nip19Coordinate::new(coordinate, [relay.clone()]);
+ let nip19 = Nip19::Coordinate(nip19_coord);
+
+ let relays = nip19_relays(&nip19);
+ assert_eq!(relays.len(), 1);
+ assert!(relays[0].to_string().contains("coord.relay"));
+ }
+
+ #[test]
+ fn nip19_relays_returns_empty_for_pubkey() {
+ let keys = Keys::generate();
+ let nip19 = Nip19::Pubkey(keys.public_key());
+
+ let relays = nip19_relays(&nip19);
+ assert!(relays.is_empty(), "pubkey nip19 should have no relays");
+ }
+}
diff --git a/src/relay_pool.rs b/src/relay_pool.rs
@@ -1,6 +1,6 @@
use crate::Error;
use nostr::prelude::RelayUrl;
-use nostr_sdk::prelude::{Client, Event, Filter, Keys, ReceiverStream};
+use nostr_sdk::prelude::{BoxedStream, Client, Filter, Keys, RelayEvent};
use std::collections::HashSet;
use std::sync::Arc;
use tokio::sync::Mutex;
@@ -21,16 +21,11 @@ pub struct RelayPool {
client: Client,
known_relays: Arc<Mutex<HashSet<String>>>,
default_relays: Arc<[RelayUrl]>,
- connect_timeout: Duration,
stats: Arc<Mutex<RelayStats>>,
}
impl RelayPool {
- pub async fn new(
- keys: Keys,
- default_relays: &[&str],
- connect_timeout: Duration,
- ) -> Result<Self, Error> {
+ pub async fn new(keys: Keys, default_relays: &[&str]) -> Result<Self, Error> {
let client = Client::builder().signer(keys).build();
let parsed_defaults: Vec<RelayUrl> = default_relays
.iter()
@@ -48,7 +43,6 @@ impl RelayPool {
client,
known_relays: Arc::new(Mutex::new(HashSet::new())),
default_relays: default_relays.clone(),
- connect_timeout,
stats: Arc::new(Mutex::new(RelayStats::default())),
};
@@ -110,7 +104,7 @@ impl RelayPool {
}
if had_new {
- self.client.connect_with_timeout(self.connect_timeout).await;
+ self.client.connect().await;
let mut stats = self.stats.lock().await;
stats.ensure_calls += 1;
@@ -147,19 +141,23 @@ impl RelayPool {
Ok(())
}
+ /// Stream events from relays, returning RelayEvent which includes source relay URL.
+ /// Takes a single Filter - callers should combine filters before calling.
pub async fn stream_events(
&self,
- filters: Vec<Filter>,
+ filter: Filter,
relays: &[RelayUrl],
timeout: Duration,
- ) -> Result<ReceiverStream<Event>, Error> {
+ ) -> Result<BoxedStream<RelayEvent>, Error> {
if relays.is_empty() {
- Ok(self.client.stream_events(filters, Some(timeout)).await?)
+ Ok(self
+ .client
+ .stream_events_with_source(filter, timeout)
+ .await?)
} else {
- let urls: Vec<String> = relays.iter().map(|r| r.to_string()).collect();
Ok(self
.client
- .stream_events_from(urls, filters, Some(timeout))
+ .stream_events_from_with_source(relays.to_vec(), filter, timeout)
.await?)
}
}
diff --git a/src/render.rs b/src/render.rs
@@ -19,8 +19,8 @@ use nostr_sdk::nips::nip19::Nip19;
use nostr_sdk::prelude::{Event, EventId, PublicKey};
use nostr_sdk::JsonUtil;
use nostrdb::{
- Block, BlockType, Blocks, FilterElement, FilterField, Mention, Ndb, Note, NoteKey, ProfileKey,
- ProfileRecord, Transaction,
+ Block, BlockType, Blocks, FilterElement, FilterField, IngestMetadata, Mention, Ndb, Note,
+ NoteKey, ProfileKey, ProfileRecord, Transaction,
};
use std::collections::{BTreeMap, BTreeSet, HashSet};
use std::sync::Arc;
@@ -71,6 +71,9 @@ impl NoteRenderData {
pub struct NoteAndProfileRenderData {
pub note_rd: NoteRenderData,
pub profile_rd: Option<ProfileRenderData>,
+ /// Source relay URL(s) where the note was fetched from.
+ /// Used for generating bech32 links with relay hints.
+ pub source_relays: Vec<RelayUrl>,
}
impl NoteAndProfileRenderData {
@@ -78,6 +81,13 @@ impl NoteAndProfileRenderData {
Self {
note_rd,
profile_rd,
+ source_relays: Vec::new(),
+ }
+ }
+
+ pub fn add_source_relay(&mut self, relay: RelayUrl) {
+ if !self.source_relays.contains(&relay) {
+ self.source_relays.push(relay);
}
}
}
@@ -261,6 +271,9 @@ pub(crate) fn convert_filter(ndb_filter: &nostrdb::Filter) -> nostr::Filter {
FilterField::Limit(limit) => {
filter.limit = Some(limit as usize);
}
+
+ // Ignore new filter fields we don't handle
+ FilterField::Search(_) | FilterField::Relays(_) | FilterField::Custom(_) => {}
}
}
@@ -280,8 +293,7 @@ fn build_address_filter(author: &[u8; 32], kind: u64, identifier: &str) -> nostr
let author_ref: [&[u8; 32]; 1] = [author];
let mut filter = nostrdb::Filter::new().authors(author_ref).kinds([kind]);
if !identifier.is_empty() {
- let ident = identifier.to_string();
- filter = filter.tags(vec![ident], 'd');
+ filter = filter.tags([identifier], 'd');
}
filter.limit(1).build()
}
@@ -295,10 +307,11 @@ fn query_note_by_address<'a>(
) -> std::result::Result<Note<'a>, nostrdb::Error> {
let mut results = ndb.query(txn, &[build_address_filter(author, kind, identifier)], 1)?;
if results.is_empty() && !identifier.is_empty() {
+ let coord_tag = coordinate_tag(author, kind, identifier);
let coord_filter = nostrdb::Filter::new()
.authors([author])
.kinds([kind])
- .tags(vec![coordinate_tag(author, kind, identifier)], 'a')
+ .tags([coord_tag.as_str()], 'a')
.limit(1)
.build();
results = ndb.query(txn, &[coord_filter], 1)?;
@@ -310,12 +323,15 @@ fn query_note_by_address<'a>(
}
}
+/// Fetches notes from relays and returns the source relay URLs.
+/// The source relays are used to generate bech32 links with relay hints.
+/// Prioritizes default relays in the returned list for better reliability.
pub async fn find_note(
relay_pool: Arc<RelayPool>,
ndb: Ndb,
filters: Vec<nostr::Filter>,
nip19: &Nip19,
-) -> Result<()> {
+) -> Result<Vec<RelayUrl>> {
use nostr_sdk::JsonUtil;
let mut relay_targets = nip19::nip19_relays(nip19);
@@ -327,35 +343,116 @@ pub async fn find_note(
debug!("finding note(s) with filters: {:?}", filters);
- let expected_events = filters.len();
+ let mut all_source_relays = Vec::new();
+ let default_relays = relay_pool.default_relays();
+
+ for filter in filters {
+ let mut streamed_events = relay_pool
+ .stream_events(
+ filter,
+ &relay_targets,
+ std::time::Duration::from_millis(2000),
+ )
+ .await?;
+
+ // Collect all responding relays, then prioritize after stream exhausts.
+ while let Some(relay_event) = streamed_events.next().await {
+ if let Err(err) = ensure_relay_hints(&relay_pool, &relay_event.event).await {
+ warn!("failed to apply relay hints: {err}");
+ }
- let mut streamed_events = relay_pool
- .stream_events(
- filters,
- &relay_targets,
- std::time::Duration::from_millis(2000),
- )
- .await?;
+ debug!("processing event {:?}", relay_event.event);
+ let relay_url = relay_event.relay_url();
+ let ingest_meta = relay_url
+ .map(|url| IngestMetadata::new().relay(url.as_str()))
+ .unwrap_or_else(IngestMetadata::new);
+ if let Err(err) = ndb.process_event_with(&relay_event.event.as_json(), ingest_meta) {
+ error!("error processing event: {err}");
+ }
- let mut num_loops = 0;
- while let Some(event) = streamed_events.next().await {
- if let Err(err) = ensure_relay_hints(&relay_pool, &event).await {
- warn!("failed to apply relay hints: {err}");
- }
+ // Skip profile events - their relays shouldn't be used as note hints
+ if relay_event.event.kind == Kind::Metadata {
+ continue;
+ }
- debug!("processing event {:?}", event);
- if let Err(err) = ndb.process_event(&event.as_json()) {
- error!("error processing event: {err}");
- }
+ let Some(relay_url) = relay_url else {
+ continue;
+ };
- num_loops += 1;
+ if all_source_relays.contains(relay_url) {
+ continue;
+ }
- if num_loops == expected_events {
- break;
+ all_source_relays.push(relay_url.clone());
}
}
- Ok(())
+ // Sort relays: default relays first (more reliable), then others.
+ // Take up to 3 for the final result.
+ const MAX_SOURCE_RELAYS: usize = 3;
+ all_source_relays.sort_by(|a, b| {
+ let a_is_default = default_relays.contains(a);
+ let b_is_default = default_relays.contains(b);
+ b_is_default.cmp(&a_is_default) // true > false, so defaults come first
+ });
+ all_source_relays.truncate(MAX_SOURCE_RELAYS);
+
+ Ok(all_source_relays)
+}
+
+/// Fetch the latest profile metadata (kind 0) from relays and update nostrdb.
+///
+/// Profile metadata is a replaceable event (NIP-01) - nostrdb keeps only the
+/// newest version by `created_at` timestamp. This function queries relays for
+/// the latest kind 0 event to ensure cached profile data stays fresh during
+/// background refreshes.
+async fn fetch_profile_metadata(
+ relay_pool: Arc<RelayPool>,
+ ndb: Ndb,
+ relays: Vec<RelayUrl>,
+ pubkey: [u8; 32],
+) {
+ use nostr_sdk::JsonUtil;
+
+ if relays.is_empty() {
+ return;
+ }
+
+ let filter = {
+ let author_ref = [&pubkey];
+ convert_filter(
+ &nostrdb::Filter::new()
+ .authors(author_ref)
+ .kinds([0])
+ .limit(1)
+ .build(),
+ )
+ };
+
+ let stream = relay_pool
+ .stream_events(filter, &relays, Duration::from_millis(2000))
+ .await;
+
+ let mut stream = match stream {
+ Ok(s) => s,
+ Err(err) => {
+ warn!("failed to stream profile metadata: {err}");
+ return;
+ }
+ };
+
+ // Process all returned events - nostrdb handles deduplication and keeps newest.
+ // Note: we skip ensure_relay_hints here because kind 0 profile metadata doesn't
+ // contain relay hints (unlike kind 1 notes which may have 'r' tags).
+ while let Some(relay_event) = stream.next().await {
+ let ingest_meta = relay_event
+ .relay_url()
+ .map(|url| IngestMetadata::new().relay(url.as_str()))
+ .unwrap_or_else(IngestMetadata::new);
+ if let Err(err) = ndb.process_event_with(&relay_event.event.as_json(), ingest_meta) {
+ error!("error processing profile metadata event: {err}");
+ }
+ }
}
pub async fn fetch_profile_feed(
@@ -365,7 +462,13 @@ pub async fn fetch_profile_feed(
) -> Result<()> {
let relay_targets = collect_profile_relays(relay_pool.clone(), ndb.clone(), pubkey).await?;
- let relay_targets_arc = Arc::new(relay_targets);
+ // Spawn metadata fetch in parallel - best-effort, don't block note refresh
+ tokio::spawn(fetch_profile_metadata(
+ relay_pool.clone(),
+ ndb.clone(),
+ relay_targets.clone(),
+ pubkey,
+ ));
let cutoff = SystemTime::now()
.checked_sub(Duration::from_secs(
@@ -377,7 +480,7 @@ pub async fn fetch_profile_feed(
let mut fetched = stream_profile_feed_once(
relay_pool.clone(),
ndb.clone(),
- relay_targets_arc.clone(),
+ &relay_targets,
pubkey,
cutoff,
)
@@ -387,7 +490,7 @@ pub async fn fetch_profile_feed(
fetched = stream_profile_feed_once(
relay_pool.clone(),
ndb.clone(),
- relay_targets_arc.clone(),
+ &relay_targets,
pubkey,
None,
)
@@ -523,8 +626,17 @@ impl RenderData {
}
}
+ // Capture source relay URLs from the fetch task
match fetch_handle.await {
- Ok(Ok(())) => Ok(()),
+ Ok(Ok(source_relays)) => {
+ // Store source relays in the render data for bech32 link generation
+ if let RenderData::Note(ref mut note_data) = self {
+ for relay in source_relays {
+ note_data.add_source_relay(relay);
+ }
+ }
+ Ok(())
+ }
Ok(Err(err)) => Err(err),
Err(join_err) => Err(Error::Generic(format!(
"relay fetch task failed: {}",
@@ -534,6 +646,89 @@ impl RenderData {
}
}
+/// Collect all unknown IDs from a note - author, mentions, quotes, reply chain.
+pub fn collect_note_unknowns(
+ ndb: &Ndb,
+ note_rd: &NoteRenderData,
+) -> Option<crate::unknowns::UnknownIds> {
+ let txn = Transaction::new(ndb).ok()?;
+ let note = note_rd.lookup(&txn, ndb).ok()?;
+
+ let mut unknowns = crate::unknowns::UnknownIds::new();
+
+ // Collect from note content, author, reply chain, mentioned profiles/events
+ unknowns.collect_from_note(ndb, &txn, ¬e);
+
+ // Also collect from quote refs (q tags and inline nevent/naddr for embedded quotes)
+ let quote_refs = crate::html::collect_all_quote_refs(ndb, &txn, ¬e);
+ if !quote_refs.is_empty() {
+ debug!("found {} quote refs in note", quote_refs.len());
+ unknowns.collect_from_quote_refs(ndb, &txn, "e_refs);
+ }
+
+ debug!("collected {} total unknowns from note", unknowns.ids_len());
+
+ if unknowns.is_empty() {
+ None
+ } else {
+ Some(unknowns)
+ }
+}
+
+/// Fetch unknown IDs (quoted events, profiles) from relays using relay hints.
+pub async fn fetch_unknowns(
+ relay_pool: &Arc<RelayPool>,
+ ndb: &Ndb,
+ unknowns: crate::unknowns::UnknownIds,
+) -> Result<()> {
+ use nostr_sdk::JsonUtil;
+
+ // Collect relay hints before consuming unknowns
+ let relay_hints = unknowns.relay_hints();
+ let relay_targets: Vec<RelayUrl> = if relay_hints.is_empty() {
+ relay_pool.default_relays().to_vec()
+ } else {
+ relay_hints.into_iter().collect()
+ };
+
+ // Build and convert filters in one go (nostrdb::Filter is not Send)
+ let nostr_filters: Vec<nostr::Filter> = {
+ let filters = unknowns.to_filters();
+ if filters.is_empty() {
+ return Ok(());
+ }
+ filters.iter().map(convert_filter).collect()
+ };
+
+ // Now we can await - nostrdb::Filter has been dropped
+ relay_pool.ensure_relays(relay_targets.clone()).await?;
+
+ debug!(
+ "fetching {} unknowns from {:?}",
+ nostr_filters.len(),
+ relay_targets
+ );
+
+ // Stream with shorter timeout since these are secondary fetches
+ for filter in nostr_filters {
+ let mut stream = relay_pool
+ .stream_events(filter, &relay_targets, Duration::from_millis(1500))
+ .await?;
+
+ while let Some(relay_event) = stream.next().await {
+ let ingest_meta = relay_event
+ .relay_url()
+ .map(|url| IngestMetadata::new().relay(url.as_str()))
+ .unwrap_or_else(IngestMetadata::new);
+ if let Err(err) = ndb.process_event_with(&relay_event.event.as_json(), ingest_meta) {
+ warn!("error processing quoted event: {err}");
+ }
+ }
+ }
+
+ Ok(())
+}
+
fn collect_relay_hints(event: &Event) -> Vec<RelayUrl> {
let mut relays = Vec::new();
for tag in event.tags.iter() {
@@ -612,34 +807,38 @@ async fn collect_profile_relays(
.build(),
);
- let mut stream = relay_pool
- .stream_events(
- vec![relay_filter, contact_filter],
- &[],
- Duration::from_millis(2000),
- )
- .await?;
- while let Some(event) = stream.next().await {
- if let Err(err) = ndb.process_event(&event.as_json()) {
- error!("error processing relay discovery event: {err}");
- }
+ // Process each filter separately since stream_events now takes a single filter
+ for filter in [relay_filter, contact_filter] {
+ let mut stream = relay_pool
+ .stream_events(filter, &[], Duration::from_millis(2000))
+ .await?;
+
+ while let Some(relay_event) = stream.next().await {
+ let ingest_meta = relay_event
+ .relay_url()
+ .map(|url| IngestMetadata::new().relay(url.as_str()))
+ .unwrap_or_else(IngestMetadata::new);
+ if let Err(err) = ndb.process_event_with(&relay_event.event.as_json(), ingest_meta) {
+ error!("error processing relay discovery event: {err}");
+ }
- let hints = collect_relay_hints(&event);
- if hints.is_empty() {
- continue;
- }
+ let hints = collect_relay_hints(&relay_event.event);
+ if hints.is_empty() {
+ continue;
+ }
- let mut fresh = Vec::new();
- for hint in hints {
- let key = hint.to_string();
- if known.insert(key) {
- targets.push(hint.clone());
- fresh.push(hint);
+ let mut fresh = Vec::new();
+ for hint in hints {
+ let key = hint.to_string();
+ if known.insert(key) {
+ targets.push(hint.clone());
+ fresh.push(hint);
+ }
}
- }
- if !fresh.is_empty() {
- relay_pool.ensure_relays(fresh).await?;
+ if !fresh.is_empty() {
+ relay_pool.ensure_relays(fresh).await?;
+ }
}
}
@@ -649,7 +848,7 @@ async fn collect_profile_relays(
async fn stream_profile_feed_once(
relay_pool: Arc<RelayPool>,
ndb: Ndb,
- relays: Arc<Vec<RelayUrl>>,
+ relays: &[RelayUrl],
pubkey: [u8; 32],
since: Option<u64>,
) -> Result<usize> {
@@ -667,16 +866,20 @@ async fn stream_profile_feed_once(
convert_filter(&builder.build())
};
let mut stream = relay_pool
- .stream_events(vec![filter], &relays, Duration::from_millis(2000))
+ .stream_events(filter, &relays, Duration::from_millis(2000))
.await?;
let mut fetched = 0usize;
- while let Some(event) = stream.next().await {
- if let Err(err) = ensure_relay_hints(&relay_pool, &event).await {
+ while let Some(relay_event) = stream.next().await {
+ if let Err(err) = ensure_relay_hints(&relay_pool, &relay_event.event).await {
warn!("failed to apply relay hints: {err}");
}
- if let Err(err) = ndb.process_event(&event.as_json()) {
+ let ingest_meta = relay_event
+ .relay_url()
+ .map(|url| IngestMetadata::new().relay(url.as_str()))
+ .unwrap_or_else(IngestMetadata::new);
+ if let Err(err) = ndb.process_event_with(&relay_event.event.as_json(), ingest_meta) {
error!("error processing profile feed event: {err}");
} else {
fetched += 1;
@@ -696,7 +899,7 @@ pub fn get_render_data(ndb: &Ndb, txn: &Transaction, nip19: &Nip19) -> Result<Re
let pk = if let Some(pk) = m_note.as_ref().map(|note| note.pubkey()) {
Some(*pk)
} else {
- nevent.author.map(|a| a.serialize())
+ nevent.author.map(|a| a.to_bytes())
};
let profile_rd = pk.as_ref().map(|pubkey| {
@@ -739,7 +942,7 @@ pub fn get_render_data(ndb: &Ndb, txn: &Transaction, nip19: &Nip19) -> Result<Re
}
Nip19::Coordinate(coordinate) => {
- let author = coordinate.public_key.serialize();
+ let author = coordinate.public_key.to_bytes();
let kind: u64 = u16::from(coordinate.kind) as u64;
let identifier = coordinate.identifier.clone();
@@ -773,7 +976,7 @@ pub fn get_render_data(ndb: &Ndb, txn: &Transaction, nip19: &Nip19) -> Result<Re
}
Nip19::Profile(nprofile) => {
- let pubkey = nprofile.public_key.serialize();
+ let pubkey = nprofile.public_key.to_bytes();
let profile_rd = if let Ok(profile_key) = ndb.get_profilekey_by_pubkey(txn, &pubkey) {
ProfileRenderData::Profile(profile_key)
} else {
@@ -784,7 +987,7 @@ pub fn get_render_data(ndb: &Ndb, txn: &Transaction, nip19: &Nip19) -> Result<Re
}
Nip19::Pubkey(public_key) => {
- let pubkey = public_key.serialize();
+ let pubkey = public_key.to_bytes();
let profile_rd = if let Ok(profile_key) = ndb.get_profilekey_by_pubkey(txn, &pubkey) {
ProfileRenderData::Profile(profile_key)
} else {
@@ -1165,7 +1368,7 @@ mod tests {
let coordinate = Coordinate::new(Kind::LongFormTextNote, keys.public_key())
.identifier(identifier_with_a);
let event_with_a_only = EventBuilder::long_form_text_note("content with a tag only")
- .tags([Tag::coordinate(coordinate)])
+ .tags([Tag::coordinate(coordinate, None)])
.sign_with_keys(&keys)
.expect("sign long-form event with coordinate tag");
diff --git a/src/sitemap.rs b/src/sitemap.rs
@@ -0,0 +1,356 @@
+//! Sitemap generation for SEO
+//!
+//! Generates XML sitemaps from cached events in nostrdb to help search engines
+//! discover and index Nostr content rendered by notecrumbs.
+
+use nostr_sdk::ToBech32;
+use nostrdb::{Filter, Ndb, Transaction};
+use std::fmt::Write;
+use std::sync::OnceLock;
+use std::time::Instant;
+
+/// Maximum URLs per sitemap (XML sitemap standard limit is 50,000)
+const MAX_SITEMAP_URLS: u64 = 10000;
+
+/// Lookback period for notes (90 days) - shorter for timely content
+const NOTES_LOOKBACK_DAYS: u64 = 90;
+
+/// Lookback period for articles (365 days) - longer for evergreen content
+const ARTICLES_LOOKBACK_DAYS: u64 = 365;
+
+/// Cached base URL (computed once at first access)
+static BASE_URL: OnceLock<String> = OnceLock::new();
+
+/// Get the base URL from environment or default
+/// Logs a warning once if not explicitly configured
+fn get_base_url() -> &'static str {
+ BASE_URL.get_or_init(|| {
+ let url = match std::env::var("NOTECRUMBS_BASE_URL") {
+ Ok(url) => url,
+ Err(_) => {
+ tracing::warn!(
+ "NOTECRUMBS_BASE_URL not set, defaulting to https://damus.io - \
+ sitemap/robots.txt may point to wrong domain"
+ );
+ "https://damus.io".to_string()
+ }
+ };
+ normalize_base_url(&url)
+ })
+}
+
+fn normalize_base_url(url: &str) -> String {
+ url.trim_end_matches('/').to_string()
+}
+
+/// Calculate Unix timestamp for N days ago
+fn days_ago(days: u64) -> u64 {
+ std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .unwrap_or_default()
+ .as_secs()
+ .saturating_sub(days * 24 * 60 * 60)
+}
+
+/// Escape special XML characters in a string
+fn xml_escape(s: &str) -> String {
+ let mut result = String::with_capacity(s.len());
+ for c in s.chars() {
+ match c {
+ '&' => result.push_str("&"),
+ '<' => result.push_str("<"),
+ '>' => result.push_str(">"),
+ '"' => result.push_str("""),
+ '\'' => result.push_str("'"),
+ _ => result.push(c),
+ }
+ }
+ result
+}
+
+/// Format a Unix timestamp as an ISO 8601 date (YYYY-MM-DD)
+fn format_lastmod(timestamp: u64) -> String {
+ use std::time::{Duration, UNIX_EPOCH};
+
+ let datetime = UNIX_EPOCH + Duration::from_secs(timestamp);
+ let secs_since_epoch = datetime
+ .duration_since(UNIX_EPOCH)
+ .unwrap_or_default()
+ .as_secs();
+
+ // Simple date formatting without external dependencies
+ let days_since_epoch = secs_since_epoch / 86400;
+ let mut year = 1970i32;
+ let mut remaining_days = days_since_epoch as i32;
+
+ loop {
+ let days_in_year = if is_leap_year(year) { 366 } else { 365 };
+ if remaining_days < days_in_year {
+ break;
+ }
+ remaining_days -= days_in_year;
+ year += 1;
+ }
+
+ let is_leap = is_leap_year(year);
+ let days_in_months: [i32; 12] = if is_leap {
+ [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
+ } else {
+ [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
+ };
+
+ let mut month = 1u32;
+ for days in days_in_months {
+ if remaining_days < days {
+ break;
+ }
+ remaining_days -= days;
+ month += 1;
+ }
+
+ let day = remaining_days + 1;
+
+ format!("{:04}-{:02}-{:02}", year, month, day)
+}
+
+fn is_leap_year(year: i32) -> bool {
+ (year % 4 == 0 && year % 100 != 0) || (year % 400 == 0)
+}
+
+/// Entry in the sitemap
+struct SitemapEntry {
+ loc: String,
+ lastmod: String,
+ priority: &'static str,
+ changefreq: &'static str,
+}
+
+/// Generate sitemap XML from cached events in nostrdb
+pub fn generate_sitemap(ndb: &Ndb) -> Result<String, nostrdb::Error> {
+ let start = Instant::now();
+ let base_url = get_base_url();
+ let txn = Transaction::new(ndb)?;
+
+ let mut entries: Vec<SitemapEntry> = Vec::new();
+ let mut notes_count: u64 = 0;
+ let mut articles_count: u64 = 0;
+ let mut profiles_count: u64 = 0;
+
+ // Add homepage
+ entries.push(SitemapEntry {
+ loc: base_url.to_string(),
+ lastmod: format_lastmod(
+ std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .unwrap_or_default()
+ .as_secs(),
+ ),
+ priority: "1.0",
+ changefreq: "daily",
+ });
+
+ // Query recent notes (kind:1 - short text notes)
+ // Use since filter to prioritize recent content for SEO freshness
+ let notes_filter = Filter::new()
+ .kinds([1])
+ .since(days_ago(NOTES_LOOKBACK_DAYS))
+ .limit(MAX_SITEMAP_URLS)
+ .build();
+
+ let results = ndb.query(&txn, &[notes_filter], MAX_SITEMAP_URLS as i32).unwrap_or_default();
+ for result in results {
+ let Ok(note) = ndb.get_note_by_key(&txn, result.note_key) else {
+ continue;
+ };
+ let Some(eid) = nostr_sdk::EventId::from_slice(note.id()).ok() else {
+ continue;
+ };
+ // to_bech32() returns Result<String, Infallible>, so unwrap is safe
+ let bech32 = eid.to_bech32().unwrap();
+ entries.push(SitemapEntry {
+ loc: format!("{}/{}", base_url, xml_escape(&bech32)),
+ lastmod: format_lastmod(note.created_at()),
+ priority: "0.8",
+ changefreq: "weekly",
+ });
+ notes_count += 1;
+ }
+
+ // Query long-form articles (kind:30023)
+ // Longer lookback for evergreen content
+ let articles_filter = Filter::new()
+ .kinds([30023])
+ .since(days_ago(ARTICLES_LOOKBACK_DAYS))
+ .limit(MAX_SITEMAP_URLS)
+ .build();
+
+ let results = ndb.query(&txn, &[articles_filter], MAX_SITEMAP_URLS as i32).unwrap_or_default();
+ for result in results {
+ let Ok(note) = ndb.get_note_by_key(&txn, result.note_key) else {
+ continue;
+ };
+
+ // Extract d-tag identifier - skip if missing or empty to avoid
+ // ambiguous URLs and potential collisions across authors
+ let identifier = note
+ .tags()
+ .iter()
+ .find(|tag| tag.count() >= 2 && tag.get_unchecked(0).variant().str() == Some("d"))
+ .and_then(|tag| tag.get_unchecked(1).variant().str());
+
+ let Some(identifier) = identifier else {
+ continue;
+ };
+ if identifier.is_empty() {
+ continue;
+ }
+
+ let Some(pk) = nostr_sdk::PublicKey::from_slice(note.pubkey()).ok() else {
+ continue;
+ };
+
+ // For addressable events, create naddr
+ let kind = nostr::Kind::from(note.kind() as u16);
+ let coord = nostr::nips::nip01::Coordinate::new(kind, pk).identifier(identifier);
+ let Ok(bech32) = coord.to_bech32() else {
+ continue;
+ };
+
+ entries.push(SitemapEntry {
+ loc: format!("{}/{}", base_url, xml_escape(&bech32)),
+ lastmod: format_lastmod(note.created_at()),
+ priority: "0.9",
+ changefreq: "weekly",
+ });
+ articles_count += 1;
+ }
+
+ // Query profiles (kind:0 - metadata)
+ // No since filter for profiles - they update less frequently
+ let profiles_filter = Filter::new()
+ .kinds([0])
+ .limit(MAX_SITEMAP_URLS)
+ .build();
+
+ let results = ndb.query(&txn, &[profiles_filter], MAX_SITEMAP_URLS as i32).unwrap_or_default();
+ for result in results {
+ let Ok(note) = ndb.get_note_by_key(&txn, result.note_key) else {
+ continue;
+ };
+ let Some(pk) = nostr_sdk::PublicKey::from_slice(note.pubkey()).ok() else {
+ continue;
+ };
+ // to_bech32() returns Result<String, Infallible>, so unwrap is safe
+ let bech32 = pk.to_bech32().unwrap();
+ entries.push(SitemapEntry {
+ loc: format!("{}/{}", base_url, xml_escape(&bech32)),
+ lastmod: format_lastmod(note.created_at()),
+ priority: "0.7",
+ changefreq: "weekly",
+ });
+ profiles_count += 1;
+ }
+
+ // Build XML
+ let mut xml = String::with_capacity(entries.len() * 200);
+ xml.push_str("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+ xml.push_str("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n");
+
+ for entry in &entries {
+ let _ = write!(
+ xml,
+ " <url>\n <loc>{}</loc>\n <lastmod>{}</lastmod>\n <changefreq>{}</changefreq>\n <priority>{}</priority>\n </url>\n",
+ entry.loc, entry.lastmod, entry.changefreq, entry.priority
+ );
+ }
+
+ xml.push_str("</urlset>\n");
+
+ // Record metrics (aggregate stats, not user-tracking)
+ let duration = start.elapsed();
+ metrics::counter!("sitemap_generations_total", 1);
+ metrics::gauge!("sitemap_generation_duration_seconds", duration.as_secs_f64());
+ metrics::gauge!("sitemap_urls_total", entries.len() as f64);
+ metrics::gauge!("sitemap_notes_count", notes_count as f64);
+ metrics::gauge!("sitemap_articles_count", articles_count as f64);
+ metrics::gauge!("sitemap_profiles_count", profiles_count as f64);
+
+ Ok(xml)
+}
+
+/// Generate robots.txt content
+pub fn generate_robots_txt() -> String {
+ let base_url = get_base_url();
+ format!(
+ "User-agent: *\n\
+ Allow: /\n\
+ Allow: /.well-known/nostr.json\n\
+ Disallow: /metrics\n\
+ Disallow: /*.json\n\
+ \n\
+ Sitemap: {}/sitemap.xml\n",
+ base_url
+ )
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_xml_escape() {
+ assert_eq!(xml_escape("hello"), "hello");
+ assert_eq!(xml_escape("a&b"), "a&b");
+ assert_eq!(xml_escape("<tag>"), "<tag>");
+ assert_eq!(xml_escape("\"quoted\""), ""quoted"");
+ }
+
+ #[test]
+ fn test_format_lastmod() {
+ // 2024-01-01 00:00:00 UTC = 1704067200
+ assert_eq!(format_lastmod(1704067200), "2024-01-01");
+ // 2023-06-15 12:00:00 UTC = 1686830400
+ assert_eq!(format_lastmod(1686830400), "2023-06-15");
+ }
+
+ #[test]
+ fn test_is_leap_year() {
+ assert!(is_leap_year(2000));
+ assert!(is_leap_year(2024));
+ assert!(!is_leap_year(1900));
+ assert!(!is_leap_year(2023));
+ }
+
+ #[test]
+ fn test_normalize_base_url() {
+ assert_eq!(normalize_base_url("https://example.com/"), "https://example.com");
+ assert_eq!(normalize_base_url("https://example.com"), "https://example.com");
+ }
+
+ #[test]
+ fn test_days_ago_range() {
+ let start = std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .unwrap_or_default()
+ .as_secs();
+ let cutoff = days_ago(1);
+ let end = std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .unwrap_or_default()
+ .as_secs();
+
+ let start_cutoff = start.saturating_sub(86400);
+ let end_cutoff = end.saturating_sub(86400);
+ assert!(cutoff >= start_cutoff);
+ assert!(cutoff <= end_cutoff);
+ }
+
+ #[test]
+ fn test_robots_txt_format() {
+ let robots = generate_robots_txt();
+ assert!(robots.contains("User-agent: *"));
+ assert!(robots.contains("Allow: /"));
+ assert!(robots.contains("Disallow: /metrics"));
+ assert!(robots.contains("Sitemap:"));
+ }
+}
diff --git a/src/unknowns.rs b/src/unknowns.rs
@@ -0,0 +1,274 @@
+//! Unknown ID collection with relay provenance for fetching missing data.
+//!
+//! Adapted from notedeck's unknowns pattern for notecrumbs' one-shot HTTP context.
+//! Collects unknown note IDs and profile pubkeys from:
+//! - Quote references (q tags, inline nevent/note/naddr)
+//! - Mentioned profiles (npub/nprofile in content)
+//! - Reply chain (e tags with reply/root markers)
+//! - Author profile
+
+use crate::html::QuoteRef;
+use nostr::RelayUrl;
+use nostrdb::{BlockType, Mention, Ndb, Note, Transaction};
+use std::collections::{HashMap, HashSet};
+
+/// An unknown ID that needs to be fetched from relays.
+#[derive(Hash, Eq, PartialEq, Clone, Debug)]
+pub enum UnknownId {
+ /// A note ID (event)
+ NoteId([u8; 32]),
+ /// A profile pubkey
+ Profile([u8; 32]),
+}
+
+/// Collection of unknown IDs with their associated relay hints.
+#[derive(Default, Debug)]
+pub struct UnknownIds {
+ ids: HashMap<UnknownId, HashSet<RelayUrl>>,
+}
+
+impl UnknownIds {
+ pub fn new() -> Self {
+ Self::default()
+ }
+
+ pub fn is_empty(&self) -> bool {
+ self.ids.is_empty()
+ }
+
+ pub fn ids_len(&self) -> usize {
+ self.ids.len()
+ }
+
+ /// Add a note ID if it's not already in ndb.
+ pub fn add_note_if_missing(
+ &mut self,
+ ndb: &Ndb,
+ txn: &Transaction,
+ id: &[u8; 32],
+ relays: impl IntoIterator<Item = RelayUrl>,
+ ) {
+ // Check if we already have this note
+ if ndb.get_note_by_id(txn, id).is_ok() {
+ return;
+ }
+
+ let unknown_id = UnknownId::NoteId(*id);
+ self.ids.entry(unknown_id).or_default().extend(relays);
+ }
+
+ /// Add a profile pubkey if it's not already in ndb.
+ pub fn add_profile_if_missing(&mut self, ndb: &Ndb, txn: &Transaction, pk: &[u8; 32]) {
+ // Check if we already have this profile
+ if ndb.get_profile_by_pubkey(txn, pk).is_ok() {
+ return;
+ }
+
+ let unknown_id = UnknownId::Profile(*pk);
+ self.ids.entry(unknown_id).or_default();
+ }
+
+ /// Collect all relay hints from unknowns.
+ pub fn relay_hints(&self) -> HashSet<RelayUrl> {
+ self.ids
+ .values()
+ .flat_map(|relays| relays.iter().cloned())
+ .collect()
+ }
+
+ /// Build nostrdb filters for fetching unknown IDs.
+ pub fn to_filters(&self) -> Vec<nostrdb::Filter> {
+ if self.ids.is_empty() {
+ return vec![];
+ }
+
+ let mut filters = Vec::new();
+
+ // Collect note IDs
+ let note_ids: Vec<&[u8; 32]> = self
+ .ids
+ .keys()
+ .filter_map(|id| match id {
+ UnknownId::NoteId(id) => Some(id),
+ _ => None,
+ })
+ .collect();
+
+ if !note_ids.is_empty() {
+ filters.push(nostrdb::Filter::new().ids(note_ids).build());
+ }
+
+ // Collect profile pubkeys
+ let pubkeys: Vec<&[u8; 32]> = self
+ .ids
+ .keys()
+ .filter_map(|id| match id {
+ UnknownId::Profile(pk) => Some(pk),
+ _ => None,
+ })
+ .collect();
+
+ if !pubkeys.is_empty() {
+ filters.push(nostrdb::Filter::new().authors(pubkeys).kinds([0]).build());
+ }
+
+ filters
+ }
+
+ /// Collect unknown IDs from quote refs.
+ pub fn collect_from_quote_refs(
+ &mut self,
+ ndb: &Ndb,
+ txn: &Transaction,
+ quote_refs: &[QuoteRef],
+ ) {
+ for quote_ref in quote_refs {
+ match quote_ref {
+ QuoteRef::Event { id, relays, .. } => {
+ self.add_note_if_missing(ndb, txn, id, relays.iter().cloned());
+ }
+ QuoteRef::Article { addr, relays, .. } => {
+ // For articles, we need to parse the address to get the author pubkey
+ // and check if we have the article. For now, just try to look it up.
+ let parts: Vec<&str> = addr.splitn(3, ':').collect();
+ if parts.len() >= 2 {
+ if let Ok(pk_bytes) = hex::decode(parts[1]) {
+ if let Ok(pk) = pk_bytes.try_into() {
+ // Add author profile if missing
+ self.add_profile_if_missing(ndb, txn, &pk);
+ }
+ }
+ }
+ // Note: For articles we'd ideally build an address filter,
+ // but for now we rely on the profile fetch to help
+ let _ = relays; // TODO: use for article fetching
+ }
+ }
+ }
+ }
+
+ /// Collect all unknown IDs from a note - author, mentioned profiles/events, reply chain.
+ ///
+ /// This is the comprehensive collection function adapted from notedeck's pattern.
+ pub fn collect_from_note(&mut self, ndb: &Ndb, txn: &Transaction, note: &Note) {
+ // 1. Author profile
+ self.add_profile_if_missing(ndb, txn, note.pubkey());
+
+ // 2. Reply chain - check e tags for root/reply markers
+ self.collect_reply_chain(ndb, txn, note);
+
+ // 3. Mentioned profiles and events from content blocks
+ self.collect_from_blocks(ndb, txn, note);
+ }
+
+ /// Collect reply chain unknowns using nostrdb's NoteReply (NIP-10 compliant).
+ fn collect_reply_chain(&mut self, ndb: &Ndb, txn: &Transaction, note: &Note) {
+ use nostrdb::NoteReply;
+
+ let reply = NoteReply::new(note.tags());
+
+ // Add root note if missing
+ if let Some(root_ref) = reply.root() {
+ let relay_hint: Vec<RelayUrl> = root_ref
+ .relay
+ .and_then(|s| RelayUrl::parse(s).ok())
+ .into_iter()
+ .collect();
+ self.add_note_if_missing(ndb, txn, root_ref.id, relay_hint);
+ }
+
+ // Add reply note if missing (and different from root)
+ if let Some(reply_ref) = reply.reply() {
+ let relay_hint: Vec<RelayUrl> = reply_ref
+ .relay
+ .and_then(|s| RelayUrl::parse(s).ok())
+ .into_iter()
+ .collect();
+ self.add_note_if_missing(ndb, txn, reply_ref.id, relay_hint);
+ }
+ }
+
+ /// Collect unknowns from content blocks (mentions).
+ fn collect_from_blocks(&mut self, ndb: &Ndb, txn: &Transaction, note: &Note) {
+ let Some(note_key) = note.key() else {
+ return;
+ };
+
+ let Ok(blocks) = ndb.get_blocks_by_key(txn, note_key) else {
+ return;
+ };
+
+ for block in blocks.iter(note) {
+ if block.blocktype() != BlockType::MentionBech32 {
+ continue;
+ }
+
+ let Some(mention) = block.as_mention() else {
+ continue;
+ };
+
+ match mention {
+ // npub - simple pubkey mention
+ Mention::Pubkey(npub) => {
+ self.add_profile_if_missing(ndb, txn, npub.pubkey());
+ }
+ // nprofile - pubkey with relay hints
+ Mention::Profile(nprofile) => {
+ if ndb.get_profile_by_pubkey(txn, nprofile.pubkey()).is_err() {
+ let relays: HashSet<RelayUrl> = nprofile
+ .relays_iter()
+ .filter_map(|s| RelayUrl::parse(s).ok())
+ .collect();
+ let unknown_id = UnknownId::Profile(*nprofile.pubkey());
+ self.ids.entry(unknown_id).or_default().extend(relays);
+ }
+ }
+ // nevent - event with relay hints
+ Mention::Event(ev) => {
+ let relays: HashSet<RelayUrl> = ev
+ .relays_iter()
+ .filter_map(|s| RelayUrl::parse(s).ok())
+ .collect();
+
+ match ndb.get_note_by_id(txn, ev.id()) {
+ Err(_) => {
+ // Event not found - add it and its author if specified
+ self.add_note_if_missing(ndb, txn, ev.id(), relays.clone());
+ if let Some(pk) = ev.pubkey() {
+ if ndb.get_profile_by_pubkey(txn, pk).is_err() {
+ let unknown_id = UnknownId::Profile(*pk);
+ self.ids.entry(unknown_id).or_default().extend(relays);
+ }
+ }
+ }
+ Ok(found_note) => {
+ // Event found but maybe we need the author profile
+ if ndb.get_profile_by_pubkey(txn, found_note.pubkey()).is_err() {
+ let unknown_id = UnknownId::Profile(*found_note.pubkey());
+ self.ids.entry(unknown_id).or_default().extend(relays);
+ }
+ }
+ }
+ }
+ // note1 - simple note mention
+ Mention::Note(note_mention) => {
+ match ndb.get_note_by_id(txn, note_mention.id()) {
+ Err(_) => {
+ self.add_note_if_missing(
+ ndb,
+ txn,
+ note_mention.id(),
+ std::iter::empty(),
+ );
+ }
+ Ok(found_note) => {
+ // Note found but maybe we need the author profile
+ self.add_profile_if_missing(ndb, txn, found_note.pubkey());
+ }
+ }
+ }
+ _ => {}
+ }
+ }
+ }
+}
