metadata.c (12547B)
1 2 #include "nostrdb.h" 3 #include "binmoji.h" 4 #include "metadata.h" 5 6 int ndb_reaction_str_is_emoji(union ndb_reaction_str str) 7 { 8 return binmoji_get_user_flag(str.binmoji) == 0; 9 } 10 11 uint16_t ndb_note_meta_entries_count(struct ndb_note_meta *meta) 12 { 13 return meta->count; 14 } 15 16 static int ndb_reaction_set_emoji(union ndb_reaction_str *str, const char *emoji) 17 { 18 struct binmoji binmoji; 19 /* TODO: parse failures? */ 20 binmoji_parse(emoji, &binmoji); 21 str->binmoji = binmoji_encode(&binmoji); 22 return 1; 23 } 24 25 static int ndb_reaction_set_str(union ndb_reaction_str *reaction, const char *str) 26 { 27 int i; 28 char c; 29 30 /* this is like memset'ing the packed string to all 0s as well */ 31 reaction->binmoji = 0; 32 33 /* set the binmoji user flag so we can catch corrupt binmojis */ 34 /* this is in the LSB so it will only touch reaction->packed.flag */ 35 reaction->binmoji = binmoji_set_user_flag(reaction->binmoji, 1); 36 assert(reaction->packed.flag != 0); 37 38 for (i = 0; i < 7; i++) { 39 c = str[i]; 40 /* string is too big */ 41 if (i == 6 && c != '\0') 42 return 0; 43 reaction->packed.str[i] = c; 44 if (c == '\0') 45 return 1; 46 } 47 48 return 0; 49 } 50 51 const char *ndb_reaction_to_str(union ndb_reaction_str *str, char buf[128]) 52 { 53 struct binmoji binmoji; 54 55 if (ndb_reaction_str_is_emoji(*str)) { 56 binmoji_decode(str->binmoji, &binmoji); 57 binmoji_to_string(&binmoji, buf, 128); 58 return (const char *)buf; 59 } else { 60 return (const char *)str->packed.str; 61 } 62 } 63 64 /* set the value of an ndb_reaction_str to an emoji or small string */ 65 int ndb_reaction_set(union ndb_reaction_str *reaction, const char *str) 66 { 67 struct binmoji binmoji; 68 char output_emoji[136]; 69 70 /* our variant of emoji detection is to simply try to create 71 * a binmoji and parse it again. if we round-trip successfully 72 * then we know its an emoji, or at least a simple string 73 */ 74 binmoji_parse(str, &binmoji); 75 reaction->binmoji = binmoji_encode(&binmoji); 76 binmoji_to_string(&binmoji, output_emoji, sizeof(output_emoji)); 77 78 /* round trip is successful, let's just use binmojis for this encoding */ 79 if (!strcmp(output_emoji, str)) 80 return 1; 81 82 /* no round trip? let's just set a non-emoji string */ 83 return ndb_reaction_set_str(reaction, str); 84 } 85 86 void ndb_note_meta_header_init(struct ndb_note_meta *meta) 87 { 88 meta->version = 1; 89 meta->flags = 0; 90 meta->count = 0; 91 meta->data_table_size = 0; 92 } 93 94 static inline size_t ndb_note_meta_entries_size(struct ndb_note_meta *meta) 95 { 96 return (sizeof(struct ndb_note_meta_entry) * meta->count); 97 } 98 99 void *ndb_note_meta_data_table(struct ndb_note_meta *meta, size_t *size) 100 { 101 return meta + ndb_note_meta_entries_size(meta); 102 } 103 104 size_t ndb_note_meta_total_size(struct ndb_note_meta *header) 105 { 106 size_t total_size = sizeof(*header) + header->data_table_size + ndb_note_meta_entries_size(header); 107 assert((total_size % 8) == 0); 108 return total_size; 109 } 110 111 struct ndb_note_meta_entry *ndb_note_meta_add_entry(struct ndb_note_meta_builder *builder) 112 { 113 struct ndb_note_meta *header = (struct ndb_note_meta *)builder->cursor.start; 114 struct ndb_note_meta_entry *entry = NULL; 115 116 assert(builder->cursor.p != builder->cursor.start); 117 118 if (!(entry = cursor_malloc(&builder->cursor, sizeof(*entry)))) 119 return NULL; 120 121 /* increase count entry count */ 122 header->count++; 123 124 return entry; 125 } 126 127 void ndb_note_meta_builder_resized(struct ndb_note_meta_builder *builder, unsigned char *buf, size_t bufsize) 128 { 129 make_cursor(buf, buf + bufsize, &builder->cursor); 130 } 131 132 int ndb_note_meta_builder_init(struct ndb_note_meta_builder *builder, unsigned char *buf, size_t bufsize) 133 { 134 ndb_note_meta_builder_resized(builder, buf, bufsize); 135 136 /* allocate some space for the header */ 137 if (!cursor_malloc(&builder->cursor, sizeof(struct ndb_note_meta))) 138 return 0; 139 140 ndb_note_meta_header_init((struct ndb_note_meta*)builder->cursor.start); 141 142 return 1; 143 } 144 145 /* note flags are stored in the header entry */ 146 uint64_t *ndb_note_meta_flags(struct ndb_note_meta *meta) 147 { 148 return &meta->flags; 149 } 150 151 /* note flags are stored in the header entry */ 152 void ndb_note_meta_set_flags(struct ndb_note_meta *meta, uint32_t flags) 153 { 154 meta->flags = flags; 155 } 156 157 static int compare_entries(const void *a, const void *b) 158 { 159 struct ndb_note_meta_entry *entry_a, *entry_b; 160 uint64_t binmoji_a, binmoji_b; 161 int res; 162 163 entry_a = (struct ndb_note_meta_entry *)a; 164 entry_b = (struct ndb_note_meta_entry *)b; 165 166 res = entry_a->type - entry_b->type; 167 168 if (res == 0 && entry_a->type == NDB_NOTE_META_REACTION) { 169 /* we sort by reaction string for stability */ 170 binmoji_a = entry_a->payload.reaction_str.binmoji; 171 binmoji_b = entry_b->payload.reaction_str.binmoji; 172 173 if (binmoji_a < binmoji_b) { 174 return -1; 175 } else if (binmoji_a > binmoji_b) { 176 return 1; 177 } else { 178 return 0; 179 } 180 } else { 181 return res; 182 } 183 } 184 185 struct ndb_note_meta_entry *ndb_note_meta_entries(struct ndb_note_meta *meta) 186 { 187 /* entries start at the end of the header record */ 188 return (struct ndb_note_meta_entry *)((unsigned char*)meta + sizeof(*meta)); 189 } 190 191 struct ndb_note_meta_entry *ndb_note_meta_entry_at(struct ndb_note_meta *meta, int i) 192 { 193 if (i >= ndb_note_meta_entries_count(meta)) 194 return NULL; 195 196 return &ndb_note_meta_entries(meta)[i]; 197 } 198 void ndb_note_meta_build(struct ndb_note_meta_builder *builder, struct ndb_note_meta **meta) 199 { 200 /* sort entries */ 201 struct ndb_note_meta_entry *entries; 202 struct ndb_note_meta *header = (struct ndb_note_meta*)builder->cursor.start; 203 204 /* not initialized */ 205 assert(builder->cursor.start != builder->cursor.p); 206 207 if (header->count > 1) { 208 entries = ndb_note_meta_entries(header); 209 /*assert(entries);*/ 210 211 /* ensure entries are always sorted so bsearch is possible for large metadata 212 * entries. probably won't need that for awhile though */ 213 214 /* this also ensures our counts entry is near the front, which will be a very 215 * hot and common entry to hit */ 216 qsort(entries, header->count, sizeof(struct ndb_note_meta_entry), compare_entries); 217 } 218 219 *meta = header; 220 return; 221 } 222 223 uint16_t *ndb_note_meta_entry_type(struct ndb_note_meta_entry *entry) 224 { 225 return &entry->type; 226 } 227 228 /* find a metadata entry, optionally matching a payload */ 229 static struct ndb_note_meta_entry *ndb_note_meta_find_entry_impl(struct ndb_note_meta *meta, uint16_t type, uint64_t *payload, int sorted) 230 { 231 struct ndb_note_meta_entry *entries, *entry; 232 int i; 233 234 if (meta->count == 0) 235 return NULL; 236 237 entries = ndb_note_meta_entries(meta); 238 assert(((intptr_t)entries - (intptr_t)meta) == 16); 239 240 /* TODO(jb55): do bsearch for large sorted entries */ 241 242 for (i = 0; i < meta->count; i++) { 243 entry = &entries[i]; 244 assert(((uintptr_t)entry % 8) == 0); 245 /* 246 assert(entry->type < 100); 247 printf("finding %d/%d q:%d q:%"PRIx64" entry_type:%d entry:%"PRIx64"\n", 248 i+1, (int)meta->count, type, payload ? *payload : 0, entry->type, entry->payload.value); 249 */ 250 if (entry->type != type) 251 continue; 252 if (payload && (*payload != entry->payload.value)) 253 continue; 254 return entry; 255 } 256 257 return NULL; 258 } 259 260 struct ndb_note_meta_entry *ndb_note_meta_find_entry(struct ndb_note_meta *meta, uint16_t type, uint64_t *payload) 261 { 262 int sorted = 1; 263 return ndb_note_meta_find_entry_impl(meta, type, payload, sorted); 264 } 265 266 struct ndb_note_meta_entry *ndb_note_meta_builder_find_entry( 267 struct ndb_note_meta_builder *builder, 268 uint16_t type, 269 uint64_t *payload) 270 { 271 /* meta building in progress is not necessarily sorted */ 272 int sorted = 0; 273 return ndb_note_meta_find_entry_impl((struct ndb_note_meta *)builder->cursor.start, type, payload, sorted); 274 } 275 276 void ndb_note_meta_reaction_set(struct ndb_note_meta_entry *entry, uint32_t count, union ndb_reaction_str str) 277 { 278 entry->type = NDB_NOTE_META_REACTION; 279 entry->aux2.flags = 0; 280 entry->aux.value = count; 281 entry->payload.reaction_str = str; 282 } 283 284 /* sets the quote repost count for this note */ 285 void ndb_note_meta_counts_set(struct ndb_note_meta_entry *entry, 286 uint32_t total_reactions, 287 uint16_t quotes, 288 uint16_t direct_replies, 289 uint32_t thread_replies, 290 uint16_t reposts) 291 { 292 entry->type = NDB_NOTE_META_COUNTS; 293 entry->aux.total_reactions = total_reactions; 294 entry->aux2.reposts = reposts; 295 entry->payload.counts.quotes = quotes; 296 entry->payload.counts.direct_replies = direct_replies; 297 entry->payload.counts.thread_replies = thread_replies; 298 } 299 300 /* clones a metadata, either adding a new entry of a specific type, or returing 301 * a reference to it 302 * 303 * [in/out] meta: pointer to an existing meta entry, can but overwritten to 304 * [out] entry: pointer to the added entry 305 * 306 * */ 307 enum ndb_meta_clone_result ndb_note_meta_clone_with_entry( 308 struct ndb_note_meta **meta, 309 struct ndb_note_meta_entry **entry, 310 uint16_t type, 311 uint64_t *payload, 312 unsigned char *buf, 313 size_t bufsize) 314 { 315 size_t size, offset; 316 struct ndb_note_meta_builder builder; 317 318 if (*meta == NULL) { 319 ndb_note_meta_builder_init(&builder, buf, bufsize); 320 *entry = ndb_note_meta_add_entry(&builder); 321 *meta = (struct ndb_note_meta*)buf; 322 323 assert(*entry); 324 325 ndb_note_meta_build(&builder, meta); 326 return NDB_META_CLONE_NEW_ENTRY; 327 } else if ((size = ndb_note_meta_total_size(*meta)) > bufsize) { 328 ndb_debug("buf size too small (%d < %d) for metadata entry\n", bufsize, size); 329 goto fail; 330 } else if ((*entry = ndb_note_meta_find_entry(*meta, type, payload))) { 331 offset = (unsigned char *)(*entry) - (unsigned char *)(*meta); 332 333 /* we have an existing entry. simply memcpy and return the new entry position */ 334 assert(offset < size); 335 assert((offset % 16) == 0); 336 assert(((uintptr_t)buf % 8) == 0); 337 338 memcpy(buf, *meta, size); 339 *meta = (struct ndb_note_meta*)buf; 340 *entry = (struct ndb_note_meta_entry*)(((unsigned char *)(*meta)) + offset); 341 return NDB_META_CLONE_EXISTING_ENTRY; 342 } else if (size + sizeof(*entry) > bufsize) { 343 /* if we don't have an existing entry, make sure we have room to add one */ 344 345 ndb_debug("note metadata is too big (%d > %d) to clone with entry\n", 346 (int)(len + sizeof(*entry)), (int)scratch_size); 347 /* no room. this is bad, if this happens we should fix it */ 348 goto fail; 349 } else { 350 /* we need to add a new entry */ 351 ndb_note_meta_builder_init(&builder, buf, bufsize); 352 353 memcpy(buf, *meta, size); 354 builder.cursor.p = buf + size; 355 356 *entry = ndb_note_meta_add_entry(&builder); 357 assert(*entry); 358 (*entry)->type = type; 359 (*entry)->payload.value = payload? *payload : 0; 360 361 *meta = (struct ndb_note_meta*)buf; 362 363 assert(*entry); 364 assert(*meta); 365 366 ndb_note_meta_build(&builder, meta); 367 368 /* we re-find here since it could have been sorted */ 369 *entry = ndb_note_meta_find_entry(*meta, type, payload); 370 assert(*entry); 371 assert(*ndb_note_meta_entry_type(*entry) == type); 372 373 return NDB_META_CLONE_NEW_ENTRY; 374 } 375 376 assert(!"should be impossible to get here"); 377 fail: 378 *entry = NULL; 379 *meta = NULL; 380 return 0; 381 } 382 383 uint32_t *ndb_note_meta_reaction_count(struct ndb_note_meta_entry *entry) 384 { 385 return &entry->aux.value; 386 } 387 388 uint16_t *ndb_note_meta_counts_direct_replies(struct ndb_note_meta_entry *entry) 389 { 390 return &entry->payload.counts.direct_replies; 391 } 392 393 uint32_t *ndb_note_meta_counts_total_reactions(struct ndb_note_meta_entry *entry) 394 { 395 return &entry->aux.total_reactions; 396 } 397 398 uint32_t *ndb_note_meta_counts_thread_replies(struct ndb_note_meta_entry *entry) 399 { 400 return &entry->payload.counts.thread_replies; 401 } 402 403 uint16_t *ndb_note_meta_counts_quotes(struct ndb_note_meta_entry *entry) 404 { 405 return &entry->payload.counts.quotes; 406 } 407 408 uint16_t *ndb_note_meta_counts_reposts(struct ndb_note_meta_entry *entry) 409 { 410 return &entry->aux2.reposts; 411 } 412 413 void ndb_note_meta_reaction_set_count(struct ndb_note_meta_entry *entry, uint32_t count) 414 { 415 entry->aux.value = count; 416 } 417 418 union ndb_reaction_str ndb_note_meta_reaction_str(struct ndb_note_meta_entry *entry) 419 { 420 return entry->payload.reaction_str; 421 } 422 423 void print_note_meta(struct ndb_note_meta *meta) 424 { 425 int count, i; 426 struct ndb_note_meta_entry *entries, *entry; 427 union ndb_reaction_str reaction; 428 char strbuf[128]; 429 430 count = ndb_note_meta_entries_count(meta); 431 entries = ndb_note_meta_entries(meta); 432 433 for (i = 0; i < count; i++) { 434 entry = &entries[i]; 435 switch (entry->type) { 436 case NDB_NOTE_META_REACTION: 437 reaction = ndb_note_meta_reaction_str(entry); 438 439 ndb_reaction_to_str(&reaction, strbuf); 440 printf("%s%d ", strbuf, *ndb_note_meta_reaction_count(entry)); 441 break; 442 case NDB_NOTE_META_COUNTS: 443 printf("reposts %d\tquotes %d\treplies %d\tall_replies %d\treactions %d\t", 444 *ndb_note_meta_counts_reposts(entry), 445 *ndb_note_meta_counts_quotes(entry), 446 *ndb_note_meta_counts_direct_replies(entry), 447 *ndb_note_meta_counts_thread_replies(entry), 448 *ndb_note_meta_counts_total_reactions(entry)); 449 break; 450 } 451 } 452 453 printf("\n"); 454 }