nostrdb

parser.c (43746B)

---

 /*
  * FlatBuffers IDL parser.
  *
  * Originally based on the numeric parser in the Luthor lexer project.
  *
  * We are moving away from TDOP approach because the grammer doesn't
  * really benefit from it. We use the same overall framework.
  */
 
 #include <stdio.h>
 #include <stdlib.h>
 #include <assert.h>
 #include <stdarg.h>
 
 #include "semantics.h"
 #include "codegen.h"
 #include "fileio.h"
 #include "pstrutil.h"
 #include "flatcc/portable/pparseint.h"
 
 void fb_default_error_out(void *err_ctx, const char *buf, size_t len)
 {
     (void)err_ctx;
 
     fwrite(buf, 1, len, stderr);
 }
 
 int fb_print_error(fb_parser_t *P, const char * format, ...)
 {
     int n;
     va_list ap;
     char buf[ERROR_BUFSIZ];
 
     va_start (ap, format);
     n = vsnprintf (buf, ERROR_BUFSIZ, format, ap);
     va_end (ap);
     if (n >= ERROR_BUFSIZ) {
         strcpy(buf + ERROR_BUFSIZ - 5, "...\n");
         n = ERROR_BUFSIZ - 1;
     }
     P->error_out(P->error_ctx, buf, (size_t)n);
     return n;
 }
 
 const char *error_find_file_of_token(fb_parser_t *P, fb_token_t *t)
 {
     /*
      * Search token in dependent buffers if not in current token
      * buffer. We can do this as a linear search because we limit the
      * number of output errors.
      */
     while (P) {
         if (P->ts <= t && P->te > t) {
             return P->schema.errorname;
         }
         P = P->dependencies;
     }
     return "";
 }
 
 void error_report(fb_parser_t *P, fb_token_t *t, const char *msg, fb_token_t *peer, const char *s, size_t len)
 {
     const char *file, *peer_file;
 
     if (t && !s) {
         s = t->text;
         len = (size_t)t->len;
     }
     if (!msg) {
         msg = "";
     }
     if (!s) {
         s = "";
         len = 0;
     }
     if (t && !peer) {
         file = error_find_file_of_token(P, t);
         fb_print_error(P, "%s:%ld:%ld: error: '%.*s': %s\n",
                 file, (long)t->linenum, (long)t->pos, len, s, msg);
     } else if (t && peer) {
         file = error_find_file_of_token(P, t);
         peer_file = error_find_file_of_token(P, peer);
         fb_print_error(P, "%s:%ld:%ld: error: '%.*s': %s: %s:%ld:%ld: '%.*s'\n",
                 file, (long)t->linenum, (long)t->pos, len, s, msg,
                 peer_file, (long)peer->linenum, (long)peer->pos, (int)peer->len, peer->text);
     } else if (!t && !peer) {
         fb_print_error(P, "error: %s\n", msg);
     } else if (peer) {
         peer_file = error_find_file_of_token(P, peer);
         fb_print_error(P, "error: %s: %s:%ld:%ld: '%.*s'\n",
                 msg,
                 peer_file, (long)peer->linenum, (long)peer->pos, (int)peer->len, peer->text);
     } else {
         fb_print_error(P, "internal error: unexpected state\n");
     }
     ++P->failed;
 }
 
 void error_ref_sym(fb_parser_t *P, fb_ref_t *ref, const char *msg, fb_symbol_t *s2)
 {
     fb_ref_t *p;
     char buf[FLATCC_MAX_IDENT_SHOW + 1];
     size_t k = FLATCC_MAX_IDENT_SHOW;
     size_t n = 0;
     size_t n0 = 0;
     int truncated = 0;
 
     p = ref;
     while (p && k > 0) {
         if (n0 > 0) {
             buf[n0] = '.';
             --k;
             ++n0;
         }
         n = (size_t)p->ident->len;
         if (k < n) {
             n = k;
             truncated = 1;
         }
         memcpy(buf + n0, p->ident->text, n);
         k -= n;
         n0 += n;
         p = p->link;
     }
     if (p) truncated = 1;
     buf[n0] = '\0';
     if (n0 > 0) {
         --n0;
     }
     if (truncated) {
         memcpy(buf + FLATCC_MAX_IDENT_SHOW + 1 - 4, "...\0", 4);
         n0 = FLATCC_MAX_IDENT_SHOW;
     }
     error_report(P, ref->ident, msg, s2 ? s2->ident : 0, buf, n0);
 }
 
 //#define LEX_DEBUG
 
 /* Flatbuffers reserve keywords. */
 #define LEX_KEYWORDS
 
 #define LEX_C_BLOCK_COMMENT
 /*
  * Flatbuffers also support /// on a single line for documentation but
  * we can handle that within the normal line comment parsing logic.
  */
 #define LEX_C99_LINE_COMMENT
 /*
  * String escapes are not defined in fb schema but it only uses strings
  * for attribute, namespace, file ext, and file id. For JSON objects we
  * use C string escapes but control characters must be detected.
  */
 #define LEX_C_STRING
 
 /* Accept numbers like -0x42 as integer literals. */
 #define LEX_HEX_NUMERIC
 
 #include "parser.h"
 
 #ifdef LEX_DEBUG
 
 static void print_token(fb_token_t *t)
 {
     lex_fprint_token(stderr, t->id, t->text, t->text + t->len, t->linenum, t->pos);
 }
 
 static void debug_token(const char *info, fb_token_t *t)
 {
     fprintf(stderr, "%s\n    ", info);
     print_token(t);
 }
 #else
 #define debug_token(info, t) ((void)0)
 #endif
 
 static void revert_metadata(fb_metadata_t **list)
 {
     REVERT_LIST(fb_metadata_t, link, list);
 }
 
 static void revert_symbols(fb_symbol_t **list)
 {
     REVERT_LIST(fb_symbol_t, link, list);
 }
 
 static void revert_names(fb_name_t **list)
 {
     REVERT_LIST(fb_name_t, link, list);
 }
 
 static inline fb_doc_t *fb_add_doc(fb_parser_t *P, fb_token_t *t)
 {
     fb_doc_t *p;
 
     p = new_elem(P, sizeof(*p));
     p->ident = t;
     p->link = P->doc;
     P->doc = p;
     return p;
 }
 
 #define fb_assign_doc(P, p) {\
     revert_symbols(&P->doc); p->doc = P->doc; P->doc = 0; }
 
 static inline fb_compound_type_t *fb_add_table(fb_parser_t *P)
 {
     fb_compound_type_t *p;
 
     p = new_elem(P, sizeof(*p));
     p->symbol.link = P->schema.symbols;
     p->symbol.kind = fb_is_table;
     P->schema.symbols = &p->symbol;
     p->scope = P->current_scope;
     fb_assign_doc(P, p);
     return p;
 }
 
 static inline fb_compound_type_t *fb_add_struct(fb_parser_t *P)
 {
     fb_compound_type_t *p;
 
     p = new_elem(P, sizeof(*p));
     p->symbol.link = P->schema.symbols;
     p->symbol.kind = fb_is_struct;
     P->schema.symbols = &p->symbol;
     p->scope = P->current_scope;
     fb_assign_doc(P, p);
     return p;
 }
 
 static inline fb_compound_type_t *fb_add_rpc_service(fb_parser_t *P)
 {
     fb_compound_type_t *p;
 
     p = new_elem(P, sizeof(*p));
     p->symbol.link = P->schema.symbols;
     p->symbol.kind = fb_is_rpc_service;
     P->schema.symbols = &p->symbol;
     p->scope = P->current_scope;
     fb_assign_doc(P, p);
     return p;
 }
 
 static inline fb_compound_type_t *fb_add_enum(fb_parser_t *P)
 {
     fb_compound_type_t *p;
 
     p = new_elem(P, sizeof(*p));
     p->symbol.link = P->schema.symbols;
     p->symbol.kind = fb_is_enum;
     P->schema.symbols = &p->symbol;
     p->scope = P->current_scope;
     fb_assign_doc(P, p);
     return p;
 }
 
 static inline fb_compound_type_t *fb_add_union(fb_parser_t *P)
 {
     fb_compound_type_t *p;
 
     p = new_elem(P, sizeof(*p));
     p->symbol.link = P->schema.symbols;
     p->symbol.kind = fb_is_union;
     P->schema.symbols = &p->symbol;
     p->scope = P->current_scope;
     fb_assign_doc(P, p);
     return p;
 }
 
 static inline fb_ref_t *fb_add_ref(fb_parser_t *P, fb_token_t *t)
 {
     fb_ref_t *p;
 
     p = new_elem(P, sizeof(*p));
     p->ident = t;
     return p;
 }
 
 static inline fb_attribute_t *fb_add_attribute(fb_parser_t *P)
 {
     fb_attribute_t *p;
 
     p = new_elem(P, sizeof(*p));
     p->name.link = P->schema.attributes;
     P->schema.attributes = &p->name;
     return p;
 }
 
 static inline fb_include_t *fb_add_include(fb_parser_t *P)
 {
     fb_include_t *p;
     p = new_elem(P, sizeof(*p));
     p->link = P->schema.includes;
     return P->schema.includes = p;
 }
 
 static inline fb_scope_t *fb_add_scope(fb_parser_t *P, fb_ref_t *name)
 {
     fb_scope_t *p;
 
     p = fb_scope_table_find(&P->schema.root_schema->scope_index, name, 0);
     if (p) {
         return p;
     }
     p = new_elem(P, sizeof(*p));
     p->name = name;
     p->prefix = P->schema.prefix;
 
     fb_scope_table_insert_item(&P->schema.root_schema->scope_index, p, ht_keep);
     return p;
 }
 
 static inline fb_metadata_t *fb_add_metadata(fb_parser_t *P, fb_metadata_t **metadata)
 {
     fb_metadata_t *p;
     p = new_elem(P, sizeof(*p));
     p->link = *metadata;
     return *metadata = p;
 }
 
 static inline fb_member_t *fb_add_member(fb_parser_t *P, fb_symbol_t **members)
 {
     fb_member_t *p;
     p = new_elem(P, sizeof(*p));
     p->symbol.link = *members;
     p->symbol.kind = fb_is_member;
     *members = (fb_symbol_t *)p;
     fb_assign_doc(P, p);
     return p;
 }
 
 static inline int is_end(fb_token_t *t)
 {
     return t->id == LEX_TOK_EOF;
 }
 
 static fb_token_t *next(fb_parser_t *P)
 {
 again:
     ++P->token;
     if (P->token == P->te) {
         /* We keep returning end of token to help binary operators etc., if any. */
         --P->token;
         assert(0);
         switch (P->token->id) {
         case LEX_TOK_EOS: case LEX_TOK_EOB: case LEX_TOK_EOF:
             P->token->id = LEX_TOK_EOF;
             return P->token;
         }
         error_tok(P, P->token, "unexpected end of input");
     }
     if (P->token->id == tok_kw_doc_comment) {
         /* Note: we can have blanks that are control characters here, such as \t. */
         fb_add_doc(P, P->token);
         goto again;
     }
     debug_token("next", P->token);
     return P->token;
 }
 
 static void recover(fb_parser_t *P, long token_id, int consume)
 {
     while (!is_end(P->token)) {
         if (P->token->id == token_id) {
             if (consume) {
                 next(P);
             }
             P->doc = 0;
             return;
         }
         next(P);
     }
 }
 
 static void recover2(fb_parser_t *P, long token_id, int consume, long token_id_2, int consume_2)
 {
     while (!is_end(P->token)) {
         if (P->token->id == token_id) {
             if (consume) {
                 next(P);
             }
             P->doc = 0;
             return;
         }
         if (P->token->id == token_id_2) {
             if (consume_2) {
                 next(P);
             }
             P->doc = 0;
             return;
         }
         next(P);
     }
 }
 
 static inline fb_token_t *optional(fb_parser_t *P, long id) {
     fb_token_t *t = 0;
     if (P->token->id == id) {
         t = P->token;
         next(P);
     }
     return t;
 }
 
 static inline fb_token_t *match(fb_parser_t *P, long id, char *msg) {
     fb_token_t *t = 0;
     if (P->token->id == id) {
         t = P->token;
         next(P);
     } else {
         error_tok(P, P->token, msg);
     }
     return t;
 }
 
 /*
  * When a keyword should also be accepted as an identifier.
  * This is useful for JSON where field naems are visible.
  * Since field names are not referenced within the schema,
  * this is generally safe. Enums can also be resererved but
  * they can then not be used as default values. Table names
  * and other type names should not be remapped as they can then
  * not by used as a type name for other fields.
  */
 #if FLATCC_ALLOW_KW_FIELDS
 static inline void remap_field_ident(fb_parser_t *P)
 {
     if (P->token->id >= LEX_TOK_KW_BASE && P->token->id < LEX_TOK_KW_END) {
         P->token->id = LEX_TOK_ID;
     }
 }
 #else
 static inline void remap_field_ident(fb_parser_t *P) { (void)P; }
 #endif
 
 #if FLATCC_ALLOW_KW_ENUMS
 static inline void remap_enum_ident(fb_parser_t *P)
 {
     if (P->token->id >= LEX_TOK_KW_BASE && P->token->id < LEX_TOK_KW_END) {
         P->token->id = LEX_TOK_ID;
     }
 }
 #else
 static inline void remap_enum_ident(fb_parser_t *P) { (void)P; }
 #endif
 
 static fb_token_t *advance(fb_parser_t *P, long id, const char *msg, fb_token_t *peer)
 {
     /*
      * `advance` is generally used at end of statements so it is a
      * convenient place to get rid of rogue doc comments we can't attach
      * to anything meaningful.
      */
     P->doc = 0;
     if (P->token->id != id) {
         error_tok_2(P, P->token, msg, peer);
         return P->token;
     }
     return next(P);
 }
 
 static void read_integer_value(fb_parser_t *P, fb_token_t *t, fb_value_t *v, int sign)
 {
     int status;
 
     v->type = vt_uint;
     /* The token does not store the sign internally. */
     parse_integer(t->text, (size_t)t->len, &v->u, &status);
     if (status != PARSE_INTEGER_UNSIGNED) {
         v->type = vt_invalid;
         error_tok(P, t, "invalid integer format");
     }
     if (sign) {
         v->i = -(int64_t)v->u;
         v->type = vt_int;
 #ifdef FLATCC_FAIL_ON_INT_SIGN_OVERFLOW
         /* Sometimes we might want this, so don't fail by default. */
         if (v->i > 0) {
             v->type = vt_invalid;
             error_tok(P, t, "sign overflow in integer format");
         }
 #endif
     }
 }
 
 static void read_hex_value(fb_parser_t *P, fb_token_t *t, fb_value_t *v, int sign)
 {
     int status;
 
     v->type = vt_uint;
     /* The token does not store the sign internally. */
     parse_hex_integer(t->text, (size_t)t->len, &v->u, &status);
     if (status != PARSE_INTEGER_UNSIGNED) {
         v->type = vt_invalid;
         error_tok(P, t, "invalid hex integer format");
     }
     if (sign) {
         v->i = -(int64_t)v->u;
         v->type = vt_int;
 #ifdef FLATCC_FAIL_ON_INT_SIGN_OVERFLOW
         /* Sometimes we might want this, so don't fail by default. */
         if (v->i > 0) {
             v->type = vt_invalid;
             error_tok(P, t, "sign overflow in hex integer format");
         }
 #endif
     }
 }
 
 static void read_float_value(fb_parser_t *P, fb_token_t *t, fb_value_t *v, int sign)
 {
     char *end;
 
     v->type = vt_float;
     v->f = strtod(t->text, &end);
     if (end != t->text + t->len) {
         v->type = vt_invalid;
         error_tok(P, t, "invalid float format");
     } else if (t->text[0] == '.') {
         v->type = vt_invalid;
         /* The FB spec requires this, in line with the JSON format. */
         error_tok(P, t, "numeric values must start with a digit");
     } else if (sign) {
         v->f = -v->f;
     }
 }
 
 /*
  * We disallow escape characters, newlines and other control characters,
  * but especially escape characters because they would require us to
  * reallocate the string and convert the escaped characters. We also
  * disallow non-utf8 characters, but we do not check for it. The tab
  * character could meaningfully be accepted, but we don't.
  *
  * String literals are only used to name attributes, namespaces,
  * file identifiers and file externsions, so we really have no need
  * for these extra featuresescape .
  *
  * JSON strings should be handled separately, if or when supported -
  * either by converting escapes and reallocating the string, or
  * simply by ignoring the escape errors and use the string unmodified.
  */
 static void parse_string_literal(fb_parser_t *P, fb_value_t *v)
 {
     fb_token_t *t;
 
     v->type = vt_string;
     v->s.s = 0;
     v->s.len = 0;
 
     for (;;) {
         t = P->token;
         switch (t->id) {
         case LEX_TOK_STRING_PART:
             if (v->s.s == 0) {
                 v->s.s = (char *)t->text;
             }
             break;
         case LEX_TOK_STRING_ESCAPE:
             v->type = vt_invalid;
             error_tok(P, t, "escape not allowed in strings");
             break;
         case LEX_TOK_STRING_CTRL:
             v->type = vt_invalid;
             error_tok_as_string(P, t, "control characters not allowed in strings", "?", 1);
             break;
         case LEX_TOK_STRING_NEWLINE:
             v->type = vt_invalid;
             error_tok(P, t, "newline not allowed in strings");
             break;
         case LEX_TOK_STRING_UNTERMINATED:
         case LEX_TOK_STRING_END:
             goto done;
 
         default:
             error_tok(P, t, "internal error: unexpected token in string");
             v->type = vt_invalid;
             goto done;
         }
         next(P);
     }
 done:
     /*
      * If we were to ignore all errors, we would get the full
      * string as is excluding delimiting quotes.
      */
     if (v->s.s) {
         v->s.len = (int)(P->token->text - v->s.s);
     }
     if (!match(P, LEX_TOK_STRING_END, "unterminated string")) {
         v->type = vt_invalid;
     }
 }
 
 /* Current token must be an identifier. */
 static void parse_ref(fb_parser_t *P, fb_ref_t **ref)
 {
     *ref = fb_add_ref(P, P->token);
     next(P);
     ref = &((*ref)->link);
     while (optional(P, '.')) {
         if (P->token->id != LEX_TOK_ID) {
             error_tok(P, P->token, "namespace prefix expected identifier");
             break;
         }
         *ref = fb_add_ref(P, P->token);
         ref = &((*ref)->link);
         next(P);
     }
 }
 
 /* `flags` */
 enum { allow_string_value = 1, allow_id_value = 2, allow_null_value = 4 };
 static void parse_value(fb_parser_t *P, fb_value_t *v, int flags, const char *error_msg)
 {
     fb_token_t *t;
     fb_token_t *sign;
 
     sign = optional(P, '-');
     t = P->token;
 
     switch (t->id) {
     case LEX_TOK_HEX:
         read_hex_value(P, t, v, sign != 0);
         break;
     case LEX_TOK_INT:
         read_integer_value(P, t, v, sign != 0);
         break;
     case LEX_TOK_FLOAT:
         read_float_value(P, t, v, sign != 0);
         break;
     case tok_kw_true:
         v->b = 1;
         v->type = vt_bool;
         break;
     case tok_kw_false:
         v->b = 0;
         v->type = vt_bool;
         break;
     case tok_kw_null:
         if (!(flags & allow_null_value)) {
             v->type = vt_invalid;
             error_tok(P, t, error_msg);
             return;
         }
         v->type = vt_null;
         break;
     case LEX_TOK_STRING_BEGIN:
         next(P);
         parse_string_literal(P, v);
         if (!(flags & allow_string_value)) {
             v->type = vt_invalid;
             error_tok(P, t, error_msg);
             return;
         }
         if (sign) {
             v->type = vt_invalid;
             error_tok(P, t, "string constants cannot be signed");
             return;
         }
         return;
     case LEX_TOK_ID:
         parse_ref(P, &v->ref);
         v->type = vt_name_ref;
         if (sign) {
             v->type = vt_invalid;
             /* Technically they could, but we do not allow it. */
             error_tok(P, t, "named values cannot be signed");
         }
         return;
     default:
         /* We might have consumed a sign, but never mind that. */
         error_tok(P, t, error_msg);
         return;
     }
     if (sign && v->type == vt_bool) {
         v->type = vt_invalid;
         error_tok(P, t, "boolean constants cannot be signed");
     }
     next(P);
 }
 
 static void parse_fixed_array_size(fb_parser_t *P, fb_token_t *ttype, fb_value_t *v)
 {
     const char *error_msg = "fixed length array length expected to be an unsigned integer";
     fb_value_t vsize;
     fb_token_t *tlen = P->token;
 
     parse_value(P, &vsize, 0, error_msg);
     if (vsize.type != vt_uint) {
         error_tok(P, tlen, error_msg);
         v->type = vt_invalid;
         return;
     }
     if (v->type == vt_invalid) return;
     switch (v->type) {
     case vt_vector_type:
         v->type = vt_fixed_array_type;
         break;
     case vt_vector_type_ref:
         v->type = vt_fixed_array_type_ref;
         break;
     case vt_vector_string_type:
         v->type = vt_fixed_array_string_type;
         break;
     case vt_invalid:
         return;
     default:
         error_tok(P, ttype, "invalid fixed length array type");
         v->type = vt_invalid;
         return;
     }
     if (vsize.u == 0) {
         error_tok(P, tlen, "fixed length array length cannot be 0");
         v->type = vt_invalid;
         return;
     }
     /*
      * This allows for safe 64-bit multiplication by elements no
      * larger than 2^32-1 and also fits into the value len field.
      * without extra size cost.
      */
     if (vsize.u > UINT32_MAX) {
         error_tok(P, tlen, "fixed length array length overflow");
         v->type = vt_invalid;
         return;
     }
     v->len = (uint32_t)vsize.u;
 }
 
 /* ':' must already be matched */
 static void parse_type(fb_parser_t *P, fb_value_t *v)
 {
     fb_token_t *t = 0;
     fb_token_t *ttype = 0;
     fb_token_t *t0 = P->token;
     int vector = 0;
 
     v->len = 1;
     v->type = vt_invalid;
     while ((t = optional(P, '['))) {
         ++vector;
     }
     if (vector > 1) {
         error_tok(P, t0, "vector type can only be one-dimensional");
     }
     ttype = P->token;
     switch (ttype->id) {
     case tok_kw_int:
     case tok_kw_bool:
     case tok_kw_byte:
     case tok_kw_long:
     case tok_kw_uint:
     case tok_kw_float:
     case tok_kw_short:
     case tok_kw_char:
     case tok_kw_ubyte:
     case tok_kw_ulong:
     case tok_kw_ushort:
     case tok_kw_double:
     case tok_kw_int8:
     case tok_kw_int16:
     case tok_kw_int32:
     case tok_kw_int64:
     case tok_kw_uint8:
     case tok_kw_uint16:
     case tok_kw_uint32:
     case tok_kw_uint64:
     case tok_kw_float32:
     case tok_kw_float64:
         v->t = P->token;
         v->type = vector ? vt_vector_type : vt_scalar_type;
         next(P);
         break;
     case tok_kw_string:
         v->t = P->token;
         v->type = vector ? vt_vector_string_type : vt_string_type;
         next(P);
         break;
     case LEX_TOK_ID:
         parse_ref(P, &v->ref);
         v->type = vector ? vt_vector_type_ref : vt_type_ref;
         break;
     case ']':
         error_tok(P, t, "vector type cannot be empty");
         break;
     default:
         error_tok(P, ttype, "invalid type specifier");
         break;
     }
     if (vector && optional(P, ':')) {
         parse_fixed_array_size(P, ttype, v);
     }
     while (optional(P, ']') && vector--) {
     }
     if (vector) {
         error_tok_2(P, t, "vector type missing ']' to match", t0);
     }
     if ((t = optional(P, ']'))) {
         error_tok_2(P, t, "extra ']' not matching", t0);
         while (optional(P, ']')) {
         }
     }
     if (ttype->id == tok_kw_char && v->type != vt_invalid) {
         if (v->type != vt_fixed_array_type) {
             error_tok(P, ttype, "char can only be used as a fixed length array type [char:<n>]");
             v->type = vt_invalid;
         }
     }
 }
 
 static fb_metadata_t *parse_metadata(fb_parser_t *P)
 {
     fb_token_t *t, *t0;
     fb_metadata_t *md = 0;
 
     if (!(t0 = optional(P, '('))) {
         return 0;
     }
     if ((t = optional(P, LEX_TOK_ID)))
     for (;;) {
         fb_add_metadata(P, &md);
         md->ident = t;
         if (optional(P, ':')) {
             parse_value(P, &md->value, allow_string_value, "scalar or string value expected");
         }
         if (P->failed >= FLATCC_MAX_ERRORS) {
             return md;
         }
         if (!optional(P, ',')) {
             break;
         }
         if (!(t = match(P, LEX_TOK_ID, "attribute name expected identifier after ','"))) {
             break;
         }
     }
     advance(P, ')', "metadata expected ')' to match", t0);
     revert_metadata(&md);
     return md;
 }
 
 static void parse_field(fb_parser_t *P, fb_member_t *fld)
 {
     fb_token_t *t;
 
     remap_field_ident(P);
     if (!(t = match(P, LEX_TOK_ID, "field expected identifier"))) {
         goto fail;
     }
     fld->symbol.ident = t;
     if (!match(P, ':', "field expected ':' before mandatory type")) {
         goto fail;
     }
     parse_type(P, &fld->type);
     if (optional(P, '=')) {
         /*
          * Because types can be named references, we do not check the
          * default assignment before the schema is fully parsed.
          * We allow the initializer to be a name in case it is an enum
          * name.
          */
         parse_value(P, &fld->value, allow_id_value | allow_null_value, "initializer must be of scalar type or null");
     }
     fld->metadata = parse_metadata(P);
     advance(P, ';', "field must be terminated with ';'", 0);
     return;
 fail:
     recover2(P, ';', 1, '}', 0);
 }
 
 static void parse_method(fb_parser_t *P, fb_member_t *fld)
 {
     fb_token_t *t;
     if (!(t = match(P, LEX_TOK_ID, "method expected identifier"))) {
         goto fail;
     }
     fld->symbol.ident = t;
     if (!match(P, '(', "method expected '(' after identifier")) {
         goto fail;
     }
     parse_type(P, &fld->req_type);
     if (!match(P, ')', "method expected ')' after request type")) {
         goto fail;
     }
     if (!match(P, ':', "method expected ':' before mandatory response type")) {
         goto fail;
     }
     parse_type(P, &fld->type);
     if ((t = optional(P, '='))) {
         error_tok(P, t, "method does not accept an initializer");
         goto fail;
     }
     fld->metadata = parse_metadata(P);
     advance(P, ';', "method must be terminated with ';'", 0);
     return;
 fail:
     recover2(P, ';', 1, '}', 0);
 }
 
 /* `enum` must already be matched. */
 static void parse_enum_decl(fb_parser_t *P, fb_compound_type_t *ct)
 {
     fb_token_t *t, *t0;
     fb_member_t *member;
 
     if (!(ct->symbol.ident = match(P, LEX_TOK_ID, "enum declaration expected identifier"))) {
         goto fail;
     }
     if (optional(P, ':')) {
         parse_type(P, &ct->type);
         if (ct->type.type != vt_scalar_type) {
             error_tok(P, ct->type.t, "integral type expected");
         } else {
             switch (ct->type.t->id) {
             case tok_kw_float:
             case tok_kw_double:
             case tok_kw_float32:
             case tok_kw_float64:
                 error_tok(P, ct->type.t, "integral type expected");
                 break;
             default:
                 break;
             }
         }
     }
     ct->metadata = parse_metadata(P);
     if (!((t0 = match(P, '{', "enum declaration expected '{'")))) {
         goto fail;
     }
     for (;;) {
         remap_enum_ident(P);
         if (!(t = match(P, LEX_TOK_ID,
                 "member identifier expected"))) {
             goto fail;
         }
         if (P->failed >= FLATCC_MAX_ERRORS) {
             goto fail;
         }
         member = fb_add_member(P, &ct->members);
         member->symbol.ident = t;
         if (optional(P, '=')) {
             t = P->token;
             parse_value(P, &member->value, 0, "integral constant expected");
             /* Leave detailed type (e.g. no floats) and range checking to a later stage. */
         }
         /*
          * Trailing comma is optional in flatc but not in grammar, we
          * follow flatc.
          */
         if (!optional(P, ',') || P->token->id == '}') {
             break;
         }
         P->doc = 0;
     }
     if (t0) {
         advance(P, '}', "enum missing closing '}' to match", t0);
     }
     revert_symbols(&ct->members);
     return;
 fail:
     recover(P, '}', 1);
 }
 
 /* `union` must already be matched. */
 static void parse_union_decl(fb_parser_t *P, fb_compound_type_t *ct)
 {
     fb_token_t *t0;
     fb_member_t *member;
     fb_ref_t *ref;
     fb_token_t *t;
 
     if (!(ct->symbol.ident = match(P, LEX_TOK_ID, "union declaration expected identifier"))) {
         goto fail;
     }
     ct->metadata = parse_metadata(P);
     if (!((t0 = match(P, '{', "union declaration expected '{'")))) {
         goto fail;
     }
     for (;;) {
         if (P->token->id != LEX_TOK_ID) {
             error_tok(P, P->token, "union expects an identifier");
             goto fail;
         }
         if (P->failed >= FLATCC_MAX_ERRORS) {
             goto fail;
         }
         t = P->token;
         member = fb_add_member(P, &ct->members);
         parse_ref(P, &ref);
         member->type.ref = ref;
         member->type.type = vt_type_ref;
         while (ref->link) {
             ref = ref->link;
         }
         /* The union member name is the unqualified reference. */
         member->symbol.ident = ref->ident;
         if (optional(P, ':')) {
             if (member->type.ref->link) {
                 error_tok(P, t, "qualified union member name cannot have an explicit type");
             }
             parse_type(P, &member->type);
             /* Leave type checking to later stage. */
         }
         if (optional(P, '=')) {
             parse_value(P, &member->value, 0, "integral constant expected");
             /* Leave detailed type (e.g. no floats) and range checking to a later stage. */
         }
         if (!optional(P, ',') || P->token->id == '}') {
             break;
         }
         P->doc = 0;
     }
     advance(P, '}', "union missing closing '}' to match", t0);
     revert_symbols(&ct->members);
     /* Add implicit `NONE` member first in the list. */
     member = fb_add_member(P, &ct->members);
     member->symbol.ident = &P->t_none;
     return;
 fail:
     recover2(P, ';', 1, '}', 0);
 }
 
 /* `struct` , `table`, or 'rpc_service' must already be matched. */
 static void parse_compound_type(fb_parser_t *P, fb_compound_type_t *ct, long token)
 {
     fb_token_t *t = 0;
 
     if (!(t = match(P, LEX_TOK_ID, "Declaration expected an identifier"))) {
         goto fail;
     }
     ct->symbol.ident = t;
     ct->metadata = parse_metadata(P);
     if (!(match(P, '{', "Declaration expected '{'"))) {
         goto fail;
     }
     t = P->token;
 
 /* Allow empty tables and structs. */
 #if 0
     if (P->token->id == '}') {
         error_tok(P, t, "table / struct declaration cannot be empty");
     }
 #endif
     while (P->token->id != '}') {
         if (token == tok_kw_rpc_service) {
             parse_method(P, fb_add_member(P, &ct->members));
         } else {
             parse_field(P, fb_add_member(P, &ct->members));
         }
         if (P->failed >= FLATCC_MAX_ERRORS) {
             goto fail;
         }
     }
     if (!optional(P, '}') && t) {
         error_tok_2(P, P->token, "Declaration missing closing '}' to match", t);
     }
     revert_symbols(&ct->members);
     return;
 fail:
     recover(P, '}', 1);
 }
 
 static void parse_namespace(fb_parser_t *P)
 {
     fb_ref_t *ref = 0;
     fb_token_t *t = P->token;
 
     if (optional(P, ';') && t) {
         /* Revert to global namespace. */
         P->current_scope = P->root_scope;
         return;
     }
     if (P->token->id != LEX_TOK_ID) {
         error_tok(P, P->token, "namespace expects an identifier");
         recover(P, ';', 1);
         return;
     }
     parse_ref(P, &ref);
     advance(P, ';', "missing ';' expected by namespace at", t);
     P->current_scope = fb_add_scope(P, ref);
 }
 
 static void parse_root_type(fb_parser_t *P, fb_root_type_t *rt)
 {
     fb_token_t *t = P->token;
 
     if (rt->name) {
         error_tok(P, P->token, "root_type already set");
     }
     parse_ref(P, &rt->name);
     rt->scope = P->current_scope;
     advance(P, ';', "missing ';' expected by root_type at", t);
 }
 
 static void parse_include(fb_parser_t *P)
 {
     fb_token_t *t = P->token;
 
     while (optional(P, tok_kw_include)) {
         if (P->opts.disable_includes) {
             error_tok(P, t, "include statements not supported by current environment");
         }
         if (P->failed >= FLATCC_MAX_ERRORS) {
             return;
         }
         if (!match(P, LEX_TOK_STRING_BEGIN,
                     "include expected a string literal as filename")) {
             recover(P, ';', 1);
         }
         parse_string_literal(P, &fb_add_include(P)->name);
         match(P, ';', "include statement expected ';'");
     }
 }
 
 static void parse_attribute(fb_parser_t *P, fb_attribute_t *a)
 {
     fb_token_t *t = P->token;
 
     if (match(P, LEX_TOK_STRING_BEGIN, "attribute expected string literal")) {
         parse_string_literal(P, &a->name.name);
         if (a->name.name.s.len == 0) {
             error_tok_as_string(P, t, "attribute name cannot be empty", 0, 0);
         }
     }
     match(P, ';', "attribute expected ';'");
 }
 
 static void parse_file_extension(fb_parser_t *P, fb_value_t *v)
 {
     if (v->type == vt_string) {
         error_tok_as_string(P, P->token, "file extension already set", v->s.s, (size_t)v->s.len);
     }
     if (!match(P, LEX_TOK_STRING_BEGIN, "file_extension expected string literal")) {
         goto fail;
     }
     parse_string_literal(P, v);
     match(P, ';', "file_extension expected ';'");
     return;
 fail:
     recover(P, ';', 1);
 }
 
 static void parse_file_identifier(fb_parser_t *P, fb_value_t *v)
 {
     fb_token_t *t;
     if (v->type != vt_missing) {
         error_tok_as_string(P, P->token, "file identifier already set", v->s.s, (size_t)v->s.len);
     }
     if (!match(P, LEX_TOK_STRING_BEGIN, "file_identifier expected string literal")) {
         goto fail;
     }
     t = P->token;
     parse_string_literal(P, v);
     if (v->s.s && v->s.len != 4) {
         v->type = vt_invalid;
         error_tok(P, t, "file_identifier must be 4 characters");
     }
     match(P, ';', "file_identifier expected ';'");
     return;
 fail:
     recover(P, ';', 1);
 }
 
 static void parse_schema_decl(fb_parser_t *P)
 {
     switch(P->token->id) {
     case tok_kw_namespace:
         next(P);
         parse_namespace(P);
         break;
     case tok_kw_file_extension:
         next(P);
         parse_file_extension(P, &P->schema.file_extension);
         break;
     case tok_kw_file_identifier:
         next(P);
         parse_file_identifier(P, &P->schema.file_identifier);
         break;
     case tok_kw_root_type:
         next(P);
         parse_root_type(P, &P->schema.root_type);
         break;
     case tok_kw_attribute:
         next(P);
         parse_attribute(P, fb_add_attribute(P));
         break;
     case tok_kw_struct:
         next(P);
         parse_compound_type(P, fb_add_struct(P), tok_kw_struct);
         break;
     case tok_kw_table:
         next(P);
         parse_compound_type(P, fb_add_table(P), tok_kw_table);
         break;
     case tok_kw_rpc_service:
         next(P);
         parse_compound_type(P, fb_add_rpc_service(P), tok_kw_rpc_service);
         break;
     case tok_kw_enum:
         next(P);
         parse_enum_decl(P, fb_add_enum(P));
         break;
     case tok_kw_union:
         next(P);
         parse_union_decl(P, fb_add_union(P));
         break;
     case tok_kw_include:
         error_tok(P, P->token, "include statements must be placed first in the schema");
         break;
     case '{':
         error_tok(P, P->token, "JSON objects in schema file is not supported - but a schema specific JSON parser can be generated");
         break;
     case LEX_TOK_CTRL:
         error_tok_as_string(P, P->token, "unexpected control character in schema definition", "?", 1);
         break;
     case LEX_TOK_COMMENT_CTRL:
         error_tok_as_string(P, P->token, "unexpected control character in comment", "?", 1);
         break;
     case LEX_TOK_COMMENT_UNTERMINATED:
         error_tok_as_string(P, P->token, "unterminated comment", "<eof>", 5);
         break;
     default:
         error_tok(P, P->token, "unexpected token in schema definition");
         break;
     }
 }
 
 static int parse_schema(fb_parser_t *P)
 {
     fb_token_t *t, *t0;
     parse_include(P);
     t = P->token;
     for (;;) {
         if (is_end(t)) {
             break;
         }
         if (P->failed >= FLATCC_MAX_ERRORS) {
             return -1;
         }
         t0 = t;
         parse_schema_decl(P);
         t = P->token;
         if (t == t0) {
             if (P->failed) {
                 return -1;
             }
             error_tok(P, t, "extra tokens in input");
             return -1;
         }
     }
     revert_names(&P->schema.attributes);
     revert_symbols(&P->schema.symbols);
     return 0;
 }
 
 static inline void clear_elem_buffers(fb_parser_t *P)
 {
     void **p, **p2;
 
     p = P->elem_buffers;
     while (p) {
         p2 = *((void**)p);
         free(p);
         p = p2;
     };
 }
 
 static void push_token(fb_parser_t *P, long id, const char *first, const char *last)
 {
     size_t offset;
     fb_token_t *t;
 
     if (P->token == P->te) {
         offset = (size_t)(P->token - P->ts);
         P->tcapacity = P->tcapacity ? 2 * P->tcapacity : 1024;
         P->ts = realloc(P->ts, (size_t)P->tcapacity * sizeof(fb_token_t));
         checkmem(P->ts);
         P->te = P->ts + P->tcapacity;
         P->token = P->ts + offset;
     }
     t = P->token;
     t->id = id;
     t->text = first;
     t->len = (long)(last - first);
     t->linenum = P->linenum;
     t->pos = (long)(first - P->line + 1);
     ++P->token;
 }
 
 /*
  * If the file contains a control character, we can get multiple
  * comments per line.
  */
 static inline void push_comment(fb_parser_t *P, const char *first, const char *last)
 {
     if (P->doc_mode) {
         push_token(P, tok_kw_doc_comment, first, last);
     }
 }
 
 static void inject_token(fb_token_t *t, const char *lex, long id)
 {
     t->id = id;
     t->text = lex;
     t->len = (long)strlen(lex);
     t->pos = 0;
     t->linenum = 0;
 }
 
 /* --- Customize lexer --- */
 
 /* Depends on the `context` argument given to the lex function. */
 #define ctx(name) (((fb_parser_t *)context)->name)
 
 #define lex_emit_newline(first, last) (ctx(linenum)++, ctx(line) = last)
 
 #define lex_emit_string_newline(first, last)                            \
     (ctx(linenum)++, ctx(line) = last,                                  \
     push_token((fb_parser_t*)context, LEX_TOK_STRING_NEWLINE, first, last))
 
 /*
  * Add emtpy comment on comment start - otherwise we miss empty lines.
  * Save is_doc becuase comment_part does not remember.
  */
 #define lex_emit_comment_begin(first, last, is_doc)                     \
     { ctx(doc_mode) = is_doc; push_comment((fb_parser_t*)context, last, last); }
 #define lex_emit_comment_part(first, last) push_comment((fb_parser_t*)context, first, last)
 #define lex_emit_comment_end(first, last) (ctx(doc_mode) = 0)
 
 /* By default emitted as lex_emit_other which would be ignored. */
 #define lex_emit_comment_unterminated(pos)                                  \
     push_token((fb_parser_t*)context, LEX_TOK_COMMENT_UNTERMINATED, pos, pos)
 
 #define lex_emit_comment_ctrl(pos)                                          \
     if (lex_isblank(*pos) || !lex_isctrl(*pos)) {                           \
         push_comment((fb_parser_t*)context, pos, pos + 1);                  \
     } else {                                                                \
         push_token((fb_parser_t*)context, LEX_TOK_CTRL,                     \
                 pos, pos + 1);                                              \
     }
 
 /*
  * Provide hook to lexer for emitting tokens. We can override many
  * things, but most default to calling lex_emit, so that is all we need
  * to handle.
  *
  * `context` is a magic name available to macros in the lexer.
  */
 #define lex_emit(token, first, last)                                    \
     push_token((fb_parser_t*)context, token, first, last)
 
 /*
  * We could just eos directly as it defaults to emit, but formally we
  * should use the eof marker which is always zero, so parser can check
  * for it easily, if needed.
  */
 #define lex_emit_eos(first, last)                                       \
     push_token((fb_parser_t*)context, LEX_TOK_EOF, first, last)
 
 /*
  * This event happens in place of eos if we exhaust the input buffer.
  * In this case we treat this as end of input, but this choice prevents
  * us from parsing across multiple buffers.
  */
 #define lex_emit_eob(pos)                                       \
     push_token((fb_parser_t*)context, LEX_TOK_EOF, pos, pos)
 
 /*
  * Luthor is our speedy generic lexer - it knows most common operators
  * and therefore allows us to fail meaningfully on those that we don't
  * support here, which is most.
  */
 #include "lex/luthor.c"
 
 #include "keywords.h"
 
 /* Root schema `rs` is null for top level parser. */
 int fb_init_parser(fb_parser_t *P, fb_options_t *opts, const char *name,
         fb_error_fun error_out, void *error_ctx, fb_root_schema_t *rs)
 {
     size_t n, name_len;
     char *s;
 
     memset(P, 0, sizeof(*P));
 
     if (error_out) {
         P->error_out = error_out;
         P->error_ctx = error_ctx;
     } else {
         P->error_out = fb_default_error_out;
     }
     if (opts) {
         memcpy(&P->opts, opts, sizeof(*opts));
     } else {
         flatcc_init_options(&P->opts);
     }
     P->schema.root_schema = rs ? rs : &P->schema.root_schema_instance;
     switch (P->opts.offset_size) {
     case 2:
     case 4:
     case 8:
         break;
     default:
         error(P, "invalid offset configured, must be 2, 4 (default), or 8");
         return -1;
     }
     switch (P->opts.voffset_size) {
     case 2:
     case 4:
     case 8:
         break;
     default:
         error(P, "invalid voffset configured, must be 2 (default), 4, or 8");
         return -1;
     }
     if (!name) {
         /* Mostly for testing, just so we always have a name. */
         name = FLATCC_DEFAULT_FILENAME;
     }
     if (name == 0) {
         name = "";
     }
     name_len = strlen(name);
     checkmem((P->schema.basename = fb_create_basename(name, name_len, opts->default_schema_ext)));
     n = strlen(P->schema.basename);
     checkmem(s = fb_copy_path_n(P->schema.basename, n));
     pstrntoupper(s, n);
     P->schema.basenameup = s;
     P->schema.name.name.s.s = s;
     P->schema.name.name.s.len = (int)n;
     checkmem((P->schema.errorname = fb_create_basename(name, name_len, "")));
     P->schema.prefix.s = "";
     P->schema.prefix.len = 0;
     if (opts->ns) {
         P->schema.prefix.s = (char *)opts->ns;
         P->schema.prefix.len = (int)strlen(opts->ns);
     }
     P->root_scope = fb_add_scope(P, 0);
     P->current_scope = P->root_scope;
     assert(P->current_scope == fb_scope_table_find(&P->schema.root_schema->scope_index, 0, 0));
     return 0;
 }
 
 /*
  * Main entry function for this specific parser type.
  * We expect a zero terminated string.
  *
  * The parser structure is uninitialized upon entry, and should be
  * cleared with `clear_flatbuffer_parser` subsequently.
  *
  * Datastructures point into the token buffer and into the input
  * buffer, so the parser and input should not be cleared prematurely.
  *
  * The input buffer must remain valid until the parser is cleared
  * because the internal represenation stores pointers into the buffer.
  *
  * `own_buffer` indicates that the the buffer should be deallocated when
  * the parser is cleaned up.
  */
 int fb_parse(fb_parser_t *P, const char *input, size_t len, int own_buffer)
 {
     static const char *id_none = "NONE";
     static const char *id_ubyte = "ubyte";
 
     P->line = input;
     P->linenum = 1;
 
     /* Used with union defaults. */
     inject_token(&P->t_none, id_none, LEX_TOK_ID);
     inject_token(&P->t_ubyte, id_ubyte, tok_kw_ubyte);
 
     if (own_buffer) {
         P->managed_input = input;
     }
     lex(input, len, 0, P);
 
     P->te = P->token;
     P->token = P->ts;
     /* Only used while processing table id's. */
     checkmem((P->tmp_field_marker = malloc(sizeof(P->tmp_field_marker[0]) * (size_t)P->opts.vt_max_count)));
     checkmem((P->tmp_field_index = malloc(sizeof(P->tmp_field_index[0]) * (size_t)P->opts.vt_max_count)));
     if (P->token->id == tok_kw_doc_comment) {
         next(P);
     }
     parse_schema(P);
     return P->failed;
 }
 
 static void __destroy_scope_item(void *item, fb_scope_t *scope)
 {
     /* Each scope points into table that is cleared separately. */
     (void)item;
 
     fb_symbol_table_clear(&scope->symbol_index);
 }
 
 void fb_clear_parser(fb_parser_t *P)
 {
     fb_symbol_t *sym;
     fb_compound_type_t *ct;
 
     for (sym = P->schema.symbols; sym; sym = sym->link) {
         switch (sym->kind) {
         case fb_is_struct:
         case fb_is_table:
         case fb_is_rpc_service:
         case fb_is_enum:
         case fb_is_union:
             ct = (fb_compound_type_t *)sym;
             fb_symbol_table_clear(&ct->index);
             fb_value_set_clear(&ct->value_set);
         }
     }
     fb_schema_table_clear(&P->schema.root_schema_instance.include_index);
     fb_name_table_clear(&P->schema.root_schema_instance.attribute_index);
     ptr_set_clear(&P->schema.visible_schema);
     if (P->tmp_field_marker) {
         free(P->tmp_field_marker);
     }
     if (P->tmp_field_index) {
         free(P->tmp_field_index);
     }
     if (P->ts) {
         free(P->ts);
     }
     if (P->schema.basename) {
         free((void *)P->schema.basename);
     }
     if (P->schema.basenameup) {
         free((void *)P->schema.basenameup);
     }
     if (P->schema.errorname) {
         free((void *)P->schema.errorname);
     }
     /*
      * P->referer_path in included files points to parent P->path, so
      * don't free it, and don't access it after this point.
      */
     if (P->path) {
         free((void *)P->path);
     }
     fb_scope_table_destroy(&P->schema.root_schema_instance.scope_index,
             __destroy_scope_item, 0);
     /* Destroy last since destructor has references into elem buffer. */
     clear_elem_buffers(P);
     if (P->managed_input) {
         free((void *)P->managed_input);
     }
     memset(P, 0, sizeof(*P));
 }