Multiple schemas parsed by flatc are now parsed independently.

It used to be such that later schemas could depend on earlier
schemas. This was a convenience from days before include files
were implemented. Nowadays they cause subtle bugs rather than being
useful, so this functionality has been removed.

You now need to explicitly include files you depend upon.

Change-Id: Id8292c3c621fc38fbd796da2d2cbdd63efc230d1
Tested: on Linux.

Author: Wouter van Oortmerssen

docs/html/md__compiler.html

@@ -63,8 +63,8 @@
 <div class="contents">
 <div class="textblock"><p>Usage: </p><pre class="fragment">flatc [ GENERATOR OPTIONS ] [ -o PATH ] [ -I PATH ] [ -S ] FILES...
       [ -- FILES...]
-</pre><p>The files are read and parsed in order, and can contain either schemas or data (see below). Later files can make use of definitions in earlier files.</p>
-<p><code>--</code> indicates that the following files are binary files in FlatBuffer format conforming to the schema(s) indicated before it. Incompatible binary files currently will give unpredictable results (!)</p>
+</pre><p>The files are read and parsed in order, and can contain either schemas or data (see below). Data files are processed according to the definitions of the most recent schema specified.</p>
+<p><code>--</code> indicates that the following files are binary files in FlatBuffer format conforming to the schema indicated before it.</p>
 <p>Depending on the flags passed, additional files may be generated for each file processed:</p>
 <p>For any schema input files, one or more generators can be specified:</p>
 <ul>

docs/source/Compiler.md

@@ -6,12 +6,11 @@ Usage:
           [ -- FILES...]
 
 The files are read and parsed in order, and can contain either schemas
-or data (see below). Later files can make use of definitions in earlier
-files.
+or data (see below). Data files are processed according to the definitions of
+the most recent schema specified.
 
 `--` indicates that the following files are binary files in
-FlatBuffer format conforming to the schema(s) indicated before it.
-Incompatible binary files currently will give unpredictable results (!)
+FlatBuffer format conforming to the schema indicated before it.
 
 Depending on the flags passed, additional files may
 be generated for each file processed:

include/flatbuffers/util.h

@@ -174,6 +174,12 @@ inline std::string StripExtension(const std::string &filepath) {
   return i != std::string::npos ? filepath.substr(0, i) : filepath;
 }
 
+// Returns the extension, if any.
+inline std::string GetExtension(const std::string &filepath) {
+  size_t i = filepath.find_last_of(".");
+  return i != std::string::npos ? filepath.substr(i + 1) : "";
+}
+
 // Return the last component of the path, after the last separator.
 inline std::string StripPath(const std::string &filepath) {
   size_t i = filepath.find_last_of(PathSeparatorSet);

src/flatc.cpp

@@ -79,7 +79,8 @@ const Generator generators[] = {
     flatbuffers::GeneralMakeRule },
 };
 
-const char *program_name = NULL;
+const char *program_name = nullptr;
+flatbuffers::Parser *parser = nullptr;
 
 static void Error(const std::string &err, bool usage, bool show_exe_name) {
   if (show_exe_name) printf("%s: ", program_name);
@@ -122,6 +123,7 @@ static void Error(const std::string &err, bool usage, bool show_exe_name) {
       "example: %s -c -b schema1.fbs schema2.fbs data.json\n",
       program_name);
   }
+  if (parser) delete parser;
   exit(1);
 }
 
@@ -205,7 +207,7 @@ int main(int argc, const char *argv[]) {
     Error("no options: specify one of -c -g -j -t -b etc.", true);
 
   // Now process the files:
-  flatbuffers::Parser parser(opts.strict_json, proto_mode);
+  parser = new flatbuffers::Parser(opts.strict_json, proto_mode);
   for (auto file_it = filenames.begin();
             file_it != filenames.end();
           ++file_it) {
@@ -216,8 +218,8 @@ int main(int argc, const char *argv[]) {
       bool is_binary = static_cast<size_t>(file_it - filenames.begin()) >=
                        binary_files_from;
       if (is_binary) {
-        parser.builder_.Clear();
-        parser.builder_.PushFlatBuffer(
+        parser->builder_.Clear();
+        parser->builder_.PushFlatBuffer(
           reinterpret_cast<const uint8_t *>(contents.c_str()),
           contents.length());
         if (!raw_binary) {
@@ -226,30 +228,37 @@ int main(int argc, const char *argv[]) {
           // does not contain a file identifier.
           // We'd expect that typically any binary used as a file would have
           // such an identifier, so by default we require them to match.
-          if (!parser.file_identifier_.length()) {
+          if (!parser->file_identifier_.length()) {
             Error("current schema has no file_identifier: cannot test if \"" +
                  *file_it +
                  "\" matches the schema, use --raw-binary to read this file"
                  " anyway.");
           } else if (!flatbuffers::BufferHasIdentifier(contents.c_str(),
-                                             parser.file_identifier_.c_str())) {
+                                             parser->file_identifier_.c_str())) {
             Error("binary \"" +
                  *file_it +
                  "\" does not have expected file_identifier \"" +
-                 parser.file_identifier_ +
+                 parser->file_identifier_ +
                  "\", use --raw-binary to read this file anyway.");
           }
         }
       } else {
+        if (flatbuffers::GetExtension(*file_it) == "fbs") {
+          // If we're processing multiple schemas, make sure to start each
+          // one from scratch. If it depends on previous schemas it must do
+          // so explicitly using an include.
+          delete parser;
+          parser = new flatbuffers::Parser(opts.strict_json, proto_mode);
+        }
         auto local_include_directory = flatbuffers::StripFileName(*file_it);
         include_directories.push_back(local_include_directory.c_str());
         include_directories.push_back(nullptr);
-        if (!parser.Parse(contents.c_str(), &include_directories[0],
+        if (!parser->Parse(contents.c_str(), &include_directories[0],
                           file_it->c_str()))
-          Error(parser.error_, false, false);
+          Error(parser->error_, false, false);
         if (schema_binary) {
-          parser.Serialize();
-          parser.file_extension_ = reflection::SchemaExtension();
+          parser->Serialize();
+          parser->file_extension_ = reflection::SchemaExtension();
         }
         include_directories.pop_back();
         include_directories.pop_back();
@@ -280,11 +289,8 @@ int main(int argc, const char *argv[]) {
       }
 
       if (proto_mode) GenerateFBS(parser, output_path, filebase, opts);
-
-      // We do not want to generate code for the definitions in this file
-      // in any files coming up next.
-      parser.MarkGenerated();
   }
 
+  delete parser;
   return 0;
 }

src/idl_parser.cpp

@@ -1072,10 +1072,9 @@ bool Parser::SetRootType(const char *name) {
 }
 
 void Parser::MarkGenerated() {
-  // Since the Parser object retains definitions across files, we must
-  // ensure we only output code for definitions once, in the file they are first
-  // declared. This function marks all existing definitions as having already
-  // been generated.
+  // This function marks all existing definitions as having already
+  // been generated, which signals no code for included files should be
+  // generated.
   for (auto it = enums_.vec.begin();
            it != enums_.vec.end(); ++it) {
     (*it)->generated = true;