ParserKeywords split into multiple files

In order to reduce parsing load on clients, the generated ParserKeywords now come in their own sorted-by-first-letter files, so that inclusion of the entire tree is unnecessary when only a handful of keywords will do. This also applies to the generated source code (which is pretty heavy to compile), which now is split into multiple files to enable parallel builds.
2016-01-18 11:53:26 +01:00
parent b73f117f41
commit c2eb0ea934
4 changed files with 120 additions and 59 deletions
--- a/opm/parser/eclipse/CMakeLists.txt
+++ b/opm/parser/eclipse/CMakeLists.txt
@@ -60,8 +60,7 @@ Parser/ParserItem.cpp
 Parser/ParserIntItem.cpp  
 Parser/ParserFloatItem.cpp
 Parser/ParserDoubleItem.cpp
-Parser/ParserStringItem.cpp  
-${PROJECT_BINARY_DIR}/generated-source/ParserKeywords.cpp
+Parser/ParserStringItem.cpp
 )

 set( generator_source
@@ -303,7 +302,7 @@ Utility/EquilWrapper.hpp
 Utility/EndscaleWrapper.hpp
 Utility/ScalecrsWrapper.hpp)

-add_library(buildParser STATIC ${rawdeck_source} ${build_parser_source} ${deck_source} ${unit_source} ${generator_source})
+add_library(buildParser ${rawdeck_source} ${build_parser_source} ${deck_source} ${unit_source} ${generator_source})
 target_link_libraries(buildParser opmjson ${Boost_LIBRARIES}  ${ERT_LIBRARIES})

 #-----------------------------------------------------------------
@@ -324,23 +323,37 @@ target_link_libraries( createDefaultKeywordList buildParser opmjson ${Boost_LIBR
 #    This target will always run - the dependency "management" is
 #    implicitly handled in the createDefaultKeywordList application.

-add_custom_target( generatedCode ALL COMMAND createDefaultKeywordList 
-                   ${PROJECT_SOURCE_DIR}/opm/parser/share/keywords 
-                   ${PROJECT_BINARY_DIR}/generated-source/ParserKeywords.cpp 
-                   ${PROJECT_BINARY_DIR}/generated-source/include/opm/parser/eclipse/Parser/ParserKeywords.hpp
-                   ${PROJECT_BINARY_DIR}/generated-source/inlineKeywordTest.cpp )
+set( generated_source ${PROJECT_BINARY_DIR}/generated-source/ParserKeywords.cpp)
+# to tune the number of files addDefaultKeywords is split into, change
+# generated_num_files. 4 is a very reasonable estimate.
+# TODO: support from command line options?
+set( generated_num_files 4 )
+math( EXPR generated_num_files_end "${generated_num_files} - 1" )
+foreach( n RANGE ${generated_num_files_end} )
+    list(APPEND generated_source ${PROJECT_BINARY_DIR}/generated-source/ParserKeywords${n}.cpp)
+    set_source_files_properties(${PROJECT_BINARY_DIR}/generated-source/ParserKeywords${n}.cpp PROPERTIES GENERATED TRUE)
+endforeach( n )

 set_source_files_properties(${PROJECT_BINARY_DIR}/generated-source/ParserKeywords.cpp PROPERTIES GENERATED TRUE)
 set_source_files_properties(${PROJECT_BINARY_DIR}/generated-source/inlcude/opm/parser/eclipse/Parser/ParserKeywords.hpp PROPERTIES GENERATED TRUE)
 set_source_files_properties(${PROJECT_BINARY_DIR}/generated-source/inlineKeywordTest.cpp PROPERTIES GENERATED TRUE)

+add_custom_target( generatedCode ALL COMMAND createDefaultKeywordList
+                   ${PROJECT_SOURCE_DIR}/opm/parser/share/keywords
+                   ${PROJECT_BINARY_DIR}/generated-source/ParserKeywords.cpp
+                   ${PROJECT_BINARY_DIR}/generated-source/include/
+                   opm/parser/eclipse/Parser/ParserKeywords
+                   ${PROJECT_BINARY_DIR}/generated-source/inlineKeywordTest.cpp
+                   ${generated_num_files}
+                )
+
 opm_add_test( runInlineKeywordTest SOURCES ${PROJECT_BINARY_DIR}/generated-source/inlineKeywordTest.cpp
              LIBRARIES opmparser ${Boost_LIBRARIES}
              DEPENDS generatedCode )

 #-----------------------------------------------------------------

-add_library(opmparser ${rawdeck_source} ${parser_source} ${deck_source} ${state_source} ${unit_source} ${log_source} ${generator_source})
+add_library(opmparser ${generated_source} ${state_source} ${rawdeck_source} ${parser_source} ${deck_source} ${unit_source} ${log_source} ${generator_source})
 add_dependencies( opmparser generatedCode )
 target_link_libraries(opmparser opmjson ${Boost_LIBRARIES}  ${ERT_LIBRARIES})
 set_target_properties(opmparser PROPERTIES VERSION ${opm-parser_VERSION_MAJOR}.${opm-parser_VERSION_MINOR}
@@ -349,8 +362,13 @@ set_target_properties(opmparser PROPERTIES VERSION ${opm-parser_VERSION_MAJOR}.$
 include( ${PROJECT_SOURCE_DIR}/cmake/Modules/install_headers.cmake )   
 install_headers( "${HEADER_FILES}" "${CMAKE_INSTALL_PREFIX}" )
 install( TARGETS opmparser DESTINATION ${CMAKE_INSTALL_LIBDIR} )
-install( FILES ${PROJECT_BINARY_DIR}/generated-source/include/opm/parser/eclipse/Parser/ParserKeywords.hpp DESTINATION "include/opm/parser/eclipse/Parser")

+
+FILE( GLOB generated_header
+        ${PROJECT_BINARY_DIR}/generated-source/include/opm/parser/eclipse/Parser/ParserKeywords.hpp
+        ${PROJECT_BINARY_DIR}/generated-source/include/opm/parser/eclipse/Parser/ParserKeywords/*.hpp
+    )
+install(FILES ${generated_header} DESTINATION include/opm/parser/eclipse/Parser)
 if (ENABLE_PYTHON)
   add_subdirectory( python )
 endif()
--- a/opm/parser/eclipse/Generator/KeywordGenerator.cpp
+++ b/opm/parser/eclipse/Generator/KeywordGenerator.cpp
@@ -73,9 +73,9 @@ namespace Opm {
        return header;
    }

-    std::string KeywordGenerator::headerHeader() {
-        std::string header = "#ifndef PARSER_KEYWORDS_HPP\n"
-            "#define PARSER_KEYWORDS_HPP\n"
+    std::string KeywordGenerator::headerHeader(const std::string& suffix) {
+        std::string header = "#ifndef PARSER_KEYWORDS_" + suffix + "_HPP\n"
+            "#define PARSER_KEYWORDS_" + suffix + "_HPP\n"
            "#include <opm/parser/eclipse/Parser/ParserKeyword.hpp>\n"
            "namespace Opm {\n"
            "namespace ParserKeywords {\n\n";
@@ -112,60 +112,100 @@ namespace Opm {
        return update;
    }

+    static bool write_file( const std::stringstream& stream, const std::string& file, bool verbose, std::string desc = "source" ) {
+        auto update = KeywordGenerator::updateFile( stream, file );
+        if( !verbose ) return update;

-    bool KeywordGenerator::updateSource(const KeywordLoader& loader , const std::string& sourceFile) const {
+        if( update )
+            std::cout << "Updated " << desc << " file written to: " << file << std::endl;
+        else
+            std::cout << "No changes to " << desc << " file: " << file << std::endl;
+
+        return update;
+    }
+
+
+    bool KeywordGenerator::updateSource(const KeywordLoader& loader , const std::string& sourceFile, int blocks ) const {
        std::stringstream newSource;

+        const int keywords = loader.size();
+        const int blocksize = (keywords / blocks) + 1;
+
+        std::vector< std::stringstream > streams( blocks );
+        for( unsigned int i = 0; i < streams.size(); ++i )
+            streams[ i ] << sourceHeader() << std::endl
+                << "void addDefaultKeywords" << i << "(Parser& p) {" << std::endl;
+
+        int bi = 0;
+        for( auto iter = loader.keyword_begin(); iter != loader.keyword_end(); ++iter ) {
+            auto block = bi++ / blocksize;
+            streams[ block ] << "p.addKeyword< ParserKeywords::"
+                << iter->second->className() << " >();" << std::endl;
+        }
+
+        for( auto& stream : streams ) stream << "}}}" << std::endl;
+
+        for( unsigned int i = 0; i < streams.size(); ++i ) {
+            auto srcfile = sourceFile;
+            updateFile( streams[i], srcfile.insert( srcfile.size() - 4, std::to_string( i ) ) );
+        }
+
        newSource << sourceHeader();
        for (auto iter = loader.keyword_begin(); iter != loader.keyword_end(); ++iter) {
            std::shared_ptr<ParserKeyword> keyword = (*iter).second;
            newSource << keyword->createCode() << std::endl;
        }
-        newSource << "}" << std::endl;
-        {
-            newSource << "void Parser::addDefaultKeywords() {" << std::endl;
-            for (auto iter = loader.keyword_begin(); iter != loader.keyword_end(); ++iter) {
-                std::shared_ptr<ParserKeyword> keyword = (*iter).second;
-                newSource << "   addKeyword<ParserKeywords::" << keyword->className() << ">();" << std::endl;
-            }
-            newSource << "}" << std::endl;
-        }
+
+        for( auto i = 0; i < blocks; ++i )
+            newSource << "void addDefaultKeywords" << i << "(Parser& p);" << std::endl;
+
        newSource << "}" << std::endl;

-        {
-            bool update = updateFile( newSource , sourceFile );
-            if (m_verbose) {
-                if (update)
-                    std::cout << "Updated source file written to: " << sourceFile << std::endl;
-                else
-                    std::cout << "No changes to source file: " << sourceFile << std::endl;
-            }
-            return update;
-        }
+        newSource << "void Parser::addDefaultKeywords() {" << std::endl;
+        for( auto i = 0; i < blocks; ++i )
+            newSource << "Opm::ParserKeywords::addDefaultKeywords" << i << "(*this);" << std::endl;
+
+        newSource << "}}" << std::endl;
+
+        return write_file( newSource, sourceFile, m_verbose, "source" );
    }

+    bool KeywordGenerator::updateHeader(const KeywordLoader& loader, const std::string& headerBuildPath, const std::string& headerFile) const {
+        bool update = false;

-    bool KeywordGenerator::updateHeader(const KeywordLoader& loader , const std::string& headerFile) const {
-        std::stringstream stream;
+        std::map< char, std::vector< const ParserKeyword* > > keywords;
+        for( auto iter = loader.keyword_begin(); iter != loader.keyword_end(); ++iter )
+            keywords[ std::toupper( iter->second->className().at(0) ) ].push_back( iter->second.get() );

-        stream << headerHeader();
-        for (auto iter = loader.keyword_begin(); iter != loader.keyword_end(); ++iter) {
-            std::shared_ptr<ParserKeyword> keyword = (*iter).second;
-            stream << keyword->createDeclaration("   ") << std::endl;
+        for( const auto& iter : keywords ) {
+            std::stringstream stream;
+
+            stream << headerHeader( std::string( 1, std::toupper( iter.first ) ) );
+            for( auto& kw : iter.second )
+                stream << kw->createDeclaration("   ") << std::endl;
+
+            stream << "}" << std::endl << "}" << std::endl;
+            stream << "#endif" << std::endl;
+
+            const auto final_path = headerBuildPath + headerFile + "/" + std::string( 1, iter.first ) + ".hpp";
+            if( write_file( stream, final_path, m_verbose, "header" ) )
+                update = true;
        }
-        stream << "}" << std::endl << "}" << std::endl;
+
+        std::stringstream stream;
+        stream << headerHeader("");
+        stream << "}}" << std::endl;
+
+        for( const auto& iter : keywords )
+            stream << "#include <"
+                << headerFile + "/"
+                << std::string( 1, std::toupper( iter.first ) ) + ".hpp>"
+                << std::endl;
+
        stream << "#endif" << std::endl;

-        {
-            bool update = updateFile( stream , headerFile );
-            if (m_verbose) {
-                if (update)
-                    std::cout << "Updated header file written to: " << headerFile << std::endl;
-                else
-                    std::cout << "No changes to header file: " << headerFile << std::endl;
-            }
-            return update;
-        }
+        const auto final_path = headerBuildPath + headerFile + ".hpp";
+        return write_file( stream, final_path, m_verbose, "header" ) || update;
    }


--- a/opm/parser/eclipse/Generator/KeywordGenerator.hpp
+++ b/opm/parser/eclipse/Generator/KeywordGenerator.hpp
@@ -38,11 +38,11 @@ namespace Opm {
        static std::string endTest();
        static std::string startTest(const std::string& test_name);
        static std::string sourceHeader();
-        static std::string headerHeader();
+        static std::string headerHeader( const std::string& );
        static bool updateFile(const std::stringstream& newContent, const std::string& filename);

-        bool updateSource(const KeywordLoader& loader, const std::string& sourceFile) const;
-        bool updateHeader(const KeywordLoader& loader, const std::string& headerFile) const;
+        bool updateSource(const KeywordLoader& loader, const std::string& sourceFile, int ) const;
+        bool updateHeader(const KeywordLoader& loader, const std::string& headerBuildPath, const std::string& headerFile) const;
        bool updateTest(const KeywordLoader& loader , const std::string& testFile) const;

    private:
--- a/opm/parser/eclipse/Parser/createDefaultKeywordList.cpp
+++ b/opm/parser/eclipse/Parser/createDefaultKeywordList.cpp
@@ -4,11 +4,13 @@


 int main(int argc , char ** argv) {
-    if (argc == 5) {
+    if (argc == 7) {
        const char * config_root = argv[1];
        const char * source_file_name = argv[2];
-        const char * header_file_name = argv[3];
-        const char * test_file_name = argv[4];
+        const char * header_file_base_path = argv[3];
+        const char * header_file_name = argv[4];
+        const char * test_file_name = argv[5];
+        const char * output_files = argv[6];
        bool verboseLoader = false;
        bool verboseGenerator = true;

@@ -16,13 +18,14 @@ int main(int argc , char ** argv) {
        Opm::KeywordGenerator generator(verboseGenerator);
        loader.loadMultipleKeywordDirectories( config_root );

-        generator.updateSource(loader , source_file_name );
-        generator.updateHeader(loader , header_file_name );
+        const int num_output_files = std::atoi( output_files );
+        generator.updateSource(loader , source_file_name, num_output_files );
+        generator.updateHeader(loader, header_file_base_path, header_file_name );
        generator.updateTest(loader , test_file_name );

        exit(0);
    } else {
-        std::cerr << "Error calling keyword generator: Expected arguments: <config_root> <source_file_name> <header_file_name> <test_file_name>" << std::endl;
+        std::cerr << "Error calling keyword generator: Expected arguments: <config_root> <source_file_name> <header_build_path> <header_file_name> <test_file_name> <num_output_files>" << std::endl;
        exit(1);
    }
 }