From ec5ae89076cc0b335a49fe2b67130c8141f9113b Mon Sep 17 00:00:00 2001 From: Will Sobel Date: Tue, 18 Apr 2023 17:38:47 +0200 Subject: [PATCH 01/18] before adapter command refactor --- CMakeLists.txt | 2 +- .../source/adapter/shdr/connector.cpp | 17 ++--- .../source/adapter/shdr/shdr_adapter.cpp | 63 ++++++++++++------- .../source/adapter/shdr/shdr_adapter.hpp | 9 +++ test/adapter_test.cpp | 29 +++++++++ test/connector_test.cpp | 9 ++- 6 files changed, 92 insertions(+), 37 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index edc49b19..4f48190f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ set(AGENT_VERSION_MAJOR 2) set(AGENT_VERSION_MINOR 2) set(AGENT_VERSION_PATCH 0) set(AGENT_VERSION_BUILD 1) -set(AGENT_VERSION_RC "_RC3") +set(AGENT_VERSION_RC "_RC4") # This minimum version is to support Visual Studio 2017 and C++ feature checking and FetchContent cmake_minimum_required(VERSION 3.16 FATAL_ERROR) diff --git a/src/mtconnect/source/adapter/shdr/connector.cpp b/src/mtconnect/source/adapter/shdr/connector.cpp index bb144eb1..f8570448 100644 --- a/src/mtconnect/source/adapter/shdr/connector.cpp +++ b/src/mtconnect/source/adapter/shdr/connector.cpp @@ -289,19 +289,12 @@ namespace mtconnect::source::adapter::shdr { LOG(trace) << "(" << m_server << ":" << m_port << ") Received line: " << line; // Check for heartbeats - if (line[0] == '*') + if (line[0] == '*' && !line.compare(0, 6, "* PONG")) { - if (!line.compare(0, 6, "* PONG")) - { - LOG(debug) << "(Port:" << m_localPort << ") Received a PONG for " << m_server << " on port " - << m_port; - if (!m_heartbeats) - startHeartbeats(line); - } - else - { - protocolCommand(line); - } + LOG(debug) << "(Port:" << m_localPort << ") Received a PONG for " << m_server << " on port " + << m_port; + if (!m_heartbeats) + startHeartbeats(line); } else { diff --git a/src/mtconnect/source/adapter/shdr/shdr_adapter.cpp b/src/mtconnect/source/adapter/shdr/shdr_adapter.cpp index 0496ff38..f9d8abf8 100644 --- a/src/mtconnect/source/adapter/shdr/shdr_adapter.cpp +++ b/src/mtconnect/source/adapter/shdr/shdr_adapter.cpp @@ -21,6 +21,10 @@ #include #include +#include +#include +#include + #include #include #include @@ -112,29 +116,25 @@ namespace mtconnect::source::adapter::shdr { { if (data == *m_terminator) { - if (m_handler && m_handler->m_processData) - m_handler->m_processData(m_body.str(), getIdentity()); + forwardData(m_body.str()); m_terminator.reset(); m_body.str(""); } else { - m_body << endl << data; + m_body << std::endl << data; } - - return; } - - if (size_t multi = data.find("--multiline--"); multi != string::npos) + else if (size_t multi = data.find("--multiline--"); multi != std::string::npos) { m_body.str(""); m_body << data.substr(0, multi); m_terminator = data.substr(multi); - return; } - - if (m_handler && m_handler->m_processData) - m_handler->m_processData(data, getIdentity()); + else + { + forwardData(data); + } } catch (std::exception &e) { @@ -163,19 +163,34 @@ namespace mtconnect::source::adapter::shdr { void ShdrAdapter::protocolCommand(const std::string &data) { NAMED_SCOPE("ShdrAdapter::protocolCommand"); - - static auto pattern = regex("\\*[ ]*([^:]+):[ ]*(.+)"); - smatch match; - + using namespace boost::algorithm; - - if (std::regex_match(data, match, pattern)) + namespace qi = boost::spirit::qi; + namespace ascii = boost::spirit::ascii; + namespace phoenix = boost::phoenix; + + using qi::char_; + using qi::lit; + using qi::lexeme; + using ascii::space; + + string command; + auto f = [&command](const auto &s) { + command = string(s.begin(), s.end()); + }; + + auto it = data.begin(); + bool res = qi::phrase_parse(it, data.end(), + ( lit("*") >> lexeme[+(char_ - ':')][f] + >> ':') + , space); + + if (res) { - auto command = to_lower_copy(match[1].str()); - auto value = match[2].str(); - + string value(it, data.end()); + ConfigOptions options; - + if (command == "conversionrequired") options[configuration::ConversionRequired] = is_true(value); else if (command == "relativetime") @@ -186,11 +201,15 @@ namespace mtconnect::source::adapter::shdr { options[configuration::Device] = value; else if (command == "shdrversion") options[configuration::ShdrVersion] = stringToInt(value, 1); - + if (options.size() > 0) setOptions(options); else if (m_handler && m_handler->m_command) m_handler->m_command(data, getIdentity()); } + else + { + LOG(warning) << "protocolCommand: Cannot parse command: " << data; + } } } // namespace mtconnect::source::adapter::shdr diff --git a/src/mtconnect/source/adapter/shdr/shdr_adapter.hpp b/src/mtconnect/source/adapter/shdr/shdr_adapter.hpp index 169d61fe..ce7dc058 100644 --- a/src/mtconnect/source/adapter/shdr/shdr_adapter.hpp +++ b/src/mtconnect/source/adapter/shdr/shdr_adapter.hpp @@ -126,6 +126,15 @@ namespace mtconnect { if (!m_pipeline.started()) m_pipeline.start(); } + + protected: + void forwardData(const std::string &data) + { + if (data[0] == '*') + protocolCommand(data); + else if (m_handler && m_handler->m_processData) + m_handler->m_processData(data, getIdentity()); + } protected: ShdrPipeline m_pipeline; diff --git a/test/adapter_test.cpp b/test/adapter_test.cpp index c30b4ea5..4b75e812 100644 --- a/test/adapter_test.cpp +++ b/test/adapter_test.cpp @@ -75,3 +75,32 @@ Another Line... --multiline--ABC---)DOC"; EXPECT_EQ(exp, data); } + +TEST(AdapterTest, should_forward_multiline_command) +{ + asio::io_context ioc; + asio::io_context::strand strand(ioc); + ConfigOptions options {{configuration::Host, "localhost"s}, {configuration::Port, 7878}}; + boost::property_tree::ptree tree; + pipeline::PipelineContextPtr context = make_shared(); + auto adapter = make_unique(ioc, context, options, tree); + + auto handler = make_unique(); + string data; + handler->m_command = [&](const string &d, const string &s) { data = d; }; + adapter->setHandler(handler); + + adapter->processData("* deviceModel: --multiline--ABC1234"); + EXPECT_TRUE(adapter->getTerminator()); + EXPECT_EQ("--multiline--ABC1234", *adapter->getTerminator()); + adapter->processData(""); + adapter->processData(" "); + adapter->processData(""); + adapter->processData("--multiline--ABC1234"); + + const auto exp = R"DOC(* deviceModel: + + +)DOC"; + EXPECT_EQ(exp, data); +} diff --git a/test/connector_test.cpp b/test/connector_test.cpp index d44fc62d..092cb202 100644 --- a/test/connector_test.cpp +++ b/test/connector_test.cpp @@ -67,8 +67,13 @@ class TestConnector : public Connector void processData(const std::string &data) override { - m_data = data; - m_list.emplace_back(m_data); + if (data[0] == '*') + protocolCommand(data); + else + { + m_data = data; + m_list.emplace_back(m_data); + } } void protocolCommand(const std::string &data) override { m_command = data; } From 9e1db273a529841897c2d1b78e7c6d73f38714e3 Mon Sep 17 00:00:00 2001 From: Will Sobel Date: Tue, 18 Apr 2023 18:06:44 +0200 Subject: [PATCH 02/18] Refactored adapter process commands to pre-parse command and value and pass through --- src/mtconnect/agent.cpp | 26 ++++++------------- .../source/adapter/adapter_pipeline.cpp | 5 ++-- .../source/adapter/adapter_pipeline.hpp | 3 ++- .../source/adapter/shdr/shdr_adapter.cpp | 5 ++-- .../source/adapter/shdr/shdr_adapter.hpp | 5 ++-- test/adapter_test.cpp | 25 ++++++++++++++---- 6 files changed, 39 insertions(+), 30 deletions(-) diff --git a/src/mtconnect/agent.cpp b/src/mtconnect/agent.cpp index 931afd9d..4d976703 100644 --- a/src/mtconnect/agent.cpp +++ b/src/mtconnect/agent.cpp @@ -1026,30 +1026,20 @@ namespace mtconnect { void AgentPipelineContract::deliverCommand(entity::EntityPtr entity) { - static auto pattern = regex("\\*[ ]*([^:]+):[ ]*(.+)"); + auto command = entity->get("command"); auto value = entity->getValue(); - smatch match; - if (std::regex_match(value, match, pattern)) - { - auto device = entity->maybeGet("device"); - auto command = boost::algorithm::to_lower_copy(match[1].str()); - auto param = match[2].str(); - auto source = entity->maybeGet("source"); + auto device = entity->maybeGet("device"); + auto source = entity->maybeGet("source"); - if (!device || !source) - { - LOG(error) << "Invalid command: " << command << ", device or source not specified"; - } - else - { - LOG(debug) << "Processing command: " << command << ": " << value; - m_agent->receiveCommand(*device, command, param, *source); - } + if (!device || !source) + { + LOG(error) << "Invalid command: " << command << ", device or source not specified"; } else { - LOG(warning) << "Cannot parse command: " << value; + LOG(debug) << "Processing command: " << command << ": " << value; + m_agent->receiveCommand(*device, command, value, *source); } } diff --git a/src/mtconnect/source/adapter/adapter_pipeline.cpp b/src/mtconnect/source/adapter/adapter_pipeline.cpp index ce6f83c8..f6e07e97 100644 --- a/src/mtconnect/source/adapter/adapter_pipeline.cpp +++ b/src/mtconnect/source/adapter/adapter_pipeline.cpp @@ -69,9 +69,10 @@ namespace mtconnect { "Message", Properties {{"VALUE", data}, {"topic", topic}, {"source", source}}); run(std::move(entity)); }; - handler->m_command = [this](const std::string &data, const std::string &source) { + handler->m_command = [this](const std::string &command, const std::string &value, const std::string &source) { auto entity = - make_shared("Command", Properties {{"VALUE", data}, {"source", source}}); + make_shared("Command", Properties { + {"command", command }, {"VALUE", value}, {"source", source}}); run(std::move(entity)); }; diff --git a/src/mtconnect/source/adapter/adapter_pipeline.hpp b/src/mtconnect/source/adapter/adapter_pipeline.hpp index 3948c862..13a25f1c 100644 --- a/src/mtconnect/source/adapter/adapter_pipeline.hpp +++ b/src/mtconnect/source/adapter/adapter_pipeline.hpp @@ -26,6 +26,7 @@ namespace mtconnect::source::adapter { struct Handler { using ProcessData = std::function; + using ProcessCommand = std::function; using ProcessMessage = std::function; using Connect = std::function; @@ -33,7 +34,7 @@ namespace mtconnect::source::adapter { /// @brief Process Data Messages ProcessData m_processData; /// @brief Process an adapter command - ProcessData m_command; + ProcessCommand m_command; /// @brief Process a message with a topic ProcessMessage m_processMessage; diff --git a/src/mtconnect/source/adapter/shdr/shdr_adapter.cpp b/src/mtconnect/source/adapter/shdr/shdr_adapter.cpp index f9d8abf8..62d4b94f 100644 --- a/src/mtconnect/source/adapter/shdr/shdr_adapter.cpp +++ b/src/mtconnect/source/adapter/shdr/shdr_adapter.cpp @@ -188,9 +188,10 @@ namespace mtconnect::source::adapter::shdr { if (res) { string value(it, data.end()); - ConfigOptions options; + boost::to_lower(command); + if (command == "conversionrequired") options[configuration::ConversionRequired] = is_true(value); else if (command == "relativetime") @@ -205,7 +206,7 @@ namespace mtconnect::source::adapter::shdr { if (options.size() > 0) setOptions(options); else if (m_handler && m_handler->m_command) - m_handler->m_command(data, getIdentity()); + m_handler->m_command(command, value, getIdentity()); } else { diff --git a/src/mtconnect/source/adapter/shdr/shdr_adapter.hpp b/src/mtconnect/source/adapter/shdr/shdr_adapter.hpp index ce7dc058..1f0660f4 100644 --- a/src/mtconnect/source/adapter/shdr/shdr_adapter.hpp +++ b/src/mtconnect/source/adapter/shdr/shdr_adapter.hpp @@ -122,11 +122,12 @@ namespace mtconnect { { for (auto &o : options) m_options.insert_or_assign(o.first, o.second); + bool started = m_pipeline.started(); m_pipeline.build(m_options); - if (!m_pipeline.started()) + if (!m_pipeline.started() && started) m_pipeline.start(); } - + protected: void forwardData(const std::string &data) { diff --git a/test/adapter_test.cpp b/test/adapter_test.cpp index 4b75e812..996f823d 100644 --- a/test/adapter_test.cpp +++ b/test/adapter_test.cpp @@ -86,8 +86,8 @@ TEST(AdapterTest, should_forward_multiline_command) auto adapter = make_unique(ioc, context, options, tree); auto handler = make_unique(); - string data; - handler->m_command = [&](const string &d, const string &s) { data = d; }; + string command, value; + handler->m_command = [&](const string &c, const string &v, const string &s) { command = c; value = v; }; adapter->setHandler(handler); adapter->processData("* deviceModel: --multiline--ABC1234"); @@ -98,9 +98,24 @@ TEST(AdapterTest, should_forward_multiline_command) adapter->processData(""); adapter->processData("--multiline--ABC1234"); - const auto exp = R"DOC(* deviceModel: - + const auto exp = R"DOC( )DOC"; - EXPECT_EQ(exp, data); + EXPECT_EQ("devicemodel", command); + EXPECT_EQ(exp, value); +} + +TEST(AdapterTest, should_set_options_from_commands) +{ + asio::io_context ioc; + asio::io_context::strand strand(ioc); + ConfigOptions options {{configuration::Host, "localhost"s}, {configuration::Port, 7878}}; + boost::property_tree::ptree tree; + pipeline::PipelineContextPtr context = make_shared(); + auto adapter = make_unique(ioc, context, options, tree); + + adapter->processData("* shdrVersion: 3"); + + auto v = GetOption(adapter->getOptions(), "ShdrVersion"); + ASSERT_EQ(int64_t(3), *v); } From 77a41fe66567097f60d12878d829f9fba276e26a Mon Sep 17 00:00:00 2001 From: Will Sobel Date: Wed, 19 Apr 2023 18:08:16 +0200 Subject: [PATCH 03/18] Added async context to the agent instead of the raw IO context. Allows for pausing of the events from the agent --- src/mtconnect/agent.cpp | 39 +++++++++++++++++-- src/mtconnect/agent.hpp | 12 ++++-- src/mtconnect/configuration/agent_config.cpp | 2 +- src/mtconnect/configuration/agent_config.hpp | 2 +- src/mtconnect/configuration/async_context.hpp | 35 ++++++++++++++++- src/mtconnect/entity/xml_parser.cpp | 3 +- src/mtconnect/entity/xml_parser.hpp | 3 +- src/mtconnect/parser/xml_parser.cpp | 23 +++++++++++ src/mtconnect/parser/xml_parser.hpp | 7 ++++ src/mtconnect/pipeline/shdr_token_mapper.cpp | 2 +- src/mtconnect/source/loopback_source.cpp | 2 +- test/agent_adapter_test.cpp | 3 +- test/agent_test_helper.hpp | 7 +++- test/asset_test.cpp | 4 +- test/component_parameters_test.cpp | 6 +-- test/config_test.cpp | 20 +++++----- test/cutting_tool_test.cpp | 18 ++++----- test/entity_parser_test.cpp | 12 +++--- test/entity_printer_test.cpp | 10 ++--- test/file_asset_test.cpp | 4 +- test/json_printer_asset_test.cpp | 2 +- test/json_printer_test.cpp | 14 +++---- test/mqtt_isolated_test.cpp | 10 +++-- test/qif_document_test.cpp | 6 +-- test/raw_material_test.cpp | 8 ++-- 25 files changed, 180 insertions(+), 74 deletions(-) diff --git a/src/mtconnect/agent.cpp b/src/mtconnect/agent.cpp index 4d976703..4b8d78e4 100644 --- a/src/mtconnect/agent.cpp +++ b/src/mtconnect/agent.cpp @@ -73,7 +73,7 @@ namespace mtconnect { static const string g_available("AVAILABLE"); // Agent public methods - Agent::Agent(boost::asio::io_context &context, const string &deviceXmlPath, + Agent::Agent(config::AsyncContext &context, const string &deviceXmlPath, const ConfigOptions &options) : m_options(options), m_context(context), @@ -360,7 +360,6 @@ namespace mtconnect { return false; } - // Fir the DeviceAdded event for each device bool changed = false; for (auto device : devices) { @@ -386,6 +385,37 @@ namespace mtconnect { throw f; } } + + DevicePtr Agent::loadDevice(const std::string &deviceXml) + { + DevicePtr device; + + try + { + auto printer = dynamic_cast(m_printers["xml"].get()); + device = m_xmlParser->parseDevice(deviceXml, printer); + + bool changed = receiveDevice(device, false); + if (changed) + loadCachedProbe(); + } + catch (runtime_error &e) + { + LOG(fatal) << "Error loading device: " + deviceXml; + LOG(fatal) << "Error detail: " << e.what(); + cerr << e.what() << endl; + throw e; + } + catch (exception &f) + { + LOG(fatal) << "Error loading device: " + deviceXml; + LOG(fatal) << "Error detail: " << f.what(); + cerr << f.what() << endl; + throw f; + } + + return device; + } bool Agent::receiveDevice(device_model::DevicePtr device, bool version) { @@ -1032,7 +1062,7 @@ namespace mtconnect { auto device = entity->maybeGet("device"); auto source = entity->maybeGet("source"); - if (!device || !source) + if ((command != "devicemodel" && !device) || !source) { LOG(error) << "Invalid command: " << command << ", device or source not specified"; } @@ -1319,6 +1349,9 @@ namespace mtconnect { } } }}, + {"devicemodel", [this](DevicePtr device, const string &value) { + loadDevice(value); + }} }; static std::unordered_map adapterDataItems { diff --git a/src/mtconnect/agent.hpp b/src/mtconnect/agent.hpp index 8483855c..1397b17f 100644 --- a/src/mtconnect/agent.hpp +++ b/src/mtconnect/agent.hpp @@ -54,6 +54,7 @@ #include "mtconnect/source/adapter/adapter.hpp" #include "mtconnect/source/loopback_source.hpp" #include "mtconnect/source/source.hpp" +#include "mtconnect/configuration/async_context.hpp" namespace mtconnect { namespace mic = boost::multi_index; @@ -90,7 +91,7 @@ namespace mtconnect { /// - VersionDeviceXmlUpdates /// - JsonVersion /// - DisableAgentDevice - Agent(boost::asio::io_context &context, const std::string &deviceXmlPath, + Agent(configuration::AsyncContext &context, const std::string &deviceXmlPath, const ConfigOptions &options); /// Destructor for the Agent. @@ -257,9 +258,14 @@ namespace mtconnect { /// @param[in] oldName The old name void deviceChanged(DevicePtr device, const std::string &oldUuid, const std::string &oldName); /// @brief Reload the devices from a device file after updates - /// @param deviceFile The device file to load + /// @param[in] deviceFile The device file to load /// @return true if successful bool reloadDevices(const std::string &deviceFile); + + /// @brief receive and parse a single device from a source + /// @param[in] deviceXml the device xml as a string + /// @return device shared pointer if successful + DevicePtr loadDevice(const std::string &deviceXml); /// @name Message when source has connected and disconnected ///@{ @@ -427,7 +433,7 @@ namespace mtconnect { protected: ConfigOptions m_options; - boost::asio::io_context &m_context; + configuration::AsyncContext &m_context; boost::asio::io_context::strand m_strand; std::shared_ptr m_loopback; diff --git a/src/mtconnect/configuration/agent_config.cpp b/src/mtconnect/configuration/agent_config.cpp index 3b546fc1..e2ce878b 100644 --- a/src/mtconnect/configuration/agent_config.cpp +++ b/src/mtconnect/configuration/agent_config.cpp @@ -102,7 +102,7 @@ namespace mtconnect::configuration { boost::log::trivial::logger_type *gAgentLogger = nullptr; AgentConfiguration::AgentConfiguration() - : m_context {make_unique()}, m_monitorTimer(m_context->getContext()) + : m_context {make_unique()}, m_monitorTimer(m_context->get()) { NAMED_SCOPE("AgentConfiguration::AgentConfiguration"); using namespace source; diff --git a/src/mtconnect/configuration/agent_config.hpp b/src/mtconnect/configuration/agent_config.hpp index 7ec330fa..e9bc7b1d 100644 --- a/src/mtconnect/configuration/agent_config.hpp +++ b/src/mtconnect/configuration/agent_config.hpp @@ -114,7 +114,7 @@ namespace mtconnect { /// @brief get the agent associated with the configuration const Agent *getAgent() const { return m_agent.get(); } /// @brief get the boost asio io context - auto &getContext() { return m_context->getContext(); } + auto &getContext() { return m_context->get(); } /// @brief get a pointer to the async io manager auto &getAsyncContext() { return *m_context.get(); } diff --git a/src/mtconnect/configuration/async_context.hpp b/src/mtconnect/configuration/async_context.hpp index 31f35c3f..ced205d9 100644 --- a/src/mtconnect/configuration/async_context.hpp +++ b/src/mtconnect/configuration/async_context.hpp @@ -39,9 +39,13 @@ namespace mtconnect::configuration { /// @brief removes the copy constructor AsyncContext(const AsyncContext &) = delete; ~AsyncContext() {} + + /// @brief Testing only: method to remove the run guard from the context + void removeGuard() { m_guard.reset(); } /// @brief get the boost asio context reference - auto &getContext() { return m_context; } + auto &get() { return m_context; } + /// @brief operator() returns a reference to the io context /// @return the io context operator boost::asio::io_context &() { return m_context; } @@ -126,6 +130,35 @@ namespace mtconnect::configuration { m_guard.emplace(m_context.get_executor()); m_context.restart(); } + + /// @brief Cover methods for asio io_context + /// @{ + + /// @brief io_context::run_for + template + auto run_for(const std::chrono::duration& rel_time) { return m_context.run_for(rel_time); } + + /// @brief io_context::run + auto run() { return m_context.run(); } + + /// @brief io_context::run_one + auto run_one() { return m_context.run_one(); } + + /// @brief io_context::run_one_for + template + auto run_one_for(const std::chrono::duration& rel_time) { return m_context.run_one_for(rel_time); } + + /// @brief io_context::run_one_until + template + auto run_one_until(const std::chrono::time_point& abs_time) { return m_context.run_one_for(abs_time); } + + /// @brief io_context::poll + auto poll() { return m_context.poll(); } + + /// @brief io_context::poll + auto get_executor() BOOST_ASIO_NOEXCEPT { return m_context.get_executor(); } + + /// @} private: void operator=(const AsyncContext &) {} diff --git a/src/mtconnect/entity/xml_parser.cpp b/src/mtconnect/entity/xml_parser.cpp index 1190994b..d700439a 100644 --- a/src/mtconnect/entity/xml_parser.cpp +++ b/src/mtconnect/entity/xml_parser.cpp @@ -234,8 +234,7 @@ namespace mtconnect::entity { return nullptr; } - EntityPtr XmlParser::parse(FactoryPtr factory, const string &document, const string &version, - ErrorList &errors, bool parseNamespaces) + EntityPtr XmlParser::parse(FactoryPtr factory, const string &document, ErrorList &errors, bool parseNamespaces) { NAMED_SCOPE("entity.xml_parser"); EntityPtr entity; diff --git a/src/mtconnect/entity/xml_parser.hpp b/src/mtconnect/entity/xml_parser.hpp index 5c8c346f..ece8917e 100644 --- a/src/mtconnect/entity/xml_parser.hpp +++ b/src/mtconnect/entity/xml_parser.hpp @@ -49,12 +49,11 @@ namespace mtconnect { /// @brief Parse a string document to an entity /// @param factory The factory to use to create the top level entity /// @param document the document as a string - /// @param version version to parse /// @param errors errors that occurred during the parsing /// @param parseNamespaces `true` if namespaces should be parsed /// @return a shared pointer to an entity if successful static EntityPtr parse(FactoryPtr factory, const std::string &document, - const std::string &version, ErrorList &errors, + ErrorList &errors, bool parseNamespaces = true); }; } // namespace entity diff --git a/src/mtconnect/parser/xml_parser.cpp b/src/mtconnect/parser/xml_parser.cpp index b45e60bd..3ce1dc97 100644 --- a/src/mtconnect/parser/xml_parser.cpp +++ b/src/mtconnect/parser/xml_parser.cpp @@ -240,6 +240,29 @@ namespace mtconnect::parser { return deviceList; } + + DevicePtr XmlParser::parseDevice(const std::string &deviceXml, printer::XmlPrinter *aPrinter) + { + DevicePtr device; + + using namespace boost::adaptors; + using namespace boost::range; + + std::unique_lock lock(m_mutex); + + try + { + entity::ErrorList errors; + auto entity = entity::XmlParser::parse(Device::getFactory(), deviceXml, errors); + } + catch (string e) + { + LOG(fatal) << "Cannot parse XML document: " << e; + throw; + } + + return device; + } XmlParser::~XmlParser() { diff --git a/src/mtconnect/parser/xml_parser.hpp b/src/mtconnect/parser/xml_parser.hpp index 6590d708..eff7ac9c 100644 --- a/src/mtconnect/parser/xml_parser.hpp +++ b/src/mtconnect/parser/xml_parser.hpp @@ -49,8 +49,15 @@ namespace mtconnect::parser { /// @brief Parses a file and returns a list of devices /// @param[in] aPath to the file /// @param[in] aPrinter the printer to obtain and set namespaces + /// @returns a list of device pointers std::list parseFile(const std::string &aPath, printer::XmlPrinter *aPrinter); + /// @brief Parses a single device fragment + /// @param[in] deviceXml device xml of a single device + /// @param[in] aPrinter the printer to obtain and set namespaces + /// @returns a shared device pointer if successful + device_model::DevicePtr parseDevice(const std::string &deviceXml, + printer::XmlPrinter *aPrinter); /// @brief Just loads the document, assumed it has already been parsed before. /// @param aDoc the XML document to parse diff --git a/src/mtconnect/pipeline/shdr_token_mapper.cpp b/src/mtconnect/pipeline/shdr_token_mapper.cpp index 5c022236..188848c5 100644 --- a/src/mtconnect/pipeline/shdr_token_mapper.cpp +++ b/src/mtconnect/pipeline/shdr_token_mapper.cpp @@ -270,7 +270,7 @@ namespace mtconnect { auto body = *token++; XmlParser parser; - res = parser.parse(Asset::getRoot(), body, "2.0", errors); + res = parser.parse(Asset::getRoot(), body, errors); if (auto asset = dynamic_pointer_cast(res)) { asset->setAssetId(assetId); diff --git a/src/mtconnect/source/loopback_source.cpp b/src/mtconnect/source/loopback_source.cpp index 05acd1de..9befdcf1 100644 --- a/src/mtconnect/source/loopback_source.cpp +++ b/src/mtconnect/source/loopback_source.cpp @@ -112,7 +112,7 @@ namespace mtconnect::source { entity::ErrorList &errors) { // Parse the asset - auto entity = entity::XmlParser::parse(asset::Asset::getRoot(), document, "1.7", errors); + auto entity = entity::XmlParser::parse(asset::Asset::getRoot(), document, errors); if (!entity) { LOG(warning) << "Asset could not be parsed"; diff --git a/test/agent_adapter_test.cpp b/test/agent_adapter_test.cpp index 386e2787..c5f2a7ae 100644 --- a/test/agent_adapter_test.cpp +++ b/test/agent_adapter_test.cpp @@ -121,7 +121,8 @@ class AgentAdapterTest : public testing::Test } m_agentTestHelper->getAgent()->stop(); - m_agentTestHelper->m_ioContext.run_for(100s); + m_agentTestHelper->m_ioContext.removeGuard(); + m_agentTestHelper->m_ioContext.run_for(10s); m_agentTestHelper.reset(); m_adapter.reset(); diff --git a/test/agent_test_helper.hpp b/test/agent_test_helper.hpp index bb83eac2..86baf7da 100644 --- a/test/agent_test_helper.hpp +++ b/test/agent_test_helper.hpp @@ -117,7 +117,9 @@ class AgentTestHelper public: using Hook = std::function; - AgentTestHelper() : m_incomingIp("127.0.0.1"), m_strand(m_ioContext), m_socket(m_ioContext) {} + AgentTestHelper() : m_incomingIp("127.0.0.1"), m_strand(m_ioContext), m_socket(m_ioContext) + { + } ~AgentTestHelper() { @@ -127,6 +129,7 @@ class AgentTestHelper if (m_agent) m_agent->stop(); m_agent.reset(); + m_ioContext.stop(); } auto session() { return m_session; } @@ -309,7 +312,7 @@ class AgentTestHelper std::unique_ptr m_agent; std::stringstream m_out; mtconnect::sink::rest_sink::RequestPtr m_request; - boost::asio::io_context m_ioContext; + mtconnect::configuration::AsyncContext m_ioContext; boost::asio::io_context::strand m_strand; boost::asio::ip::tcp::socket m_socket; mtconnect::sink::rest_sink::Response m_response; diff --git a/test/asset_test.cpp b/test/asset_test.cpp index 82d3aebf..49e383f3 100644 --- a/test/asset_test.cpp +++ b/test/asset_test.cpp @@ -74,7 +74,7 @@ TEST_F(AssetTest, TestExtendedAsset) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(Asset::getRoot(), doc, "1.7", errors); + auto entity = parser.parse(Asset::getRoot(), doc, errors); ASSERT_EQ(0, errors.size()); auto asset = dynamic_cast(entity.get()); ASSERT_NE(nullptr, asset); @@ -102,7 +102,7 @@ TEST_F(AssetTest, asset_should_parse_and_load_if_asset_id_is_missing_from_xml) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(Asset::getRoot(), doc, "1.7", errors); + auto entity = parser.parse(Asset::getRoot(), doc, errors); ASSERT_EQ(0, errors.size()); auto asset = dynamic_cast(entity.get()); ASSERT_NE(nullptr, asset); diff --git a/test/component_parameters_test.cpp b/test/component_parameters_test.cpp index 61fc0c16..5b5c5294 100644 --- a/test/component_parameters_test.cpp +++ b/test/component_parameters_test.cpp @@ -80,7 +80,7 @@ TEST_F(ComponentParametersTest, should_parse_simple_parameter_set) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(Asset::getRoot(), doc, "2.2", errors); + auto entity = parser.parse(Asset::getRoot(), doc, errors); ASSERT_EQ(0, errors.size()); auto asset = dynamic_cast(entity.get()); @@ -133,7 +133,7 @@ TEST_F(ComponentParametersTest, should_parse_simple_parameter_set) hdoc.insert(68, string(" hash=\"" + hash1 + "\"")); ASSERT_EQ(content, hdoc); - auto entity2 = parser.parse(Asset::getRoot(), content, "2.2", errors); + auto entity2 = parser.parse(Asset::getRoot(), content, errors); ASSERT_EQ(hash1, entity2->hash()); @@ -162,7 +162,7 @@ TEST_F(ComponentParametersTest, should_parse_two_parameter_sets) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(Asset::getRoot(), doc, "2.2", errors); + auto entity = parser.parse(Asset::getRoot(), doc, errors); ASSERT_EQ(0, errors.size()); auto asset = dynamic_cast(entity.get()); diff --git a/test/config_test.cpp b/test/config_test.cpp index 00d7b690..630eb8b5 100644 --- a/test/config_test.cpp +++ b/test/config_test.cpp @@ -993,7 +993,7 @@ Port = 0 ASSERT_TRUE(dataItem); ASSERT_EQ("SPINDLE_SPEED", dataItem->getType()); - boost::asio::steady_timer timer1(context.getContext()); + boost::asio::steady_timer timer1(context.get()); timer1.expires_from_now(1s); timer1.async_wait([this, &devices, agent](boost::system::error_code ec) { if (ec) @@ -1012,7 +1012,7 @@ Port = 0 } }); - boost::asio::steady_timer timer2(context.getContext()); + boost::asio::steady_timer timer2(context.get()); timer2.expires_from_now(6s); timer2.async_wait([this, agent, &chg](boost::system::error_code ec) { if (!ec) @@ -1072,7 +1072,7 @@ Port = 0 ASSERT_TRUE(dataItem); ASSERT_EQ("SPINDLE_SPEED", dataItem->getType()); - boost::asio::steady_timer timer1(context.getContext()); + boost::asio::steady_timer timer1(context.get()); timer1.expires_from_now(1s); timer1.async_wait([this, &devices](boost::system::error_code ec) { if (ec) @@ -1085,7 +1085,7 @@ Port = 0 } }); - boost::asio::steady_timer timer2(context.getContext()); + boost::asio::steady_timer timer2(context.get()); timer2.expires_from_now(6s); timer2.async_wait([this, &chg](boost::system::error_code ec) { if (!ec) @@ -1138,7 +1138,7 @@ Port = 0 auto instance = rest->instanceId(); - boost::asio::steady_timer timer1(context.getContext()); + boost::asio::steady_timer timer1(context.get()); timer1.expires_from_now(1s); timer1.async_wait([this, &config](boost::system::error_code ec) { if (ec) @@ -1154,7 +1154,7 @@ Port = 0 auto th = thread([this, agent, instance, &context]() { this_thread::sleep_for(5s); - boost::asio::steady_timer timer1(context.getContext()); + boost::asio::steady_timer timer1(context.get()); timer1.expires_from_now(1s); timer1.async_wait([this, agent, instance](boost::system::error_code ec) { if (!ec) @@ -1215,7 +1215,7 @@ Port = 0 DataItemPtr di; - boost::asio::steady_timer timer1(context.getContext()); + boost::asio::steady_timer timer1(context.get()); timer1.expires_from_now(1s); timer1.async_wait([this, &devices](boost::system::error_code ec) { if (ec) @@ -1229,7 +1229,7 @@ Port = 0 } }); - boost::asio::steady_timer timer2(context.getContext()); + boost::asio::steady_timer timer2(context.get()); timer2.expires_from_now(6s); timer2.async_wait([this](boost::system::error_code ec) { if (!ec) @@ -1332,7 +1332,7 @@ Port = 0 ASSERT_NE(nullptr, printer); ASSERT_EQ("1.2", *printer->getSchemaVersion()); - boost::asio::steady_timer timer1(context.getContext()); + boost::asio::steady_timer timer1(context.get()); timer1.expires_from_now(1s); timer1.async_wait([this, &devices, agent](boost::system::error_code ec) { if (ec) @@ -1355,7 +1355,7 @@ Port = 0 auto th = thread([this, agent, instance, &context]() { this_thread::sleep_for(5s); - boost::asio::steady_timer timer1(context.getContext()); + boost::asio::steady_timer timer1(context.get()); timer1.expires_from_now(1s); timer1.async_wait([this, agent, instance](boost::system::error_code ec) { if (!ec) diff --git a/test/cutting_tool_test.cpp b/test/cutting_tool_test.cpp index db06c5d0..85e7afe1 100644 --- a/test/cutting_tool_test.cpp +++ b/test/cutting_tool_test.cpp @@ -101,7 +101,7 @@ TEST_F(CuttingToolTest, TestMinmalArchetype) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(Asset::getRoot(), doc, "1.7", errors); + auto entity = parser.parse(Asset::getRoot(), doc, errors); ASSERT_EQ(0, errors.size()); auto asset = dynamic_cast(entity.get()); @@ -160,7 +160,7 @@ TEST_F(CuttingToolTest, TestMeasurements) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(Asset::getRoot(), doc, "1.7", errors); + auto entity = parser.parse(Asset::getRoot(), doc, errors); ASSERT_EQ(0, errors.size()); auto asset = dynamic_cast(entity.get()); @@ -235,7 +235,7 @@ TEST_F(CuttingToolTest, TestItems) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(Asset::getRoot(), doc, "1.7", errors); + auto entity = parser.parse(Asset::getRoot(), doc, errors); ASSERT_EQ(0, errors.size()); auto asset = dynamic_cast(entity.get()); @@ -354,7 +354,7 @@ TEST_F(CuttingToolTest, TestMinmalTool) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(Asset::getRoot(), doc, "1.7", errors); + auto entity = parser.parse(Asset::getRoot(), doc, errors); ASSERT_EQ(0, errors.size()); auto asset = dynamic_cast(entity.get()); @@ -410,7 +410,7 @@ TEST_F(CuttingToolTest, TestMinmalToolError) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(Asset::getRoot(), doc, "1.7", errors); + auto entity = parser.parse(Asset::getRoot(), doc, errors); ASSERT_EQ(2, errors.size()); ASSERT_EQ( "CuttingToolLifeCycle(CutterStatus): Property CutterStatus is required and not provided", @@ -437,7 +437,7 @@ TEST_F(CuttingToolTest, TestMeasurementsError) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(Asset::getRoot(), doc, "1.7", errors); + auto entity = parser.parse(Asset::getRoot(), doc, errors); ASSERT_EQ(6, errors.size()); auto it = errors.begin(); EXPECT_EQ("FunctionalLength(VALUE): Property VALUE is required and not provided", @@ -522,7 +522,7 @@ TEST_F(CuttingToolTest, test_extended_cutting_item) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(Asset::getRoot(), doc, "1.7", errors); + auto entity = parser.parse(Asset::getRoot(), doc, errors); ASSERT_EQ(0, errors.size()); auto asset = dynamic_cast(entity.get()); @@ -663,7 +663,7 @@ TEST_F(CuttingToolTest, test_xmlns_with_top_element_alias) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(Asset::getRoot(), doc, "1.7", errors); + auto entity = parser.parse(Asset::getRoot(), doc, errors); ASSERT_EQ(0, errors.size()); // Round trip test @@ -728,7 +728,7 @@ TEST_F(CuttingToolTest, element_order_should_place_cutter_status_before_locus) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(Asset::getRoot(), doc, "2.0", errors); + auto entity = parser.parse(Asset::getRoot(), doc, errors); ASSERT_EQ(0, errors.size()); // Round trip test diff --git a/test/entity_parser_test.cpp b/test/entity_parser_test.cpp index 91a8f029..1c130b87 100644 --- a/test/entity_parser_test.cpp +++ b/test/entity_parser_test.cpp @@ -118,7 +118,7 @@ TEST_F(EntityParserTest, TestParseSimpleDocument) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(root, doc, "1.7", errors); + auto entity = parser.parse(root, doc, errors); ASSERT_EQ(0, errors.size()); ASSERT_EQ("FileArchetype", entity->getName()); @@ -163,7 +163,7 @@ TEST_F(EntityParserTest, TestRecursiveEntityLists) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(root, doc, "1.7", errors); + auto entity = parser.parse(root, doc, errors); ASSERT_EQ(0, errors.size()); ASSERT_EQ("Device", entity->getName()); @@ -212,7 +212,7 @@ TEST_F(EntityParserTest, TestRecursiveEntityListFailure) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(root, doc, "1.7", errors); + auto entity = parser.parse(root, doc, errors); ASSERT_EQ(1, errors.size()); ASSERT_FALSE(entity); ASSERT_EQ(string("Device(uuid): Property uuid is required and not provided"), @@ -236,7 +236,7 @@ TEST_F(EntityParserTest, TestRecursiveEntityListMissingComponents) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(root, doc, "1.7", errors); + auto entity = parser.parse(root, doc, errors); ASSERT_EQ(2, errors.size()); ASSERT_TRUE(entity); ASSERT_EQ(string("Components(Component): Entity list requirement Component must have at least 1 " @@ -280,7 +280,7 @@ TEST_F(EntityParserTest, TestRawContent) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(root, doc, "1.7", errors); + auto entity = parser.parse(root, doc, errors); auto expected = R"DOC( And some text @@ -310,7 +310,7 @@ TEST_F(EntityParserTest, check_proper_line_truncation) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(root, doc, "1.7", errors); + auto entity = parser.parse(root, doc, errors); ASSERT_EQ("Description", entity->getName()); ASSERT_EQ("And some text", entity->getValue()); } diff --git a/test/entity_printer_test.cpp b/test/entity_printer_test.cpp index d8b844b5..20caa8c1 100644 --- a/test/entity_printer_test.cpp +++ b/test/entity_printer_test.cpp @@ -103,7 +103,7 @@ TEST_F(EntityPrinterTest, TestParseSimpleDocument) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(root, doc, "1.7", errors); + auto entity = parser.parse(root, doc, errors); ASSERT_EQ(0, errors.size()); entity::XmlPrinter printer; @@ -129,7 +129,7 @@ TEST_F(EntityPrinterTest, TestFileArchetypeWithDescription) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(root, doc, "1.7", errors); + auto entity = parser.parse(root, doc, errors); ASSERT_EQ(0, errors.size()); entity::XmlPrinter printer; @@ -176,7 +176,7 @@ TEST_F(EntityPrinterTest, TestRecursiveEntityLists) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(root, doc, "1.7", errors); + auto entity = parser.parse(root, doc, errors); ASSERT_EQ(0, errors.size()); entity::XmlPrinter printer; @@ -222,7 +222,7 @@ TEST_F(EntityPrinterTest, TestEntityOrder) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(root, doc, "1.7", errors); + auto entity = parser.parse(root, doc, errors); ASSERT_EQ(0, errors.size()); entity::XmlPrinter printer; @@ -264,7 +264,7 @@ TEST_F(EntityPrinterTest, TestRawContent) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(root, doc, "1.7", errors); + auto entity = parser.parse(root, doc, errors); ASSERT_EQ(0, errors.size()); entity::XmlPrinter printer; diff --git a/test/file_asset_test.cpp b/test/file_asset_test.cpp index 5a8c0aba..a2d57c93 100644 --- a/test/file_asset_test.cpp +++ b/test/file_asset_test.cpp @@ -83,7 +83,7 @@ TEST_F(FileAssetTest, TestMinmalArchetype) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(Asset::getRoot(), doc, "1.7", errors); + auto entity = parser.parse(Asset::getRoot(), doc, errors); ASSERT_EQ(0, errors.size()); auto asset = dynamic_cast(entity.get()); @@ -141,7 +141,7 @@ TEST_F(FileAssetTest, TestMinmalFile) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(Asset::getRoot(), doc, "1.7", errors); + auto entity = parser.parse(Asset::getRoot(), doc, errors); ASSERT_EQ(0, errors.size()); auto asset = dynamic_cast(entity.get()); diff --git a/test/json_printer_asset_test.cpp b/test/json_printer_asset_test.cpp index 3ae3dfd8..b58be6e8 100644 --- a/test/json_printer_asset_test.cpp +++ b/test/json_printer_asset_test.cpp @@ -76,7 +76,7 @@ class JsonPrinterAssetTest : public testing::Test AssetPtr parseAsset(const std::string &xml, entity::ErrorList &errors) { - auto entity = m_parser->parse(Asset::getRoot(), xml, "1.7", errors); + auto entity = m_parser->parse(Asset::getRoot(), xml, errors); AssetPtr asset; for (auto &error : errors) { diff --git a/test/json_printer_test.cpp b/test/json_printer_test.cpp index f86a78ea..90912b3d 100644 --- a/test/json_printer_test.cpp +++ b/test/json_printer_test.cpp @@ -167,7 +167,7 @@ TEST_F(JsonPrinterTest, Header) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(root, doc, "1.7", errors); + auto entity = parser.parse(root, doc, errors); ASSERT_EQ(0, errors.size()); entity::JsonEntityPrinter jprinter(1, true); @@ -190,7 +190,7 @@ TEST_F(JsonPrinterTest, Devices) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(root, doc, "1.7", errors); + auto entity = parser.parse(root, doc, errors); ASSERT_EQ(0, errors.size()); entity::JsonEntityPrinter jprinter(1, true); @@ -212,7 +212,7 @@ TEST_F(JsonPrinterTest, Components) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(root, doc, "1.7", errors); + auto entity = parser.parse(root, doc, errors); ASSERT_EQ(0, errors.size()); entity::JsonEntityPrinter jprinter(1, true); @@ -243,7 +243,7 @@ TEST_F(JsonPrinterTest, TopLevelDataItems) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(root, doc, "1.7", errors); + auto entity = parser.parse(root, doc, errors); ASSERT_EQ(0, errors.size()); entity::JsonEntityPrinter jprinter(1, true); @@ -265,7 +265,7 @@ TEST_F(JsonPrinterTest, data_items_using_version_2) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(root, doc, "1.7", errors); + auto entity = parser.parse(root, doc, errors); ASSERT_EQ(0, errors.size()); entity::JsonEntityPrinter jprinter(2, true); @@ -302,7 +302,7 @@ TEST_F(JsonPrinterTest, ElementListWithProperty) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(root, doc, "1.7", errors); + auto entity = parser.parse(root, doc, errors); ASSERT_EQ(0, errors.size()); entity::JsonEntityPrinter jprinter(1, true); @@ -340,7 +340,7 @@ TEST_F(JsonPrinterTest, elements_with_property_list_version_2) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(root, doc, "1.7", errors); + auto entity = parser.parse(root, doc, errors); ASSERT_EQ(0, errors.size()); entity::JsonEntityPrinter jprinter(2, true); diff --git a/test/mqtt_isolated_test.cpp b/test/mqtt_isolated_test.cpp index b5db41c4..3bfaf993 100644 --- a/test/mqtt_isolated_test.cpp +++ b/test/mqtt_isolated_test.cpp @@ -220,7 +220,7 @@ TEST_F(MqttIsolatedUnitTest, mqtt_tcp_client_should_receive_loopback_publication std::uint16_t pid_sub1; - auto client = mqtt::make_async_client(m_agentTestHelper->m_ioContext, "localhost", m_port); + auto client = mqtt::make_async_client(m_agentTestHelper->m_ioContext.get(), "localhost", m_port); client->set_client_id("cliendId1"); client->set_clean_session(true); @@ -320,7 +320,8 @@ TEST_F(MqttIsolatedUnitTest, mqtt_tcp_client_should_receive_loopback_publication client->async_connect(); - m_agentTestHelper->m_ioContext.run(); + m_agentTestHelper->m_ioContext.removeGuard(); + m_agentTestHelper->m_ioContext.run_for(2s); ASSERT_TRUE(received); } @@ -446,7 +447,7 @@ TEST_F(MqttIsolatedUnitTest, mqtt_tcp_client_authentication) std::uint16_t pid_sub1; - auto client = mqtt::make_async_client(m_agentTestHelper->m_ioContext, "localhost", m_port); + auto client = mqtt::make_async_client(m_agentTestHelper->m_ioContext.get(), "localhost", m_port); client->set_client_id("cliendId1"); client->set_clean_session(true); @@ -545,7 +546,8 @@ TEST_F(MqttIsolatedUnitTest, mqtt_tcp_client_authentication) client->async_connect(); - m_agentTestHelper->m_ioContext.run(); + m_agentTestHelper->m_ioContext.removeGuard(); + m_agentTestHelper->m_ioContext.run_for(2s); ASSERT_TRUE(received); } diff --git a/test/qif_document_test.cpp b/test/qif_document_test.cpp index 84af60bb..8955bd71 100644 --- a/test/qif_document_test.cpp +++ b/test/qif_document_test.cpp @@ -117,7 +117,7 @@ TEST_F(QIFDocumentTest, minimal_qif_definition) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(Asset::getRoot(), doc, "2.0", errors); + auto entity = parser.parse(Asset::getRoot(), doc, errors); ASSERT_EQ(0, errors.size()); auto asset = dynamic_cast(entity.get()); @@ -183,7 +183,7 @@ TEST_F(QIFDocumentTest, test_qif_xml_round_trip) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(Asset::getRoot(), doc, "2.0", errors); + auto entity = parser.parse(Asset::getRoot(), doc, errors); ASSERT_EQ(0, errors.size()); auto asset = dynamic_cast(entity.get()); @@ -227,7 +227,7 @@ TEST_F(QIFDocumentTest, should_generate_json) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(Asset::getRoot(), doc, "2.0", errors); + auto entity = parser.parse(Asset::getRoot(), doc, errors); ASSERT_EQ(0, errors.size()); auto asset = dynamic_cast(entity.get()); diff --git a/test/raw_material_test.cpp b/test/raw_material_test.cpp index d6c600fe..558dd262 100644 --- a/test/raw_material_test.cpp +++ b/test/raw_material_test.cpp @@ -101,7 +101,7 @@ TEST_F(RawMaterialTest, minimal_raw_material_definition) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(Asset::getRoot(), doc, "2.0", errors); + auto entity = parser.parse(Asset::getRoot(), doc, errors); ASSERT_EQ(0, errors.size()); auto asset = dynamic_cast(entity.get()); @@ -143,7 +143,7 @@ TEST_F(RawMaterialTest, should_parse_raw_material_and_material) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(Asset::getRoot(), doc, "2.0", errors); + auto entity = parser.parse(Asset::getRoot(), doc, errors); ASSERT_EQ(0, errors.size()); auto asset = dynamic_cast(entity.get()); @@ -194,7 +194,7 @@ TEST_F(RawMaterialTest, should_round_trip_xml) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(Asset::getRoot(), doc, "2.0", errors); + auto entity = parser.parse(Asset::getRoot(), doc, errors); ASSERT_EQ(0, errors.size()); entity::XmlPrinter printer; @@ -233,7 +233,7 @@ TEST_F(RawMaterialTest, should_generate_json) ErrorList errors; entity::XmlParser parser; - auto entity = parser.parse(Asset::getRoot(), doc, "2.0", errors); + auto entity = parser.parse(Asset::getRoot(), doc, errors); ASSERT_EQ(0, errors.size()); entity::JsonEntityPrinter jsonPrinter(1, true); From f4ef019cd4e4e6c9570df7a6a0162947d5bfcd7c Mon Sep 17 00:00:00 2001 From: Will Sobel Date: Thu, 20 Apr 2023 23:10:27 +0200 Subject: [PATCH 04/18] Fixed issues with adding device to empty device set. --- samples/empty.xml | 9 ++ src/mtconnect/agent.cpp | 76 +++++++++------- src/mtconnect/agent.hpp | 15 ++-- src/mtconnect/configuration/agent_config.cpp | 6 +- src/mtconnect/configuration/agent_config.hpp | 2 +- src/mtconnect/configuration/async_context.hpp | 29 +++--- src/mtconnect/entity/xml_parser.cpp | 3 +- src/mtconnect/entity/xml_parser.hpp | 3 +- src/mtconnect/parser/xml_parser.cpp | 62 ++++++++----- src/mtconnect/pipeline/guard.hpp | 6 +- src/mtconnect/pipeline/pipeline.hpp | 12 +-- src/mtconnect/pipeline/pipeline_context.hpp | 2 +- .../source/adapter/adapter_pipeline.cpp | 8 +- .../source/adapter/adapter_pipeline.hpp | 3 +- .../source/adapter/shdr/shdr_adapter.cpp | 33 +++---- .../source/adapter/shdr/shdr_adapter.hpp | 4 +- src/mtconnect/source/source.hpp | 4 + test/adapter_test.cpp | 9 +- test/agent_test_helper.hpp | 4 +- test/config_test.cpp | 90 +++++++++++++++++++ 20 files changed, 256 insertions(+), 124 deletions(-) create mode 100644 samples/empty.xml diff --git a/samples/empty.xml b/samples/empty.xml new file mode 100644 index 00000000..5e336730 --- /dev/null +++ b/samples/empty.xml @@ -0,0 +1,9 @@ + + +
+ + diff --git a/src/mtconnect/agent.cpp b/src/mtconnect/agent.cpp index 4b8d78e4..765afaf9 100644 --- a/src/mtconnect/agent.cpp +++ b/src/mtconnect/agent.cpp @@ -385,36 +385,43 @@ namespace mtconnect { throw f; } } - - DevicePtr Agent::loadDevice(const std::string &deviceXml) + + void Agent::loadDevice(const string &deviceXml, + const optional source) { - DevicePtr device; - - try - { - auto printer = dynamic_cast(m_printers["xml"].get()); - device = m_xmlParser->parseDevice(deviceXml, printer); - - bool changed = receiveDevice(device, false); - if (changed) - loadCachedProbe(); - } - catch (runtime_error &e) - { - LOG(fatal) << "Error loading device: " + deviceXml; - LOG(fatal) << "Error detail: " << e.what(); - cerr << e.what() << endl; - throw e; - } - catch (exception &f) - { - LOG(fatal) << "Error loading device: " + deviceXml; - LOG(fatal) << "Error detail: " << f.what(); - cerr << f.what() << endl; - throw f; - } - - return device; + m_context.pause([=](config::AsyncContext &context) { + try + { + auto printer = dynamic_cast(m_printers["xml"].get()); + auto device = m_xmlParser->parseDevice(deviceXml, printer); + + if (device) + { + bool changed = receiveDevice(device, true); + if (changed) + loadCachedProbe(); + + if (source) + { + auto s = findSource(*source); + if (s) + s->setOptions({{ config::Device, device->getName() }}); + } + } + } + catch (runtime_error &e) + { + LOG(error) << "Error loading device: " + deviceXml; + LOG(error) << "Error detail: " << e.what(); + cerr << e.what() << endl; + } + catch (exception &f) + { + LOG(error) << "Error loading device: " + deviceXml; + LOG(error) << "Error detail: " << f.what(); + cerr << f.what() << endl; + } + }); } bool Agent::receiveDevice(device_model::DevicePtr device, bool version) @@ -1348,11 +1355,8 @@ namespace mtconnect { di->setConverter(conv); } } - }}, - {"devicemodel", [this](DevicePtr device, const string &value) { - loadDevice(value); - }} - }; + }} + }; static std::unordered_map adapterDataItems { {"adapterversion", "_adapter_software_version"}, @@ -1372,6 +1376,10 @@ namespace mtconnect { deviceChanged(device, oldUuid, oldName); } } + else if (command == "devicemodel") + { + loadDevice(value, source); + } else { auto action = deviceCommands.find(command); diff --git a/src/mtconnect/agent.hpp b/src/mtconnect/agent.hpp index 1397b17f..2c0e82b4 100644 --- a/src/mtconnect/agent.hpp +++ b/src/mtconnect/agent.hpp @@ -40,6 +40,7 @@ #include "mtconnect/buffer/checkpoint.hpp" #include "mtconnect/buffer/circular_buffer.hpp" #include "mtconnect/config.hpp" +#include "mtconnect/configuration/async_context.hpp" #include "mtconnect/configuration/hook_manager.hpp" #include "mtconnect/configuration/service.hpp" #include "mtconnect/device_model/agent_device.hpp" @@ -54,7 +55,6 @@ #include "mtconnect/source/adapter/adapter.hpp" #include "mtconnect/source/loopback_source.hpp" #include "mtconnect/source/source.hpp" -#include "mtconnect/configuration/async_context.hpp" namespace mtconnect { namespace mic = boost::multi_index; @@ -168,14 +168,15 @@ namespace mtconnect { // Source and Sink /// @brief Find a source by name - /// @param[in] name the name to find + /// @param[in] name the identity to find /// @return A shared pointer to the source if found, otherwise nullptr source::SourcePtr findSource(const std::string &name) const { for (auto &s : m_sources) - if (s->getIdentity() == name) + { + if (s->getIdentity() == name || s->getName() == name) return s; - + } return nullptr; } /// @brief Find a sink by name @@ -261,11 +262,11 @@ namespace mtconnect { /// @param[in] deviceFile The device file to load /// @return true if successful bool reloadDevices(const std::string &deviceFile); - + /// @brief receive and parse a single device from a source /// @param[in] deviceXml the device xml as a string - /// @return device shared pointer if successful - DevicePtr loadDevice(const std::string &deviceXml); + /// @param[in] source the source loading the device + void loadDevice(const std::string &deviceXml, const std::optional source = std::nullopt); /// @name Message when source has connected and disconnected ///@{ diff --git a/src/mtconnect/configuration/agent_config.cpp b/src/mtconnect/configuration/agent_config.cpp index e2ce878b..ff4ea577 100644 --- a/src/mtconnect/configuration/agent_config.cpp +++ b/src/mtconnect/configuration/agent_config.cpp @@ -896,10 +896,14 @@ namespace mtconnect::configuration { adapterOptions, ptree {}); m_agent->addSource(source, false); } - else + else if (m_agent->getDevices().size() > 1) { throw runtime_error("Adapters must be defined if more than one device is present"); } + else + { + LOG(warning) << "Starting with no devices or adapters"; + } } #ifdef WITH_PYTHON diff --git a/src/mtconnect/configuration/agent_config.hpp b/src/mtconnect/configuration/agent_config.hpp index e9bc7b1d..32fcffba 100644 --- a/src/mtconnect/configuration/agent_config.hpp +++ b/src/mtconnect/configuration/agent_config.hpp @@ -112,7 +112,7 @@ namespace mtconnect { /// @param[in] agent the agent the configuration will take ownership of void setAgent(std::unique_ptr &agent) { m_agent = std::move(agent); } /// @brief get the agent associated with the configuration - const Agent *getAgent() const { return m_agent.get(); } + Agent *getAgent() const { return m_agent.get(); } /// @brief get the boost asio io context auto &getContext() { return m_context->get(); } /// @brief get a pointer to the async io manager diff --git a/src/mtconnect/configuration/async_context.hpp b/src/mtconnect/configuration/async_context.hpp index ced205d9..97bc6c37 100644 --- a/src/mtconnect/configuration/async_context.hpp +++ b/src/mtconnect/configuration/async_context.hpp @@ -39,13 +39,13 @@ namespace mtconnect::configuration { /// @brief removes the copy constructor AsyncContext(const AsyncContext &) = delete; ~AsyncContext() {} - + /// @brief Testing only: method to remove the run guard from the context void removeGuard() { m_guard.reset(); } /// @brief get the boost asio context reference auto &get() { return m_context; } - + /// @brief operator() returns a reference to the io context /// @return the io context operator boost::asio::io_context &() { return m_context; } @@ -130,14 +130,17 @@ namespace mtconnect::configuration { m_guard.emplace(m_context.get_executor()); m_context.restart(); } - + /// @brief Cover methods for asio io_context /// @{ /// @brief io_context::run_for template - auto run_for(const std::chrono::duration& rel_time) { return m_context.run_for(rel_time); } - + auto run_for(const std::chrono::duration &rel_time) + { + return m_context.run_for(rel_time); + } + /// @brief io_context::run auto run() { return m_context.run(); } @@ -146,18 +149,24 @@ namespace mtconnect::configuration { /// @brief io_context::run_one_for template - auto run_one_for(const std::chrono::duration& rel_time) { return m_context.run_one_for(rel_time); } + auto run_one_for(const std::chrono::duration &rel_time) + { + return m_context.run_one_for(rel_time); + } /// @brief io_context::run_one_until template - auto run_one_until(const std::chrono::time_point& abs_time) { return m_context.run_one_for(abs_time); } - + auto run_one_until(const std::chrono::time_point &abs_time) + { + return m_context.run_one_for(abs_time); + } + /// @brief io_context::poll auto poll() { return m_context.poll(); } - + /// @brief io_context::poll auto get_executor() BOOST_ASIO_NOEXCEPT { return m_context.get_executor(); } - + /// @} private: diff --git a/src/mtconnect/entity/xml_parser.cpp b/src/mtconnect/entity/xml_parser.cpp index d700439a..e027e362 100644 --- a/src/mtconnect/entity/xml_parser.cpp +++ b/src/mtconnect/entity/xml_parser.cpp @@ -234,7 +234,8 @@ namespace mtconnect::entity { return nullptr; } - EntityPtr XmlParser::parse(FactoryPtr factory, const string &document, ErrorList &errors, bool parseNamespaces) + EntityPtr XmlParser::parse(FactoryPtr factory, const string &document, ErrorList &errors, + bool parseNamespaces) { NAMED_SCOPE("entity.xml_parser"); EntityPtr entity; diff --git a/src/mtconnect/entity/xml_parser.hpp b/src/mtconnect/entity/xml_parser.hpp index ece8917e..8e66a5f3 100644 --- a/src/mtconnect/entity/xml_parser.hpp +++ b/src/mtconnect/entity/xml_parser.hpp @@ -52,8 +52,7 @@ namespace mtconnect { /// @param errors errors that occurred during the parsing /// @param parseNamespaces `true` if namespaces should be parsed /// @return a shared pointer to an entity if successful - static EntityPtr parse(FactoryPtr factory, const std::string &document, - ErrorList &errors, + static EntityPtr parse(FactoryPtr factory, const std::string &document, ErrorList &errors, bool parseNamespaces = true); }; } // namespace entity diff --git a/src/mtconnect/parser/xml_parser.cpp b/src/mtconnect/parser/xml_parser.cpp index 3ce1dc97..f634d571 100644 --- a/src/mtconnect/parser/xml_parser.cpp +++ b/src/mtconnect/parser/xml_parser.cpp @@ -190,30 +190,32 @@ namespace mtconnect::parser { devices = xmlXPathEval(BAD_CAST path.c_str(), xpathCtx); - if (!devices) - throw(string) xpathCtx->lastError.message; - - xmlNodeSetPtr nodeset = devices->nodesetval; - - if (!nodeset || !nodeset->nodeNr) - throw(string) "Could not find Device in XML configuration"; - - entity::ErrorList errors; - for (int i = 0; i != nodeset->nodeNr; ++i) + if (!devices || !devices->nodesetval || !devices->nodesetval->nodeNr) { - auto device = - entity::XmlParser::parseXmlNode(Device::getRoot(), nodeset->nodeTab[i], errors); - if (device) - deviceList.emplace_back(dynamic_pointer_cast(device)); - - if (!errors.empty()) + LOG(warning) << "No devices in Devices.xml file – expecting dynamic configuration"; + } + else + { + xmlNodeSetPtr nodeset = devices->nodesetval; + + entity::ErrorList errors; + for (int i = 0; i != nodeset->nodeNr; ++i) { - for (auto &e : errors) - LOG(warning) << "Error parsing device: " << e->what(); + auto device = + entity::XmlParser::parseXmlNode(Device::getRoot(), nodeset->nodeTab[i], errors); + if (device) + deviceList.emplace_back(dynamic_pointer_cast(device)); + + if (!errors.empty()) + { + for (auto &e : errors) + LOG(warning) << "Error parsing device: " << e->what(); + } } } - xmlXPathFreeObject(devices); + if (devices) + xmlXPathFreeObject(devices); xmlXPathFreeContext(xpathCtx); } catch (string e) @@ -240,27 +242,39 @@ namespace mtconnect::parser { return deviceList; } - + DevicePtr XmlParser::parseDevice(const std::string &deviceXml, printer::XmlPrinter *aPrinter) { DevicePtr device; - + using namespace boost::adaptors; using namespace boost::range; std::unique_lock lock(m_mutex); - + try { entity::ErrorList errors; - auto entity = entity::XmlParser::parse(Device::getFactory(), deviceXml, errors); + auto entity = entity::XmlParser::parse(Device::getRoot(), deviceXml, errors); + if (errors.size() > 0) + { + LOG(warning) << "Errors parsing Device: " << deviceXml; + for (auto &e : errors) + { + LOG(warning) << " " << e->what(); + } + } + else + { + device = dynamic_pointer_cast(entity); + } } catch (string e) { LOG(fatal) << "Cannot parse XML document: " << e; throw; } - + return device; } diff --git a/src/mtconnect/pipeline/guard.hpp b/src/mtconnect/pipeline/guard.hpp index 6b252605..5c66c935 100644 --- a/src/mtconnect/pipeline/guard.hpp +++ b/src/mtconnect/pipeline/guard.hpp @@ -170,7 +170,7 @@ namespace mtconnect { { return check(matches(entity), entity); } - + /// @brief set the alternative action if this guard does not match auto &operator||(Guard other) { @@ -194,7 +194,7 @@ namespace mtconnect { { return check(matches(entity), entity); } - + /// @brief set the alternative action if this guard does not match auto &operator||(Guard other) { @@ -244,7 +244,7 @@ namespace mtconnect { { return B::check(matches(entity), entity); } - + /// @brief set the alternative action if this guard does not match auto &operator||(Guard other) { diff --git a/src/mtconnect/pipeline/pipeline.hpp b/src/mtconnect/pipeline/pipeline.hpp index 20a0044f..1ee84923 100644 --- a/src/mtconnect/pipeline/pipeline.hpp +++ b/src/mtconnect/pipeline/pipeline.hpp @@ -155,8 +155,7 @@ namespace mtconnect { return true; } - - + /// @brief add a transform after the target. /// @param[in] target the named transforms to insert this transform after /// @param[in] transform the transform to add before @@ -183,7 +182,7 @@ namespace mtconnect { return true; } - + /// @brief splices the transform as the first option in targets next list. /// @param[in] target the named transforms to insert this transform after /// @param[in] transform the transform to add before @@ -284,10 +283,7 @@ namespace mtconnect { /// @brief Sends the entity through the pipeline /// @param[in] entity the entity to send through the pipeline /// @return the entity returned from the transform - entity::EntityPtr run(entity::EntityPtr &&entity) - { - return m_start->next(std::move(entity)); - } + entity::EntityPtr run(entity::EntityPtr &&entity) { return m_start->next(std::move(entity)); } /// @brief Bind the transform to the start /// @param[in] transform the transform to bind @@ -301,7 +297,7 @@ namespace mtconnect { /// @brief check if the pipeline has a pipeline context /// @returns `true` if there is a context bool hasContext() const { return bool(m_context); } - + /// @brief check if the pipeline has a pipeline contract /// @returns `true` if there is a contract bool hasContract() const { return bool(m_context) && bool(m_context->m_contract); } diff --git a/src/mtconnect/pipeline/pipeline_context.hpp b/src/mtconnect/pipeline/pipeline_context.hpp index 99d506a0..bd451f15 100644 --- a/src/mtconnect/pipeline/pipeline_context.hpp +++ b/src/mtconnect/pipeline/pipeline_context.hpp @@ -73,7 +73,7 @@ namespace mtconnect::pipeline { using SharedState = std::unordered_map; SharedState m_sharedState; }; - + /// @brief Alias for a shared pointer to the pipeline context using PipelineContextPtr = std::shared_ptr; } // namespace mtconnect::pipeline diff --git a/src/mtconnect/source/adapter/adapter_pipeline.cpp b/src/mtconnect/source/adapter/adapter_pipeline.cpp index f6e07e97..b902164f 100644 --- a/src/mtconnect/source/adapter/adapter_pipeline.cpp +++ b/src/mtconnect/source/adapter/adapter_pipeline.cpp @@ -69,10 +69,10 @@ namespace mtconnect { "Message", Properties {{"VALUE", data}, {"topic", topic}, {"source", source}}); run(std::move(entity)); }; - handler->m_command = [this](const std::string &command, const std::string &value, const std::string &source) { - auto entity = - make_shared("Command", Properties { - {"command", command }, {"VALUE", value}, {"source", source}}); + handler->m_command = [this](const std::string &command, const std::string &value, + const std::string &source) { + auto entity = make_shared( + "Command", Properties {{"command", command}, {"VALUE", value}, {"source", source}}); run(std::move(entity)); }; diff --git a/src/mtconnect/source/adapter/adapter_pipeline.hpp b/src/mtconnect/source/adapter/adapter_pipeline.hpp index 13a25f1c..6f1660cf 100644 --- a/src/mtconnect/source/adapter/adapter_pipeline.hpp +++ b/src/mtconnect/source/adapter/adapter_pipeline.hpp @@ -26,7 +26,8 @@ namespace mtconnect::source::adapter { struct Handler { using ProcessData = std::function; - using ProcessCommand = std::function; + using ProcessCommand = std::function; using ProcessMessage = std::function; using Connect = std::function; diff --git a/src/mtconnect/source/adapter/shdr/shdr_adapter.cpp b/src/mtconnect/source/adapter/shdr/shdr_adapter.cpp index 62d4b94f..1b119b6d 100644 --- a/src/mtconnect/source/adapter/shdr/shdr_adapter.cpp +++ b/src/mtconnect/source/adapter/shdr/shdr_adapter.cpp @@ -19,11 +19,10 @@ #include "shdr_adapter.hpp" #include -#include - -#include #include #include +#include +#include #include #include @@ -163,35 +162,31 @@ namespace mtconnect::source::adapter::shdr { void ShdrAdapter::protocolCommand(const std::string &data) { NAMED_SCOPE("ShdrAdapter::protocolCommand"); - + using namespace boost::algorithm; namespace qi = boost::spirit::qi; namespace ascii = boost::spirit::ascii; namespace phoenix = boost::phoenix; + using ascii::space; using qi::char_; - using qi::lit; using qi::lexeme; - using ascii::space; - + using qi::lit; + string command; - auto f = [&command](const auto &s) { - command = string(s.begin(), s.end()); - }; - + auto f = [&command](const auto &s) { command = string(s.begin(), s.end()); }; + auto it = data.begin(); - bool res = qi::phrase_parse(it, data.end(), - ( lit("*") >> lexeme[+(char_ - ':')][f] - >> ':') - , space); - + bool res = + qi::phrase_parse(it, data.end(), (lit("*") >> lexeme[+(char_ - ':')][f] >> ':'), space); + if (res) { string value(it, data.end()); ConfigOptions options; - + boost::to_lower(command); - + if (command == "conversionrequired") options[configuration::ConversionRequired] = is_true(value); else if (command == "relativetime") @@ -202,7 +197,7 @@ namespace mtconnect::source::adapter::shdr { options[configuration::Device] = value; else if (command == "shdrversion") options[configuration::ShdrVersion] = stringToInt(value, 1); - + if (options.size() > 0) setOptions(options); else if (m_handler && m_handler->m_command) diff --git a/src/mtconnect/source/adapter/shdr/shdr_adapter.hpp b/src/mtconnect/source/adapter/shdr/shdr_adapter.hpp index 1f0660f4..5187aa6c 100644 --- a/src/mtconnect/source/adapter/shdr/shdr_adapter.hpp +++ b/src/mtconnect/source/adapter/shdr/shdr_adapter.hpp @@ -118,7 +118,7 @@ namespace mtconnect { /// @brief Change the options for the adapter /// @param[in] options the set of options - void setOptions(const ConfigOptions &options) + void setOptions(const ConfigOptions &options) override { for (auto &o : options) m_options.insert_or_assign(o.first, o.second); @@ -127,7 +127,7 @@ namespace mtconnect { if (!m_pipeline.started() && started) m_pipeline.start(); } - + protected: void forwardData(const std::string &data) { diff --git a/src/mtconnect/source/source.hpp b/src/mtconnect/source/source.hpp index 38c6a71c..98ee9f2c 100644 --- a/src/mtconnect/source/source.hpp +++ b/src/mtconnect/source/source.hpp @@ -86,6 +86,10 @@ namespace mtconnect { /// @brief get the source's strand /// @return the asio strand boost::asio::io_context::strand &getStrand(); + + /// @brief changes the options in the source + /// @param[in] options the options to update + virtual void setOptions(const ConfigOptions &options) { } protected: std::string m_name; diff --git a/test/adapter_test.cpp b/test/adapter_test.cpp index 996f823d..1ac8c8e8 100644 --- a/test/adapter_test.cpp +++ b/test/adapter_test.cpp @@ -87,9 +87,12 @@ TEST(AdapterTest, should_forward_multiline_command) auto handler = make_unique(); string command, value; - handler->m_command = [&](const string &c, const string &v, const string &s) { command = c; value = v; }; + handler->m_command = [&](const string &c, const string &v, const string &s) { + command = c; + value = v; + }; adapter->setHandler(handler); - + adapter->processData("* deviceModel: --multiline--ABC1234"); EXPECT_TRUE(adapter->getTerminator()); EXPECT_EQ("--multiline--ABC1234", *adapter->getTerminator()); @@ -115,7 +118,7 @@ TEST(AdapterTest, should_set_options_from_commands) auto adapter = make_unique(ioc, context, options, tree); adapter->processData("* shdrVersion: 3"); - + auto v = GetOption(adapter->getOptions(), "ShdrVersion"); ASSERT_EQ(int64_t(3), *v); } diff --git a/test/agent_test_helper.hpp b/test/agent_test_helper.hpp index 86baf7da..2f3291bc 100644 --- a/test/agent_test_helper.hpp +++ b/test/agent_test_helper.hpp @@ -117,9 +117,7 @@ class AgentTestHelper public: using Hook = std::function; - AgentTestHelper() : m_incomingIp("127.0.0.1"), m_strand(m_ioContext), m_socket(m_ioContext) - { - } + AgentTestHelper() : m_incomingIp("127.0.0.1"), m_strand(m_ioContext), m_socket(m_ioContext) {} ~AgentTestHelper() { diff --git a/test/config_test.cpp b/test/config_test.cpp index 630eb8b5..541b26cf 100644 --- a/test/config_test.cpp +++ b/test/config_test.cpp @@ -1385,5 +1385,95 @@ Port = 0 m_config->start(); th.join(); } + + TEST_F(ConfigTest, should_add_a_new_device_when_deviceModel_received_from_adapter) + { + using namespace mtconnect::source::adapter; + + fs::path root {createTempDirectory("6")}; + + fs::path devices(root / "Devices.xml"); + fs::path config {root / "agent.cfg"}; + { + ofstream cfg(config.string()); + cfg << R"DOC( +VersionDeviceXmlUpdates = true +Port = 0 + +Adapters { + Device { + } +} +)DOC"; + cfg << "Devices = " << devices << endl; + } + + copyFile("empty.xml", devices, 0min); + + boost::program_options::variables_map options; + boost::program_options::variable_value value(boost::optional(config.string()), false); + options.insert(make_pair("config-file"s, value)); + + m_config->initialize(options); + auto &asyncContext = m_config->getAsyncContext(); + + auto agent = m_config->getAgent(); + const auto &printer = agent->getPrinter("xml"); + ASSERT_NE(nullptr, printer); + + auto sp = agent->findSource("_localhost_7878"); + ASSERT_TRUE(sp); + + auto adapter = dynamic_pointer_cast(sp); + ASSERT_TRUE(adapter); + + boost::asio::steady_timer timer1(asyncContext.get()); + timer1.expires_from_now(100ms); + timer1.async_wait([this, &adapter](boost::system::error_code ec) { + if (ec) + { + m_config->stop(); + } + else + { + adapter->processData("* deviceModel: --multiline--AAAAA"); + adapter->processData(R"( + + + + + + + + + + + + + + +)"); + adapter->processData("--multiline--AAAAA"); + } + }); + + m_config->start(); + } + + TEST_F(ConfigTest, should_update_a_device_when_received_from_adapter) + { + fs::path root {createTempDirectory("6")}; + + fs::path devices(root / "Devices.xml"); + fs::path config {root / "agent.cfg"}; + { + ofstream cfg(config.string()); + cfg << R"DOC( +VersionDeviceXmlUpdates = true +Port = 0 +)DOC"; + cfg << "Devices = " << devices << endl; + } + } } // namespace From 125ce6777c7694b3cd2146fcc3a6cfaf237e44ec Mon Sep 17 00:00:00 2001 From: Will Sobel Date: Sat, 22 Apr 2023 12:04:28 +0200 Subject: [PATCH 05/18] First cut adding dynamic device via adapter. --- src/mtconnect/agent.cpp | 22 ++++----- src/mtconnect/agent.hpp | 3 +- src/mtconnect/parser/xml_parser.cpp | 6 +-- src/mtconnect/source/source.hpp | 4 +- test/config_test.cpp | 75 ++++++++++++++++++++++++----- 5 files changed, 80 insertions(+), 30 deletions(-) diff --git a/src/mtconnect/agent.cpp b/src/mtconnect/agent.cpp index 765afaf9..20292422 100644 --- a/src/mtconnect/agent.cpp +++ b/src/mtconnect/agent.cpp @@ -386,26 +386,25 @@ namespace mtconnect { } } - void Agent::loadDevice(const string &deviceXml, - const optional source) + void Agent::loadDevice(const string &deviceXml, const optional source) { m_context.pause([=](config::AsyncContext &context) { try - { + { auto printer = dynamic_cast(m_printers["xml"].get()); auto device = m_xmlParser->parseDevice(deviceXml, printer); - + if (device) { bool changed = receiveDevice(device, true); if (changed) loadCachedProbe(); - + if (source) { auto s = findSource(*source); if (s) - s->setOptions({{ config::Device, device->getName() }}); + s->setOptions({{config::Device, device->getName()}}); } } } @@ -545,11 +544,14 @@ namespace mtconnect { void Agent::versionDeviceXml() { + using namespace std::chrono; + if (m_versionDeviceXml) { + // update with a new version of the device.xml, saving the old one // with a date time stamp - auto ext = "."s + getCurrentTime(LOCAL); + auto ext = date::format(".%Y-%m-%dT%H+%M+%SZ", date::floor(system_clock::now())); fs::path file(m_deviceXmlPath); fs::path backup(m_deviceXmlPath + ext); if (!fs::exists(backup)) @@ -1333,8 +1335,7 @@ namespace mtconnect { {"description", mem_fn(&Device::setDescriptionValue)}, {"nativename", [](DevicePtr device, const string &name) { device->setProperty("nativeName", name); }}, - {"calibration", - [](DevicePtr device, const string &value) { + {"calibration", [](DevicePtr device, const string &value) { istringstream line(value); // Look for name|factor|offset triples @@ -1355,8 +1356,7 @@ namespace mtconnect { di->setConverter(conv); } } - }} - }; + }}}; static std::unordered_map adapterDataItems { {"adapterversion", "_adapter_software_version"}, diff --git a/src/mtconnect/agent.hpp b/src/mtconnect/agent.hpp index 2c0e82b4..515888d4 100644 --- a/src/mtconnect/agent.hpp +++ b/src/mtconnect/agent.hpp @@ -266,7 +266,8 @@ namespace mtconnect { /// @brief receive and parse a single device from a source /// @param[in] deviceXml the device xml as a string /// @param[in] source the source loading the device - void loadDevice(const std::string &deviceXml, const std::optional source = std::nullopt); + void loadDevice(const std::string &deviceXml, + const std::optional source = std::nullopt); /// @name Message when source has connected and disconnected ///@{ diff --git a/src/mtconnect/parser/xml_parser.cpp b/src/mtconnect/parser/xml_parser.cpp index f634d571..14b4199c 100644 --- a/src/mtconnect/parser/xml_parser.cpp +++ b/src/mtconnect/parser/xml_parser.cpp @@ -197,15 +197,15 @@ namespace mtconnect::parser { else { xmlNodeSetPtr nodeset = devices->nodesetval; - + entity::ErrorList errors; for (int i = 0; i != nodeset->nodeNr; ++i) { auto device = - entity::XmlParser::parseXmlNode(Device::getRoot(), nodeset->nodeTab[i], errors); + entity::XmlParser::parseXmlNode(Device::getRoot(), nodeset->nodeTab[i], errors); if (device) deviceList.emplace_back(dynamic_pointer_cast(device)); - + if (!errors.empty()) { for (auto &e : errors) diff --git a/src/mtconnect/source/source.hpp b/src/mtconnect/source/source.hpp index 98ee9f2c..cdacfb9f 100644 --- a/src/mtconnect/source/source.hpp +++ b/src/mtconnect/source/source.hpp @@ -86,10 +86,10 @@ namespace mtconnect { /// @brief get the source's strand /// @return the asio strand boost::asio::io_context::strand &getStrand(); - + /// @brief changes the options in the source /// @param[in] options the options to update - virtual void setOptions(const ConfigOptions &options) { } + virtual void setOptions(const ConfigOptions &options) {} protected: std::string m_name; diff --git a/test/config_test.cpp b/test/config_test.cpp index 541b26cf..8b8a9c25 100644 --- a/test/config_test.cpp +++ b/test/config_test.cpp @@ -75,8 +75,12 @@ namespace { fs::path createTempDirectory(const string &ext) { fs::path root {fs::path(TEST_BIN_ROOT_DIR) / ("config_test_" + ext)}; - if (!fs::exists(root)) - fs::create_directory(root); + if (fs::exists(root)) + { + fs::remove_all(root); + } + + fs::create_directory(root); chdir(root.string().c_str()); m_config->updateWorkingDirectory(); // m_config->setDebug(false); @@ -1385,11 +1389,11 @@ Port = 0 m_config->start(); th.join(); } - + TEST_F(ConfigTest, should_add_a_new_device_when_deviceModel_received_from_adapter) { using namespace mtconnect::source::adapter; - + fs::path root {createTempDirectory("6")}; fs::path devices(root / "Devices.xml"); @@ -1420,16 +1424,54 @@ Adapters { auto agent = m_config->getAgent(); const auto &printer = agent->getPrinter("xml"); ASSERT_NE(nullptr, printer); - + auto sp = agent->findSource("_localhost_7878"); ASSERT_TRUE(sp); - + auto adapter = dynamic_pointer_cast(sp); ASSERT_TRUE(adapter); - boost::asio::steady_timer timer1(asyncContext.get()); - timer1.expires_from_now(100ms); - timer1.async_wait([this, &adapter](boost::system::error_code ec) { + auto validate = [this, &agent](boost::system::error_code ec) { + using namespace std::filesystem; + using namespace std::chrono; + using namespace boost::algorithm; + + if (!ec) + { + // Check for backup file + auto ext = date::format(".%Y-%m-%dT%H+", date::floor(system_clock::now())); + auto dit = directory_iterator("."); + auto found = false; + for (auto &f : dit) + { + auto fe = f.path().extension().string(); + if (starts_with(fe, ext)) + { + found = true; + break; + } + } + ASSERT_TRUE(found) << "Cannot find backup device file with extension: " << ext << '*'; + + auto device = agent->getDeviceByName("LinuxCNC"); + ASSERT_TRUE(device) << "Cannot find LinuxCNC device"; + + const auto &components = device->getChildren(); + ASSERT_EQ(1, components->size()); + + auto cont = device->getComponentById("cont"); + ASSERT_TRUE(cont) << "Cannot find Component with id cont"; + + auto exec = device->getDeviceDataItem("exec"); + ASSERT_TRUE(exec) << "Cannot find DataItem with id exec"; + + } + m_config->stop(); + }; + + boost::asio::steady_timer timer2(asyncContext.get()); + + auto send = [this, &adapter, &timer2, validate](boost::system::error_code ec) { if (ec) { m_config->stop(); @@ -1446,7 +1488,7 @@ Adapters { - + @@ -1454,15 +1496,22 @@ Adapters { )"); adapter->processData("--multiline--AAAAA"); + + timer2.expires_from_now(500ms); + timer2.async_wait(validate); } - }); - + }; + + boost::asio::steady_timer timer1(asyncContext.get()); + timer1.expires_from_now(100ms); + timer1.async_wait(send); + m_config->start(); } TEST_F(ConfigTest, should_update_a_device_when_received_from_adapter) { - fs::path root {createTempDirectory("6")}; + fs::path root {createTempDirectory("7")}; fs::path devices(root / "Devices.xml"); fs::path config {root / "agent.cfg"}; From 34e5127b70bdd266212c9ad199049b8cc2220f01 Mon Sep 17 00:00:00 2001 From: Will Sobel Date: Sat, 22 Apr 2023 21:43:53 +0200 Subject: [PATCH 06/18] Before unique ids --- src/mtconnect/agent.cpp | 5 +++- src/mtconnect/device_model/component.hpp | 13 +++++++++ .../device_model/data_item/data_item.cpp | 11 +------- src/mtconnect/device_model/device.hpp | 6 +++++ .../source/adapter/adapter_pipeline.cpp | 2 +- test/config_test.cpp | 27 ++++++++++--------- 6 files changed, 39 insertions(+), 25 deletions(-) diff --git a/src/mtconnect/agent.cpp b/src/mtconnect/agent.cpp index 20292422..33804cfe 100644 --- a/src/mtconnect/agent.cpp +++ b/src/mtconnect/agent.cpp @@ -404,7 +404,10 @@ namespace mtconnect { { auto s = findSource(*source); if (s) - s->setOptions({{config::Device, device->getName()}}); + { + const auto &name = device->getComponentName(); + s->setOptions({{config::Device, *name}}); + } } } } diff --git a/src/mtconnect/device_model/component.hpp b/src/mtconnect/device_model/component.hpp index fb91fce6..671a77c1 100644 --- a/src/mtconnect/device_model/component.hpp +++ b/src/mtconnect/device_model/component.hpp @@ -244,6 +244,19 @@ namespace mtconnect { return nullptr; } + + /// @brief Get the component topic path as a list + /// + /// Recurses to root and then appends getTopicName + /// @param[in,out] pth the path list to append to + void path(std::list &pth) + { + auto p = getParent(); + if (p) + p->path(pth); + + pth.push_back(getTopicName()); + } protected: void setParent(ComponentPtr parent) { m_parent = parent; } diff --git a/src/mtconnect/device_model/data_item/data_item.cpp b/src/mtconnect/device_model/data_item/data_item.cpp index 89842b47..c99160d1 100644 --- a/src/mtconnect/device_model/data_item/data_item.cpp +++ b/src/mtconnect/device_model/data_item/data_item.cpp @@ -297,21 +297,12 @@ namespace mtconnect { } } - inline void path(std::list &pth, ComponentPtr c) - { - auto p = c->getParent(); - if (p) - path(pth, p); - - pth.push_back(c->getTopicName()); - } - void DataItem::makeTopic() { std::list pth; auto comp = m_component.lock(); - path(pth, comp); + comp->path(pth); { auto cp = m_composition.lock(); if (cp) diff --git a/src/mtconnect/device_model/device.hpp b/src/mtconnect/device_model/device.hpp index 97e62af9..0cd4059e 100644 --- a/src/mtconnect/device_model/device.hpp +++ b/src/mtconnect/device_model/device.hpp @@ -235,6 +235,12 @@ namespace mtconnect { /// @brief get the topic for this device /// @return the uuid of the device const std::string getTopicName() const override { return *m_uuid; } + + /// @brief Converts all the ids to unique ids by hasing the topics + /// + /// Converts the id attribute to a unique value and caches the original value + /// in case it is required later + protected: void cachePointers(DataItemPtr dataItem); diff --git a/src/mtconnect/source/adapter/adapter_pipeline.cpp b/src/mtconnect/source/adapter/adapter_pipeline.cpp index b902164f..bc9dc8cc 100644 --- a/src/mtconnect/source/adapter/adapter_pipeline.cpp +++ b/src/mtconnect/source/adapter/adapter_pipeline.cpp @@ -83,7 +83,7 @@ namespace mtconnect { { clear(); m_options = options; - + m_identity = GetOption(m_options, configuration::AdapterIdentity).value_or("unknown"); } diff --git a/test/config_test.cpp b/test/config_test.cpp index 8b8a9c25..28cb35d9 100644 --- a/test/config_test.cpp +++ b/test/config_test.cpp @@ -1431,7 +1431,7 @@ Adapters { auto adapter = dynamic_pointer_cast(sp); ASSERT_TRUE(adapter); - auto validate = [this, &agent](boost::system::error_code ec) { + auto validate = [&](boost::system::error_code ec) { using namespace std::filesystem; using namespace std::chrono; using namespace boost::algorithm; @@ -1441,17 +1441,10 @@ Adapters { // Check for backup file auto ext = date::format(".%Y-%m-%dT%H+", date::floor(system_clock::now())); auto dit = directory_iterator("."); - auto found = false; - for (auto &f : dit) - { - auto fe = f.path().extension().string(); - if (starts_with(fe, ext)) - { - found = true; - break; - } - } - ASSERT_TRUE(found) << "Cannot find backup device file with extension: " << ext << '*'; + auto it = find_if(dit, end(dit), [&ext](const auto &de) { + return starts_with(de.path().extension().string(), ext); + }); + ASSERT_NE(end(dit), it) << "Cannot find backup device file with extension: " << ext << '*'; auto device = agent->getDeviceByName("LinuxCNC"); ASSERT_TRUE(device) << "Cannot find LinuxCNC device"; @@ -1464,7 +1457,9 @@ Adapters { auto exec = device->getDeviceDataItem("exec"); ASSERT_TRUE(exec) << "Cannot find DataItem with id exec"; - + + auto pipeline = dynamic_cast(adapter->getPipeline()); + ASSERT_EQ("LinuxCNC", pipeline->getDevice()); } m_config->stop(); }; @@ -1523,6 +1518,12 @@ Port = 0 )DOC"; cfg << "Devices = " << devices << endl; } + + GTEST_SKIP(); } + TEST_F(ConfigTest, should_update_the_ids_of_all_entities) + { + GTEST_SKIP(); + } } // namespace From 987b45f78150c75fc1ab048ac01678b2352cfadb Mon Sep 17 00:00:00 2001 From: Will Sobel Date: Tue, 25 Apr 2023 16:08:25 +0200 Subject: [PATCH 07/18] First draft with working versioning and unique ids --- samples/dyn_load.xml | 23 +++ src/mtconnect/agent.cpp | 35 +++- src/mtconnect/agent.hpp | 134 +++++++------- src/mtconnect/configuration/agent_config.cpp | 1 + src/mtconnect/configuration/async_context.hpp | 2 +- .../configuration/config_options.hpp | 1 + src/mtconnect/device_model/component.cpp | 7 + src/mtconnect/device_model/component.hpp | 6 + .../device_model/data_item/data_item.cpp | 2 +- .../device_model/data_item/data_item.hpp | 31 ++++ src/mtconnect/device_model/device.cpp | 63 +++++++ src/mtconnect/device_model/device.hpp | 39 +++- src/mtconnect/entity/entity.cpp | 108 ++++++++++- src/mtconnect/entity/entity.hpp | 19 ++ src/mtconnect/entity/factory.hpp | 4 + src/mtconnect/entity/json_printer.hpp | 19 +- src/mtconnect/entity/xml_printer.cpp | 11 +- src/mtconnect/entity/xml_printer.hpp | 5 +- src/mtconnect/printer/json_printer.cpp | 5 +- src/mtconnect/printer/json_printer.hpp | 3 +- src/mtconnect/printer/printer.hpp | 3 +- src/mtconnect/printer/xml_printer.cpp | 5 +- src/mtconnect/printer/xml_printer.hpp | 3 +- src/mtconnect/sink/rest_sink/server.hpp | 1 + src/mtconnect/utilities.hpp | 49 +++++ test/config_test.cpp | 164 ++++++++++++++++- test/entity_printer_test.cpp | 78 ++++++++ test/json_printer_test.cpp | 174 ++++++++++++++++++ 28 files changed, 902 insertions(+), 93 deletions(-) create mode 100644 samples/dyn_load.xml diff --git a/samples/dyn_load.xml b/samples/dyn_load.xml new file mode 100644 index 00000000..ecbfe510 --- /dev/null +++ b/samples/dyn_load.xml @@ -0,0 +1,23 @@ + + +
+ + + + + + + + + + + + + + + + + + + + diff --git a/src/mtconnect/agent.cpp b/src/mtconnect/agent.cpp index 33804cfe..09db8563 100644 --- a/src/mtconnect/agent.cpp +++ b/src/mtconnect/agent.cpp @@ -83,7 +83,7 @@ namespace mtconnect { m_deviceXmlPath(deviceXmlPath), m_circularBuffer(GetOption(options, config::BufferSize).value_or(17), GetOption(options, config::CheckpointFrequency).value_or(1000)), - m_pretty(GetOption(options, mtconnect::configuration::Pretty).value_or(false)) + m_pretty(IsOptionSet(options, mtconnect::configuration::Pretty)) { using namespace asset; @@ -97,8 +97,8 @@ namespace mtconnect { m_assetStorage = make_unique( GetOption(options, mtconnect::configuration::MaxAssets).value_or(1024)); - m_versionDeviceXml = - GetOption(options, mtconnect::configuration::VersionDeviceXmlUpdates).value_or(false); + m_versionDeviceXml = IsOptionSet(options, mtconnect::configuration::VersionDeviceXmlUpdates); + m_createUniqueIds = IsOptionSet(options, config::CreateUniqueIds); auto jsonVersion = uint32_t(GetOption(options, mtconnect::configuration::JsonVersion).value_or(2)); @@ -145,6 +145,9 @@ namespace mtconnect { for (auto device : devices) addDevice(device); + if (m_versionDeviceXml && m_createUniqueIds) + versionDeviceXml(); + loadCachedProbe(); m_initialized = true; @@ -480,7 +483,18 @@ namespace mtconnect { device->addDataItem(odi, errors); if (auto odi = oldDev->getAssetCount(), ndi = device->getAssetCount(); odi && !ndi) device->addDataItem(odi, errors); + + if (errors.size() > 0) + { + LOG(error) << "Error adding device required data items for " << *device->getUuid() << ':'; + for (const auto &e : errors) + LOG(error) << " " << e->what(); + return false; + } + + if (m_createUniqueIds) + createUniqueIds(device); verifyDevice(device); LOG(info) << "Checking if device " << *uuid << " has changed"; @@ -565,7 +579,8 @@ namespace mtconnect { std::list list; copy_if(m_deviceIndex.begin(), m_deviceIndex.end(), back_inserter(list), [](DevicePtr d) { return dynamic_cast(d.get()) == nullptr; }); - auto probe = printer.printProbe(0, 0, 0, 0, 0, list); + auto probe = printer.printProbe(0, 0, 0, 0, 0, list, nullptr, + true); ofstream devices(file.string()); devices << probe; @@ -859,6 +874,7 @@ namespace mtconnect { // TODO: Redo Resolve Reference with entity // device->resolveReferences(); + createUniqueIds(device); verifyDevice(device); if (m_observationsInitialized) @@ -917,6 +933,7 @@ namespace mtconnect { if (changed) { + createUniqueIds(device); if (m_intSchemaVersion >= SCHEMA_VERSION(2, 2)) device->addHash(); @@ -951,6 +968,16 @@ namespace mtconnect { } } } + + void Agent::createUniqueIds(DevicePtr device) + { + if (m_createUniqueIds && !dynamic_pointer_cast(device)) + { + device->createUniqueIds(m_idMap); + device->updateReferences(m_idMap); + } + } + void Agent::loadCachedProbe() { diff --git a/src/mtconnect/agent.hpp b/src/mtconnect/agent.hpp index 515888d4..d8ead32f 100644 --- a/src/mtconnect/agent.hpp +++ b/src/mtconnect/agent.hpp @@ -93,55 +93,55 @@ namespace mtconnect { /// - DisableAgentDevice Agent(configuration::AsyncContext &context, const std::string &deviceXmlPath, const ConfigOptions &options); - + /// Destructor for the Agent. /// > Note: Does not stop the agent. ~Agent(); - + /// @brief Hook callback type using Hook = std::function; - + /// @brief Functions to run before the agent begins the initialization process. /// @return configuration::HookManager& auto &beforeInitializeHooks() { return m_beforeInitializeHooks; } - + /// @brief Function that run after all agent initialization is complete /// @return configuration::HookManager& auto &afterInitializeHooks() { return m_afterInitializeHooks; } - + /// @brief Hooks to run when before the agent starts all the soures and sinks /// @return configuration::HookManager& auto &beforeStartHooks() { return m_beforeStartHooks; } - + /// @brief Hooks before the agent stops all the sources and sinks /// @return configuration::HookManager& auto &beforeStopHooks() { return m_beforeStopHooks; } - + /// @brief the agent given a pipeline context /// @param context: the pipeline context shared between all pipelines void initialize(pipeline::PipelineContextPtr context); - + /// @brief initial UNAVAILABLE observations for all data items /// unless they have constant values. void initialDataItemObservations(); - + // Start and stop /// @brief Starts all the sources and sinks void start(); /// @brief Stops all the sources and syncs. void stop(); - + /// @brief Get the boost asio io context /// @return boost::asio::io_context auto &getContext() { return m_context; } - + /// @brief Create a contract for pipelines to access agent information /// @return A contract between the pipeline and this agent std::unique_ptr makePipelineContract(); /// @brief Gets the pipe context shared by all pipelines /// @return A shared pipeline context auto getPipelineContext() { return m_pipelineContext; } - + /// @brief Makes a unique sink contract /// @return A contract between the sink and the agent sink::SinkContractPtr makeSinkContract(); @@ -156,7 +156,7 @@ namespace mtconnect { /// get latest and historical data. /// @return A const reference to the circular buffer const auto &getCircularBuffer() const { return m_circularBuffer; } - + /// @brief Adds an adapter to the agent /// @param[in] source: shared pointer to the source being added /// @param[in] start: starts the source if start is true, otherwise delayed start @@ -165,7 +165,7 @@ namespace mtconnect { /// @param[in] sink shared pointer to the the sink being added /// @param[in] start: starts the source if start is true, otherwise delayed start void addSink(sink::SinkPtr sink, bool start = false); - + // Source and Sink /// @brief Find a source by name /// @param[in] name the identity to find @@ -187,21 +187,21 @@ namespace mtconnect { for (auto &s : m_sinks) if (s->getName() == name) return s; - + return nullptr; } - + /// @brief Get the list of all sources /// @return The list of all source in the agent const auto &getSources() const { return m_sources; } /// @brief Get the list of all sinks /// @return The list of all sinks in the agent const auto &getSinks() const { return m_sinks; } - + /// @brief Get the MTConnect schema version the agent is supporting /// @return The MTConnect schema version as a string const auto &getSchemaVersion() const { return m_schemaVersion; } - + /// @brief Find a device by name /// @param[in] name The name of the device to find /// @return A shared pointer to the device @@ -222,7 +222,7 @@ namespace mtconnect { boost::push_back(list, m_deviceIndex); return list; } - + /// @brief Get a pointer to the default device /// /// The default device is the first device that is not the Agent device. @@ -236,7 +236,7 @@ namespace mtconnect { if (device->getName() != "Agent") return device; } - + return nullptr; } /// @deprecated use `getDefaultDevice()` instead @@ -244,11 +244,11 @@ namespace mtconnect { /// @return A shared pointer to the default device /// @note Cover method for `getDefaultDevice()` DevicePtr defaultDevice() const { return getDefaultDevice(); } - + /// @brief Get a pointer to the asset storage object /// @return A pointer to the asset storage object asset::AssetStorage *getAssetStorage() { return m_assetStorage.get(); } - + /// @brief Add a device to the agent /// @param[in] device The device to add. /// @note This method is not fully implemented after agent initialization @@ -262,16 +262,16 @@ namespace mtconnect { /// @param[in] deviceFile The device file to load /// @return true if successful bool reloadDevices(const std::string &deviceFile); - + /// @brief receive and parse a single device from a source /// @param[in] deviceXml the device xml as a string /// @param[in] source the source loading the device void loadDevice(const std::string &deviceXml, const std::optional source = std::nullopt); - + /// @name Message when source has connected and disconnected ///@{ - + /// @brief Called when source begins trying to connect /// @param source The source identity void connecting(const std::string &source); @@ -287,9 +287,9 @@ namespace mtconnect { /// @param[in] autoAvailable `true` if the source should automatically set available to /// `AVAILABLE` void connected(const std::string &source, const StringList &devices, bool autoAvailable); - + ///@} - + /// @brief Called when a source receives a command from a data source /// @param[in] device The device name associated with this source /// @param[in] command The command being sent @@ -297,7 +297,7 @@ namespace mtconnect { /// @param[in] source The identity of the source void receiveCommand(const std::string &device, const std::string &command, const std::string &value, const std::string &source); - + /// @brief Method to get a data item for a device /// @param[in] deviceName The name or uuid of the device /// @param[in] dataItemName The name or id of the data item @@ -310,7 +310,7 @@ namespace mtconnect { auto dev = findDeviceByUUIDorName(deviceName); return (dev) ? dev->getDeviceDataItem(dataItemName) : nullptr; } - + /// @brief Get a data item by its id. /// @param id Unique id of the data item /// @return Shared pointer to the data item if found @@ -321,10 +321,10 @@ namespace mtconnect { return diPos->second.lock(); return nullptr; } - + /// @name Pipeline related methods to receive data from sources ///@{ - + /// @brief Receive an observation /// @param[in] observation A shared pointer to the observation void receiveObservation(observation::ObservationPtr observation); @@ -359,21 +359,21 @@ namespace mtconnect { /// @param device The device related to the asset /// @param asset The asset void notifyAssetRemoved(DevicePtr device, const asset::AssetPtr &asset); - + ///@} - + /// @brief Method called by source when it cannot continue /// @param identity identity of the source void sourceFailed(const std::string &identity); - + /// @name For testing ///@{ - + /// @brief Returns a shared pointer to the agent device /// @return shared pointer to the agent device auto getAgentDevice() { return m_agentDevice; } ///@} - + /// @brief Get a pointer to the printer for a mime type /// @param type The mime type /// @return pointer to the printer or nullptr if it does not exist @@ -386,11 +386,11 @@ namespace mtconnect { else return nullptr; } - + /// @brief Get the map of available printers /// @return A const reference to the printer map const auto &getPrinters() const { return m_printers; } - + /// @brief Prefixes the path with the device and rewrites the composed /// paths by repeating the prefix. The resulting path is valid /// XPath. @@ -410,10 +410,18 @@ namespace mtconnect { /// @return The rewritten path properly prefixed std::string devicesAndPath(const std::optional &path, const DevicePtr device) const; - + + /// @brief Creates unique ids for the device model and maps to the originals + /// + /// Also updates the agents data item map by adding the new ids. Duplicate original + /// ids will be last in wins. + /// + /// @param[in] device device to modify + void createUniqueIds(DevicePtr device); + protected: friend class AgentPipelineContract; - + // Initialization methods void createAgentDevice(); std::list loadXMLDeviceFile(const std::string &config); @@ -422,47 +430,47 @@ namespace mtconnect { std::optional> skip = std::nullopt); void loadCachedProbe(); void versionDeviceXml(); - + // Asset count management void updateAssetCounts(const DevicePtr &device, const std::optional type); - + observation::ObservationPtr getLatest(const std::string &id) { return m_circularBuffer.getLatest().getObservation(id); } - + observation::ObservationPtr getLatest(const DataItemPtr &di) { return getLatest(di->getId()); } - + protected: ConfigOptions m_options; configuration::AsyncContext &m_context; boost::asio::io_context::strand m_strand; - + std::shared_ptr m_loopback; - + bool m_started {false}; - + // Asset Management std::unique_ptr m_assetStorage; - + // Unique id based on the time of creation bool m_initialized {false}; bool m_observationsInitialized {false}; - + // Sources and Sinks source::SourceList m_sources; sink::SinkList m_sinks; - + // Pipeline pipeline::PipelineContextPtr m_pipelineContext; - + // Pointer to the configuration file for node access std::unique_ptr m_xmlParser; PrinterMap m_printers; - + // Agent Device device_model::AgentDevicePtr m_agentDevice; - + /// @brief tag for Device multi-index unsorted struct BySeq {}; @@ -472,7 +480,7 @@ namespace mtconnect { /// @brief tag for Device multi-index by UUID struct ByUuid {}; - + /// @brief Device UUID extractor for multi-index struct ExtractDeviceUuid { @@ -480,7 +488,7 @@ namespace mtconnect { const result_type &operator()(const DevicePtr &d) const { return *d->getUuid(); } result_type operator()(const DevicePtr &d) { return *d->getUuid(); } }; - + /// @brief Device name extractor for multi-index struct ExtractDeviceName { @@ -488,20 +496,22 @@ namespace mtconnect { const result_type &operator()(const DevicePtr &d) const { return *d->getComponentName(); } result_type operator()(DevicePtr &d) { return *d->getComponentName(); } }; - + /// @brief Devuce multi-index using DeviceIndex = mic::multi_index_container< - DevicePtr, mic::indexed_by>, - mic::hashed_unique, ExtractDeviceUuid>, - mic::hashed_unique, ExtractDeviceName>>>; - + DevicePtr, mic::indexed_by>, + mic::hashed_unique, ExtractDeviceUuid>, + mic::hashed_unique, ExtractDeviceName>>>; + DeviceIndex m_deviceIndex; std::unordered_map m_dataItemMap; - + std::unordered_map m_idMap; + // Xml Config std::optional m_schemaVersion; std::string m_deviceXmlPath; - bool m_versionDeviceXml = false; + bool m_versionDeviceXml { false }; + bool m_createUniqueIds { false }; int32_t m_intSchemaVersion = IntDefaultSchemaVersion(); // Circular Buffer diff --git a/src/mtconnect/configuration/agent_config.cpp b/src/mtconnect/configuration/agent_config.cpp index ff4ea577..c0f8e1d6 100644 --- a/src/mtconnect/configuration/agent_config.cpp +++ b/src/mtconnect/configuration/agent_config.cpp @@ -645,6 +645,7 @@ namespace mtconnect::configuration { {configuration::MaxAssets, int(DEFAULT_MAX_ASSETS)}, {configuration::CheckpointFrequency, 1000}, {configuration::LegacyTimeout, 600s}, + {configuration::CreateUniqueIds, false}, {configuration::ReconnectInterval, 10000ms}, {configuration::IgnoreTimestamps, false}, {configuration::ConversionRequired, true}, diff --git a/src/mtconnect/configuration/async_context.hpp b/src/mtconnect/configuration/async_context.hpp index 97bc6c37..0446685e 100644 --- a/src/mtconnect/configuration/async_context.hpp +++ b/src/mtconnect/configuration/async_context.hpp @@ -131,7 +131,7 @@ namespace mtconnect::configuration { m_context.restart(); } - /// @brief Cover methods for asio io_context + /// @name Cover methods for asio io_context /// @{ /// @brief io_context::run_for diff --git a/src/mtconnect/configuration/config_options.hpp b/src/mtconnect/configuration/config_options.hpp index 7116d858..a012d399 100644 --- a/src/mtconnect/configuration/config_options.hpp +++ b/src/mtconnect/configuration/config_options.hpp @@ -68,6 +68,7 @@ namespace mtconnect { DECLARE_CONFIGURATION(TlsOnly); DECLARE_CONFIGURATION(TlsPrivateKey); DECLARE_CONFIGURATION(TlsVerifyClientCertificate); + DECLARE_CONFIGURATION(CreateUniqueIds); DECLARE_CONFIGURATION(VersionDeviceXmlUpdates); DECLARE_CONFIGURATION(WorkerThreads); ///@} diff --git a/src/mtconnect/device_model/component.cpp b/src/mtconnect/device_model/component.cpp index 1c9b6a13..507bdddb 100644 --- a/src/mtconnect/device_model/component.cpp +++ b/src/mtconnect/device_model/component.cpp @@ -160,5 +160,12 @@ namespace mtconnect { device->registerDataItem(dataItem); } } + + std::optional Component::createUniqueId(std::unordered_map &idMap, const boost::uuids::detail::sha1 &psha1) + { + auto newId = Entity::createUniqueId(idMap, psha1); + m_id = *newId; + return newId; + } } // namespace device_model } // namespace mtconnect diff --git a/src/mtconnect/device_model/component.hpp b/src/mtconnect/device_model/component.hpp index 671a77c1..2bb8c1fd 100644 --- a/src/mtconnect/device_model/component.hpp +++ b/src/mtconnect/device_model/component.hpp @@ -43,6 +43,7 @@ namespace mtconnect { using DevicePtr = std::shared_ptr; using DataItemPtr = std::shared_ptr; + using WeakDataItemPtr = std::weak_ptr; /// @brief MTConnect Component Entity class AGENT_LIB_API Component : public entity::Entity @@ -257,6 +258,11 @@ namespace mtconnect { pth.push_back(getTopicName()); } + + /// @brief create unique ids recursively + /// @param[in] itemMap data item id to data item map + /// @param[in] sha the parents sha1 + std::optional createUniqueId(std::unordered_map &idMap, const boost::uuids::detail::sha1 &sha1) override; protected: void setParent(ComponentPtr parent) { m_parent = parent; } diff --git a/src/mtconnect/device_model/data_item/data_item.cpp b/src/mtconnect/device_model/data_item/data_item.cpp index c99160d1..c04bc52a 100644 --- a/src/mtconnect/device_model/data_item/data_item.cpp +++ b/src/mtconnect/device_model/data_item/data_item.cpp @@ -243,7 +243,7 @@ namespace mtconnect { bool DataItem::hasName(const string &name) const { - return m_id == name || (m_name && *m_name == name) || (m_source && *m_source == name); + return m_id == name || (m_name && *m_name == name) || (m_source && *m_source == name) || (m_originalId && *m_originalId == name); } // Sort by: Device, Component, Category, DataItem diff --git a/src/mtconnect/device_model/data_item/data_item.hpp b/src/mtconnect/device_model/data_item/data_item.hpp index 3138a8a9..1e8e420b 100644 --- a/src/mtconnect/device_model/data_item/data_item.hpp +++ b/src/mtconnect/device_model/data_item/data_item.hpp @@ -40,6 +40,7 @@ namespace mtconnect { } namespace device_model { class Composition; + struct UpdateDataItemId; /// @brief DataItem related entities namespace data_item { @@ -221,15 +222,45 @@ namespace mtconnect { bool operator<(const DataItem &another) const; bool operator==(const DataItem &another) const { return m_id == another.m_id; } + /// @brief Return the category as a char * const char *getCategoryText() const { return m_categoryText; } + /// @brief create unique ids recursively + /// @param[in] itemMap data item id to data item map + /// @param[in] sha the parents sha1 + std::optional createUniqueId(std::unordered_map &idMap, const boost::uuids::detail::sha1 &sha1) override + { + m_originalId.emplace(m_id); + auto pref = m_id == m_preferredName; + m_id = *Entity::createUniqueId(idMap, sha1); + if (pref) + m_preferredName = m_id; + m_observatonProperties.insert_or_assign("dataItemId", m_id); + return m_id; + } + + /// @brief Get a reference to the optional original id + /// @returns optional original id + const auto &getOriginalId() const { return m_originalId; } + + void updateReferences(const std::unordered_map idMap) override + { + Entity::updateReferences(idMap); + if (hasProperty("compositionId")) + m_observatonProperties.insert_or_assign("compositionId", get("compositionId")); + } + + protected: double simpleFactor(const std::string &units); std::map buildAttributes() const; + friend struct device_model::UpdateDataItemId; + protected: // Unique ID for each component std::string m_id; + std::optional m_originalId; // Name for itself std::optional m_name; diff --git a/src/mtconnect/device_model/device.cpp b/src/mtconnect/device_model/device.cpp index d2fbf3b7..4d571f10 100644 --- a/src/mtconnect/device_model/device.cpp +++ b/src/mtconnect/device_model/device.cpp @@ -152,6 +152,9 @@ namespace mtconnect { { if (auto it = m_dataItems.get().find(name); it != m_dataItems.get().end()) return it->lock(); + + if (auto it = m_dataItems.get().find(name); it != m_dataItems.get().end()) + return it->lock(); if (auto it = m_dataItems.get().find(name); it != m_dataItems.get().end()) return it->lock(); @@ -161,5 +164,65 @@ namespace mtconnect { return nullptr; } + + struct UpdateDataItemId { + const string &m_id; + UpdateDataItemId(const string &id) : m_id(id) {} + + void operator()(WeakDataItemPtr &ptr) + { + auto di = ptr.lock(); + di->m_id = m_id; + } + }; + + void Device::createUniqueIds(std::unordered_map &idMap) + { + boost::uuids::detail::sha1 sha; + sha.process_bytes(m_uuid->data(), m_uuid->size()); + + Component::createUniqueId(idMap, sha); + + for (auto it = m_dataItems.begin(); it != m_dataItems.end(); it++) + { + auto di = it->lock(); + const auto &oldId = di->getOriginalId(); + if (oldId) + { + auto id = idMap.find(*oldId); + if (id != idMap.end()) + { + m_dataItems.modify(it, UpdateDataItemId(id->second)); + } + else + { + LOG(error) << "Cannot find id " << *oldId << " in data item map"; + } + } + else + { + LOG(error) << "DataItem id for " << di->getId() << " was not made unique"; + } + } + + for (auto p : m_componentsById) + { + auto comp = p.second.lock(); + if (comp) + { + auto orig = comp->maybeGet("originalId"); + if (orig && m_componentsById.count(*orig) == 0) + { + m_componentsById.emplace(*orig, comp); + } + if (m_componentsById.count(comp->getId()) == 0) + { + m_componentsById.emplace(comp->getId(), comp); + } + } + + } + } + } // namespace device_model } // namespace mtconnect diff --git a/src/mtconnect/device_model/device.hpp b/src/mtconnect/device_model/device.hpp index 0cd4059e..df0d96ca 100644 --- a/src/mtconnect/device_model/device.hpp +++ b/src/mtconnect/device_model/device.hpp @@ -53,6 +53,9 @@ namespace mtconnect { /// @brief multi-index tag: Data items indexed by id struct ById {}; + /// @brief multi-index tag: Data items indexed by optional original id + struct ByOriginalId + {}; /// @brief multi-index tag: Data items index by Source struct BySource {}; @@ -66,6 +69,29 @@ namespace mtconnect { using result_type = std::string; const result_type &operator()(const WeakDataItemPtr d) const { return d.lock()->getId(); } }; + /// @brief multi-index data item id extractor + /// + /// falls back to id if orginal id is not given + + struct ExtractOriginalId + { + using result_type = std::string; + const result_type operator()(const WeakDataItemPtr &d) const + { + const static result_type none {}; + if (d.expired()) + return none; + else + { + auto di = d.lock(); + auto id = di->getOriginalId(); + if (id) + return *id; + else + return di->getId(); + } + } + }; /// @brief multi-index data item name extractor /// /// falls back to id if name is not given @@ -125,11 +151,12 @@ namespace mtconnect { } } }; - + /// @brief Mapping of device names to data items using DataItemIndex = mic::multi_index_container< WeakDataItemPtr, mic::indexed_by, ExtractId>, + mic::hashed_unique, ExtractOriginalId>, mic::hashed_non_unique, ExtractSource>, mic::hashed_non_unique, ExtractName>, mic::ordered_non_unique, ExtractType>>>; @@ -165,7 +192,13 @@ namespace mtconnect { /// @brief Add a data item to the device /// @param[in] dataItem shared pointer to the data item void addDeviceDataItem(DataItemPtr dataItem); - /// @brief get a data item by source, name, and id + /// @brief get a data item by multiple indexes + /// + /// Looks for a data item using the following idexes in the following order: + /// 1. id + /// 2. original id (set when creating unique ids) + /// 3. name + /// 4. source /// @param[in] name the source, name, or id of the data item /// @return shared pointer to the data item if found DataItemPtr getDeviceDataItem(const std::string &name) const; @@ -240,7 +273,7 @@ namespace mtconnect { /// /// Converts the id attribute to a unique value and caches the original value /// in case it is required later - + void createUniqueIds(std::unordered_map &idMap); protected: void cachePointers(DataItemPtr dataItem); diff --git a/src/mtconnect/entity/entity.cpp b/src/mtconnect/entity/entity.cpp index 557e7ac0..82693b31 100644 --- a/src/mtconnect/entity/entity.cpp +++ b/src/mtconnect/entity/entity.cpp @@ -16,7 +16,7 @@ // #include - +#include #include "factory.hpp" using namespace std; @@ -131,7 +131,7 @@ namespace mtconnect::entity { for (const auto &e : m_properties) { // Skip hash - if (!skip.contains(e.first)) + if (!skip.contains(e.first) && !isHidden(e.first)) { const auto &value = e.second; sha1.process_bytes(e.first.c_str(), e.first.size()); @@ -140,5 +140,109 @@ namespace mtconnect::entity { } } } + + struct UniqueIdVisitor { + std::unordered_map &m_idMap; + const boost::uuids::detail::sha1 &m_sha1; + UniqueIdVisitor(std::unordered_map &idMap, + const boost::uuids::detail::sha1 &sha1) + : m_idMap(idMap), m_sha1(sha1) + {} + + void operator()(EntityPtr &p) + { + p->createUniqueId(m_idMap, m_sha1); + } + + void operator()(EntityList &l) + { + for (auto &e : l) + e->createUniqueId(m_idMap, m_sha1); + } + + template + void operator()(const T &) {} + }; + + std::optional Entity::createUniqueId(std::unordered_map &idMap, const boost::uuids::detail::sha1 &sha1) + { + optional res; + + auto it = m_properties.find("id"); + if (it != m_properties.end()) + { + std::string newId, oldId; + auto origId = maybeGet("originalId"); + if (!origId) + { + oldId = std::get(it->second); + m_properties.emplace("originalId", oldId); + newId = makeUniqueId(sha1, oldId); + it->second = newId; + } + else + { + oldId = *origId; + newId = std::get(it->second); + } + idMap.emplace(oldId, newId); + res.emplace(newId); + } + + UniqueIdVisitor visitor(idMap, sha1); + + // Recurse properties + for (auto &p : m_properties) + { + std::visit(visitor, p.second); + } + + return res; + } + + struct ReferenceIdVisitor { + const std::unordered_map &m_idMap; + ReferenceIdVisitor(const std::unordered_map &idMap) + : m_idMap(idMap) + {} + + void operator()(EntityPtr &p) + { + p->updateReferences(m_idMap); + } + + void operator()(EntityList &l) + { + for (auto &e : l) + e->updateReferences(m_idMap); + } + + template + void operator()(T &) {} + }; + + void Entity::updateReferences(std::unordered_map idMap) + { + using namespace boost::algorithm; + for (auto &prop : m_properties) + { + if (prop.first != "originalId" && (iends_with(prop.first, "idref") || (prop.first.length() > 2 && iends_with(prop.first, "id")))) + { + auto it = idMap.find(std::get(prop.second)); + if (it != idMap.end()) + { + prop.second = it->second; + } + } + } + + ReferenceIdVisitor visitor(idMap); + + // Recurse all + for (auto &p : m_properties) + { + std::visit(visitor, p.second); + } + } } // namespace mtconnect::entity diff --git a/src/mtconnect/entity/entity.hpp b/src/mtconnect/entity/entity.hpp index f5eebb56..07a6d099 100644 --- a/src/mtconnect/entity/entity.hpp +++ b/src/mtconnect/entity/entity.hpp @@ -105,6 +105,16 @@ namespace mtconnect { /// @brief method to return the entities identity. defaults to `id`. /// @return the identity virtual const entity::Value &getIdentity() const { return getProperty("id"); } + + /// @brief create unique ids recursively + /// @param[in,out] idMap data item id to data item map + /// @param[in] sha the parents sha1 + /// @returns optional string value of the new id + virtual std::optional createUniqueId(std::unordered_map &idMap, const boost::uuids::detail::sha1 &sha1); + + /// @brief update all id references to the new ids recursively + /// @param[in] idMap map of old ids to new ids + virtual void updateReferences(const std::unordered_map idMap); /// @brief checks if there is entity is a list that has additional properties /// @return `true` if this a a list with additional attributes @@ -122,6 +132,15 @@ namespace mtconnect { /// @brief checkis if this is an entity only containing a list /// @returns `true` if this entity contains a list bool isList() const { return m_properties.count("LIST") > 0; } + /// @brief check if the property is hidden for normal serialization + /// @param[in] name the name of the property + /// + /// Used for serializing properties for configuration purposes where additional attibutes + /// may be included in the document. The only property is `originalId` for now. + /// + /// This may be extended to include a property set in the future. + /// @returns `true` if the property is hidden + bool isHidden(const std::string &name) const { return name == "originalId"; } /// @brief get the name of the entity /// @return name const auto &getName() const { return m_name; } diff --git a/src/mtconnect/entity/factory.hpp b/src/mtconnect/entity/factory.hpp index db769f4c..e1d91bc3 100644 --- a/src/mtconnect/entity/factory.hpp +++ b/src/mtconnect/entity/factory.hpp @@ -359,6 +359,10 @@ namespace mtconnect { /// @brief scan requirements and register factories refrenced in the requirements void registerEntityRequirements() { + if (!m_simpleProperties.count("originalId")) + { + m_requirements.emplace_back("originalId", false); + } for (auto &r : m_requirements) { m_properties.emplace(r.getName()); diff --git a/src/mtconnect/entity/json_printer.hpp b/src/mtconnect/entity/json_printer.hpp index 795ba776..b78cea93 100644 --- a/src/mtconnect/entity/json_printer.hpp +++ b/src/mtconnect/entity/json_printer.hpp @@ -106,7 +106,8 @@ namespace mtconnect::entity { /// @param version the supported MTConnect serialization version /// - Version 1 has a repreated objects in arrays for collections of objects /// - Version 2 combines arrays of objects by type - JsonPrinter(T &writer, uint32_t version) : m_version(version), m_writer(writer) {}; + JsonPrinter(T &writer, uint32_t version, + bool includeHidden = false) : m_version(version), m_writer(writer), m_includeHidden(includeHidden) {}; /// @brief create a json object from an entity /// @@ -134,8 +135,11 @@ namespace mtconnect::entity { for (auto &prop : entity->getProperties()) { - visitor.m_key = &prop.first; - visit(visitor, prop.second); + if (m_includeHidden || !entity->isHidden(prop.first)) + { + visitor.m_key = &prop.first; + visit(visitor, prop.second); + } } } @@ -268,6 +272,7 @@ namespace mtconnect::entity { protected: uint32_t m_version; T &m_writer; + bool m_includeHidden { false }; }; /// @brief Serialization wrapper to turn an entity into a json string. @@ -275,7 +280,8 @@ namespace mtconnect::entity { { public: /// @brief Create a printer for with a JSON vesion and flag to pretty print - JsonEntityPrinter(uint32_t version, bool pretty = false) : m_version(version), m_pretty(pretty) + JsonEntityPrinter(uint32_t version, bool pretty = false, + bool includeHidden = false) : m_version(version), m_pretty(pretty), m_includeHidden(includeHidden) {} /// @brief wrapper around the JsonPrinter print method that creates the correct printer @@ -287,7 +293,7 @@ namespace mtconnect::entity { using namespace rapidjson; StringBuffer output; RenderJson(output, m_pretty, [&](auto &writer) { - JsonPrinter printer(writer, m_version); + JsonPrinter printer(writer, m_version, m_includeHidden); printer.printEntity(entity); }); @@ -306,7 +312,7 @@ namespace mtconnect::entity { using namespace rapidjson; StringBuffer output; RenderJson(output, m_pretty, [&](auto &writer) { - JsonPrinter printer(writer, m_version); + JsonPrinter printer(writer, m_version, m_includeHidden); printer.print(entity); }); @@ -316,5 +322,6 @@ namespace mtconnect::entity { protected: uint32_t m_version; bool m_pretty; + bool m_includeHidden { false }; }; } // namespace mtconnect::entity diff --git a/src/mtconnect/entity/xml_printer.cpp b/src/mtconnect/entity/xml_printer.cpp index 4eadbd19..1ba68e0c 100644 --- a/src/mtconnect/entity/xml_printer.cpp +++ b/src/mtconnect/entity/xml_printer.cpp @@ -207,10 +207,13 @@ namespace mtconnect { for (const auto &prop : properties) { auto &key = prop.first; - if (islower(key.getName()[0]) || attrs.count(key) > 0) - attributes.emplace_back(prop); - else - elements.emplace_back(prop); + if (m_includeHidden || !entity->isHidden(key)) + { + if (islower(key.getName()[0]) || attrs.count(key) > 0) + attributes.emplace_back(prop); + else + elements.emplace_back(prop); + } } // Reorder elements if they need to be specially ordered. diff --git a/src/mtconnect/entity/xml_printer.hpp b/src/mtconnect/entity/xml_printer.hpp index 959fc8ec..b32e1289 100644 --- a/src/mtconnect/entity/xml_printer.hpp +++ b/src/mtconnect/entity/xml_printer.hpp @@ -34,7 +34,7 @@ namespace mtconnect { class AGENT_LIB_API XmlPrinter { public: - XmlPrinter() = default; + XmlPrinter(bool includeHidden = false) : m_includeHidden(includeHidden) {} /// @brief convert an entity to a XML document using `libxml2` /// @param writer libxml2 `xmlTextWriterPtr` @@ -42,6 +42,9 @@ namespace mtconnect { /// @param namespaces a set of namespaces to use in the document void print(xmlTextWriterPtr writer, const EntityPtr entity, const std::unordered_set &namespaces); + + protected: + bool m_includeHidden { false }; }; } // namespace entity } // namespace mtconnect diff --git a/src/mtconnect/printer/json_printer.cpp b/src/mtconnect/printer/json_printer.cpp index 2e36fde6..2fb86018 100644 --- a/src/mtconnect/printer/json_printer.cpp +++ b/src/mtconnect/printer/json_printer.cpp @@ -180,13 +180,14 @@ namespace mtconnect::printer { const uint64_t nextSeq, const unsigned int assetBufferSize, const unsigned int assetCount, const std::list &devices, - const std::map *count) const + const std::map *count, + bool includeHidden) const { defaultSchemaVersion(); StringBuffer output; RenderJson(output, m_pretty, [&](auto &writer) { - entity::JsonPrinter printer(writer, m_jsonVersion); + entity::JsonPrinter printer(writer, m_jsonVersion, includeHidden); AutoJsonObject top(writer); AutoJsonObject obj(writer, "MTConnectDevices"); diff --git a/src/mtconnect/printer/json_printer.hpp b/src/mtconnect/printer/json_printer.hpp index 8b417e09..85de4f69 100644 --- a/src/mtconnect/printer/json_printer.hpp +++ b/src/mtconnect/printer/json_printer.hpp @@ -36,7 +36,8 @@ namespace mtconnect::printer { std::string printProbe(const uint64_t instanceId, const unsigned int bufferSize, const uint64_t nextSeq, const unsigned int assetBufferSize, const unsigned int assetCount, const std::list &devices, - const std::map *count = nullptr) const override; + const std::map *count = nullptr, + bool includeHidden = false) const override; std::string printSample(const uint64_t instanceId, const unsigned int bufferSize, const uint64_t nextSeq, const uint64_t firstSeq, const uint64_t lastSeq, diff --git a/src/mtconnect/printer/printer.hpp b/src/mtconnect/printer/printer.hpp index 56810ba6..d22cb1e8 100644 --- a/src/mtconnect/printer/printer.hpp +++ b/src/mtconnect/printer/printer.hpp @@ -86,7 +86,8 @@ namespace mtconnect { const uint64_t instanceId, const unsigned int bufferSize, const uint64_t nextSeq, const unsigned int assetBufferSize, const unsigned int assetCount, const std::list &devices, - const std::map *count = nullptr) const = 0; + const std::map *count = nullptr, + bool includeHidden = false) const = 0; /// @brief Print a MTConnect Streams document /// @param[in] instanceId the instance id /// @param[in] bufferSize the buffer size diff --git a/src/mtconnect/printer/xml_printer.cpp b/src/mtconnect/printer/xml_printer.cpp index ec0ffea5..43fe3f3b 100644 --- a/src/mtconnect/printer/xml_printer.cpp +++ b/src/mtconnect/printer/xml_printer.cpp @@ -377,7 +377,8 @@ namespace mtconnect::printer { string XmlPrinter::printProbe(const uint64_t instanceId, const unsigned int bufferSize, const uint64_t nextSeq, const unsigned int assetBufferSize, const unsigned int assetCount, const list &deviceList, - const std::map *count) const + const std::map *count, + bool includeHidden) const { string ret; @@ -390,7 +391,7 @@ namespace mtconnect::printer { { AutoElement devices(writer, "Devices"); - entity::XmlPrinter printer; + entity::XmlPrinter printer(includeHidden); for (auto &device : deviceList) printer.print(writer, device, m_deviceNsSet); diff --git a/src/mtconnect/printer/xml_printer.hpp b/src/mtconnect/printer/xml_printer.hpp index 955dcf90..a453b82f 100644 --- a/src/mtconnect/printer/xml_printer.hpp +++ b/src/mtconnect/printer/xml_printer.hpp @@ -49,7 +49,8 @@ namespace mtconnect { std::string printProbe(const uint64_t instanceId, const unsigned int bufferSize, const uint64_t nextSeq, const unsigned int assetBufferSize, const unsigned int assetCount, const std::list &devices, - const std::map *count = nullptr) const override; + const std::map *count = nullptr, + bool includeHidden = false) const override; std::string printSample(const uint64_t instanceId, const unsigned int bufferSize, const uint64_t nextSeq, const uint64_t firstSeq, diff --git a/src/mtconnect/sink/rest_sink/server.hpp b/src/mtconnect/sink/rest_sink/server.hpp index c7c84557..bcd6c8e1 100644 --- a/src/mtconnect/sink/rest_sink/server.hpp +++ b/src/mtconnect/sink/rest_sink/server.hpp @@ -245,6 +245,7 @@ namespace mtconnect::sink::rest_sink { /// @name Swagger Support /// @{ + /// /// @brief Add swagger routings to the Agent void addSwaggerRoutings(); /// @brief generate swagger API from routings diff --git a/src/mtconnect/utilities.hpp b/src/mtconnect/utilities.hpp index 4ef4862a..7fa1d0c4 100644 --- a/src/mtconnect/utilities.hpp +++ b/src/mtconnect/utilities.hpp @@ -24,6 +24,8 @@ #include #include #include +#include +#include #include #include @@ -725,4 +727,51 @@ namespace mtconnect { /// @param[in] context the boost asio io_context for resolving the address /// @param[in] onlyV4 only consider IPV4 addresses if `true` std::string GetBestHostAddress(boost::asio::io_context &context, bool onlyV4 = false); + + /// @brief Function to create a unique id given a sha1 namespace and an id. + /// + /// Creates a base 64 encoded version of the string and removes any illegal characters + /// for an ID. If the first character is not a legal start character, maps the first 2 characters + /// to the legal ID start char set. + /// + /// @param[in] sha the sha1 namespace to use as context + /// @param[in] id the id to use transform + /// @returns Returns the first 16 characters of the base 64 encoded sha1 + inline std::string makeUniqueId(const boost::uuids::detail::sha1 &sha, const std::string &id) + { + using namespace std; + + boost::uuids::detail::sha1 sha1(sha); + + constexpr string_view startc( "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_"); + constexpr auto isIDStartChar = [](unsigned char c) -> bool { + return isalpha(c) || c == '_'; + }; + constexpr auto isIDChar = [isIDStartChar](unsigned char c) -> bool { + return isIDStartChar(c) || isdigit(c) || c == '.' || c == '-'; + }; + + sha1.process_bytes(id.data(), id.length()); + unsigned int digest[5]; + sha1.get_digest(digest); + + string s(32, ' '); + auto len = boost::beast::detail::base64::encode(s.data(), digest, sizeof(digest)); + + s.erase(len - 1); + std::remove_if(++(s.begin()), s.end(), not_fn(isIDChar)); + + // Check if the character is legal. + if (!isIDStartChar(s[0])) + { + // Change the start character to a legal character + uint32_t c = s[0] + s[1]; + s.erase(0, 1); + s[0] = startc[c % startc.size()]; + } + + s.erase(16); + + return s; + } } // namespace mtconnect diff --git a/test/config_test.cpp b/test/config_test.cpp index 28cb35d9..3f1f69fc 100644 --- a/test/config_test.cpp +++ b/test/config_test.cpp @@ -1506,6 +1506,8 @@ Adapters { TEST_F(ConfigTest, should_update_a_device_when_received_from_adapter) { + using namespace mtconnect::source::adapter; + fs::path root {createTempDirectory("7")}; fs::path devices(root / "Devices.xml"); @@ -1514,16 +1516,174 @@ Adapters { ofstream cfg(config.string()); cfg << R"DOC( VersionDeviceXmlUpdates = true +CreateUniqueIds = true Port = 0 )DOC"; cfg << "Devices = " << devices << endl; } + + copyFile("dyn_load.xml", devices, 0min); + + boost::program_options::variables_map options; + boost::program_options::variable_value value(boost::optional(config.string()), false); + options.insert(make_pair("config-file"s, value)); + + m_config->initialize(options); + auto &asyncContext = m_config->getAsyncContext(); + + auto agent = m_config->getAgent(); + auto device = agent->getDeviceByName("LinuxCNC"); + ASSERT_TRUE(device); - GTEST_SKIP(); + const auto &printer = agent->getPrinter("xml"); + ASSERT_NE(nullptr, printer); + + auto sp = agent->findSource("_localhost_7878"); + ASSERT_TRUE(sp); + + auto adapter = dynamic_pointer_cast(sp); + ASSERT_TRUE(adapter); + + auto validate = [&](boost::system::error_code ec) { + using namespace std::filesystem; + using namespace std::chrono; + using namespace boost::algorithm; + + if (!ec) + { + // Check for backup file + auto device = agent->getDeviceByName("LinuxCNC"); + ASSERT_TRUE(device) << "Cannot find LinuxCNC device"; + + const auto &components = device->getChildren(); + ASSERT_EQ(1, components->size()); + + auto cont = device->getComponentById("cont"); + ASSERT_TRUE(cont) << "Cannot find Component with id cont"; + + auto devDIs = device->getDataItems(); + ASSERT_TRUE(devDIs); + ASSERT_EQ(5, devDIs->size()); + + auto dataItems = cont->getDataItems(); + ASSERT_TRUE(dataItems); + ASSERT_EQ(2, dataItems->size()); + + auto it = dataItems->begin(); + ASSERT_EQ("exc", (*it)->get("originalId")); + it++; + ASSERT_EQ("mode", (*it)->get("originalId")); + + + auto estop = device->getDeviceDataItem("estop"); + ASSERT_TRUE(estop) << "Cannot find DataItem with id estop"; + + auto exec = device->getDeviceDataItem("exc"); + ASSERT_TRUE(exec) << "Cannot find DataItem with id exc"; + + auto pipeline = dynamic_cast(adapter->getPipeline()); + ASSERT_EQ("LinuxCNC", pipeline->getDevice()); + } + m_config->stop(); + }; + + boost::asio::steady_timer timer2(asyncContext.get()); + + auto send = [this, &adapter, &timer2, validate](boost::system::error_code ec) { + if (ec) + { + m_config->stop(); + } + else + { + adapter->processData("* deviceModel: --multiline--AAAAA"); + adapter->processData(R"( + + + + + + + + + + + + + + + +)"); + adapter->processData("--multiline--AAAAA"); + + timer2.expires_from_now(500ms); + timer2.async_wait(validate); + } + }; + + boost::asio::steady_timer timer1(asyncContext.get()); + timer1.expires_from_now(100ms); + timer1.async_wait(send); + + m_config->start(); + } TEST_F(ConfigTest, should_update_the_ids_of_all_entities) { - GTEST_SKIP(); + fs::path root {createTempDirectory("8")}; + + fs::path devices(root / "Devices.xml"); + fs::path config {root / "agent.cfg"}; + { + ofstream cfg(config.string()); + cfg << R"DOC( +VersionDeviceXmlUpdates = true +CreateUniqueIds = true +Port = 0 +)DOC"; + cfg << "Devices = " << devices << endl; + } + + copyFile("dyn_load.xml", devices, 0min); + + boost::program_options::variables_map options; + boost::program_options::variable_value value(boost::optional(config.string()), false); + options.insert(make_pair("config-file"s, value)); + + m_config->initialize(options); + + auto agent = m_config->getAgent(); + auto device = agent->getDeviceByName("LinuxCNC"); + ASSERT_TRUE(device); + + auto deviceId = device->getId(); + + ASSERT_NE("d", deviceId); + ASSERT_EQ("d", device->get("originalId")); + + // Get the data item by its old id + auto exec = device->getDeviceDataItem("exec"); + ASSERT_TRUE(exec); + ASSERT_TRUE(exec->getOriginalId()); + ASSERT_EQ("exec", *exec->getOriginalId()); + + // Re-initialize the agent with the modified device.xml with the unique ids aready created + // This tests if the originalId in the device xml file does the ritght thing when mapping ids + m_config = std::make_unique(); + m_config->setDebug(true); + m_config->initialize(options); + + auto agent2 = m_config->getAgent(); + + auto device2 = agent2->getDeviceByName("LinuxCNC"); + ASSERT_TRUE(device2); + ASSERT_EQ(deviceId, device2->getId()); + + auto exec2 = device->getDeviceDataItem("exec"); + ASSERT_TRUE(exec2); + ASSERT_EQ(exec->getId(), exec2->getId()); + ASSERT_TRUE(exec2->getOriginalId()); + ASSERT_EQ("exec", *exec2->getOriginalId()); } } // namespace diff --git a/test/entity_printer_test.cpp b/test/entity_printer_test.cpp index 20caa8c1..a285d683 100644 --- a/test/entity_printer_test.cpp +++ b/test/entity_printer_test.cpp @@ -280,6 +280,84 @@ TEST_F(EntityPrinterTest, TestRawContent) ASSERT_EQ(expected, m_writer->getContent()); } +TEST_F(EntityPrinterTest, should_honor_include_hidden_parameter) +{ + auto component = make_shared(Requirements { + Requirement("id", true), + Requirement("name", false), + Requirement("uuid", false), + }); + + auto components = make_shared( + Requirements({Requirement("Component", ENTITY, component, 1, Requirement::Infinite)})); + components->registerMatchers(); + components->registerFactory(regex(".+"), component); + + component->addRequirements({Requirement("Components", ENTITY_LIST, components, false)}); + + auto device = make_shared(*component); + device->addRequirements(Requirements { + Requirement("name", true), + Requirement("uuid", true), + }); + + auto root = make_shared(Requirements {Requirement("Device", ENTITY, device)}); + + auto doc = string { + "\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + "\n"}; + + ErrorList errors; + entity::XmlParser parser; + + auto entity = parser.parse(root, doc, errors); + ASSERT_EQ(0, errors.size()); + + boost::uuids::detail::sha1 sha1; + unordered_map idMap; + + entity->createUniqueId(idMap, sha1); + + entity::XmlPrinter printer(false); + printer.print(*m_writer, entity, {}); + + ASSERT_EQ(R"( + + + + + + + + + +)", m_writer->getContent()); + + m_writer = make_unique(true); + entity::XmlPrinter printer2(true); + printer2.print(*m_writer, entity, {}); + + ASSERT_EQ(R"( + + + + + + + + + +)", m_writer->getContent()); +} + class EntityPrinterNamespaceTest : public EntityPrinterTest { protected: diff --git a/test/json_printer_test.cpp b/test/json_printer_test.cpp index 90912b3d..80c3c2c6 100644 --- a/test/json_printer_test.cpp +++ b/test/json_printer_test.cpp @@ -354,3 +354,177 @@ TEST_F(JsonPrinterTest, elements_with_property_list_version_2) ASSERT_EQ("2", jdoc.at("/Root/CuttingItems/list/CuttingItem/1/itemId"_json_pointer).get()); } + +TEST_F(JsonPrinterTest, should_honor_include_hidden_parameter) +{ + auto root = createFileArchetypeFactory(); + auto doc = deviceModel(); + + ErrorList errors; + entity::XmlParser parser; + + auto entity = parser.parse(root, doc, errors); + ASSERT_EQ(0, errors.size()); + + boost::uuids::detail::sha1 sha1; + unordered_map idMap; + + entity->createUniqueId(idMap, sha1); + + entity::JsonEntityPrinter jprinter(1, true, false); + auto jdoc = jprinter.print(entity); + + ASSERT_EQ(R"({ + "MTConnectDevices": { + "Devices": [ + { + "Device": { + "Components": [ + { + "Systems": { + "Components": [ + { + "Electric": { + "id": "Pm2JhGKEeAYzVA8c" + } + }, + { + "Heating": { + "id": "culKrBObwYWb6x0g" + } + } + ], + "Description": { + "value": "Hey Will", + "model": "abc" + }, + "id": "_cNZEyq5kGkgppmh" + } + } + ], + "DataItems": [ + { + "DataItem": { + "category": "EVENT", + "id": "FFZeJQRwQvAdUJX4", + "name": "avail", + "type": "AVAILABILITY" + } + }, + { + "DataItem": { + "category": "EVENT", + "id": "T0qItk3igtyip1XX", + "type": "ASSET_CHANGED" + } + }, + { + "DataItem": { + "category": "EVENT", + "id": "LWOt9yZtpFPWjL7v", + "type": "ASSET_REMOVED" + } + } + ], + "id": "DFYX7ls4d4to2Lhb", + "name": "foo", + "uuid": "xxx" + } + } + ], + "Header": { + "assetBufferSize": 8096, + "assetCount": 60, + "bufferSize": 131072, + "creationTime": "2021-01-07T18:34:15Z", + "deviceModelChangeTime": "2021-01-07T18:34:15Z", + "instanceId": 1609418103, + "sender": "DMZ-MTCNCT", + "version": "1.6.0.6" + } + } +})", jdoc); + + entity::JsonEntityPrinter jprinter2(1, true, true); + jdoc = jprinter2.print(entity); + + ASSERT_EQ(R"({ + "MTConnectDevices": { + "Devices": [ + { + "Device": { + "Components": [ + { + "Systems": { + "Components": [ + { + "Electric": { + "id": "Pm2JhGKEeAYzVA8c", + "originalId": "e1" + } + }, + { + "Heating": { + "id": "culKrBObwYWb6x0g", + "originalId": "h1" + } + } + ], + "Description": { + "value": "Hey Will", + "model": "abc" + }, + "id": "_cNZEyq5kGkgppmh", + "originalId": "s1" + } + } + ], + "DataItems": [ + { + "DataItem": { + "category": "EVENT", + "id": "FFZeJQRwQvAdUJX4", + "name": "avail", + "originalId": "avail", + "type": "AVAILABILITY" + } + }, + { + "DataItem": { + "category": "EVENT", + "id": "T0qItk3igtyip1XX", + "originalId": "d1_asset_chg", + "type": "ASSET_CHANGED" + } + }, + { + "DataItem": { + "category": "EVENT", + "id": "LWOt9yZtpFPWjL7v", + "originalId": "d1_asset_rem", + "type": "ASSET_REMOVED" + } + } + ], + "id": "DFYX7ls4d4to2Lhb", + "name": "foo", + "originalId": "d1", + "uuid": "xxx" + } + } + ], + "Header": { + "assetBufferSize": 8096, + "assetCount": 60, + "bufferSize": 131072, + "creationTime": "2021-01-07T18:34:15Z", + "deviceModelChangeTime": "2021-01-07T18:34:15Z", + "instanceId": 1609418103, + "sender": "DMZ-MTCNCT", + "version": "1.6.0.6" + } + } +})", jdoc); + + +} From d8b9ea6faefc0eea9795424705a3d3082bb5688e Mon Sep 17 00:00:00 2001 From: Will Sobel Date: Tue, 25 Apr 2023 16:34:01 +0200 Subject: [PATCH 08/18] Formatted w/ clang --- src/mtconnect/agent.cpp | 18 +-- src/mtconnect/agent.hpp | 128 +++++++++--------- src/mtconnect/device_model/component.cpp | 6 +- src/mtconnect/device_model/component.hpp | 7 +- .../device_model/data_item/data_item.cpp | 3 +- .../device_model/data_item/data_item.hpp | 14 +- src/mtconnect/device_model/device.cpp | 21 +-- src/mtconnect/device_model/device.hpp | 4 +- src/mtconnect/entity/entity.cpp | 67 +++++---- src/mtconnect/entity/entity.hpp | 10 +- src/mtconnect/entity/json_printer.hpp | 12 +- src/mtconnect/entity/xml_printer.hpp | 4 +- src/mtconnect/printer/printer.hpp | 10 +- src/mtconnect/sink/rest_sink/server.hpp | 2 +- .../source/adapter/adapter_pipeline.cpp | 2 +- src/mtconnect/utilities.hpp | 20 ++- test/config_test.cpp | 30 ++-- test/entity_printer_test.cpp | 64 ++++----- test/json_printer_test.cpp | 16 +-- 19 files changed, 220 insertions(+), 218 deletions(-) diff --git a/src/mtconnect/agent.cpp b/src/mtconnect/agent.cpp index 09db8563..cf2a85d8 100644 --- a/src/mtconnect/agent.cpp +++ b/src/mtconnect/agent.cpp @@ -147,7 +147,7 @@ namespace mtconnect { if (m_versionDeviceXml && m_createUniqueIds) versionDeviceXml(); - + loadCachedProbe(); m_initialized = true; @@ -483,7 +483,7 @@ namespace mtconnect { device->addDataItem(odi, errors); if (auto odi = oldDev->getAssetCount(), ndi = device->getAssetCount(); odi && !ndi) device->addDataItem(odi, errors); - + if (errors.size() > 0) { LOG(error) << "Error adding device required data items for " << *device->getUuid() << ':'; @@ -491,7 +491,6 @@ namespace mtconnect { LOG(error) << " " << e->what(); return false; } - if (m_createUniqueIds) createUniqueIds(device); @@ -562,13 +561,12 @@ namespace mtconnect { void Agent::versionDeviceXml() { using namespace std::chrono; - + if (m_versionDeviceXml) { - // update with a new version of the device.xml, saving the old one // with a date time stamp - auto ext = date::format(".%Y-%m-%dT%H+%M+%SZ", date::floor(system_clock::now())); + auto ext = date::format(".%Y-%m-%dT%H+%M+%SZ", date::floor(system_clock::now())); fs::path file(m_deviceXmlPath); fs::path backup(m_deviceXmlPath + ext); if (!fs::exists(backup)) @@ -579,8 +577,7 @@ namespace mtconnect { std::list list; copy_if(m_deviceIndex.begin(), m_deviceIndex.end(), back_inserter(list), [](DevicePtr d) { return dynamic_cast(d.get()) == nullptr; }); - auto probe = printer.printProbe(0, 0, 0, 0, 0, list, nullptr, - true); + auto probe = printer.printProbe(0, 0, 0, 0, 0, list, nullptr, true); ofstream devices(file.string()); devices << probe; @@ -968,17 +965,16 @@ namespace mtconnect { } } } - + void Agent::createUniqueIds(DevicePtr device) { if (m_createUniqueIds && !dynamic_pointer_cast(device)) { device->createUniqueIds(m_idMap); device->updateReferences(m_idMap); - } + } } - void Agent::loadCachedProbe() { NAMED_SCOPE("Agent::loadCachedProbe"); diff --git a/src/mtconnect/agent.hpp b/src/mtconnect/agent.hpp index d8ead32f..1b04a9ba 100644 --- a/src/mtconnect/agent.hpp +++ b/src/mtconnect/agent.hpp @@ -93,55 +93,55 @@ namespace mtconnect { /// - DisableAgentDevice Agent(configuration::AsyncContext &context, const std::string &deviceXmlPath, const ConfigOptions &options); - + /// Destructor for the Agent. /// > Note: Does not stop the agent. ~Agent(); - + /// @brief Hook callback type using Hook = std::function; - + /// @brief Functions to run before the agent begins the initialization process. /// @return configuration::HookManager& auto &beforeInitializeHooks() { return m_beforeInitializeHooks; } - + /// @brief Function that run after all agent initialization is complete /// @return configuration::HookManager& auto &afterInitializeHooks() { return m_afterInitializeHooks; } - + /// @brief Hooks to run when before the agent starts all the soures and sinks /// @return configuration::HookManager& auto &beforeStartHooks() { return m_beforeStartHooks; } - + /// @brief Hooks before the agent stops all the sources and sinks /// @return configuration::HookManager& auto &beforeStopHooks() { return m_beforeStopHooks; } - + /// @brief the agent given a pipeline context /// @param context: the pipeline context shared between all pipelines void initialize(pipeline::PipelineContextPtr context); - + /// @brief initial UNAVAILABLE observations for all data items /// unless they have constant values. void initialDataItemObservations(); - + // Start and stop /// @brief Starts all the sources and sinks void start(); /// @brief Stops all the sources and syncs. void stop(); - + /// @brief Get the boost asio io context /// @return boost::asio::io_context auto &getContext() { return m_context; } - + /// @brief Create a contract for pipelines to access agent information /// @return A contract between the pipeline and this agent std::unique_ptr makePipelineContract(); /// @brief Gets the pipe context shared by all pipelines /// @return A shared pipeline context auto getPipelineContext() { return m_pipelineContext; } - + /// @brief Makes a unique sink contract /// @return A contract between the sink and the agent sink::SinkContractPtr makeSinkContract(); @@ -156,7 +156,7 @@ namespace mtconnect { /// get latest and historical data. /// @return A const reference to the circular buffer const auto &getCircularBuffer() const { return m_circularBuffer; } - + /// @brief Adds an adapter to the agent /// @param[in] source: shared pointer to the source being added /// @param[in] start: starts the source if start is true, otherwise delayed start @@ -165,7 +165,7 @@ namespace mtconnect { /// @param[in] sink shared pointer to the the sink being added /// @param[in] start: starts the source if start is true, otherwise delayed start void addSink(sink::SinkPtr sink, bool start = false); - + // Source and Sink /// @brief Find a source by name /// @param[in] name the identity to find @@ -187,21 +187,21 @@ namespace mtconnect { for (auto &s : m_sinks) if (s->getName() == name) return s; - + return nullptr; } - + /// @brief Get the list of all sources /// @return The list of all source in the agent const auto &getSources() const { return m_sources; } /// @brief Get the list of all sinks /// @return The list of all sinks in the agent const auto &getSinks() const { return m_sinks; } - + /// @brief Get the MTConnect schema version the agent is supporting /// @return The MTConnect schema version as a string const auto &getSchemaVersion() const { return m_schemaVersion; } - + /// @brief Find a device by name /// @param[in] name The name of the device to find /// @return A shared pointer to the device @@ -222,7 +222,7 @@ namespace mtconnect { boost::push_back(list, m_deviceIndex); return list; } - + /// @brief Get a pointer to the default device /// /// The default device is the first device that is not the Agent device. @@ -236,7 +236,7 @@ namespace mtconnect { if (device->getName() != "Agent") return device; } - + return nullptr; } /// @deprecated use `getDefaultDevice()` instead @@ -244,11 +244,11 @@ namespace mtconnect { /// @return A shared pointer to the default device /// @note Cover method for `getDefaultDevice()` DevicePtr defaultDevice() const { return getDefaultDevice(); } - + /// @brief Get a pointer to the asset storage object /// @return A pointer to the asset storage object asset::AssetStorage *getAssetStorage() { return m_assetStorage.get(); } - + /// @brief Add a device to the agent /// @param[in] device The device to add. /// @note This method is not fully implemented after agent initialization @@ -262,16 +262,16 @@ namespace mtconnect { /// @param[in] deviceFile The device file to load /// @return true if successful bool reloadDevices(const std::string &deviceFile); - + /// @brief receive and parse a single device from a source /// @param[in] deviceXml the device xml as a string /// @param[in] source the source loading the device void loadDevice(const std::string &deviceXml, const std::optional source = std::nullopt); - + /// @name Message when source has connected and disconnected ///@{ - + /// @brief Called when source begins trying to connect /// @param source The source identity void connecting(const std::string &source); @@ -287,9 +287,9 @@ namespace mtconnect { /// @param[in] autoAvailable `true` if the source should automatically set available to /// `AVAILABLE` void connected(const std::string &source, const StringList &devices, bool autoAvailable); - + ///@} - + /// @brief Called when a source receives a command from a data source /// @param[in] device The device name associated with this source /// @param[in] command The command being sent @@ -297,7 +297,7 @@ namespace mtconnect { /// @param[in] source The identity of the source void receiveCommand(const std::string &device, const std::string &command, const std::string &value, const std::string &source); - + /// @brief Method to get a data item for a device /// @param[in] deviceName The name or uuid of the device /// @param[in] dataItemName The name or id of the data item @@ -310,7 +310,7 @@ namespace mtconnect { auto dev = findDeviceByUUIDorName(deviceName); return (dev) ? dev->getDeviceDataItem(dataItemName) : nullptr; } - + /// @brief Get a data item by its id. /// @param id Unique id of the data item /// @return Shared pointer to the data item if found @@ -321,10 +321,10 @@ namespace mtconnect { return diPos->second.lock(); return nullptr; } - + /// @name Pipeline related methods to receive data from sources ///@{ - + /// @brief Receive an observation /// @param[in] observation A shared pointer to the observation void receiveObservation(observation::ObservationPtr observation); @@ -359,21 +359,21 @@ namespace mtconnect { /// @param device The device related to the asset /// @param asset The asset void notifyAssetRemoved(DevicePtr device, const asset::AssetPtr &asset); - + ///@} - + /// @brief Method called by source when it cannot continue /// @param identity identity of the source void sourceFailed(const std::string &identity); - + /// @name For testing ///@{ - + /// @brief Returns a shared pointer to the agent device /// @return shared pointer to the agent device auto getAgentDevice() { return m_agentDevice; } ///@} - + /// @brief Get a pointer to the printer for a mime type /// @param type The mime type /// @return pointer to the printer or nullptr if it does not exist @@ -386,11 +386,11 @@ namespace mtconnect { else return nullptr; } - + /// @brief Get the map of available printers /// @return A const reference to the printer map const auto &getPrinters() const { return m_printers; } - + /// @brief Prefixes the path with the device and rewrites the composed /// paths by repeating the prefix. The resulting path is valid /// XPath. @@ -410,7 +410,7 @@ namespace mtconnect { /// @return The rewritten path properly prefixed std::string devicesAndPath(const std::optional &path, const DevicePtr device) const; - + /// @brief Creates unique ids for the device model and maps to the originals /// /// Also updates the agents data item map by adding the new ids. Duplicate original @@ -418,10 +418,10 @@ namespace mtconnect { /// /// @param[in] device device to modify void createUniqueIds(DevicePtr device); - + protected: friend class AgentPipelineContract; - + // Initialization methods void createAgentDevice(); std::list loadXMLDeviceFile(const std::string &config); @@ -430,47 +430,47 @@ namespace mtconnect { std::optional> skip = std::nullopt); void loadCachedProbe(); void versionDeviceXml(); - + // Asset count management void updateAssetCounts(const DevicePtr &device, const std::optional type); - + observation::ObservationPtr getLatest(const std::string &id) { return m_circularBuffer.getLatest().getObservation(id); } - + observation::ObservationPtr getLatest(const DataItemPtr &di) { return getLatest(di->getId()); } - + protected: ConfigOptions m_options; configuration::AsyncContext &m_context; boost::asio::io_context::strand m_strand; - + std::shared_ptr m_loopback; - + bool m_started {false}; - + // Asset Management std::unique_ptr m_assetStorage; - + // Unique id based on the time of creation bool m_initialized {false}; bool m_observationsInitialized {false}; - + // Sources and Sinks source::SourceList m_sources; sink::SinkList m_sinks; - + // Pipeline pipeline::PipelineContextPtr m_pipelineContext; - + // Pointer to the configuration file for node access std::unique_ptr m_xmlParser; PrinterMap m_printers; - + // Agent Device device_model::AgentDevicePtr m_agentDevice; - + /// @brief tag for Device multi-index unsorted struct BySeq {}; @@ -480,7 +480,7 @@ namespace mtconnect { /// @brief tag for Device multi-index by UUID struct ByUuid {}; - + /// @brief Device UUID extractor for multi-index struct ExtractDeviceUuid { @@ -488,7 +488,7 @@ namespace mtconnect { const result_type &operator()(const DevicePtr &d) const { return *d->getUuid(); } result_type operator()(const DevicePtr &d) { return *d->getUuid(); } }; - + /// @brief Device name extractor for multi-index struct ExtractDeviceName { @@ -496,22 +496,22 @@ namespace mtconnect { const result_type &operator()(const DevicePtr &d) const { return *d->getComponentName(); } result_type operator()(DevicePtr &d) { return *d->getComponentName(); } }; - + /// @brief Devuce multi-index using DeviceIndex = mic::multi_index_container< - DevicePtr, mic::indexed_by>, - mic::hashed_unique, ExtractDeviceUuid>, - mic::hashed_unique, ExtractDeviceName>>>; - + DevicePtr, mic::indexed_by>, + mic::hashed_unique, ExtractDeviceUuid>, + mic::hashed_unique, ExtractDeviceName>>>; + DeviceIndex m_deviceIndex; std::unordered_map m_dataItemMap; std::unordered_map m_idMap; - + // Xml Config std::optional m_schemaVersion; std::string m_deviceXmlPath; - bool m_versionDeviceXml { false }; - bool m_createUniqueIds { false }; + bool m_versionDeviceXml {false}; + bool m_createUniqueIds {false}; int32_t m_intSchemaVersion = IntDefaultSchemaVersion(); // Circular Buffer diff --git a/src/mtconnect/device_model/component.cpp b/src/mtconnect/device_model/component.cpp index 507bdddb..4fdfcc04 100644 --- a/src/mtconnect/device_model/component.cpp +++ b/src/mtconnect/device_model/component.cpp @@ -160,8 +160,10 @@ namespace mtconnect { device->registerDataItem(dataItem); } } - - std::optional Component::createUniqueId(std::unordered_map &idMap, const boost::uuids::detail::sha1 &psha1) + + std::optional Component::createUniqueId( + std::unordered_map &idMap, + const boost::uuids::detail::sha1 &psha1) { auto newId = Entity::createUniqueId(idMap, psha1); m_id = *newId; diff --git a/src/mtconnect/device_model/component.hpp b/src/mtconnect/device_model/component.hpp index 2bb8c1fd..a020208f 100644 --- a/src/mtconnect/device_model/component.hpp +++ b/src/mtconnect/device_model/component.hpp @@ -245,7 +245,7 @@ namespace mtconnect { return nullptr; } - + /// @brief Get the component topic path as a list /// /// Recurses to root and then appends getTopicName @@ -258,11 +258,12 @@ namespace mtconnect { pth.push_back(getTopicName()); } - + /// @brief create unique ids recursively /// @param[in] itemMap data item id to data item map /// @param[in] sha the parents sha1 - std::optional createUniqueId(std::unordered_map &idMap, const boost::uuids::detail::sha1 &sha1) override; + std::optional createUniqueId(std::unordered_map &idMap, + const boost::uuids::detail::sha1 &sha1) override; protected: void setParent(ComponentPtr parent) { m_parent = parent; } diff --git a/src/mtconnect/device_model/data_item/data_item.cpp b/src/mtconnect/device_model/data_item/data_item.cpp index c04bc52a..85ac2381 100644 --- a/src/mtconnect/device_model/data_item/data_item.cpp +++ b/src/mtconnect/device_model/data_item/data_item.cpp @@ -243,7 +243,8 @@ namespace mtconnect { bool DataItem::hasName(const string &name) const { - return m_id == name || (m_name && *m_name == name) || (m_source && *m_source == name) || (m_originalId && *m_originalId == name); + return m_id == name || (m_name && *m_name == name) || (m_source && *m_source == name) || + (m_originalId && *m_originalId == name); } // Sort by: Device, Component, Category, DataItem diff --git a/src/mtconnect/device_model/data_item/data_item.hpp b/src/mtconnect/device_model/data_item/data_item.hpp index 1e8e420b..4f04be05 100644 --- a/src/mtconnect/device_model/data_item/data_item.hpp +++ b/src/mtconnect/device_model/data_item/data_item.hpp @@ -228,7 +228,9 @@ namespace mtconnect { /// @brief create unique ids recursively /// @param[in] itemMap data item id to data item map /// @param[in] sha the parents sha1 - std::optional createUniqueId(std::unordered_map &idMap, const boost::uuids::detail::sha1 &sha1) override + std::optional createUniqueId( + std::unordered_map &idMap, + const boost::uuids::detail::sha1 &sha1) override { m_originalId.emplace(m_id); auto pref = m_id == m_preferredName; @@ -242,21 +244,21 @@ namespace mtconnect { /// @brief Get a reference to the optional original id /// @returns optional original id const auto &getOriginalId() const { return m_originalId; } - - void updateReferences(const std::unordered_map idMap) override + + void updateReferences(const std::unordered_map idMap) override { Entity::updateReferences(idMap); if (hasProperty("compositionId")) - m_observatonProperties.insert_or_assign("compositionId", get("compositionId")); + m_observatonProperties.insert_or_assign("compositionId", + get("compositionId")); } - protected: double simpleFactor(const std::string &units); std::map buildAttributes() const; friend struct device_model::UpdateDataItemId; - + protected: // Unique ID for each component std::string m_id; diff --git a/src/mtconnect/device_model/device.cpp b/src/mtconnect/device_model/device.cpp index 4d571f10..1de61472 100644 --- a/src/mtconnect/device_model/device.cpp +++ b/src/mtconnect/device_model/device.cpp @@ -152,8 +152,9 @@ namespace mtconnect { { if (auto it = m_dataItems.get().find(name); it != m_dataItems.get().end()) return it->lock(); - - if (auto it = m_dataItems.get().find(name); it != m_dataItems.get().end()) + + if (auto it = m_dataItems.get().find(name); + it != m_dataItems.get().end()) return it->lock(); if (auto it = m_dataItems.get().find(name); it != m_dataItems.get().end()) @@ -164,25 +165,26 @@ namespace mtconnect { return nullptr; } - - struct UpdateDataItemId { + + struct UpdateDataItemId + { const string &m_id; UpdateDataItemId(const string &id) : m_id(id) {} - + void operator()(WeakDataItemPtr &ptr) { auto di = ptr.lock(); di->m_id = m_id; } }; - + void Device::createUniqueIds(std::unordered_map &idMap) { boost::uuids::detail::sha1 sha; sha.process_bytes(m_uuid->data(), m_uuid->size()); - + Component::createUniqueId(idMap, sha); - + for (auto it = m_dataItems.begin(); it != m_dataItems.end(); it++) { auto di = it->lock(); @@ -204,7 +206,7 @@ namespace mtconnect { LOG(error) << "DataItem id for " << di->getId() << " was not made unique"; } } - + for (auto p : m_componentsById) { auto comp = p.second.lock(); @@ -220,7 +222,6 @@ namespace mtconnect { m_componentsById.emplace(comp->getId(), comp); } } - } } diff --git a/src/mtconnect/device_model/device.hpp b/src/mtconnect/device_model/device.hpp index df0d96ca..1e61ec31 100644 --- a/src/mtconnect/device_model/device.hpp +++ b/src/mtconnect/device_model/device.hpp @@ -151,7 +151,7 @@ namespace mtconnect { } } }; - + /// @brief Mapping of device names to data items using DataItemIndex = mic::multi_index_container< WeakDataItemPtr, @@ -268,7 +268,7 @@ namespace mtconnect { /// @brief get the topic for this device /// @return the uuid of the device const std::string getTopicName() const override { return *m_uuid; } - + /// @brief Converts all the ids to unique ids by hasing the topics /// /// Converts the id attribute to a unique value and caches the original value diff --git a/src/mtconnect/entity/entity.cpp b/src/mtconnect/entity/entity.cpp index 82693b31..777eb306 100644 --- a/src/mtconnect/entity/entity.cpp +++ b/src/mtconnect/entity/entity.cpp @@ -15,8 +15,10 @@ // limitations under the License. // -#include #include + +#include + #include "factory.hpp" using namespace std; @@ -140,31 +142,31 @@ namespace mtconnect::entity { } } } - - struct UniqueIdVisitor { + + struct UniqueIdVisitor + { std::unordered_map &m_idMap; const boost::uuids::detail::sha1 &m_sha1; UniqueIdVisitor(std::unordered_map &idMap, const boost::uuids::detail::sha1 &sha1) - : m_idMap(idMap), m_sha1(sha1) + : m_idMap(idMap), m_sha1(sha1) {} - - void operator()(EntityPtr &p) - { - p->createUniqueId(m_idMap, m_sha1); - } - + + void operator()(EntityPtr &p) { p->createUniqueId(m_idMap, m_sha1); } + void operator()(EntityList &l) { for (auto &e : l) e->createUniqueId(m_idMap, m_sha1); } - template - void operator()(const T &) {} + template + void operator()(const T &) + {} }; - - std::optional Entity::createUniqueId(std::unordered_map &idMap, const boost::uuids::detail::sha1 &sha1) + + std::optional Entity::createUniqueId( + std::unordered_map &idMap, const boost::uuids::detail::sha1 &sha1) { optional res; @@ -188,37 +190,34 @@ namespace mtconnect::entity { idMap.emplace(oldId, newId); res.emplace(newId); } - + UniqueIdVisitor visitor(idMap, sha1); - + // Recurse properties for (auto &p : m_properties) { std::visit(visitor, p.second); } - + return res; } - - struct ReferenceIdVisitor { + + struct ReferenceIdVisitor + { const std::unordered_map &m_idMap; - ReferenceIdVisitor(const std::unordered_map &idMap) - : m_idMap(idMap) - {} - - void operator()(EntityPtr &p) - { - p->updateReferences(m_idMap); - } - + ReferenceIdVisitor(const std::unordered_map &idMap) : m_idMap(idMap) {} + + void operator()(EntityPtr &p) { p->updateReferences(m_idMap); } + void operator()(EntityList &l) { for (auto &e : l) e->updateReferences(m_idMap); } - template - void operator()(T &) {} + template + void operator()(T &) + {} }; void Entity::updateReferences(std::unordered_map idMap) @@ -226,7 +225,8 @@ namespace mtconnect::entity { using namespace boost::algorithm; for (auto &prop : m_properties) { - if (prop.first != "originalId" && (iends_with(prop.first, "idref") || (prop.first.length() > 2 && iends_with(prop.first, "id")))) + if (prop.first != "originalId" && (iends_with(prop.first, "idref") || + (prop.first.length() > 2 && iends_with(prop.first, "id")))) { auto it = idMap.find(std::get(prop.second)); if (it != idMap.end()) @@ -235,14 +235,13 @@ namespace mtconnect::entity { } } } - + ReferenceIdVisitor visitor(idMap); - + // Recurse all for (auto &p : m_properties) { std::visit(visitor, p.second); } - } } // namespace mtconnect::entity diff --git a/src/mtconnect/entity/entity.hpp b/src/mtconnect/entity/entity.hpp index 07a6d099..5bdf4105 100644 --- a/src/mtconnect/entity/entity.hpp +++ b/src/mtconnect/entity/entity.hpp @@ -105,16 +105,18 @@ namespace mtconnect { /// @brief method to return the entities identity. defaults to `id`. /// @return the identity virtual const entity::Value &getIdentity() const { return getProperty("id"); } - + /// @brief create unique ids recursively /// @param[in,out] idMap data item id to data item map /// @param[in] sha the parents sha1 /// @returns optional string value of the new id - virtual std::optional createUniqueId(std::unordered_map &idMap, const boost::uuids::detail::sha1 &sha1); - + virtual std::optional createUniqueId( + std::unordered_map &idMap, + const boost::uuids::detail::sha1 &sha1); + /// @brief update all id references to the new ids recursively /// @param[in] idMap map of old ids to new ids - virtual void updateReferences(const std::unordered_map idMap); + virtual void updateReferences(const std::unordered_map idMap); /// @brief checks if there is entity is a list that has additional properties /// @return `true` if this a a list with additional attributes diff --git a/src/mtconnect/entity/json_printer.hpp b/src/mtconnect/entity/json_printer.hpp index b78cea93..3cb57f16 100644 --- a/src/mtconnect/entity/json_printer.hpp +++ b/src/mtconnect/entity/json_printer.hpp @@ -106,8 +106,8 @@ namespace mtconnect::entity { /// @param version the supported MTConnect serialization version /// - Version 1 has a repreated objects in arrays for collections of objects /// - Version 2 combines arrays of objects by type - JsonPrinter(T &writer, uint32_t version, - bool includeHidden = false) : m_version(version), m_writer(writer), m_includeHidden(includeHidden) {}; + JsonPrinter(T &writer, uint32_t version, bool includeHidden = false) + : m_version(version), m_writer(writer), m_includeHidden(includeHidden) {}; /// @brief create a json object from an entity /// @@ -272,7 +272,7 @@ namespace mtconnect::entity { protected: uint32_t m_version; T &m_writer; - bool m_includeHidden { false }; + bool m_includeHidden {false}; }; /// @brief Serialization wrapper to turn an entity into a json string. @@ -280,8 +280,8 @@ namespace mtconnect::entity { { public: /// @brief Create a printer for with a JSON vesion and flag to pretty print - JsonEntityPrinter(uint32_t version, bool pretty = false, - bool includeHidden = false) : m_version(version), m_pretty(pretty), m_includeHidden(includeHidden) + JsonEntityPrinter(uint32_t version, bool pretty = false, bool includeHidden = false) + : m_version(version), m_pretty(pretty), m_includeHidden(includeHidden) {} /// @brief wrapper around the JsonPrinter print method that creates the correct printer @@ -322,6 +322,6 @@ namespace mtconnect::entity { protected: uint32_t m_version; bool m_pretty; - bool m_includeHidden { false }; + bool m_includeHidden {false}; }; } // namespace mtconnect::entity diff --git a/src/mtconnect/entity/xml_printer.hpp b/src/mtconnect/entity/xml_printer.hpp index b32e1289..9d556bbf 100644 --- a/src/mtconnect/entity/xml_printer.hpp +++ b/src/mtconnect/entity/xml_printer.hpp @@ -42,9 +42,9 @@ namespace mtconnect { /// @param namespaces a set of namespaces to use in the document void print(xmlTextWriterPtr writer, const EntityPtr entity, const std::unordered_set &namespaces); - + protected: - bool m_includeHidden { false }; + bool m_includeHidden {false}; }; } // namespace entity } // namespace mtconnect diff --git a/src/mtconnect/printer/printer.hpp b/src/mtconnect/printer/printer.hpp index d22cb1e8..ccd004b9 100644 --- a/src/mtconnect/printer/printer.hpp +++ b/src/mtconnect/printer/printer.hpp @@ -82,11 +82,11 @@ namespace mtconnect { /// @param[in] devices a list of devices /// @param[in] count optional asset count and type association /// @return the MTConnect Devices document - virtual std::string printProbe( - const uint64_t instanceId, const unsigned int bufferSize, const uint64_t nextSeq, - const unsigned int assetBufferSize, const unsigned int assetCount, - const std::list &devices, - const std::map *count = nullptr, + virtual std::string printProbe(const uint64_t instanceId, const unsigned int bufferSize, + const uint64_t nextSeq, const unsigned int assetBufferSize, + const unsigned int assetCount, + const std::list &devices, + const std::map *count = nullptr, bool includeHidden = false) const = 0; /// @brief Print a MTConnect Streams document /// @param[in] instanceId the instance id diff --git a/src/mtconnect/sink/rest_sink/server.hpp b/src/mtconnect/sink/rest_sink/server.hpp index bcd6c8e1..07c5c4b5 100644 --- a/src/mtconnect/sink/rest_sink/server.hpp +++ b/src/mtconnect/sink/rest_sink/server.hpp @@ -245,7 +245,7 @@ namespace mtconnect::sink::rest_sink { /// @name Swagger Support /// @{ - /// + /// /// @brief Add swagger routings to the Agent void addSwaggerRoutings(); /// @brief generate swagger API from routings diff --git a/src/mtconnect/source/adapter/adapter_pipeline.cpp b/src/mtconnect/source/adapter/adapter_pipeline.cpp index bc9dc8cc..b902164f 100644 --- a/src/mtconnect/source/adapter/adapter_pipeline.cpp +++ b/src/mtconnect/source/adapter/adapter_pipeline.cpp @@ -83,7 +83,7 @@ namespace mtconnect { { clear(); m_options = options; - + m_identity = GetOption(m_options, configuration::AdapterIdentity).value_or("unknown"); } diff --git a/src/mtconnect/utilities.hpp b/src/mtconnect/utilities.hpp index 7fa1d0c4..8d6247fd 100644 --- a/src/mtconnect/utilities.hpp +++ b/src/mtconnect/utilities.hpp @@ -21,11 +21,11 @@ #pragma once #include +#include #include #include #include #include -#include #include #include @@ -727,26 +727,24 @@ namespace mtconnect { /// @param[in] context the boost asio io_context for resolving the address /// @param[in] onlyV4 only consider IPV4 addresses if `true` std::string GetBestHostAddress(boost::asio::io_context &context, bool onlyV4 = false); - + /// @brief Function to create a unique id given a sha1 namespace and an id. /// /// Creates a base 64 encoded version of the string and removes any illegal characters /// for an ID. If the first character is not a legal start character, maps the first 2 characters /// to the legal ID start char set. - /// + /// /// @param[in] sha the sha1 namespace to use as context /// @param[in] id the id to use transform /// @returns Returns the first 16 characters of the base 64 encoded sha1 inline std::string makeUniqueId(const boost::uuids::detail::sha1 &sha, const std::string &id) { using namespace std; - + boost::uuids::detail::sha1 sha1(sha); - - constexpr string_view startc( "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_"); - constexpr auto isIDStartChar = [](unsigned char c) -> bool { - return isalpha(c) || c == '_'; - }; + + constexpr string_view startc("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_"); + constexpr auto isIDStartChar = [](unsigned char c) -> bool { return isalpha(c) || c == '_'; }; constexpr auto isIDChar = [isIDStartChar](unsigned char c) -> bool { return isIDStartChar(c) || isdigit(c) || c == '.' || c == '-'; }; @@ -769,9 +767,9 @@ namespace mtconnect { s.erase(0, 1); s[0] = startc[c % startc.size()]; } - + s.erase(16); - + return s; } } // namespace mtconnect diff --git a/test/config_test.cpp b/test/config_test.cpp index 3f1f69fc..3bc3a37a 100644 --- a/test/config_test.cpp +++ b/test/config_test.cpp @@ -79,7 +79,7 @@ namespace { { fs::remove_all(root); } - + fs::create_directory(root); chdir(root.string().c_str()); m_config->updateWorkingDirectory(); @@ -1435,17 +1435,17 @@ Adapters { using namespace std::filesystem; using namespace std::chrono; using namespace boost::algorithm; - + if (!ec) { // Check for backup file - auto ext = date::format(".%Y-%m-%dT%H+", date::floor(system_clock::now())); + auto ext = date::format(".%Y-%m-%dT%H+", date::floor(system_clock::now())); auto dit = directory_iterator("."); auto it = find_if(dit, end(dit), [&ext](const auto &de) { return starts_with(de.path().extension().string(), ext); }); ASSERT_NE(end(dit), it) << "Cannot find backup device file with extension: " << ext << '*'; - + auto device = agent->getDeviceByName("LinuxCNC"); ASSERT_TRUE(device) << "Cannot find LinuxCNC device"; @@ -1454,11 +1454,11 @@ Adapters { auto cont = device->getComponentById("cont"); ASSERT_TRUE(cont) << "Cannot find Component with id cont"; - + auto exec = device->getDeviceDataItem("exec"); ASSERT_TRUE(exec) << "Cannot find DataItem with id exec"; - - auto pipeline = dynamic_cast(adapter->getPipeline()); + + auto pipeline = dynamic_cast(adapter->getPipeline()); ASSERT_EQ("LinuxCNC", pipeline->getDevice()); } m_config->stop(); @@ -1534,7 +1534,7 @@ Port = 0 auto agent = m_config->getAgent(); auto device = agent->getDeviceByName("LinuxCNC"); ASSERT_TRUE(device); - + const auto &printer = agent->getPrinter("xml"); ASSERT_NE(nullptr, printer); @@ -1548,7 +1548,7 @@ Port = 0 using namespace std::filesystem; using namespace std::chrono; using namespace boost::algorithm; - + if (!ec) { // Check for backup file @@ -1560,7 +1560,7 @@ Port = 0 auto cont = device->getComponentById("cont"); ASSERT_TRUE(cont) << "Cannot find Component with id cont"; - + auto devDIs = device->getDataItems(); ASSERT_TRUE(devDIs); ASSERT_EQ(5, devDIs->size()); @@ -1568,20 +1568,19 @@ Port = 0 auto dataItems = cont->getDataItems(); ASSERT_TRUE(dataItems); ASSERT_EQ(2, dataItems->size()); - + auto it = dataItems->begin(); ASSERT_EQ("exc", (*it)->get("originalId")); it++; ASSERT_EQ("mode", (*it)->get("originalId")); - auto estop = device->getDeviceDataItem("estop"); ASSERT_TRUE(estop) << "Cannot find DataItem with id estop"; auto exec = device->getDeviceDataItem("exc"); ASSERT_TRUE(exec) << "Cannot find DataItem with id exc"; - auto pipeline = dynamic_cast(adapter->getPipeline()); + auto pipeline = dynamic_cast(adapter->getPipeline()); ASSERT_EQ("LinuxCNC", pipeline->getDevice()); } m_config->stop(); @@ -1626,7 +1625,6 @@ Port = 0 timer1.async_wait(send); m_config->start(); - } TEST_F(ConfigTest, should_update_the_ids_of_all_entities) @@ -1675,11 +1673,11 @@ Port = 0 m_config->initialize(options); auto agent2 = m_config->getAgent(); - + auto device2 = agent2->getDeviceByName("LinuxCNC"); ASSERT_TRUE(device2); ASSERT_EQ(deviceId, device2->getId()); - + auto exec2 = device->getDeviceDataItem("exec"); ASSERT_TRUE(exec2); ASSERT_EQ(exec->getId(), exec2->getId()); diff --git a/test/entity_printer_test.cpp b/test/entity_printer_test.cpp index a285d683..558f4e8e 100644 --- a/test/entity_printer_test.cpp +++ b/test/entity_printer_test.cpp @@ -283,52 +283,52 @@ TEST_F(EntityPrinterTest, TestRawContent) TEST_F(EntityPrinterTest, should_honor_include_hidden_parameter) { auto component = make_shared(Requirements { - Requirement("id", true), - Requirement("name", false), - Requirement("uuid", false), + Requirement("id", true), + Requirement("name", false), + Requirement("uuid", false), }); - + auto components = make_shared( - Requirements({Requirement("Component", ENTITY, component, 1, Requirement::Infinite)})); + Requirements({Requirement("Component", ENTITY, component, 1, Requirement::Infinite)})); components->registerMatchers(); components->registerFactory(regex(".+"), component); - + component->addRequirements({Requirement("Components", ENTITY_LIST, components, false)}); - + auto device = make_shared(*component); device->addRequirements(Requirements { - Requirement("name", true), - Requirement("uuid", true), + Requirement("name", true), + Requirement("uuid", true), }); - + auto root = make_shared(Requirements {Requirement("Device", ENTITY, device)}); - + auto doc = string { - "\n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - "\n"}; - + "\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + "\n"}; + ErrorList errors; entity::XmlParser parser; - + auto entity = parser.parse(root, doc, errors); ASSERT_EQ(0, errors.size()); - + boost::uuids::detail::sha1 sha1; unordered_map idMap; - + entity->createUniqueId(idMap, sha1); - + entity::XmlPrinter printer(false); printer.print(*m_writer, entity, {}); - + ASSERT_EQ(R"( @@ -339,12 +339,13 @@ TEST_F(EntityPrinterTest, should_honor_include_hidden_parameter) -)", m_writer->getContent()); - +)", + m_writer->getContent()); + m_writer = make_unique(true); entity::XmlPrinter printer2(true); printer2.print(*m_writer, entity, {}); - + ASSERT_EQ(R"( @@ -355,7 +356,8 @@ TEST_F(EntityPrinterTest, should_honor_include_hidden_parameter) -)", m_writer->getContent()); +)", + m_writer->getContent()); } class EntityPrinterNamespaceTest : public EntityPrinterTest diff --git a/test/json_printer_test.cpp b/test/json_printer_test.cpp index 80c3c2c6..8aa813e8 100644 --- a/test/json_printer_test.cpp +++ b/test/json_printer_test.cpp @@ -368,12 +368,12 @@ TEST_F(JsonPrinterTest, should_honor_include_hidden_parameter) boost::uuids::detail::sha1 sha1; unordered_map idMap; - + entity->createUniqueId(idMap, sha1); - + entity::JsonEntityPrinter jprinter(1, true, false); auto jdoc = jprinter.print(entity); - + ASSERT_EQ(R"({ "MTConnectDevices": { "Devices": [ @@ -443,8 +443,9 @@ TEST_F(JsonPrinterTest, should_honor_include_hidden_parameter) "version": "1.6.0.6" } } -})", jdoc); - +})", + jdoc); + entity::JsonEntityPrinter jprinter2(1, true, true); jdoc = jprinter2.print(entity); @@ -524,7 +525,6 @@ TEST_F(JsonPrinterTest, should_honor_include_hidden_parameter) "version": "1.6.0.6" } } -})", jdoc); - - +})", + jdoc); } From b5092d4b161c1999619469fdc233bf7dc83e758c Mon Sep 17 00:00:00 2001 From: Will Sobel Date: Tue, 25 Apr 2023 16:57:25 +0200 Subject: [PATCH 09/18] Fixed version to include microseconds --- src/mtconnect/agent.cpp | 2 +- test/config_test.cpp | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/mtconnect/agent.cpp b/src/mtconnect/agent.cpp index cf2a85d8..f174df67 100644 --- a/src/mtconnect/agent.cpp +++ b/src/mtconnect/agent.cpp @@ -566,7 +566,7 @@ namespace mtconnect { { // update with a new version of the device.xml, saving the old one // with a date time stamp - auto ext = date::format(".%Y-%m-%dT%H+%M+%SZ", date::floor(system_clock::now())); + auto ext = date::format(".%Y-%m-%dT%H+%M+%SZ", date::floor(system_clock::now())); fs::path file(m_deviceXmlPath); fs::path backup(m_deviceXmlPath + ext); if (!fs::exists(backup)) diff --git a/test/config_test.cpp b/test/config_test.cpp index 3bc3a37a..008124f1 100644 --- a/test/config_test.cpp +++ b/test/config_test.cpp @@ -1441,10 +1441,11 @@ Adapters { // Check for backup file auto ext = date::format(".%Y-%m-%dT%H+", date::floor(system_clock::now())); auto dit = directory_iterator("."); - auto it = find_if(dit, end(dit), [&ext](const auto &de) { - return starts_with(de.path().extension().string(), ext); + std::list entries; + copy_if(dit, end(dit), back_inserter(entries), [&ext](const auto &de) { + return contains(de.path().string(), ext); }); - ASSERT_NE(end(dit), it) << "Cannot find backup device file with extension: " << ext << '*'; + ASSERT_EQ(1, entries.size()); auto device = agent->getDeviceByName("LinuxCNC"); ASSERT_TRUE(device) << "Cannot find LinuxCNC device"; @@ -1552,6 +1553,14 @@ Port = 0 if (!ec) { // Check for backup file + auto ext = date::format(".%Y-%m-%dT%H+", date::floor(system_clock::now())); + auto dit = directory_iterator("."); + std::list entries; + copy_if(dit, end(dit), back_inserter(entries), [&ext](const auto &de) { + return contains(de.path().string(), ext); + }); + ASSERT_EQ(2, entries.size()); + auto device = agent->getDeviceByName("LinuxCNC"); ASSERT_TRUE(device) << "Cannot find LinuxCNC device"; From 1a74ed5f87311eff0669fb073782ccb70c512090 Mon Sep 17 00:00:00 2001 From: Will Sobel Date: Tue, 25 Apr 2023 18:20:54 +0200 Subject: [PATCH 10/18] Added test for duplicate ids in dynamically loaded device --- src/mtconnect/agent.cpp | 68 ++++++++++++---------- src/mtconnect/agent.hpp | 1 - test/config_test.cpp | 126 +++++++++++++++++++++++++++++++++++++--- 3 files changed, 156 insertions(+), 39 deletions(-) diff --git a/src/mtconnect/agent.cpp b/src/mtconnect/agent.cpp index f174df67..2f677d91 100644 --- a/src/mtconnect/agent.cpp +++ b/src/mtconnect/agent.cpp @@ -492,9 +492,8 @@ namespace mtconnect { return false; } - if (m_createUniqueIds) - createUniqueIds(device); verifyDevice(device); + createUniqueIds(device); LOG(info) << "Checking if device " << *uuid << " has changed"; if (*device != *oldDev) @@ -566,7 +565,8 @@ namespace mtconnect { { // update with a new version of the device.xml, saving the old one // with a date time stamp - auto ext = date::format(".%Y-%m-%dT%H+%M+%SZ", date::floor(system_clock::now())); + auto ext = + date::format(".%Y-%m-%dT%H+%M+%SZ", date::floor(system_clock::now())); fs::path file(m_deviceXmlPath); fs::path backup(m_deviceXmlPath + ext); if (!fs::exists(backup)) @@ -863,39 +863,32 @@ namespace mtconnect { } else { - // Check if we are already initialized. If so, the device will need to be - // verified, additional data items added, and initial values set. - // if (!m_initialized) - { - m_deviceIndex.push_back(device); + m_deviceIndex.push_back(device); - // TODO: Redo Resolve Reference with entity - // device->resolveReferences(); - createUniqueIds(device); - verifyDevice(device); + // TODO: Redo Resolve Reference with entity + // device->resolveReferences(); + verifyDevice(device); + createUniqueIds(device); - if (m_observationsInitialized) - { - initializeDataItems(device); + if (m_observationsInitialized) + { + initializeDataItems(device); - // Check for single valued constrained data items. - if (m_agentDevice && device != m_agentDevice) + // Check for single valued constrained data items. + if (m_agentDevice && device != m_agentDevice) + { + entity::Properties props {{"VALUE", uuid}}; + if (m_intSchemaVersion >= SCHEMA_VERSION(2, 2)) { - entity::Properties props {{"VALUE", uuid}}; - if (m_intSchemaVersion >= SCHEMA_VERSION(2, 2)) - { - const auto &hash = device->getProperty("hash"); - if (hash.index() != EMPTY) - props.insert_or_assign("hash", hash); - } - - auto d = m_agentDevice->getDeviceDataItem("device_added"); - m_loopback->receive(d, props); + const auto &hash = device->getProperty("hash"); + if (hash.index() != EMPTY) + props.insert_or_assign("hash", hash); } + + auto d = m_agentDevice->getDeviceDataItem("device_added"); + m_loopback->receive(d, props); } } - // else - // LOG(warning) << "Adding device " << uuid << " after initialialization not supported yet"; } if (m_intSchemaVersion >= SCHEMA_VERSION(2, 2)) @@ -970,8 +963,21 @@ namespace mtconnect { { if (m_createUniqueIds && !dynamic_pointer_cast(device)) { - device->createUniqueIds(m_idMap); - device->updateReferences(m_idMap); + std::unordered_map idMap; + + device->createUniqueIds(idMap); + device->updateReferences(idMap); + + // Update the data item map. + for (auto &id : idMap) + { + auto di = device->getDeviceDataItem(id.second); + if (auto it = m_dataItemMap.find(id.first); it != m_dataItemMap.end()) + { + m_dataItemMap.erase(it); + m_dataItemMap.emplace(id.second, di); + } + } } } diff --git a/src/mtconnect/agent.hpp b/src/mtconnect/agent.hpp index 1b04a9ba..4b4cbc6d 100644 --- a/src/mtconnect/agent.hpp +++ b/src/mtconnect/agent.hpp @@ -505,7 +505,6 @@ namespace mtconnect { DeviceIndex m_deviceIndex; std::unordered_map m_dataItemMap; - std::unordered_map m_idMap; // Xml Config std::optional m_schemaVersion; diff --git a/test/config_test.cpp b/test/config_test.cpp index 008124f1..1571d840 100644 --- a/test/config_test.cpp +++ b/test/config_test.cpp @@ -1442,9 +1442,8 @@ Adapters { auto ext = date::format(".%Y-%m-%dT%H+", date::floor(system_clock::now())); auto dit = directory_iterator("."); std::list entries; - copy_if(dit, end(dit), back_inserter(entries), [&ext](const auto &de) { - return contains(de.path().string(), ext); - }); + copy_if(dit, end(dit), back_inserter(entries), + [&ext](const auto &de) { return contains(de.path().string(), ext); }); ASSERT_EQ(1, entries.size()); auto device = agent->getDeviceByName("LinuxCNC"); @@ -1556,11 +1555,10 @@ Port = 0 auto ext = date::format(".%Y-%m-%dT%H+", date::floor(system_clock::now())); auto dit = directory_iterator("."); std::list entries; - copy_if(dit, end(dit), back_inserter(entries), [&ext](const auto &de) { - return contains(de.path().string(), ext); - }); + copy_if(dit, end(dit), back_inserter(entries), + [&ext](const auto &de) { return contains(de.path().string(), ext); }); ASSERT_EQ(2, entries.size()); - + auto device = agent->getDeviceByName("LinuxCNC"); ASSERT_TRUE(device) << "Cannot find LinuxCNC device"; @@ -1693,4 +1691,118 @@ Port = 0 ASSERT_TRUE(exec2->getOriginalId()); ASSERT_EQ("exec", *exec2->getOriginalId()); } + + TEST_F(ConfigTest, should_add_a_new_device_with_duplicate_ids) + { + using namespace mtconnect::source::adapter; + + fs::path root {createTempDirectory("9")}; + + fs::path devices(root / "Devices.xml"); + fs::path config {root / "agent.cfg"}; + { + ofstream cfg(config.string()); + cfg << R"DOC( +VersionDeviceXmlUpdates = true +CreateUniqueIds = true +Port = 0 +)DOC"; + cfg << "Devices = " << devices << endl; + } + + copyFile("dyn_load.xml", devices, 0min); + + boost::program_options::variables_map options; + boost::program_options::variable_value value(boost::optional(config.string()), false); + options.insert(make_pair("config-file"s, value)); + + m_config->initialize(options); + auto &asyncContext = m_config->getAsyncContext(); + + auto agent = m_config->getAgent(); + auto device = agent->getDeviceByName("LinuxCNC"); + ASSERT_TRUE(device); + + const auto &printer = agent->getPrinter("xml"); + ASSERT_NE(nullptr, printer); + + auto sp = agent->findSource("_localhost_7878"); + ASSERT_TRUE(sp); + + auto adapter = dynamic_pointer_cast(sp); + ASSERT_TRUE(adapter); + + auto validate = [&](boost::system::error_code ec) { + using namespace std::filesystem; + using namespace std::chrono; + using namespace boost::algorithm; + + if (!ec) + { + // Check for backup file + auto ext = date::format(".%Y-%m-%dT%H+", date::floor(system_clock::now())); + auto dit = directory_iterator("."); + std::list entries; + copy_if(dit, end(dit), back_inserter(entries), + [&ext](const auto &de) { return contains(de.path().string(), ext); }); + ASSERT_EQ(2, entries.size()); + + ASSERT_EQ(3, agent->getDevices().size()); + + auto device1 = agent->getDeviceByName("LinuxCNC"); + ASSERT_TRUE(device1) << "Cannot find LinuxCNC device"; + + auto device2 = agent->getDeviceByName("AnotherCNC"); + ASSERT_TRUE(device2) << "Cannot find LinuxCNC device"; + + auto pipeline = dynamic_cast(adapter->getPipeline()); + ASSERT_EQ("AnotherCNC", pipeline->getDevice()); + } + m_config->stop(); + }; + + boost::asio::steady_timer timer2(asyncContext.get()); + + auto send = [this, &adapter, &timer2, validate](boost::system::error_code ec) { + if (ec) + { + m_config->stop(); + } + else + { + auto pipeline = dynamic_cast(adapter->getPipeline()); + ASSERT_EQ("LinuxCNC", pipeline->getDevice()); + + adapter->processData("* deviceModel: --multiline--AAAAA"); + adapter->processData(R"( + + + + + + + + + + + + + + + +)"); + adapter->processData("--multiline--AAAAA"); + + timer2.expires_from_now(500ms); + timer2.async_wait(validate); + } + }; + + boost::asio::steady_timer timer1(asyncContext.get()); + timer1.expires_from_now(100ms); + timer1.async_wait(send); + + m_config->start(); + } + } // namespace From 25b064bbe35e6f94422205d6e7f15795a1d16b54 Mon Sep 17 00:00:00 2001 From: Will Sobel Date: Tue, 25 Apr 2023 18:39:54 +0200 Subject: [PATCH 11/18] Cleaned up docs for createUniqueId --- src/mtconnect/device_model/component.cpp | 9 --------- src/mtconnect/device_model/component.hpp | 10 ++++++---- src/mtconnect/device_model/data_item/data_item.hpp | 3 --- src/mtconnect/entity/entity.hpp | 4 ++-- 4 files changed, 8 insertions(+), 18 deletions(-) diff --git a/src/mtconnect/device_model/component.cpp b/src/mtconnect/device_model/component.cpp index 4fdfcc04..1c9b6a13 100644 --- a/src/mtconnect/device_model/component.cpp +++ b/src/mtconnect/device_model/component.cpp @@ -160,14 +160,5 @@ namespace mtconnect { device->registerDataItem(dataItem); } } - - std::optional Component::createUniqueId( - std::unordered_map &idMap, - const boost::uuids::detail::sha1 &psha1) - { - auto newId = Entity::createUniqueId(idMap, psha1); - m_id = *newId; - return newId; - } } // namespace device_model } // namespace mtconnect diff --git a/src/mtconnect/device_model/component.hpp b/src/mtconnect/device_model/component.hpp index a020208f..a554e4c3 100644 --- a/src/mtconnect/device_model/component.hpp +++ b/src/mtconnect/device_model/component.hpp @@ -259,11 +259,13 @@ namespace mtconnect { pth.push_back(getTopicName()); } - /// @brief create unique ids recursively - /// @param[in] itemMap data item id to data item map - /// @param[in] sha the parents sha1 std::optional createUniqueId(std::unordered_map &idMap, - const boost::uuids::detail::sha1 &sha1) override; + const boost::uuids::detail::sha1 &sha1) override + { + auto newId = Entity::createUniqueId(idMap, sha1); + m_id = *newId; + return newId; + } protected: void setParent(ComponentPtr parent) { m_parent = parent; } diff --git a/src/mtconnect/device_model/data_item/data_item.hpp b/src/mtconnect/device_model/data_item/data_item.hpp index 4f04be05..35662d97 100644 --- a/src/mtconnect/device_model/data_item/data_item.hpp +++ b/src/mtconnect/device_model/data_item/data_item.hpp @@ -225,9 +225,6 @@ namespace mtconnect { /// @brief Return the category as a char * const char *getCategoryText() const { return m_categoryText; } - /// @brief create unique ids recursively - /// @param[in] itemMap data item id to data item map - /// @param[in] sha the parents sha1 std::optional createUniqueId( std::unordered_map &idMap, const boost::uuids::detail::sha1 &sha1) override diff --git a/src/mtconnect/entity/entity.hpp b/src/mtconnect/entity/entity.hpp index 5bdf4105..beca10ba 100644 --- a/src/mtconnect/entity/entity.hpp +++ b/src/mtconnect/entity/entity.hpp @@ -107,8 +107,8 @@ namespace mtconnect { virtual const entity::Value &getIdentity() const { return getProperty("id"); } /// @brief create unique ids recursively - /// @param[in,out] idMap data item id to data item map - /// @param[in] sha the parents sha1 + /// @param[in,out] idMap old entity id to new entity id map + /// @param[in] sha the root sha1 /// @returns optional string value of the new id virtual std::optional createUniqueId( std::unordered_map &idMap, From 26f440e1f8474ed02788204d91ab0cd78238e330 Mon Sep 17 00:00:00 2001 From: Will Sobel Date: Tue, 25 Apr 2023 19:19:16 +0200 Subject: [PATCH 12/18] Added skip test to verify behavior if versioning of xml files and monitoring is turned on --- test/config_test.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/config_test.cpp b/test/config_test.cpp index 1571d840..1b58b283 100644 --- a/test/config_test.cpp +++ b/test/config_test.cpp @@ -1805,4 +1805,9 @@ Port = 0 m_config->start(); } + TEST_F(ConfigTest, should_not_reload_when_monitor_files_is_on) + { + GTEST_SKIP() << "Need to either disable one or make it work"; + } + } // namespace From 10e715432bcb2bbf7080c94fc2b302b9ba3959e8 Mon Sep 17 00:00:00 2001 From: Will Sobel Date: Wed, 26 Apr 2023 15:54:37 +0200 Subject: [PATCH 13/18] Fixed default version number --- src/mtconnect/agent.cpp | 2 +- src/mtconnect/utilities.hpp | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/mtconnect/agent.cpp b/src/mtconnect/agent.cpp index 2f677d91..a035156f 100644 --- a/src/mtconnect/agent.cpp +++ b/src/mtconnect/agent.cpp @@ -715,7 +715,7 @@ namespace mtconnect { auto devices = m_xmlParser->parseFile( configXmlPath, dynamic_cast(m_printers["xml"].get())); - if (!m_schemaVersion && m_xmlParser->getSchemaVersion()) + if (!m_schemaVersion && m_xmlParser->getSchemaVersion() && !m_xmlParser->getSchemaVersion()->empty()) { m_schemaVersion = m_xmlParser->getSchemaVersion(); m_intSchemaVersion = IntSchemaVersion(*m_schemaVersion); diff --git a/src/mtconnect/utilities.hpp b/src/mtconnect/utilities.hpp index 8d6247fd..4b435c08 100644 --- a/src/mtconnect/utilities.hpp +++ b/src/mtconnect/utilities.hpp @@ -716,11 +716,18 @@ namespace mtconnect { /// @param s the version inline int32_t IntSchemaVersion(const std::string &s) { - int major, minor; + int major { 0 }, minor { 0 }; char c; std::stringstream vstr(s); vstr >> major >> c >> minor; - return SCHEMA_VERSION(major, minor); + if (major == 0) + { + return IntDefaultSchemaVersion(); + } + else + { + return SCHEMA_VERSION(major, minor); + } } /// @brief Retrieve the best Host IP address from the network interfaces. From 4ad14be4969f9d91de2ceb52444926ab8a8d56e9 Mon Sep 17 00:00:00 2001 From: Will Sobel Date: Wed, 26 Apr 2023 17:07:07 +0200 Subject: [PATCH 14/18] Fixed issues found with ubuntu build. --- samples/dyn_load.xml | 2 +- src/mtconnect/agent.cpp | 30 ++++++++++++++++++--------- src/mtconnect/printer/xml_printer.cpp | 2 ++ test/config_test.cpp | 7 ++++++- 4 files changed, 29 insertions(+), 12 deletions(-) diff --git a/samples/dyn_load.xml b/samples/dyn_load.xml index ecbfe510..d8cd7a5f 100644 --- a/samples/dyn_load.xml +++ b/samples/dyn_load.xml @@ -1,5 +1,5 @@ - +
diff --git a/src/mtconnect/agent.cpp b/src/mtconnect/agent.cpp index a035156f..59ced40c 100644 --- a/src/mtconnect/agent.cpp +++ b/src/mtconnect/agent.cpp @@ -559,6 +559,8 @@ namespace mtconnect { void Agent::versionDeviceXml() { + NAMED_SCOPE("Agent::versionDeviceXml"); + using namespace std::chrono; if (m_versionDeviceXml) @@ -572,16 +574,22 @@ namespace mtconnect { if (!fs::exists(backup)) fs::rename(file, backup); - printer::XmlPrinter printer(true); - - std::list list; - copy_if(m_deviceIndex.begin(), m_deviceIndex.end(), back_inserter(list), - [](DevicePtr d) { return dynamic_cast(d.get()) == nullptr; }); - auto probe = printer.printProbe(0, 0, 0, 0, 0, list, nullptr, true); - - ofstream devices(file.string()); - devices << probe; - devices.close(); + auto printer = getPrinter("xml"); + if (printer != nullptr) + { + std::list list; + copy_if(m_deviceIndex.begin(), m_deviceIndex.end(), back_inserter(list), + [](DevicePtr d) { return dynamic_cast(d.get()) == nullptr; }); + auto probe = printer->printProbe(0, 0, 0, 0, 0, list, nullptr, true); + + ofstream devices(file.string()); + devices << probe; + devices.close(); + } + else + { + LOG(error) << "Cannot find xml printer"; + } } } @@ -1110,6 +1118,8 @@ namespace mtconnect { else { LOG(debug) << "Processing command: " << command << ": " << value; + if (!device) + device.emplace(""); m_agent->receiveCommand(*device, command, value, *source); } } diff --git a/src/mtconnect/printer/xml_printer.cpp b/src/mtconnect/printer/xml_printer.cpp index 43fe3f3b..8cc1c350 100644 --- a/src/mtconnect/printer/xml_printer.cpp +++ b/src/mtconnect/printer/xml_printer.cpp @@ -579,6 +579,8 @@ namespace mtconnect::printer { } string rootName = "MTConnect" + xmlType; + if (!m_schemaVersion) + defaultSchemaVersion(); string xmlns = "urn:mtconnect.org:" + rootName + ":" + *m_schemaVersion; string location; diff --git a/test/config_test.cpp b/test/config_test.cpp index 1b58b283..8f9fccc2 100644 --- a/test/config_test.cpp +++ b/test/config_test.cpp @@ -1807,7 +1807,12 @@ Port = 0 TEST_F(ConfigTest, should_not_reload_when_monitor_files_is_on) { - GTEST_SKIP() << "Need to either disable one or make it work"; + GTEST_SKIP(); + } + + TEST_F(ConfigTest, should_map_references_to_new_ids) + { + GTEST_SKIP(); } } // namespace From 82d06ed24de83972572c92378e8137664a9de52e Mon Sep 17 00:00:00 2001 From: Will Sobel Date: Wed, 26 Apr 2023 18:34:46 +0200 Subject: [PATCH 15/18] Added pretty flag to printer methods. Pretty print the versioned Device XML files --- src/mtconnect/agent.cpp | 2 +- src/mtconnect/printer/json_printer.cpp | 20 ++++++++++++-------- src/mtconnect/printer/json_printer.hpp | 12 ++++++++---- src/mtconnect/printer/printer.hpp | 15 ++++++++++----- src/mtconnect/printer/xml_printer.cpp | 20 ++++++++++++-------- src/mtconnect/printer/xml_printer.hpp | 12 ++++++++---- 6 files changed, 51 insertions(+), 30 deletions(-) diff --git a/src/mtconnect/agent.cpp b/src/mtconnect/agent.cpp index 59ced40c..a6c68a07 100644 --- a/src/mtconnect/agent.cpp +++ b/src/mtconnect/agent.cpp @@ -580,7 +580,7 @@ namespace mtconnect { std::list list; copy_if(m_deviceIndex.begin(), m_deviceIndex.end(), back_inserter(list), [](DevicePtr d) { return dynamic_cast(d.get()) == nullptr; }); - auto probe = printer->printProbe(0, 0, 0, 0, 0, list, nullptr, true); + auto probe = printer->printProbe(0, 0, 0, 0, 0, list, nullptr, true, true); ofstream devices(file.string()); devices << probe; diff --git a/src/mtconnect/printer/json_printer.cpp b/src/mtconnect/printer/json_printer.cpp index 2fb86018..4a51d597 100644 --- a/src/mtconnect/printer/json_printer.cpp +++ b/src/mtconnect/printer/json_printer.cpp @@ -124,12 +124,13 @@ namespace mtconnect::printer { } std::string JsonPrinter::printErrors(const uint64_t instanceId, const unsigned int bufferSize, - const uint64_t nextSeq, const ProtoErrorList &list) const + const uint64_t nextSeq, const ProtoErrorList &list, + bool pretty) const { defaultSchemaVersion(); StringBuffer output; - RenderJson(output, m_pretty, [&](auto &writer) { + RenderJson(output, m_pretty || pretty, [&](auto &writer) { AutoJsonObject obj(writer); { AutoJsonObject obj(writer, "MTConnectError"); @@ -181,12 +182,13 @@ namespace mtconnect::printer { const unsigned int assetCount, const std::list &devices, const std::map *count, - bool includeHidden) const + bool includeHidden, + bool pretty) const { defaultSchemaVersion(); StringBuffer output; - RenderJson(output, m_pretty, [&](auto &writer) { + RenderJson(output, m_pretty || pretty, [&](auto &writer) { entity::JsonPrinter printer(writer, m_jsonVersion, includeHidden); AutoJsonObject top(writer); @@ -208,12 +210,13 @@ namespace mtconnect::printer { std::string JsonPrinter::printAssets(const uint64_t instanceId, const unsigned int bufferSize, const unsigned int assetCount, - const asset::AssetList &asset) const + const asset::AssetList &asset, + bool pretty) const { defaultSchemaVersion(); StringBuffer output; - RenderJson(output, m_pretty, [&](auto &writer) { + RenderJson(output, m_pretty || pretty, [&](auto &writer) { entity::JsonPrinter printer(writer, m_jsonVersion); AutoJsonObject top(writer); @@ -407,12 +410,13 @@ namespace mtconnect::printer { std::string JsonPrinter::printSample(const uint64_t instanceId, const unsigned int bufferSize, const uint64_t nextSeq, const uint64_t firstSeq, - const uint64_t lastSeq, ObservationList &observations) const + const uint64_t lastSeq, ObservationList &observations, + bool pretty) const { defaultSchemaVersion(); StringBuffer output; - RenderJson(output, m_pretty, [&](auto &writer) { + RenderJson(output, m_pretty || pretty, [&](auto &writer) { AutoJsonObject top(writer); AutoJsonObject obj(writer, "MTConnectStreams"); obj.AddPairs("jsonVersion", m_jsonVersion, "schemaVersion", *m_schemaVersion); diff --git a/src/mtconnect/printer/json_printer.hpp b/src/mtconnect/printer/json_printer.hpp index 85de4f69..9f892620 100644 --- a/src/mtconnect/printer/json_printer.hpp +++ b/src/mtconnect/printer/json_printer.hpp @@ -31,20 +31,24 @@ namespace mtconnect::printer { ~JsonPrinter() override = default; std::string printErrors(const uint64_t instanceId, const unsigned int bufferSize, - const uint64_t nextSeq, const ProtoErrorList &list) const override; + const uint64_t nextSeq, const ProtoErrorList &list, + bool pretty = false) const override; std::string printProbe(const uint64_t instanceId, const unsigned int bufferSize, const uint64_t nextSeq, const unsigned int assetBufferSize, const unsigned int assetCount, const std::list &devices, const std::map *count = nullptr, - bool includeHidden = false) const override; + bool includeHidden = false, + bool pretty = false) const override; std::string printSample(const uint64_t instanceId, const unsigned int bufferSize, const uint64_t nextSeq, const uint64_t firstSeq, const uint64_t lastSeq, - observation::ObservationList &results) const override; + observation::ObservationList &results, + bool pretty = false) const override; std::string printAssets(const uint64_t anInstanceId, const unsigned int bufferSize, const unsigned int assetCount, - const asset::AssetList &asset) const override; + const asset::AssetList &asset, + bool pretty = false) const override; std::string mimeType() const override { return "application/mtconnect+json"; } uint32_t getJsonVersion() const { return m_jsonVersion; } diff --git a/src/mtconnect/printer/printer.hpp b/src/mtconnect/printer/printer.hpp index ccd004b9..7c0361bb 100644 --- a/src/mtconnect/printer/printer.hpp +++ b/src/mtconnect/printer/printer.hpp @@ -61,7 +61,8 @@ namespace mtconnect { /// @return the error document virtual std::string printError(const uint64_t instanceId, const unsigned int bufferSize, const uint64_t nextSeq, const std::string &errorCode, - const std::string &errorText) const + const std::string &errorText, + bool pretty = false) const { return printErrors(instanceId, bufferSize, nextSeq, {{errorCode, errorText}}); } @@ -72,7 +73,8 @@ namespace mtconnect { /// @param[in] list the list of errors /// @return the MTConnect Error document virtual std::string printErrors(const uint64_t instanceId, const unsigned int bufferSize, - const uint64_t nextSeq, const ProtoErrorList &list) const = 0; + const uint64_t nextSeq, const ProtoErrorList &list, + bool pretty = false) const = 0; /// @brief Generate an MTConnect Devices document /// @param[in] instanceId the instance id /// @param[in] bufferSize the buffer size @@ -87,7 +89,8 @@ namespace mtconnect { const unsigned int assetCount, const std::list &devices, const std::map *count = nullptr, - bool includeHidden = false) const = 0; + bool includeHidden = false, + bool pretty = false) const = 0; /// @brief Print a MTConnect Streams document /// @param[in] instanceId the instance id /// @param[in] bufferSize the buffer size @@ -99,7 +102,8 @@ namespace mtconnect { virtual std::string printSample(const uint64_t instanceId, const unsigned int bufferSize, const uint64_t nextSeq, const uint64_t firstSeq, const uint64_t lastSeq, - observation::ObservationList &results) const = 0; + observation::ObservationList &results, + bool pretty = false) const = 0; /// @brief Generate an MTConnect Assets document /// @param[in] anInstanceId the instance id /// @param[in] bufferSize the buffer size @@ -108,7 +112,8 @@ namespace mtconnect { /// @return the MTConnect Assets document virtual std::string printAssets(const uint64_t anInstanceId, const unsigned int bufferSize, const unsigned int assetCount, - asset::AssetList const &asset) const = 0; + asset::AssetList const &asset, + bool pretty = false) const = 0; /// @brief get the mime type for the documents /// @return the mime type virtual std::string mimeType() const = 0; diff --git a/src/mtconnect/printer/xml_printer.cpp b/src/mtconnect/printer/xml_printer.cpp index 8cc1c350..5ac74e9f 100644 --- a/src/mtconnect/printer/xml_printer.cpp +++ b/src/mtconnect/printer/xml_printer.cpp @@ -340,13 +340,14 @@ namespace mtconnect::printer { } std::string XmlPrinter::printErrors(const uint64_t instanceId, const unsigned int bufferSize, - const uint64_t nextSeq, const ProtoErrorList &list) const + const uint64_t nextSeq, const ProtoErrorList &list, + bool pretty) const { string ret; try { - XmlWriter writer(m_pretty); + XmlWriter writer(m_pretty || pretty); initXmlDoc(writer, eERROR, instanceId, bufferSize, 0, 0, nextSeq, nextSeq - 1); @@ -378,13 +379,14 @@ namespace mtconnect::printer { const uint64_t nextSeq, const unsigned int assetBufferSize, const unsigned int assetCount, const list &deviceList, const std::map *count, - bool includeHidden) const + bool includeHidden, + bool pretty) const { string ret; try { - XmlWriter writer(m_pretty); + XmlWriter writer(m_pretty || pretty); initXmlDoc(writer, eDEVICES, instanceId, bufferSize, assetBufferSize, assetCount, nextSeq, 0, nextSeq - 1, count); @@ -414,13 +416,14 @@ namespace mtconnect::printer { string XmlPrinter::printSample(const uint64_t instanceId, const unsigned int bufferSize, const uint64_t nextSeq, const uint64_t firstSeq, - const uint64_t lastSeq, ObservationList &observations) const + const uint64_t lastSeq, ObservationList &observations, + bool pretty) const { string ret; try { - XmlWriter writer(m_pretty); + XmlWriter writer(m_pretty || pretty); initXmlDoc(writer, eSTREAMS, instanceId, bufferSize, 0, 0, nextSeq, firstSeq, lastSeq); @@ -493,12 +496,13 @@ namespace mtconnect::printer { } string XmlPrinter::printAssets(const uint64_t instanceId, const unsigned int bufferSize, - const unsigned int assetCount, const AssetList &asset) const + const unsigned int assetCount, const AssetList &asset, + bool pretty) const { string ret; try { - XmlWriter writer(m_pretty); + XmlWriter writer(m_pretty || pretty); initXmlDoc(writer, eASSETS, instanceId, 0u, bufferSize, assetCount, 0ull); { diff --git a/src/mtconnect/printer/xml_printer.hpp b/src/mtconnect/printer/xml_printer.hpp index a453b82f..2ecad151 100644 --- a/src/mtconnect/printer/xml_printer.hpp +++ b/src/mtconnect/printer/xml_printer.hpp @@ -44,21 +44,25 @@ namespace mtconnect { ~XmlPrinter() override = default; std::string printErrors(const uint64_t instanceId, const unsigned int bufferSize, - const uint64_t nextSeq, const ProtoErrorList &list) const override; + const uint64_t nextSeq, const ProtoErrorList &list, + bool pretty = false) const override; std::string printProbe(const uint64_t instanceId, const unsigned int bufferSize, const uint64_t nextSeq, const unsigned int assetBufferSize, const unsigned int assetCount, const std::list &devices, const std::map *count = nullptr, - bool includeHidden = false) const override; + bool includeHidden = false, + bool pretty = false) const override; std::string printSample(const uint64_t instanceId, const unsigned int bufferSize, const uint64_t nextSeq, const uint64_t firstSeq, const uint64_t lastSeq, - observation::ObservationList &results) const override; + observation::ObservationList &results, + bool pretty = false) const override; std::string printAssets(const uint64_t anInstanceId, const unsigned int bufferSize, const unsigned int assetCount, - const asset::AssetList &asset) const override; + const asset::AssetList &asset, + bool pretty = false) const override; std::string mimeType() const override { return "text/xml"; } /// @brief Add a Devices XML device namespace From c3285aa240a7a6784152dcf76a5572ad8fade696 Mon Sep 17 00:00:00 2001 From: Will Sobel Date: Wed, 26 Apr 2023 19:47:19 +0200 Subject: [PATCH 16/18] Added pretty flag to REST API calls --- src/mtconnect/agent.cpp | 5 +- src/mtconnect/printer/json_printer.cpp | 6 +- src/mtconnect/printer/json_printer.hpp | 6 +- src/mtconnect/printer/printer.hpp | 12 +- src/mtconnect/printer/xml_printer.cpp | 3 +- src/mtconnect/printer/xml_printer.hpp | 9 +- src/mtconnect/sink/rest_sink/rest_service.cpp | 96 ++++++++++------ src/mtconnect/sink/rest_sink/rest_service.hpp | 107 ++++++++++-------- src/mtconnect/utilities.hpp | 2 +- test/config_test.cpp | 10 +- 10 files changed, 138 insertions(+), 118 deletions(-) diff --git a/src/mtconnect/agent.cpp b/src/mtconnect/agent.cpp index a6c68a07..d962648f 100644 --- a/src/mtconnect/agent.cpp +++ b/src/mtconnect/agent.cpp @@ -581,7 +581,7 @@ namespace mtconnect { copy_if(m_deviceIndex.begin(), m_deviceIndex.end(), back_inserter(list), [](DevicePtr d) { return dynamic_cast(d.get()) == nullptr; }); auto probe = printer->printProbe(0, 0, 0, 0, 0, list, nullptr, true, true); - + ofstream devices(file.string()); devices << probe; devices.close(); @@ -723,7 +723,8 @@ namespace mtconnect { auto devices = m_xmlParser->parseFile( configXmlPath, dynamic_cast(m_printers["xml"].get())); - if (!m_schemaVersion && m_xmlParser->getSchemaVersion() && !m_xmlParser->getSchemaVersion()->empty()) + if (!m_schemaVersion && m_xmlParser->getSchemaVersion() && + !m_xmlParser->getSchemaVersion()->empty()) { m_schemaVersion = m_xmlParser->getSchemaVersion(); m_intSchemaVersion = IntSchemaVersion(*m_schemaVersion); diff --git a/src/mtconnect/printer/json_printer.cpp b/src/mtconnect/printer/json_printer.cpp index 4a51d597..9ab0afbf 100644 --- a/src/mtconnect/printer/json_printer.cpp +++ b/src/mtconnect/printer/json_printer.cpp @@ -182,8 +182,7 @@ namespace mtconnect::printer { const unsigned int assetCount, const std::list &devices, const std::map *count, - bool includeHidden, - bool pretty) const + bool includeHidden, bool pretty) const { defaultSchemaVersion(); @@ -209,8 +208,7 @@ namespace mtconnect::printer { } std::string JsonPrinter::printAssets(const uint64_t instanceId, const unsigned int bufferSize, - const unsigned int assetCount, - const asset::AssetList &asset, + const unsigned int assetCount, const asset::AssetList &asset, bool pretty) const { defaultSchemaVersion(); diff --git a/src/mtconnect/printer/json_printer.hpp b/src/mtconnect/printer/json_printer.hpp index 9f892620..e3045389 100644 --- a/src/mtconnect/printer/json_printer.hpp +++ b/src/mtconnect/printer/json_printer.hpp @@ -38,16 +38,14 @@ namespace mtconnect::printer { const uint64_t nextSeq, const unsigned int assetBufferSize, const unsigned int assetCount, const std::list &devices, const std::map *count = nullptr, - bool includeHidden = false, - bool pretty = false) const override; + bool includeHidden = false, bool pretty = false) const override; std::string printSample(const uint64_t instanceId, const unsigned int bufferSize, const uint64_t nextSeq, const uint64_t firstSeq, const uint64_t lastSeq, observation::ObservationList &results, bool pretty = false) const override; std::string printAssets(const uint64_t anInstanceId, const unsigned int bufferSize, - const unsigned int assetCount, - const asset::AssetList &asset, + const unsigned int assetCount, const asset::AssetList &asset, bool pretty = false) const override; std::string mimeType() const override { return "application/mtconnect+json"; } diff --git a/src/mtconnect/printer/printer.hpp b/src/mtconnect/printer/printer.hpp index 7c0361bb..14a856f5 100644 --- a/src/mtconnect/printer/printer.hpp +++ b/src/mtconnect/printer/printer.hpp @@ -61,8 +61,7 @@ namespace mtconnect { /// @return the error document virtual std::string printError(const uint64_t instanceId, const unsigned int bufferSize, const uint64_t nextSeq, const std::string &errorCode, - const std::string &errorText, - bool pretty = false) const + const std::string &errorText, bool pretty = false) const { return printErrors(instanceId, bufferSize, nextSeq, {{errorCode, errorText}}); } @@ -89,8 +88,7 @@ namespace mtconnect { const unsigned int assetCount, const std::list &devices, const std::map *count = nullptr, - bool includeHidden = false, - bool pretty = false) const = 0; + bool includeHidden = false, bool pretty = false) const = 0; /// @brief Print a MTConnect Streams document /// @param[in] instanceId the instance id /// @param[in] bufferSize the buffer size @@ -101,8 +99,7 @@ namespace mtconnect { /// @return the MTConnect Streams document virtual std::string printSample(const uint64_t instanceId, const unsigned int bufferSize, const uint64_t nextSeq, const uint64_t firstSeq, - const uint64_t lastSeq, - observation::ObservationList &results, + const uint64_t lastSeq, observation::ObservationList &results, bool pretty = false) const = 0; /// @brief Generate an MTConnect Assets document /// @param[in] anInstanceId the instance id @@ -111,8 +108,7 @@ namespace mtconnect { /// @param[in] asset the list of assets /// @return the MTConnect Assets document virtual std::string printAssets(const uint64_t anInstanceId, const unsigned int bufferSize, - const unsigned int assetCount, - asset::AssetList const &asset, + const unsigned int assetCount, asset::AssetList const &asset, bool pretty = false) const = 0; /// @brief get the mime type for the documents /// @return the mime type diff --git a/src/mtconnect/printer/xml_printer.cpp b/src/mtconnect/printer/xml_printer.cpp index 5ac74e9f..887d59ba 100644 --- a/src/mtconnect/printer/xml_printer.cpp +++ b/src/mtconnect/printer/xml_printer.cpp @@ -378,8 +378,7 @@ namespace mtconnect::printer { string XmlPrinter::printProbe(const uint64_t instanceId, const unsigned int bufferSize, const uint64_t nextSeq, const unsigned int assetBufferSize, const unsigned int assetCount, const list &deviceList, - const std::map *count, - bool includeHidden, + const std::map *count, bool includeHidden, bool pretty) const { string ret; diff --git a/src/mtconnect/printer/xml_printer.hpp b/src/mtconnect/printer/xml_printer.hpp index 2ecad151..bb4e0c2d 100644 --- a/src/mtconnect/printer/xml_printer.hpp +++ b/src/mtconnect/printer/xml_printer.hpp @@ -51,17 +51,14 @@ namespace mtconnect { const uint64_t nextSeq, const unsigned int assetBufferSize, const unsigned int assetCount, const std::list &devices, const std::map *count = nullptr, - bool includeHidden = false, - bool pretty = false) const override; + bool includeHidden = false, bool pretty = false) const override; std::string printSample(const uint64_t instanceId, const unsigned int bufferSize, const uint64_t nextSeq, const uint64_t firstSeq, - const uint64_t lastSeq, - observation::ObservationList &results, + const uint64_t lastSeq, observation::ObservationList &results, bool pretty = false) const override; std::string printAssets(const uint64_t anInstanceId, const unsigned int bufferSize, - const unsigned int assetCount, - const asset::AssetList &asset, + const unsigned int assetCount, const asset::AssetList &asset, bool pretty = false) const override; std::string mimeType() const override { return "text/xml"; } diff --git a/src/mtconnect/sink/rest_sink/rest_service.cpp b/src/mtconnect/sink/rest_sink/rest_service.cpp index 66fe262d..9dfb7e6c 100644 --- a/src/mtconnect/sink/rest_sink/rest_service.cpp +++ b/src/mtconnect/sink/rest_sink/rest_service.cpp @@ -105,6 +105,7 @@ namespace mtconnect { {"to", QUERY, "Sequence number at to stop reporting observations"}, {"from", QUERY, "Sequence number at to start reporting observations"}, {"interval", QUERY, "Time in ms between publishing data–starts streaming"}, + {"pretty", QUERY, "Instructs the result to be pretty printed"}, {"heartbeat", QUERY, "Time in ms between publishing a empty document when no data has changed"}}); @@ -400,31 +401,36 @@ namespace mtconnect { // Probe auto handler = [&](SessionPtr session, const RequestPtr request) -> bool { auto device = request->parameter("device"); + auto pretty = *request->parameter("pretty"); + auto printer = printerForAccepts(request->m_accepts); if (device && !ends_with(request->m_path, string("probe")) && m_sinkContract->findDeviceByUUIDorName(*device) == nullptr) return false; - respond(session, probeRequest(printer, device)); + respond(session, probeRequest(printer, device, pretty)); return true; }; - m_server->addRouting({boost::beast::http::verb::get, "/probe", handler}) + m_server->addRouting({boost::beast::http::verb::get, "/probe?pretty={bool:false}", handler}) .document("MTConnect probe request", "Provides metadata service for the MTConnect Devices information model for all " "devices."); - m_server->addRouting({boost::beast::http::verb::get, "/{device}/probe", handler}) + m_server + ->addRouting( + {boost::beast::http::verb::get, "/{device}/probe?pretty={bool:false}", handler}) .document("MTConnect probe request", "Provides metadata service for the MTConnect Devices information model for " "device identified by `device` matching `name` or `uuid`."); // Must be last - m_server->addRouting({boost::beast::http::verb::get, "/", handler}) + m_server->addRouting({boost::beast::http::verb::get, "/?pretty={bool:false}", handler}) .document("MTConnect probe request", "Provides metadata service for the MTConnect Devices information model for all " "devices."); - m_server->addRouting({boost::beast::http::verb::get, "/{device}", handler}) + m_server + ->addRouting({boost::beast::http::verb::get, "/{device}??pretty={bool:false}", handler}) .document("MTConnect probe request", "Provides metadata service for the MTConnect Devices information model for " "device identified by `device` matching `name` or `uuid`."); @@ -466,7 +472,9 @@ namespace mtconnect { return true; }; - string qp("type={string}&removed={bool:false}&count={integer:100}&device={string}"); + string qp( + "type={string}&removed={bool:false}&" + "count={integer:100}&device={string}&pretty={bool:false}"); m_server->addRouting({boost::beast::http::verb::get, "/assets?" + qp, handler}) .document("MTConnect assets request", "Returns up to `count` assets"); m_server->addRouting({boost::beast::http::verb::get, "/asset?" + qp, handler}) @@ -568,19 +576,23 @@ namespace mtconnect { { streamCurrentRequest(session, printerForAccepts(request->m_accepts), *interval, request->parameter("device"), - request->parameter("path")); + request->parameter("path"), + *request->parameter("pretty")); } else { respond(session, currentRequest(printerForAccepts(request->m_accepts), request->parameter("device"), request->parameter("at"), - request->parameter("path"))); + request->parameter("path"), + *request->parameter("pretty"))); } return true; }; - string qp("path={string}&at={unsigned_integer}&interval={integer}"); + string qp( + "path={string}&at={unsigned_integer}&" + "interval={integer}&pretty={bool:false}"); m_server->addRouting({boost::beast::http::verb::get, "/current?" + qp, handler}) .document("MTConnect current request", "Gets a stapshot of the state of all the observations for all devices " @@ -602,7 +614,7 @@ namespace mtconnect { session, printerForAccepts(request->m_accepts), *interval, *request->parameter("heartbeat"), *request->parameter("count"), request->parameter("device"), request->parameter("from"), - request->parameter("path")); + request->parameter("path"), *request->parameter("pretty")); } else { @@ -610,7 +622,8 @@ namespace mtconnect { sampleRequest( printerForAccepts(request->m_accepts), *request->parameter("count"), request->parameter("device"), request->parameter("from"), - request->parameter("to"), request->parameter("path"))); + request->parameter("to"), request->parameter("path"), + *request->parameter("pretty"))); } return true; }; @@ -618,7 +631,8 @@ namespace mtconnect { string qp( "path={string}&from={unsigned_integer}&" "interval={integer}&count={integer:100}&" - "heartbeat={integer:10000}&to={unsigned_integer}"); + "heartbeat={integer:10000}&to={unsigned_integer}&" + "pretty={bool:false}"); m_server->addRouting({boost::beast::http::verb::get, "/sample?" + qp, handler}) .document("MTConnect sample request", "Gets a time series of at maximum `count` observations for all devices " @@ -685,7 +699,7 @@ namespace mtconnect { // ------------------------------------------- ResponsePtr RestService::probeRequest(const Printer *printer, - const std::optional &device) + const std::optional &device, bool pretty) { NAMED_SCOPE("RestService::probeRequest"); @@ -709,14 +723,14 @@ namespace mtconnect { m_sinkContract->getCircularBuffer().getSequence(), uint32_t(m_sinkContract->getAssetStorage()->getMaxAssets()), uint32_t(m_sinkContract->getAssetStorage()->getCount()), deviceList, - &counts), + &counts, false, pretty), printer->mimeType()); } ResponsePtr RestService::currentRequest(const Printer *printer, const std::optional &device, const std::optional &at, - const std::optional &path) + const std::optional &path, bool pretty) { using namespace rest_sink; DevicePtr dev {nullptr}; @@ -732,7 +746,8 @@ namespace mtconnect { } // Check if there is a frequency to stream data or not - return make_unique(rest_sink::status::ok, fetchCurrentData(printer, filter, at), + return make_unique(rest_sink::status::ok, + fetchCurrentData(printer, filter, at, pretty), printer->mimeType()); } @@ -740,7 +755,7 @@ namespace mtconnect { const std::optional &device, const std::optional &from, const std::optional &to, - const std::optional &path) + const std::optional &path, bool pretty) { using namespace rest_sink; DevicePtr dev {nullptr}; @@ -761,7 +776,8 @@ namespace mtconnect { return make_unique( rest_sink::status::ok, - fetchSampleData(printer, filter, count, from, to, end, endOfBuffer), printer->mimeType()); + fetchSampleData(printer, filter, count, from, to, end, endOfBuffer, nullptr, pretty), + printer->mimeType()); } struct AsyncSampleResponse @@ -787,13 +803,14 @@ namespace mtconnect { ChangeObserver m_observer; chrono::system_clock::time_point m_last; boost::asio::steady_timer m_timer; + bool m_pretty {false}; }; void RestService::streamSampleRequest(rest_sink::SessionPtr session, const Printer *printer, const int interval, const int heartbeatIn, const int count, const std::optional &device, const std::optional &from, - const std::optional &path) + const std::optional &path, bool pretty) { NAMED_SCOPE("RestService::streamSampleRequest"); @@ -814,6 +831,7 @@ namespace mtconnect { asyncResponse->m_printer = printer; asyncResponse->m_heartbeat = std::chrono::milliseconds(heartbeatIn); asyncResponse->m_service = getptr(); + asyncResponse->m_pretty = pretty; checkPath(asyncResponse->m_printer, path, dev, asyncResponse->m_filter); @@ -968,7 +986,8 @@ namespace mtconnect { // sent. content = fetchSampleData(asyncResponse->m_printer, asyncResponse->m_filter, asyncResponse->m_count, asyncResponse->m_sequence, nullopt, end, - asyncResponse->m_endOfBuffer, &asyncResponse->m_observer); + asyncResponse->m_endOfBuffer, &asyncResponse->m_observer, + asyncResponse->m_pretty); // Even if we are at the end of the buffer, or within range. If we are filtering, // we will need to make sure we are not spinning when there are no valid events @@ -1003,12 +1022,13 @@ namespace mtconnect { const Printer *m_printer {nullptr}; FilterSetOpt m_filter; boost::asio::steady_timer m_timer; + bool m_pretty {false}; }; void RestService::streamCurrentRequest(SessionPtr session, const Printer *printer, const int interval, const std::optional &device, - const std::optional &path) + const std::optional &path, bool pretty) { checkRange(printer, interval, 0, numeric_limits().max(), "interval"); DevicePtr dev {nullptr}; @@ -1026,6 +1046,7 @@ namespace mtconnect { asyncResponse->m_interval = chrono::milliseconds {interval}; asyncResponse->m_printer = printer; asyncResponse->m_service = getptr(); + asyncResponse->m_pretty = pretty; asyncResponse->m_session->beginStreaming( printer->mimeType(), boost::asio::bind_executor(m_strand, [this, asyncResponse]() { @@ -1061,7 +1082,8 @@ namespace mtconnect { } asyncResponse->m_session->writeChunk( - fetchCurrentData(asyncResponse->m_printer, asyncResponse->m_filter, nullopt), + fetchCurrentData(asyncResponse->m_printer, asyncResponse->m_filter, nullopt, + asyncResponse->m_pretty), boost::asio::bind_executor(m_strand, [this, asyncResponse]() { asyncResponse->m_timer.expires_from_now(asyncResponse->m_interval); asyncResponse->m_timer.async_wait(boost::asio::bind_executor( @@ -1072,7 +1094,7 @@ namespace mtconnect { ResponsePtr RestService::assetRequest(const Printer *printer, const int32_t count, const bool removed, const std::optional &type, - const std::optional &device) + const std::optional &device, bool pretty) { using namespace rest_sink; @@ -1088,14 +1110,14 @@ namespace mtconnect { m_sinkContract->getAssetStorage()->getAssets(list, count, !removed, uuid, type); return make_unique( status::ok, - printer->printAssets(m_instanceId, - uint32_t(m_sinkContract->getAssetStorage()->getMaxAssets()), - uint32_t(m_sinkContract->getAssetStorage()->getCount()), list), + printer->printAssets( + m_instanceId, uint32_t(m_sinkContract->getAssetStorage()->getMaxAssets()), + uint32_t(m_sinkContract->getAssetStorage()->getCount()), list, pretty), printer->mimeType()); } ResponsePtr RestService::assetIdsRequest(const Printer *printer, - const std::list &ids) + const std::list &ids, bool pretty) { using namespace rest_sink; @@ -1109,16 +1131,16 @@ namespace mtconnect { auto message = str.str().substr(0, str.str().size() - 2); return make_unique(status::not_found, - printError(printer, "ASSET_NOT_FOUND", message), + printError(printer, "ASSET_NOT_FOUND", message, pretty), printer->mimeType()); } else { return make_unique( status::ok, - printer->printAssets(m_instanceId, - uint32_t(m_sinkContract->getAssetStorage()->getMaxAssets()), - uint32_t(m_sinkContract->getAssetStorage()->getCount()), list), + printer->printAssets( + m_instanceId, uint32_t(m_sinkContract->getAssetStorage()->getMaxAssets()), + uint32_t(m_sinkContract->getAssetStorage()->getCount()), list, pretty), printer->mimeType()); } } @@ -1304,13 +1326,13 @@ namespace mtconnect { } string RestService::printError(const Printer *printer, const string &errorCode, - const string &text) const + const string &text, bool pretty) const { LOG(debug) << "Returning error " << errorCode << ": " << text; if (printer) return printer->printError( m_instanceId, m_sinkContract->getCircularBuffer().getBufferSize(), - m_sinkContract->getCircularBuffer().getSequence(), errorCode, text); + m_sinkContract->getCircularBuffer().getSequence(), errorCode, text, pretty); else return errorCode + ": " + text; } @@ -1385,7 +1407,7 @@ namespace mtconnect { // ------------------------------------------- string RestService::fetchCurrentData(const Printer *printer, const FilterSetOpt &filterSet, - const optional &at) + const optional &at, bool pretty) { ObservationList observations; SequenceNumber_t firstSeq, seq; @@ -1409,14 +1431,14 @@ namespace mtconnect { } return printer->printSample(m_instanceId, m_sinkContract->getCircularBuffer().getBufferSize(), - seq, firstSeq, seq - 1, observations); + seq, firstSeq, seq - 1, observations, pretty); } string RestService::fetchSampleData(const Printer *printer, const FilterSetOpt &filterSet, int count, const std::optional &from, const std::optional &to, SequenceNumber_t &end, bool &endOfBuffer, - ChangeObserver *observer) + ChangeObserver *observer, bool pretty) { std::unique_ptr observations; SequenceNumber_t firstSeq, lastSeq; @@ -1449,7 +1471,7 @@ namespace mtconnect { } return printer->printSample(m_instanceId, m_sinkContract->getCircularBuffer().getBufferSize(), - end, firstSeq, lastSeq, *observations); + end, firstSeq, lastSeq, *observations, pretty); } } // namespace sink::rest_sink diff --git a/src/mtconnect/sink/rest_sink/rest_service.hpp b/src/mtconnect/sink/rest_sink/rest_service.hpp index 2368641a..83e27864 100644 --- a/src/mtconnect/sink/rest_sink/rest_service.hpp +++ b/src/mtconnect/sink/rest_sink/rest_service.hpp @@ -91,65 +91,75 @@ namespace mtconnect { ///@{ /// @brief Handler for a probe request - /// @param p printer for doc generation - /// @param device optional device name or uuid + /// @param[in] p printer for doc generation + /// @param[in] device optional device name or uuid + /// @param[in] pretty `true` to ensure response is formatted /// @return MTConnect Devices response ResponsePtr probeRequest(const printer::Printer *p, - const std::optional &device = std::nullopt); + const std::optional &device = std::nullopt, + bool pretty = false); /// @brief Handler for a current request - /// @param p printer for doc generation - /// @param device optional device name or uuid - /// @param at optional sequence number to take the snapshot - /// @param path an xpath to filter + /// @param[in] p printer for doc generation + /// @param[in] device optional device name or uuid + /// @param[in] at optional sequence number to take the snapshot + /// @param[in] path an xpath to filter + /// @param[in] pretty `true` to ensure response is formatted /// @return MTConnect Streams response ResponsePtr currentRequest(const printer::Printer *p, const std::optional &device = std::nullopt, const std::optional &at = std::nullopt, - const std::optional &path = std::nullopt); + const std::optional &path = std::nullopt, + bool pretty = false); /// @brief Handler for a sample request - /// @param p printer for doc generation - /// @param count maximum number of observations - /// @param device optional device name or uuid - /// @param from optional starting sequence number - /// @param to optional ending sequence number - /// @param path an xpath for filtering + /// @param[in] p printer for doc generation + /// @param[in] count maximum number of observations + /// @param[in] device optional device name or uuid + /// @param[in] from optional starting sequence number + /// @param[in] to optional ending sequence number + /// @param[in] path an xpath for filtering + /// @param[in] pretty `true` to ensure response is formatted /// @return MTConnect Streams response ResponsePtr sampleRequest(const printer::Printer *p, const int count = 100, const std::optional &device = std::nullopt, const std::optional &from = std::nullopt, const std::optional &to = std::nullopt, - const std::optional &path = std::nullopt); + const std::optional &path = std::nullopt, + bool pretty = false); /// @brief Handler for a streaming sample - /// @param session session to stream data to - /// @param p printer for doc generation - /// @param interval the minimum interval between sending documents in ms - /// @param heartbeat how often to send an empty document if no activity in ms - /// @param count the maxumum number of observations - /// @param device optional device name or uuid - /// @param from optional starting sequence number - /// @param path optional path for filtering + /// @param[in] session session to stream data to + /// @param[in] p printer for doc generation + /// @param[in] interval the minimum interval between sending documents in ms + /// @param[in] heartbeat how often to send an empty document if no activity in ms + /// @param[in] count the maxumum number of observations + /// @param[in] device optional device name or uuid + /// @param[in] from optional starting sequence number + /// @param[in] path optional path for filtering + /// @param[in] pretty `true` to ensure response is formatted void streamSampleRequest(SessionPtr session, const printer::Printer *p, const int interval, const int heartbeat, const int count = 100, const std::optional &device = std::nullopt, const std::optional &from = std::nullopt, - const std::optional &path = std::nullopt); + const std::optional &path = std::nullopt, + bool pretty = false); /// @brief Handler for a streaming current - /// @param session session to stream data to - /// @param p printer for doc generation - /// @param interval the minimum interval between sending documents in ms - /// @param device optional device name or uuid - /// @param path optional path for filtering + /// @param[in] session session to stream data to + /// @param[in] p printer for doc generation + /// @param[in] interval the minimum interval between sending documents in ms + /// @param[in] device optional device name or uuid + /// @param[in] path optional path for filtering + /// @param[in] pretty `true` to ensure response is formatted void streamCurrentRequest(SessionPtr session, const printer::Printer *p, const int interval, const std::optional &device = std::nullopt, - const std::optional &path = std::nullopt); + const std::optional &path = std::nullopt, + bool pretty = false); /// @brief Handler for put/post observation - /// @param p printer for response generation - /// @param device device - /// @param observations key/value pairs for the observations - /// @param time optional timestamp + /// @param[in] p printer for response generation + /// @param[in] device device + /// @param[in] observations key/value pairs for the observations + /// @param[in] time optional timestamp /// @return `` if succeeds ResponsePtr putObservationRequest(const printer::Printer *p, const std::string &device, const QueryMap observations, @@ -181,21 +191,25 @@ namespace mtconnect { ///@{ /// @brief Asset request handler for assets by type or device - /// @param p printer for the response document - /// @param count maximum number of assets to return - /// @param removed `true` if response should include removed assets - /// @param type optional type of asset to filter - /// @param device optional device name or uuid + /// @param[in] p printer for the response document + /// @param[in] count maximum number of assets to return + /// @param[in] removed `true` if response should include removed assets + /// @param[in] type optional type of asset to filter + /// @param[in] device optional device name or uuid + /// @param[in] pretty `true` to ensure response is formatted /// @return MTConnect Assets response document ResponsePtr assetRequest(const printer::Printer *p, const int32_t count, const bool removed, const std::optional &type = std::nullopt, - const std::optional &device = std::nullopt); + const std::optional &device = std::nullopt, + bool pretty = false); /// @brief Asset request handler using a list of asset ids - /// @param p printer for the response document - /// @param ids list of asset ids + /// @param[in] p printer for the response document + /// @param[in] ids list of asset ids + /// @param[in] pretty `true` to ensure response is formatted /// @return MTConnect Assets response document - ResponsePtr assetIdsRequest(const printer::Printer *p, const std::list &ids); + ResponsePtr assetIdsRequest(const printer::Printer *p, const std::list &ids, + bool pretty = false); /// @brief Asset request handler to update an asset /// @param p printer for the response document @@ -242,7 +256,7 @@ namespace mtconnect { /// @param text descriptive error text /// @return MTConnect Error document std::string printError(const printer::Printer *printer, const std::string &errorCode, - const std::string &text) const; + const std::string &text, bool pretty = false) const; /// @name For testing only ///@{ @@ -281,14 +295,15 @@ namespace mtconnect { // Current Data Collection std::string fetchCurrentData(const printer::Printer *printer, const FilterSetOpt &filterSet, - const std::optional &at); + const std::optional &at, bool pretty = false); // Sample data collection std::string fetchSampleData(const printer::Printer *printer, const FilterSetOpt &filterSet, int count, const std::optional &from, const std::optional &to, SequenceNumber_t &end, bool &endOfBuffer, - observation::ChangeObserver *observer = nullptr); + observation::ChangeObserver *observer = nullptr, + bool pretty = false); // Verification methods template diff --git a/src/mtconnect/utilities.hpp b/src/mtconnect/utilities.hpp index 4b435c08..07e48f10 100644 --- a/src/mtconnect/utilities.hpp +++ b/src/mtconnect/utilities.hpp @@ -716,7 +716,7 @@ namespace mtconnect { /// @param s the version inline int32_t IntSchemaVersion(const std::string &s) { - int major { 0 }, minor { 0 }; + int major {0}, minor {0}; char c; std::stringstream vstr(s); vstr >> major >> c >> minor; diff --git a/test/config_test.cpp b/test/config_test.cpp index 8f9fccc2..86630978 100644 --- a/test/config_test.cpp +++ b/test/config_test.cpp @@ -1805,14 +1805,8 @@ Port = 0 m_config->start(); } - TEST_F(ConfigTest, should_not_reload_when_monitor_files_is_on) - { - GTEST_SKIP(); - } + TEST_F(ConfigTest, should_not_reload_when_monitor_files_is_on) { GTEST_SKIP(); } - TEST_F(ConfigTest, should_map_references_to_new_ids) - { - GTEST_SKIP(); - } + TEST_F(ConfigTest, should_map_references_to_new_ids) { GTEST_SKIP(); } } // namespace From 926c098d654c135d9a4afeb0684c0e78c6711434 Mon Sep 17 00:00:00 2001 From: Will Sobel Date: Wed, 26 Apr 2023 23:41:18 +0200 Subject: [PATCH 17/18] Fixed default for boolean values --- src/mtconnect/sink/rest_sink/server.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/mtconnect/sink/rest_sink/server.cpp b/src/mtconnect/sink/rest_sink/server.cpp index a1728ade..24cf34ae 100644 --- a/src/mtconnect/sink/rest_sink/server.cpp +++ b/src/mtconnect/sink/rest_sink/server.cpp @@ -270,9 +270,10 @@ namespace mtconnect::sink::rest_sink { { obj.Key("default"); visit( - overloaded {[](const std::monostate &) {}, [&obj](const std::string &s) { obj.Add(s); }, - [&obj](int32_t i) { obj.Add(i); }, [&obj](uint64_t i) { obj.Add(i); }, - [&obj](double d) { obj.Add(d); }}, + overloaded {[](const std::monostate &) {}, [&obj](const std::string &s) { obj.Add(s); }, + [&obj](int32_t i) { obj.Add(i); }, [&obj](uint64_t i) { obj.Add(i); }, + [&obj](double d) { obj.Add(d); }, + [&obj](bool b) { obj.Add(b); }}, param.m_default); } } From c3688a3b0725ab847c84f1995dc52d39f60a8ea9 Mon Sep 17 00:00:00 2001 From: Will Sobel Date: Wed, 26 Apr 2023 23:42:03 +0200 Subject: [PATCH 18/18] Formatted with clang --- src/mtconnect/sink/rest_sink/server.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/mtconnect/sink/rest_sink/server.cpp b/src/mtconnect/sink/rest_sink/server.cpp index 24cf34ae..cfafe518 100644 --- a/src/mtconnect/sink/rest_sink/server.cpp +++ b/src/mtconnect/sink/rest_sink/server.cpp @@ -270,10 +270,9 @@ namespace mtconnect::sink::rest_sink { { obj.Key("default"); visit( - overloaded {[](const std::monostate &) {}, [&obj](const std::string &s) { obj.Add(s); }, - [&obj](int32_t i) { obj.Add(i); }, [&obj](uint64_t i) { obj.Add(i); }, - [&obj](double d) { obj.Add(d); }, - [&obj](bool b) { obj.Add(b); }}, + overloaded {[](const std::monostate &) {}, [&obj](const std::string &s) { obj.Add(s); }, + [&obj](int32_t i) { obj.Add(i); }, [&obj](uint64_t i) { obj.Add(i); }, + [&obj](double d) { obj.Add(d); }, [&obj](bool b) { obj.Add(b); }}, param.m_default); } }