From f68083d3c4b1b839536030066b38f61fa04396b3 Mon Sep 17 00:00:00 2001 From: Anel Husakovic Date: Thu, 15 Feb 2024 15:23:48 +0100 Subject: [PATCH 1/2] MDEV-33420: HASHICORP_KEY_MANAGEMENT fails on Windows with libcurl installed - When libcurl is installed in path out of default path, like on Windows, `include_directories` failed to find `curl/curl.h`. - Fix cmake by using modern syntax with imported target and `find_package` - Fix warnings treated as the errors - Remove `HASHICORP_HAVE_EXCEPTIONS` macro and related code - Add package to Server component in Windows - Closes PR #3068 - Reviewer: --- .../hashicorp_key_management/CMakeLists.txt | 6 +- .../hashicorp_key_management_plugin.cc | 65 ------------------- win/packaging/CPackWixConfig.cmake | 2 +- 3 files changed, 3 insertions(+), 70 deletions(-) diff --git a/plugin/hashicorp_key_management/CMakeLists.txt b/plugin/hashicorp_key_management/CMakeLists.txt index bd1eee844ab90..bef7e65be2715 100644 --- a/plugin/hashicorp_key_management/CMakeLists.txt +++ b/plugin/hashicorp_key_management/CMakeLists.txt @@ -1,18 +1,16 @@ -INCLUDE(FindCURL) +FIND_PACKAGE(CURL) IF(NOT CURL_FOUND) # Can't build plugin RETURN() ENDIF() -INCLUDE_DIRECTORIES(${CURL_INCLUDE_DIR}) - set(CPACK_RPM_hashicorp-key-management_PACKAGE_SUMMARY "Hashicorp Key Management plugin for MariaDB" PARENT_SCOPE) set(CPACK_RPM_hashicorp-key-management_PACKAGE_DESCRIPTION "This encryption plugin uses Hashicorp Vault for storing encryption keys for MariaDB Data-at-Rest encryption." PARENT_SCOPE) MYSQL_ADD_PLUGIN(HASHICORP_KEY_MANAGEMENT hashicorp_key_management_plugin.cc - LINK_LIBRARIES ${CURL_LIBRARIES} + LINK_LIBRARIES CURL::libcurl CONFIG hashicorp_key_management.cnf COMPONENT hashicorp-key-management MODULE_ONLY) diff --git a/plugin/hashicorp_key_management/hashicorp_key_management_plugin.cc b/plugin/hashicorp_key_management/hashicorp_key_management_plugin.cc index bdc2f7345f1aa..3365346fdf950 100644 --- a/plugin/hashicorp_key_management/hashicorp_key_management_plugin.cc +++ b/plugin/hashicorp_key_management/hashicorp_key_management_plugin.cc @@ -29,12 +29,6 @@ #include #include -#if defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND) -#define HASHICORP_HAVE_EXCEPTIONS 1 -#else -#define HASHICORP_HAVE_EXCEPTIONS 0 -#endif - #define HASHICORP_DEBUG_LOGGING 0 #define PLUGIN_ERROR_HEADER "hashicorp: " @@ -209,15 +203,6 @@ unsigned int if (key_version == ENCRYPTION_KEY_VERSION_INVALID) { clock_t timestamp; -#if HASHICORP_HAVE_EXCEPTIONS - try - { - VER_INFO &ver_info = latest_version_cache.at(key_id); - version = ver_info.key_version; - timestamp = ver_info.timestamp; - } - catch (const std::out_of_range &e) -#else VER_MAP::const_iterator ver_iter = latest_version_cache.find(key_id); if (ver_iter != latest_version_cache.end()) { @@ -225,7 +210,6 @@ unsigned int timestamp = ver_iter->second.timestamp; } else -#endif { mtx.unlock(); return ENCRYPTION_KEY_VERSION_INVALID; @@ -246,13 +230,6 @@ unsigned int } } KEY_INFO info; -#if HASHICORP_HAVE_EXCEPTIONS - try - { - info = key_info_cache.at(KEY_ID_AND_VERSION(key_id, version)); - } - catch (const std::out_of_range &e) -#else KEY_MAP::const_iterator key_iter = key_info_cache.find(KEY_ID_AND_VERSION(key_id, version)); if (key_iter != key_info_cache.end()) @@ -260,7 +237,6 @@ unsigned int info = key_iter->second; } else -#endif { mtx.unlock(); return ENCRYPTION_KEY_VERSION_INVALID; @@ -305,20 +281,12 @@ unsigned int HCData::cache_get_version (unsigned int key_id) { unsigned int version; mtx.lock(); -#if HASHICORP_HAVE_EXCEPTIONS - try - { - version = latest_version_cache.at(key_id).key_version; - } - catch (const std::out_of_range &e) -#else VER_MAP::const_iterator ver_iter = latest_version_cache.find(key_id); if (ver_iter != latest_version_cache.end()) { version = ver_iter->second.key_version; } else -#endif { version = ENCRYPTION_KEY_VERSION_INVALID; } @@ -331,15 +299,6 @@ unsigned int HCData::cache_check_version (unsigned int key_id) unsigned int version; clock_t timestamp; mtx.lock(); -#if HASHICORP_HAVE_EXCEPTIONS - try - { - VER_INFO &ver_info = latest_version_cache.at(key_id); - version = ver_info.key_version; - timestamp = ver_info.timestamp; - } - catch (const std::out_of_range &e) -#else VER_MAP::const_iterator ver_iter = latest_version_cache.find(key_id); if (ver_iter != latest_version_cache.end()) { @@ -347,7 +306,6 @@ unsigned int HCData::cache_check_version (unsigned int key_id) timestamp = ver_iter->second.timestamp; } else -#endif { mtx.unlock(); #if HASHICORP_DEBUG_LOGGING @@ -978,29 +936,6 @@ struct st_mariadb_encryption hashicorp_key_management_plugin= { 0, 0, 0, 0, 0 }; -#ifdef _MSC_VER - -static int setenv (const char *name, const char *value, int overwrite) -{ - if (!overwrite) - { - size_t len= 0; - int rc= getenv_s(&len, NULL, 0, name); - if (rc) - { - return rc; - } - if (len) - { - errno = EINVAL; - return EINVAL; - } - } - return _putenv_s(name, value); -} - -#endif - #define MAX_URL_SIZE 32768 int HCData::init () diff --git a/win/packaging/CPackWixConfig.cmake b/win/packaging/CPackWixConfig.cmake index 79a638b9d5afa..3cc8cee88d02c 100644 --- a/win/packaging/CPackWixConfig.cmake +++ b/win/packaging/CPackWixConfig.cmake @@ -53,7 +53,7 @@ add_component(Backup DESCRIPTION "Installs backup utilities(mariabackup and mbstream)") #Miscellaneous hidden components, part of server / or client programs -foreach(comp connect-engine connect-engine-jdbc ClientPlugins aws-key-management rocksdb-engine) +foreach(comp connect-engine connect-engine-jdbc ClientPlugins aws-key-management rocksdb-engine plugin-hashicorp-key-management) add_component(${comp} GROUP MySQLServer HIDDEN) endforeach() From 149b285cce869e842ae79de93238b291060e0a67 Mon Sep 17 00:00:00 2001 From: Anel Husakovic Date: Fri, 29 Dec 2023 10:47:29 +0100 Subject: [PATCH 2/2] MDEV-30432: Refactor connect to use libcurl instead of cpprestsdk/curl - Windows BB of this MDEV depends on MDEV-33420 so apply changes of PR#3068 before - Remove GetRest occurance (cpprestsdk references) - Drop execv(curl) references - Link to libcurl (Unix/Windows) - Use IMPORTED target, so that INCLUDE_DIRECTORIES be called implicity - Add libcurl feature to ConnectSE to issue HTPP request - This patch closes MDEV-26727 (tested with Docker) - Reviewer: <> --- storage/connect/CMakeLists.txt | 38 +- storage/connect/mysql-test/connect/t/rest.inc | 2 +- .../connect/mysql-test/connect/t/rest.test | 3 +- storage/connect/plgdbsem.h | 1 - storage/connect/restget.cpp | 90 ---- storage/connect/tabrest.cpp | 426 ++++++++---------- storage/connect/tabrest.h | 59 ++- 7 files changed, 242 insertions(+), 377 deletions(-) delete mode 100644 storage/connect/restget.cpp diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt index b8ae3a9f4f028..3a0c3f3dfd79e 100644 --- a/storage/connect/CMakeLists.txt +++ b/storage/connect/CMakeLists.txt @@ -360,31 +360,21 @@ ENDIF(CONNECT_WITH_MONGO) OPTION(CONNECT_WITH_REST "Compile CONNECT storage engine with REST support" ON) IF(CONNECT_WITH_REST) -# MESSAGE(STATUS "=====> REST support is ON") - SET(CONNECT_SOURCES ${CONNECT_SOURCES} tabrest.cpp tabrest.h) - add_definitions(-DREST_SUPPORT) -# FIND_PACKAGE(cpprestsdk QUIET) -# IF (cpprestsdk_FOUND) -# IF(UNIX) -## INCLUDE_DIRECTORIES(${CPPRESTSDK_INCLUDE_DIR}) -## If needed edit next line to set the path to libcpprest.so -# SET(REST_LIBRARY -lcpprest) -# MESSAGE (STATUS ${REST_LIBRARY}) -# ELSE(NOT UNIX) -## Next line sets debug compile mode matching cpprest_2_10d.dll -## when it was binary installed (can be change later in Visual Studio) -## Comment it out if not needed depending on your cpprestsdk installation. -# SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MDd") -# ENDIF(UNIX) -## IF(REST_LIBRARY) why this? how about Windows -# SET(CONNECT_SOURCES ${CONNECT_SOURCES} restget.cpp) -# add_definitions(-DREST_SOURCE) -## ENDIF() -## ELSE(NOT cpprestsdk_FOUND) -# MESSAGE(STATUS "=====> cpprestsdk package not found") -# ENDIF (cpprestsdk_FOUND) + SET(REST_LIBRARY) + FIND_PACKAGE(CURL) + IF (CURL_FOUND) + SET_PACKAGE_PROPERTIES(Curl PROPERTIES TYPE REQUIRED + PURPOSE "Required for the CONNECT_WITH_REST feature") + SET(REST_LIBRARY CURL::libcurl) + MESSAGE (STATUS ${REST_LIBRARY}) + SET(CONNECT_SOURCES ${CONNECT_SOURCES} tabrest.cpp tabrest.h) + add_definitions(-DREST_SUPPORT) + ADD_FEATURE_INFO(CONNECT_REST "ON" "Support for REST API in the CONNECT storage engine") + ELSE() + MESSAGE_ONCE(CONNECT_NO_CURL "libcurl-dev header not found.") + ADD_FEATURE_INFO(CONNECT_REST "OFF" "Support for REST API in the CONNECT storage engine") + ENDIF() ENDIF(CONNECT_WITH_REST) -ADD_FEATURE_INFO(CONNECT_REST CONNECT_WITH_REST "Support for REST API in the CONNECT storage engine") # # XMAP diff --git a/storage/connect/mysql-test/connect/t/rest.inc b/storage/connect/mysql-test/connect/t/rest.inc index 6848e4b696502..6ac98ba9ed52d 100644 --- a/storage/connect/mysql-test/connect/t/rest.inc +++ b/storage/connect/mysql-test/connect/t/rest.inc @@ -10,7 +10,7 @@ if (!`SELECT count(*) FROM INFORMATION_SCHEMA.TABLES AND CREATE_OPTIONS LIKE "%`table_type`='JSON'%"`) { DROP TABLE IF EXISTS t1; - Skip Need Curl or Casablanca; + Skip Need libcurl; } DROP TABLE t1; --enable_query_log diff --git a/storage/connect/mysql-test/connect/t/rest.test b/storage/connect/mysql-test/connect/t/rest.test index 67066ed4639e9..200f617492fcb 100644 --- a/storage/connect/mysql-test/connect/t/rest.test +++ b/storage/connect/mysql-test/connect/t/rest.test @@ -12,6 +12,5 @@ SELECT * FROM t1; DROP TABLE t1; # -# Clean up +# Clean up is done automatically # ---remove_file $MYSQLD_DATADIR/test/users.json diff --git a/storage/connect/plgdbsem.h b/storage/connect/plgdbsem.h index 4371f90a21d54..6295d7105d6f8 100644 --- a/storage/connect/plgdbsem.h +++ b/storage/connect/plgdbsem.h @@ -404,7 +404,6 @@ typedef class VCTDEF *PVCTDEF; typedef class PIVOTDEF *PPIVOTDEF; typedef class DOMDEF *PDOMDEF; typedef class DIRDEF *PDIRDEF; -typedef class RESTDEF *PRESTDEF; typedef class OEMDEF *POEMDEF; typedef class COLCRT *PCOLCRT; typedef class COLDEF *PCOLDEF; diff --git a/storage/connect/restget.cpp b/storage/connect/restget.cpp deleted file mode 100644 index 29dae23078066..0000000000000 --- a/storage/connect/restget.cpp +++ /dev/null @@ -1,90 +0,0 @@ -/************* Restget C++ Program Source Code File (.CPP) *************/ -/* Adapted from the sample program of the Casablanca tutorial. */ -/* Copyright Olivier Bertrand 2019. */ -/***********************************************************************/ -#include -#include - -using namespace utility::conversions; // String conversions utilities -using namespace web; // Common features like URIs. -using namespace web::http; // Common HTTP functionality -using namespace web::http::client; // HTTP client features -using namespace concurrency::streams; // Asynchronous streams - -typedef const char* PCSZ; - -extern "C" int restGetFile(char* m, bool xt, PCSZ http, PCSZ uri, PCSZ fn); - -/***********************************************************************/ -/* Make a local copy of the requested file. */ -/***********************************************************************/ -int restGetFile(char *m, bool xt, PCSZ http, PCSZ uri, PCSZ fn) -{ - int rc = 0; - auto fileStream = std::make_shared(); - - if (!http || !fn) { - //strcpy(g->Message, "Missing http or filename"); - strcpy(m, "Missing http or filename"); - return 2; - } // endif - - if (xt) - fprintf(stderr, "restGetFile: fn=%s\n", fn); - - // Open stream to output file. - pplx::task requestTask = fstream::open_ostream(to_string_t(fn)) - .then([=](ostream outFile) { - *fileStream= outFile; - - if (xt) - fprintf(stderr, "Outfile isopen=%d\n", outFile.is_open()); - - // Create http_client to send the request. - http_client client(to_string_t(http)); - - if (uri) { - // Build request URI and start the request. - uri_builder builder(to_string_t(uri)); - return client.request(methods::GET, builder.to_string()); - } else - return client.request(methods::GET); - }) - - // Handle response headers arriving. - .then([=](http_response response) { - if (xt) - fprintf(stderr, "Received response status code:%u\n", - response.status_code()); - - // Write response body into the file. - return response.body().read_to_end(fileStream->streambuf()); - }) - - // Close the file stream. - .then([=](size_t n) { - if (xt) - fprintf(stderr, "Return size=%zu\n", n); - - return fileStream->close(); - }); - - // Wait for all the outstanding I/O to complete and handle any exceptions - try { - if (xt) - fprintf(stderr, "Waiting\n"); - - requestTask.wait(); - } catch (const std::exception &e) { - if (xt) - fprintf(stderr, "Error exception: %s\n", e.what()); - - sprintf(m, "Error exception: %s", e.what()); - rc= 1; - } // end try/catch - - if (xt) - fprintf(stderr, "restget done: rc=%d\n", rc); - - return rc; -} // end of restGetFile diff --git a/storage/connect/tabrest.cpp b/storage/connect/tabrest.cpp index e75e200690560..7413f6029b9ef 100644 --- a/storage/connect/tabrest.cpp +++ b/storage/connect/tabrest.cpp @@ -36,6 +36,7 @@ #include "tabjson.h" #include "tabfmt.h" #include "tabrest.h" +#include #if defined(connect_EXPORTS) #define PUSH_WARNING(M) push_warning(current_thd, Sql_condition::WARN_LEVEL_NOTE, ER_UNKNOWN_ERROR, M) @@ -43,170 +44,6 @@ #define PUSH_WARNING(M) htrc(M) #endif -static XGETREST getRestFnc = NULL; -static int Xcurl(PGLOBAL g, PCSZ Http, PCSZ Uri, PCSZ filename); - -/***********************************************************************/ -/* Xcurl: retrieve the REST answer by executing cURL. */ -/***********************************************************************/ -int Xcurl(PGLOBAL g, PCSZ Http, PCSZ Uri, PCSZ filename) -{ - char buf[512]; - int rc = 0; - - if (strchr(filename, '"')) { - strcpy(g->Message, "Invalid file name"); - return 1; - } // endif filename - - if (Uri) { - if (*Uri == '/' || Http[strlen(Http) - 1] == '/') - my_snprintf(buf, sizeof(buf)-1, "%s%s", Http, Uri); - else - my_snprintf(buf, sizeof(buf)-1, "%s/%s", Http, Uri); - - } else - my_snprintf(buf, sizeof(buf)-1, "%s", Http); - -#if defined(_WIN32) - char cmd[1024]; - STARTUPINFO si; - PROCESS_INFORMATION pi; - - sprintf(cmd, "curl \"%s\" -o \"%s\"", buf, filename); - - ZeroMemory(&si, sizeof(si)); - si.cb = sizeof(si); - ZeroMemory(&pi, sizeof(pi)); - - // Start the child process. - if (CreateProcess(NULL, cmd, NULL, NULL, FALSE, 0, NULL, NULL, &si, &pi)) { - // Wait until child process exits. - WaitForSingleObject(pi.hProcess, INFINITE); - - // Close process and thread handles. - CloseHandle(pi.hProcess); - CloseHandle(pi.hThread); - } else { - snprintf(g->Message, sizeof(g->Message), "CreateProcess curl failed (%d)", GetLastError()); - rc = 1; - } // endif CreateProcess -#else // !_WIN32 - char fn[600]; - pid_t pID; - - // Check if curl package is availabe by executing subprocess - FILE *f= popen("command -v curl", "r"); - - if (!f) { - strcpy(g->Message, "Problem in allocating memory."); - return 1; - } else { - char temp_buff[50]; - size_t len = fread(temp_buff,1, 50, f); - - if(!len) { - strcpy(g->Message, "Curl not installed."); - return 1; - } else - pclose(f); - - } // endif f - -#ifdef HAVE_VFORK - pID = vfork(); -#else - pID = fork(); -#endif - sprintf(fn, "-o%s", filename); - - if (pID == 0) { - // Code executed by child process - execlp("curl", "curl", buf, fn, (char*)NULL); - - // If execlp() is successful, we should not reach this next line. - strcpy(g->Message, "Unsuccessful execlp from vfork()"); - exit(1); - } else if (pID < 0) { - // failed to fork - strcpy(g->Message, "Failed to fork"); - rc = 1; - } else { - // Parent process - wait(NULL); // Wait for the child to terminate - } // endif pID -#endif // !_WIN32 - - return rc; -} // end of Xcurl - -/***********************************************************************/ -/* GetREST: load the Rest lib and get the Rest function. */ -/***********************************************************************/ -XGETREST GetRestFunction(PGLOBAL g) -{ - if (getRestFnc) - return getRestFnc; - -#if !defined(REST_SOURCE) - if (trace(515)) - htrc("Looking for GetRest library\n"); - -#if defined(_WIN32) || defined(_WINDOWS) - HANDLE Hdll; - const char* soname = "GetRest.dll"; // Module name - - if (!(Hdll = LoadLibrary(soname))) { - char buf[256]; - DWORD rc = GetLastError(); - - snprintf(g->Message, sizeof(g->Message), MSG(DLL_LOAD_ERROR), rc, soname); - FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, NULL, rc, 0, - (LPTSTR)buf, sizeof(buf), NULL); - strcat(strcat(g->Message, ": "), buf); - return NULL; - } // endif Hdll - -// Get the function returning an instance of the external DEF class - if (!(getRestFnc = (XGETREST)GetProcAddress((HINSTANCE)Hdll, "restGetFile"))) { - char buf[256]; - DWORD rc = GetLastError(); - - snprintf(g->Message, sizeof(g->Message), MSG(PROCADD_ERROR), rc, "restGetFile"); - FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, NULL, rc, 0, - (LPTSTR)buf, sizeof(buf), NULL); - strcat(strcat(g->Message, ": "), buf); - FreeLibrary((HMODULE)Hdll); - return NULL; - } // endif getRestFnc -#else // !_WIN32 - void* Hso; - const char* error = NULL; - const char* soname = "GetRest.so"; // Module name - - // Load the desired shared library - if (!(Hso = dlopen(soname, RTLD_LAZY))) { - error = dlerror(); - snprintf(g->Message, sizeof(g->Message), MSG(SHARED_LIB_ERR), soname, SVP(error)); - return NULL; - } // endif Hdll - -// Get the function returning an instance of the external DEF class - if (!(getRestFnc = (XGETREST)dlsym(Hso, "restGetFile"))) { - error = dlerror(); - snprintf(g->Message, sizeof(g->Message), MSG(GET_FUNC_ERR), "restGetFile", SVP(error)); - dlclose(Hso); - return NULL; - } // endif getdef -#endif // !_WIN32 -#else // REST_SOURCE - getRestFnc = restGetFile; -#endif // REST_SOURCE - - return getRestFnc; -} // end of GetRestFunction /***********************************************************************/ /* Return the columns definition to MariaDB. */ @@ -214,63 +51,63 @@ XGETREST GetRestFunction(PGLOBAL g) PQRYRES RESTColumns(PGLOBAL g, PTOS tp, char *tab, char *db, bool info) { PQRYRES qrp= NULL; + RESTDEF restObject; char filename[_MAX_PATH + 1]; // MAX PATH ??? - int rc; + int rc; PCSZ http, uri, fn, ftype; - XGETREST grf = NULL; - bool curl = GetBooleanTableOption(g, tp, "Curl", false); - if (!curl && !(grf = GetRestFunction(g))) - curl = true; http = GetStringTableOption(g, tp, "Http", NULL); uri = GetStringTableOption(g, tp, "Uri", NULL); ftype = GetStringTableOption(g, tp, "Type", "JSON"); - fn = GetStringTableOption(g, tp, "Filename", NULL); - - if (!fn) { - int n, m = strlen(ftype) + 1; - - strcat(strcpy(filename, tab), "."); - n = strlen(filename); - - // Fold ftype to lower case - for (int i = 0; i < m; i++) - filename[n + i] = tolower(ftype[i]); - - fn = filename; - tp->subtype = PlugDup(g, fn); - snprintf(g->Message, sizeof(g->Message), "No file name. Table will use %s", fn); - PUSH_WARNING(g->Message); - } // endif fn + fn = GetStringTableOption(g, tp, "Filename", NULL); + + if (!fn) + { + int n, m = strlen(ftype) + 1; + strcat(strcpy(filename, tab), "."); + n = strlen(filename); + // Fold ftype to lower case + for (int i = 0; i < m; i++) + filename[n + i] = tolower(ftype[i]); + fn = filename; + tp->subtype = PlugDup(g, fn); + snprintf(g->Message, sizeof(g->Message), "No file name. Table will use %s", fn); + PUSH_WARNING(g->Message); + } // We used the file name relative to recorded datapath - PlugSetPath(filename, fn, db); - remove(filename); - - // Retrieve the file from the web and copy it locally - if (curl) - rc = Xcurl(g, http, uri, filename); - else - rc = grf(g->Message, trace(515), http, uri, filename); - - if (rc) { - strcpy(g->Message, "Cannot access to curl nor casablanca"); - return NULL; - } else if (!stricmp(ftype, "JSON")) + PlugSetPath(filename, fn, db); + restObject.Http= http; + restObject.Uri= uri; + restObject.Fn= filename; + remove(filename); + // Retrieve the file from the web using curl and copy it locally + if (restObject.curl_init(g)) + { + snprintf(g->Message, sizeof(g->Message), "Initialization of curl failed."); + return NULL; + } + rc = restObject.curl_run(g); + if (rc) + { + snprintf(g->Message, sizeof(g->Message), "Cannot access to curl."); + return NULL; + } + else if (!stricmp(ftype, "JSON")) qrp = JSONColumns(g, db, NULL, tp, info); else if (!stricmp(ftype, "CSV")) qrp = CSVColumns(g, NULL, tp, info); #if defined(XML_SUPPORT) - else if (!stricmp(ftype, "XML")) - qrp = XMLColumns(g, db, tab, tp, info); + else if (!stricmp(ftype, "XML")) + qrp = XMLColumns(g, db, tab, tp, info); #endif // XML_SUPPORT - else + else snprintf(g->Message, sizeof(g->Message), "Usupported file type %s", ftype); - return qrp; } // end of RESTColumns + /* -------------------------- Class RESTDEF -------------------------- */ /***********************************************************************/ @@ -278,18 +115,12 @@ PQRYRES RESTColumns(PGLOBAL g, PTOS tp, char *tab, char *db, bool info) /***********************************************************************/ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) { - char filename[_MAX_PATH + 1]; + char filename[_MAX_PATH + 1]; int rc = 0, n; - bool xt = trace(515); - LPCSTR ftype; - XGETREST grf = NULL; - bool curl = GetBoolCatInfo("Curl", false); - - if (!curl && !(grf = GetRestFunction(g))) - curl = true; + bool xt = trace(515); + LPCSTR ftype; ftype = GetStringCatInfo(g, "Type", "JSON"); - if (xt) htrc("ftype = %s am = %s\n", ftype, SVP(am)); @@ -299,9 +130,11 @@ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) #endif // XML_SUPPORT : (!stricmp(ftype, "CSV")) ? 3 : 0; - if (n == 0) { + if (n == 0) + { htrc("DefineAM: Unsupported REST table type %s\n", ftype); - snprintf(g->Message, sizeof(g->Message), "Unsupported REST table type %s", ftype); + snprintf(g->Message, sizeof(g->Message), + "Unsupported REST table type %s", ftype); return true; } // endif n @@ -311,27 +144,26 @@ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) // We used the file name relative to recorded datapath PlugSetPath(filename, Fn, GetPath()); - remove(filename); - - // Retrieve the file from the web and copy it locally - if (curl) { - rc = Xcurl(g, Http, Uri, filename); - xtrc(515, "Return from Xcurl: rc=%d\n", rc); - } else { - rc = grf(g->Message, xt, Http, Uri, filename); - xtrc(515, "Return from restGetFile: rc=%d\n", rc); - } // endelse - - if (rc) { - // strcpy(g->Message, "Cannot access to curl nor casablanca"); - return true; - } else switch (n) { - case 1: Tdp = new (g) JSONDEF; break; + Fn= filename; + remove(filename); + if (curl_init(g)) + { + snprintf(g->Message, sizeof(g->Message), "Initialization of curl failed."); + return true; + } + if (curl_run(g)) + return true; + else switch (n) + { + case 1: + Tdp = new (g) JSONDEF; break; #if defined(XML_SUPPORT) - case 2: Tdp = new (g) XMLDEF; break; + case 2: + Tdp = new (g) XMLDEF; break; #endif // XML_SUPPORT - case 3: Tdp = new (g) CSVDEF; break; - default: Tdp = NULL; + case 3: + Tdp = new (g) CSVDEF; break; + default: Tdp = NULL; } // endswitch n // Do make the table/view definition @@ -345,6 +177,7 @@ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) return (Tdp == NULL); } // end of DefineAM + /***********************************************************************/ /* GetTable: makes a new Table Description Block. */ /***********************************************************************/ @@ -353,12 +186,133 @@ PTDB RESTDEF::GetTable(PGLOBAL g, MODE m) if (trace(515)) htrc("REST GetTable mode=%d\n", m); - if (m != MODE_READ && m != MODE_READX && m != MODE_ANY) { - strcpy(g->Message, "REST tables are currently read only"); + if (m != MODE_READ && m != MODE_READX && m != MODE_ANY) + { + snprintf(g->Message, sizeof(g->Message), "REST tables are currently read only"); return NULL; - } // endif m + } return Tdp->GetTable(g, m); // Leave file type do the job } // end of GetTable + +/***********************************************************************/ +/* curl_init: Initilize curl */ +/***********************************************************************/ +int RESTDEF::curl_init(PGLOBAL g) +{ + CURLcode curl_res = curl_global_init(CURL_GLOBAL_ALL); + if (curl_res != CURLE_OK) + { + snprintf(g->Message, sizeof(g->Message), + "unable to initialize curl library, " + "curl returned this error code: %u " + "with the following error message: %s", + curl_res, curl_easy_strerror(curl_res)); + return 1; + } + curl_inited = true; + return 0; +} + + +/***********************************************************************/ +/* curl_deinit: Cleanup curl */ +/***********************************************************************/ +void RESTDEF::curl_deinit() +{ + if (curl_inited) + { + curl_global_cleanup(); + curl_inited = false; + } +} + + +/***********************************************************************/ +/* WriteMemoryCallback: Curl callback function */ +/***********************************************************************/ +static size_t WriteMemoryCallback(void *contents, + size_t size __attribute__((unused)), + size_t nmemb, void *userp) +{ + struct MemoryStruct *mem = (struct MemoryStruct *)userp; + char *ptr = (char *)realloc(mem->memory, mem->size + nmemb + 1); + if (ptr == NULL) + return 0; + mem->memory = ptr; + memcpy(&(mem->memory[mem->size]), contents, nmemb); + mem->size += nmemb; + mem->memory[mem->size] = 0; + return nmemb; +} + + +/***********************************************************************/ +/* curl_run: Retrieve the REST answer by executing cURL. */ +/***********************************************************************/ +int RESTDEF::curl_run(PGLOBAL g) +{ + CURL *curl = curl_easy_init(); + CURLcode curl_res = CURLE_OK; + char buf[512]; + long http_code = 0; + char curl_errbuf[CURL_ERROR_SIZE]; + if (!curl) + { + snprintf(g->Message, sizeof(g->Message), "Cannot initilize curl session."); + return 1; + } + curl_errbuf[0] = '\0'; + if (Uri) + { + if (*Uri == '/' || Http[strlen(Http) - 1] == '/') + my_snprintf(buf, sizeof(buf)-1, "%s%s", Http, Uri); + else + my_snprintf(buf, sizeof(buf)-1, "%s/%s", Http, Uri); + } + else + my_snprintf(buf, sizeof(buf)-1, "%s", Http); + + struct MemoryStruct chunk; + chunk.memory = (char *)malloc(1); + chunk.size = 0; + + if ((curl_res= curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errbuf)) != + CURLE_OK || + (curl_res= curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, + WriteMemoryCallback)) != + CURLE_OK || + (curl_res= curl_easy_setopt(curl, CURLOPT_WRITEDATA, + (void *)&chunk)) != + CURLE_OK || + (curl_res = curl_easy_setopt(curl, CURLOPT_URL, buf)) != CURLE_OK || + (curl_res = curl_easy_perform(curl)) != CURLE_OK || + (curl_res = curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, + &http_code)) != CURLE_OK) + { + curl_easy_cleanup(curl); + free(chunk.memory); + if (curl_res) + { + snprintf(g->Message, sizeof(g->Message), + "curl returned this error code: %u " + "with the following error message: %s", curl_res, + curl_errbuf[0] ? curl_errbuf : curl_easy_strerror(curl_res)); + return 1; + } + } + curl_easy_cleanup(curl); + FILE *f= fopen(Fn, "wb"); + fprintf(f, "%s", chunk.memory); + fclose(f); + free(chunk.memory); + bool is_error = http_code < 200 || http_code >= 300; + if (is_error) + { + snprintf(g->Message, sizeof(g->Message), "server error"); + return 1; + } + return 0; +} /* ---------------------- End of Class RESTDEF ----------------------- */ diff --git a/storage/connect/tabrest.h b/storage/connect/tabrest.h index 901d9102e9503..7df5bed567123 100644 --- a/storage/connect/tabrest.h +++ b/storage/connect/tabrest.h @@ -10,39 +10,52 @@ #define stricmp strcasecmp #endif // !_WIN32 -typedef int(__stdcall* XGETREST) (char*, bool, PCSZ, PCSZ, PCSZ); - /***********************************************************************/ /* Functions used by REST. */ /***********************************************************************/ -XGETREST GetRestFunction(PGLOBAL g); -#if defined(REST_SOURCE) -extern "C" int restGetFile(char* m, bool xt, PCSZ http, PCSZ uri, PCSZ fn); -#endif // REST_SOURCE #if defined(MARIADB) PQRYRES RESTColumns(PGLOBAL g, PTOS tp, char* tab, char* db, bool info); #endif // !MARIADB /***********************************************************************/ -/* Restest table. */ +/* Data structure for curl callback function */ /***********************************************************************/ -class RESTDEF : public TABDEF { /* Table description */ -public: - // Constructor - RESTDEF(void) { Tdp = NULL; Http = Uri = Fn = NULL; } - - // Implementation - virtual const char *GetType(void) { return "REST"; } +struct MemoryStruct { + char *memory; + size_t size; +}; - // Methods - virtual bool DefineAM(PGLOBAL g, LPCSTR am, int poff); - virtual PTDB GetTable(PGLOBAL g, MODE m); -protected: - // Members - PRELDEF Tdp; - PCSZ Http; /* Web connection HTTP */ - PCSZ Uri; /* Web connection URI */ - PCSZ Fn; /* The intermediate file name */ +/***********************************************************************/ +/* Restest table. */ +/***********************************************************************/ +class RESTDEF : public TABDEF { /* Table description */ +private: + bool curl_inited; +public: +// Constructor + RESTDEF() + :curl_inited(false), + Tdp(NULL), + Http(NULL), + Uri(NULL), + Fn(NULL) + {} + int curl_init (PGLOBAL g); + void curl_deinit (); + // Methods + virtual const char *GetType(void) { return "REST"; } + virtual bool DefineAM(PGLOBAL g, LPCSTR am, int poff); + virtual PTDB GetTable(PGLOBAL g, MODE m); + int curl_run(PGLOBAL g); + // Members + PRELDEF Tdp; + PCSZ Http; /* Web connection HTTP */ + PCSZ Uri; /* Web connection URI */ + PCSZ Fn; /* The intermediate file name */ + ~RESTDEF() + { + curl_deinit(); + } }; // end of class RESTDEF