From 4437e057d973dea4139de43bba66725c6aad0b1f Mon Sep 17 00:00:00 2001 From: Igor Gaponenko Date: Tue, 23 Jan 2024 07:24:14 +0000 Subject: [PATCH] Added a connectin pool for the sync HTTP client --- src/http/CMakeLists.txt | 2 + src/http/Client.cc | 20 +++++++-- src/http/Client.h | 13 +++++- src/http/ClientConnPool.cc | 69 +++++++++++++++++++++++++++++++ src/http/ClientConnPool.h | 83 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 182 insertions(+), 5 deletions(-) create mode 100644 src/http/ClientConnPool.cc create mode 100644 src/http/ClientConnPool.h diff --git a/src/http/CMakeLists.txt b/src/http/CMakeLists.txt index 4a62013fa..ed22ff5c6 100644 --- a/src/http/CMakeLists.txt +++ b/src/http/CMakeLists.txt @@ -3,7 +3,9 @@ add_library(http SHARED) target_sources(http PRIVATE AsyncReq.cc Client.cc + ClientConnPool.cc ClientConfig.cc + ClientConnPool.cc Exceptions.cc MetaModule.cc Method.cc diff --git a/src/http/Client.cc b/src/http/Client.cc index 4dd33d65b..7526efa93 100644 --- a/src/http/Client.cc +++ b/src/http/Client.cc @@ -23,6 +23,7 @@ #include "http/Client.h" // Qserv headers +#include "http/ClientConnPool.h" #include "http/Exceptions.h" // Standard headers @@ -44,10 +45,15 @@ size_t forwardToClient(char* ptr, size_t size, size_t nmemb, void* client) { } Client::Client(http::Method method, string const& url, string const& data, vector const& headers, - ClientConfig const& clientConfig) - : _method(method), _url(url), _data(data), _headers(headers), _clientConfig(clientConfig) { + ClientConfig const& clientConfig, shared_ptr const& connPool) + : _method(method), + _url(url), + _data(data), + _headers(headers), + _clientConfig(clientConfig), + _connPool(connPool) { _hcurl = curl_easy_init(); - assert(_hcurl != nullptr); // curl_easy_init() failed to allocate memory, etc. + assert(_hcurl != nullptr); } Client::~Client() { @@ -146,6 +152,14 @@ void Client::_setConnOptions() { curl_easy_setopt(_hcurl, CURLOPT_TCP_KEEPINTVL, _clientConfig.tcpKeepIntvl)); } } + if (_connPool != nullptr) { + _curlEasyErrorChecked("curl_easy_setopt(CURLOPT_SHARE)", + curl_easy_setopt(_hcurl, CURLOPT_SHARE, _connPool->shareCurl())); + if (_connPool->maxConnections() > 0) { + _curlEasyErrorChecked("curl_easy_setopt(CURLOPT_MAXCONNECTS)", + curl_easy_setopt(_hcurl, CURLOPT_MAXCONNECTS, _connPool->maxConnections())); + } + } } void Client::_setSslCertOptions() { diff --git a/src/http/Client.h b/src/http/Client.h index 442363ab0..6d3dc8162 100644 --- a/src/http/Client.h +++ b/src/http/Client.h @@ -23,6 +23,7 @@ // System headers #include +#include #include #include @@ -34,6 +35,11 @@ #include "http/ClientConfig.h" #include "http/Method.h" +// Forward declarations +namespace lsst::qserv::http { +class ClientConnPool; +} // namespace lsst::qserv::http + // This header declarations namespace lsst::qserv::http { @@ -83,10 +89,12 @@ class Client { * @param data Optional data to be sent with a request (depends on the HTTP headers). * @param headers Optional HTTP headers to be send with a request. * @param clientConfig Optional configuration parameters of the reader. + * @param connPool Optional connection pool */ Client(http::Method method, std::string const& url, std::string const& data = std::string(), std::vector const& headers = std::vector(), - ClientConfig const& clientConfig = ClientConfig()); + ClientConfig const& clientConfig = ClientConfig(), + std::shared_ptr const& connPool = nullptr); /** * Begin processing a request. The whole content of the remote data source @@ -136,7 +144,7 @@ class Client { * * @param scope A location from which the method was called (used for error reporting). * @param errnum A result reported by the CURL library function. - * @throw std::runtime_error If the error-code is not CURL_OK. + * @throw std::runtime_error If the error-code is not CURLE_OK. */ void _curlEasyErrorChecked(std::string const& scope, CURLcode errnum); @@ -166,6 +174,7 @@ class Client { std::string const _data; std::vector const _headers; ClientConfig const _clientConfig; + std::shared_ptr const _connPool; CallbackType _onDataRead; ///< set by method read() before pulling the data diff --git a/src/http/ClientConnPool.cc b/src/http/ClientConnPool.cc new file mode 100644 index 000000000..47b659525 --- /dev/null +++ b/src/http/ClientConnPool.cc @@ -0,0 +1,69 @@ +/* + * LSST Data Management System + * + * This product includes software developed by the + * LSST Project (http://www.lsst.org/). + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the LSST License Statement and + * the GNU General Public License along with this program. If not, + * see . + */ + +// Class header +#include "http/ClientConnPool.h" + +// Qserv headers +#include "http/Exceptions.h" + +// Standard headers +#include + +using namespace std; + +namespace lsst::qserv::http { + +mutex ClientConnPool::_accessShareCurlMtx; + +ClientConnPool::ClientConnPool(long maxConnections) : _maxConnections(maxConnections) { + _shareCurl = curl_share_init(); + assert(_shareCurl != nullptr); + _errorChecked("curl_share_setopt(CURLSHOPT_LOCKFUNC)", + curl_share_setopt(_shareCurl, CURLSHOPT_LOCKFUNC, &ClientConnPool::_share_lock_cb)); + _errorChecked("curl_share_setopt(CURLSHOPT_UNLOCKFUNC)", + curl_share_setopt(_shareCurl, CURLSHOPT_UNLOCKFUNC, &ClientConnPool::_share_unlock_cb)); + _errorChecked("curl_share_setopt(CURLSHOPT_SHARE, CURL_LOCK_DATA_CONNECT)", + curl_share_setopt(_shareCurl, CURLSHOPT_SHARE, CURL_LOCK_DATA_CONNECT)); +} + +ClientConnPool::~ClientConnPool() { curl_share_cleanup(_shareCurl); } + +void ClientConnPool::_share_lock_cb(CURL* handle, curl_lock_data data, curl_lock_access access, + void* userptr) { + ClientConnPool::_accessShareCurlMtx.lock(); +} + +void ClientConnPool::_share_unlock_cb(CURL* handle, curl_lock_data data, curl_lock_access access, + void* userptr) { + ClientConnPool::_accessShareCurlMtx.unlock(); +} + +void ClientConnPool::_errorChecked(string const& scope, CURLSHcode errnum) { + if (errnum != CURLSHE_OK) { + string const errorStr = curl_share_strerror(errnum); + long const httpResponseCode = 0; + http::raiseRetryAllowedError(scope, " error: '" + errorStr + "', errnum: " + to_string(errnum), + httpResponseCode); + } +} + +} // namespace lsst::qserv::http diff --git a/src/http/ClientConnPool.h b/src/http/ClientConnPool.h new file mode 100644 index 000000000..bf3bb5f48 --- /dev/null +++ b/src/http/ClientConnPool.h @@ -0,0 +1,83 @@ +/* + * LSST Data Management System + * + * This product includes software developed by the + * LSST Project (http://www.lsst.org/). + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the LSST License Statement and + * the GNU General Public License along with this program. If not, + * see . + */ +#ifndef LSST_QSERV_HTTP_CLIENTCONNPOOL_H +#define LSST_QSERV_HTTP_CLIENTCONNPOOL_H + +// System headers +#include +#include + +// Third-party headers +#include "curl/curl.h" + +// This header declarations +namespace lsst::qserv::http { +/** + * Class ClientConnPool is a helper class utilizing the libcurl's context + * sharing mechanism for building a configurable pool of the TCP connections. + * Note that this implementation doesn't directly manage connections. The connections + * are owned and managed by the library itself. A role of the class is to provide + * a synchronization context for acquring/releasing these connections in the multi-thread + * environment. + * + * The implementation is based on: https://curl.se/libcurl/c/libcurl-share.html + */ +class ClientConnPool { +public: + /** + * Initialize the pool + * @param maxConnections The maximum number of connections alloqwed in the pool. + */ + explicit ClientConnPool(long maxConnections = 0); + ClientConnPool(ClientConnPool const&) = delete; + ClientConnPool& operator=(ClientConnPool const&) = delete; + + ~ClientConnPool(); + + long maxConnections() const { return _maxConnections; } + CURLSH* shareCurl() { return _shareCurl; } + +private: + // These callback functions are required by libcurl to allow easy-based instances + // of the class Client acquire/release connections from the pool. + + static void _share_lock_cb(CURL* handle, curl_lock_data data, curl_lock_access access, void* userptr); + static void _share_unlock_cb(CURL* handle, curl_lock_data data, curl_lock_access access, void* userptr); + + /** + * Check for an error condition. + * + * @param scope A location from which the method was called (used for error reporting). + * @param errnum A result reported by the CURL library function. + * @throw std::runtime_error If the error-code is not CURLSHE_OK. + */ + void _errorChecked(std::string const& scope, CURLSHcode errnum); + + /// The miutex is shared by all instances of the pool. + static std::mutex _accessShareCurlMtx; + + long const _maxConnections = 0; + CURLSH* _shareCurl; +}; + +} // namespace lsst::qserv::http + +#endif // LSST_QSERV_HTTP_CLIENTCONNPOOL_H