Added a connectin pool for the sync HTTP client

src/http/CMakeLists.txt

@@ -3,7 +3,9 @@ add_library(http SHARED)
 target_sources(http PRIVATE
     AsyncReq.cc
     Client.cc
+    ClientConnPool.cc
     ClientConfig.cc
+    ClientConnPool.cc
     Exceptions.cc
     MetaModule.cc
     Method.cc

src/http/Client.cc

@@ -23,6 +23,7 @@
 #include "http/Client.h"
 
 // Qserv headers
+#include "http/ClientConnPool.h"
 #include "http/Exceptions.h"
 
 // Standard headers
@@ -44,10 +45,15 @@ size_t forwardToClient(char* ptr, size_t size, size_t nmemb, void* client) {
 }
 
 Client::Client(http::Method method, string const& url, string const& data, vector<string> const& headers,
-               ClientConfig const& clientConfig)
-        : _method(method), _url(url), _data(data), _headers(headers), _clientConfig(clientConfig) {
+               ClientConfig const& clientConfig, shared_ptr<ClientConnPool> const& connPool)
+        : _method(method),
+          _url(url),
+          _data(data),
+          _headers(headers),
+          _clientConfig(clientConfig),
+          _connPool(connPool) {
     _hcurl = curl_easy_init();
-    assert(_hcurl != nullptr);  // curl_easy_init() failed to allocate memory, etc.
+    assert(_hcurl != nullptr);
 }
 
 Client::~Client() {
@@ -146,6 +152,14 @@ void Client::_setConnOptions() {
                     curl_easy_setopt(_hcurl, CURLOPT_TCP_KEEPINTVL, _clientConfig.tcpKeepIntvl));
         }
     }
+    if (_connPool != nullptr) {
+        _curlEasyErrorChecked("curl_easy_setopt(CURLOPT_SHARE)",
+                              curl_easy_setopt(_hcurl, CURLOPT_SHARE, _connPool->shareCurl()));
+        if (_connPool->maxConnections() > 0) {
+            _curlEasyErrorChecked("curl_easy_setopt(CURLOPT_MAXCONNECTS)",
+                                  curl_easy_setopt(_hcurl, CURLOPT_MAXCONNECTS, _connPool->maxConnections()));
+        }
+    }
 }
 
 void Client::_setSslCertOptions() {

src/http/Client.h

@@ -23,6 +23,7 @@
 
 // System headers
 #include <functional>
+#include <memory>
 #include <string>
 #include <vector>
 
@@ -34,6 +35,11 @@
 #include "http/ClientConfig.h"
 #include "http/Method.h"
 
+// Forward declarations
+namespace lsst::qserv::http {
+class ClientConnPool;
+}  // namespace lsst::qserv::http
+
 // This header declarations
 namespace lsst::qserv::http {
 
@@ -83,10 +89,12 @@ class Client {
      * @param data Optional data to be sent with a request (depends on the HTTP headers).
      * @param headers Optional HTTP headers to be send with a request.
      * @param clientConfig Optional configuration parameters of the reader.
+     * @param connPool Optional connection pool
      */
     Client(http::Method method, std::string const& url, std::string const& data = std::string(),
            std::vector<std::string> const& headers = std::vector<std::string>(),
-           ClientConfig const& clientConfig = ClientConfig());
+           ClientConfig const& clientConfig = ClientConfig(),
+           std::shared_ptr<ClientConnPool> const& connPool = nullptr);
 
     /**
      * Begin processing a request. The whole content of the remote data source
@@ -136,7 +144,7 @@ class Client {
      *
      * @param scope A location from which the method was called (used for error reporting).
      * @param errnum A result reported by the CURL library function.
-     * @throw std::runtime_error If the error-code is not CURL_OK.
+     * @throw std::runtime_error If the error-code is not CURLE_OK.
      */
     void _curlEasyErrorChecked(std::string const& scope, CURLcode errnum);
 
@@ -166,6 +174,7 @@ class Client {
     std::string const _data;
     std::vector<std::string> const _headers;
     ClientConfig const _clientConfig;
+    std::shared_ptr<ClientConnPool> const _connPool;
 
     CallbackType _onDataRead;  ///< set by method read() before pulling the data
 

src/http/ClientConnPool.cc

@@ -0,0 +1,69 @@
+/*
+ * LSST Data Management System
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program.  If not,
+ * see <http://www.lsstcorp.org/LegalNotices/>.
+ */
+
+// Class header
+#include "http/ClientConnPool.h"
+
+// Qserv headers
+#include "http/Exceptions.h"
+
+// Standard headers
+#include <cassert>
+
+using namespace std;
+
+namespace lsst::qserv::http {
+
+mutex ClientConnPool::_accessShareCurlMtx;
+
+ClientConnPool::ClientConnPool(long maxConnections) : _maxConnections(maxConnections) {
+    _shareCurl = curl_share_init();
+    assert(_shareCurl != nullptr);
+    _errorChecked("curl_share_setopt(CURLSHOPT_LOCKFUNC)",
+                  curl_share_setopt(_shareCurl, CURLSHOPT_LOCKFUNC, &ClientConnPool::_share_lock_cb));
+    _errorChecked("curl_share_setopt(CURLSHOPT_UNLOCKFUNC)",
+                  curl_share_setopt(_shareCurl, CURLSHOPT_UNLOCKFUNC, &ClientConnPool::_share_unlock_cb));
+    _errorChecked("curl_share_setopt(CURLSHOPT_SHARE, CURL_LOCK_DATA_CONNECT)",
+                  curl_share_setopt(_shareCurl, CURLSHOPT_SHARE, CURL_LOCK_DATA_CONNECT));
+}
+
+ClientConnPool::~ClientConnPool() { curl_share_cleanup(_shareCurl); }
+
+void ClientConnPool::_share_lock_cb(CURL* handle, curl_lock_data data, curl_lock_access access,
+                                    void* userptr) {
+    ClientConnPool::_accessShareCurlMtx.lock();
+}
+
+void ClientConnPool::_share_unlock_cb(CURL* handle, curl_lock_data data, curl_lock_access access,
+                                      void* userptr) {
+    ClientConnPool::_accessShareCurlMtx.unlock();
+}
+
+void ClientConnPool::_errorChecked(string const& scope, CURLSHcode errnum) {
+    if (errnum != CURLSHE_OK) {
+        string const errorStr = curl_share_strerror(errnum);
+        long const httpResponseCode = 0;
+        http::raiseRetryAllowedError(scope, " error: '" + errorStr + "', errnum: " + to_string(errnum),
+                                     httpResponseCode);
+    }
+}
+
+}  // namespace lsst::qserv::http

src/http/ClientConnPool.h

@@ -0,0 +1,83 @@
+/*
+ * LSST Data Management System
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program.  If not,
+ * see <http://www.lsstcorp.org/LegalNotices/>.
+ */
+#ifndef LSST_QSERV_HTTP_CLIENTCONNPOOL_H
+#define LSST_QSERV_HTTP_CLIENTCONNPOOL_H
+
+// System headers
+#include <mutex>
+#include <string>
+
+// Third-party headers
+#include "curl/curl.h"
+
+// This header declarations
+namespace lsst::qserv::http {
+/**
+ * Class ClientConnPool is a helper class utilizing the libcurl's context
+ * sharing mechanism for building a configurable pool of the TCP connections.
+ * Note that this implementation doesn't directly manage connections. The connections
+ * are owned and managed by the library itself. A role of the class is to provide
+ * a synchronization context for acquring/releasing these connections in the multi-thread
+ * environment.
+ *
+ * The implementation is based on: https://curl.se/libcurl/c/libcurl-share.html
+ */
+class ClientConnPool {
+public:
+    /**
+     * Initialize the pool
+     * @param maxConnections The maximum number of connections alloqwed in the pool.
+     */
+    explicit ClientConnPool(long maxConnections = 0);
+    ClientConnPool(ClientConnPool const&) = delete;
+    ClientConnPool& operator=(ClientConnPool const&) = delete;
+
+    ~ClientConnPool();
+
+    long maxConnections() const { return _maxConnections; }
+    CURLSH* shareCurl() { return _shareCurl; }
+
+private:
+    // These callback functions are required by libcurl to allow easy-based instances
+    // of the class Client acquire/release connections from the pool.
+
+    static void _share_lock_cb(CURL* handle, curl_lock_data data, curl_lock_access access, void* userptr);
+    static void _share_unlock_cb(CURL* handle, curl_lock_data data, curl_lock_access access, void* userptr);
+
+    /**
+     * Check for an error condition.
+     *
+     * @param scope A location from which the method was called (used for error reporting).
+     * @param errnum A result reported by the CURL library function.
+     * @throw std::runtime_error If the error-code is not CURLSHE_OK.
+     */
+    void _errorChecked(std::string const& scope, CURLSHcode errnum);
+
+    /// The miutex is shared by all instances of the pool.
+    static std::mutex _accessShareCurlMtx;
+
+    long const _maxConnections = 0;
+    CURLSH* _shareCurl;
+};
+
+}  // namespace lsst::qserv::http
+
+#endif  // LSST_QSERV_HTTP_CLIENTCONNPOOL_H