diff --git a/src/global/ResourceUnit.cc b/src/global/ResourceUnit.cc index 0dc548cbc..64144b843 100644 --- a/src/global/ResourceUnit.cc +++ b/src/global/ResourceUnit.cc @@ -83,9 +83,6 @@ std::string ResourceUnit::path() const { case UNKNOWN: ss << _pathSep << "UNKNOWN_RESOURCE_UNIT"; break; - case WORKER: - ss << _workerId; - break; default: ::abort(); break; @@ -107,8 +104,6 @@ std::string ResourceUnit::prefix(UnitType const& r) { return "chk"; case UNKNOWN: return "UNKNOWN"; - case WORKER: - return "worker"; case QUERY: return "query"; case GARBAGE: @@ -121,10 +116,6 @@ std::string ResourceUnit::makePath(int chunk, std::string const& db) { return _pathSep + prefix(UnitType::DBCHUNK) + _pathSep + db + _pathSep + std::to_string(chunk); } -std::string ResourceUnit::makeWorkerPath(std::string const& id) { - return _pathSep + prefix(UnitType::WORKER) + _pathSep + id; -} - void ResourceUnit::setAsDbChunk(std::string const& db, int chunk) { _unitType = DBCHUNK; _db = db; @@ -173,17 +164,6 @@ void ResourceUnit::_setFromPath(std::string const& path) { } _chunk = t.tokenAsInt(); _ingestLeafAndKeys(t.token()); - } else if (rTypeString == prefix(WORKER)) { - _unitType = WORKER; - if (_markGarbageIfDone(t)) { - return; - } - t.next(); - _workerId = t.token(); - if (_workerId.empty()) { - _unitType = GARBAGE; - return; - } } else if (rTypeString == prefix(QUERY)) { _unitType = QUERY; if (!t.done()) { diff --git a/src/global/ResourceUnit.h b/src/global/ResourceUnit.h index b141c27c5..ad4a1ef0b 100644 --- a/src/global/ResourceUnit.h +++ b/src/global/ResourceUnit.h @@ -45,7 +45,7 @@ namespace lsst::qserv { class ResourceUnit { public: class Checker; - enum UnitType { GARBAGE, DBCHUNK, UNKNOWN, WORKER, QUERY }; + enum UnitType { GARBAGE, DBCHUNK, UNKNOWN, QUERY }; ResourceUnit() = default; explicit ResourceUnit(std::string const& path); @@ -61,7 +61,6 @@ class ResourceUnit { UnitType unitType() const { return _unitType; } std::string const& db() const { return _db; } int chunk() const { return _chunk; } - std::string const& workerId() const { return _workerId; } /// Lookup extended path variables (?k=val syntax) std::string var(std::string const& key) const; @@ -72,9 +71,6 @@ class ResourceUnit { /// @return the path of the database/chunk resource static std::string makePath(int chunk, std::string const& db); - /// @return the path of the worker-specific resource - static std::string makeWorkerPath(std::string const& id); - // Setup a path of a certain type. void setAsDbChunk(std::string const& db, int chunk = DUMMY_CHUNK); @@ -88,7 +84,6 @@ class ResourceUnit { UnitType _unitType = UnitType::GARBAGE; //< Type of unit std::string _db; //< for DBCHUNK type int _chunk = -1; //< for DBCHUNK type - std::string _workerId; //< for WORKER type typedef std::map VarMap; VarMap _vars; //< Key-value specifiers diff --git a/src/global/testResourceUnit.cc b/src/global/testResourceUnit.cc index e0fd22279..85ca378f7 100644 --- a/src/global/testResourceUnit.cc +++ b/src/global/testResourceUnit.cc @@ -81,13 +81,6 @@ BOOST_AUTO_TEST_CASE(DbChunk) { BOOST_CHECK_EQUAL(r[1].path(), "/chk/bar/968"); } -BOOST_AUTO_TEST_CASE(Worker) { - std::string const id = "worker-1"; - ResourceUnit res("/worker/" + id); - BOOST_CHECK_EQUAL(res.unitType(), ResourceUnit::WORKER); - BOOST_CHECK_EQUAL(res.workerId(), id); -} - BOOST_AUTO_TEST_CASE(Query) { ResourceUnit const res1("/query"); BOOST_CHECK_EQUAL(res1.unitType(), ResourceUnit::QUERY); diff --git a/src/proto/worker.proto b/src/proto/worker.proto index 687562069..27575440c 100644 --- a/src/proto/worker.proto +++ b/src/proto/worker.proto @@ -135,171 +135,26 @@ message Result { // Byte N+1, extent = ProtoHeader.size, Result msg // (successive Result msgs indicated by size markers in previous Result msgs) - -//////////////////////////////////////////////////////////////// -// Protocol definition for the worker management commands +///////////////////////////////////////////////////////////////// +// Protocol definition for the query management requests. These +// requests do not require any response messages to be explicitly +// sent by workers. // -// ATTENTION: each message sent or received must be preceeded by +// ATTENTION: each message sent to a worker must be preceeded by // an int32 size (network-byte-ordered) word carrying a size // of the message. //////////////////////////////////////////////////////////////// -// All requests sent to worker management resources should start with -// sending this message carrying an identifier of a command. Specific -// commands may require additional parameters which should be sent as -// separate messages (of the corresponding types). -message WorkerCommandH { - enum Command { - TEST_ECHO = 1; // Return back a value sent to the command processor. - ADD_CHUNK_GROUP = 2; // Add a group of collocated chunks. - REMOVE_CHUNK_GROUP = 3; // Remove a group of collocated chunks. - UPDATE_CHUNK_LIST = 4; // Update (rebuild and/or reload) the list of available chunks. - GET_CHUNK_LIST = 5; // Return a list of chunks known to a worker. - SET_CHUNK_LIST = 6; // Set a new list of chunks. - GET_STATUS = 7; // Return various status info on a worker. - GET_DATABASE_STATUS = 8; // Return various info on the worker database service. - GET_CONFIG = 9; // Return configuration parameters of the worker database service. - } - required Command command = 1; -} - -// The completion status to be sent back with responses to the worker commands. +// The completion status to be sent back with responses to the query management requests. message WorkerCommandStatus { enum Code { - SUCCESS = 1; // The sccessful completion of a request. - INVALID = 2; // Invalid parameters of the request. - IN_USE = 3; // The request is rejected because one of the chunks is in use. - ERROR = 4; // An error occurred during command execution. + SUCCESS = 1; // The successful completion of a request. + ERROR = 2; // An error occurred during request execution. } - optional Code code = 1 [default = SUCCESS]; + optional Code code = 3 [default = SUCCESS]; optional string error = 2 [default = ""]; // Optional error message (depends on the code) } -// This message must be sent after the command header to provide -// a service with a value to be echoed back in response to -// the 'TEST_ECHO' command. -message WorkerCommandTestEchoM { - required string value = 1; // The input string to be returned back by the service -} - -// The message to be sent back in response to the 'TEST_ECHO' command -message WorkerCommandTestEchoR { - required WorkerCommandStatus status = 4; // Completion status of the operation - required string value = 3; // The original value returned by the operation -} - -// The message type embedded into the relevant contexts below -message WorkerCommandChunk { - required string db = 1; - required uint32 chunk = 2; - optional uint32 use_count = 3 [default = 0]; // depends on a context -} - -// This message must be sent after the command header for the 'ADD_CHUNK_GROUP' -// or 'REMOVE_CHUNK_GROUP' command to tell the service which chunks needs to be -// added or removed. -message WorkerCommandChunkGroupM { - required uint32 chunk = 1; - repeated string dbs = 2; - optional bool force = 3 [ default = false]; -} - -// The message to be sent back in response to the 'ADD_CHUNK_GROUP' -// or 'REMOVE_CHUNK_GROUP' commands. -message WorkerCommandChunkGroupR { - required WorkerCommandStatus status = 3; // Completion status of the operation -} - -// This message must be sent after the command header for the 'UPDATE_CHUNK_LIST' -// command. -message WorkerCommandUpdateChunkListM { - required bool rebuild = 1; // Rebuild the list from actual tables existing in the database - required bool reload = 2; // Reload the new list into a worker -} - -// The message to be sent back in response to the 'UPDATE_CHUNK_LIST' -// command. -message WorkerCommandUpdateChunkListR { - required WorkerCommandStatus status = 5; // Completion status of the operation - repeated WorkerCommandChunk added = 3; // Chunks that were added - repeated WorkerCommandChunk removed = 4; // Chunks that were removed -} - -// The message to be sent back in response to the 'GET_CHUNK_LIST' -// command. -message WorkerCommandGetChunkListR { - required WorkerCommandStatus status = 4; // Completion status of the operation - repeated WorkerCommandChunk chunks = 3; -} - -// This message must be sent after the command header for the 'SET_CHUNK_LIST' -// to tell the service which chunks needs to be set. -message WorkerCommandSetChunkListM { - repeated WorkerCommandChunk chunks = 1; - optional bool force = 2 [ default = false]; - repeated string databases = 3; // The operation involves databases listed here -} - -// The message to be sent back in response to the 'SET_CHUNK_LIST' -// command. -message WorkerCommandSetChunkListR { - required WorkerCommandStatus status = 4; // Completion status of the operation - repeated WorkerCommandChunk chunks = 3; // The previous list of chunks -} - -// This message must be sent after the command header for the 'GET_STATUS' -// to customize a scope of the request. -message WorkerCommandGetStatusM { - // Include detailed info on the tasks - optional bool include_tasks = 1 [ default = false]; - - // Narrow a scope of the request that involves tasks (the above-define parameter - // 'include_tasks = true') to a specific collection of queries. - repeated uint64 query_ids = 2; - - // An optional selector for the state(s) of tasks. If the collection - // is empty then the selector is disregarded. Values to be put into - // the collection are the integer representation of the following transient - // C++ enum type 'wbase::TaskState'. These values are expected to be - // validated and upconverted into the enum class by the worker. - repeated uint64 task_states = 3; - - // An optional limit for the number of tasks to be reported. - // Note that no specific ordering for tasks is assumed when applying - // this limit. - optional uint32 max_tasks = 4 [ default = 0 ]; -} - -// The message to be sent back in response to the 'GET_STATUS' command -message WorkerCommandGetStatusR { - required WorkerCommandStatus status = 2; // Completion status of the operation - required string info = 1; // Status info serialized from a JSON object -} - -// The message to be sent back in response to the 'GET_DATABASE_STATUS' command -// -message WorkerCommandGetDbStatusR { - required WorkerCommandStatus status = 1; // Completion status of the operation - required string info = 2; // Status info serialized from a JSON object -} - -// The message to be sent back in response to the 'GET_CONFIG' command -// -message WorkerCommandGetConfigR { - required WorkerCommandStatus status = 1; // Completion status of the operation - required string info = 2; // Configuration info serialized from a JSON object -} - -///////////////////////////////////////////////////////////////// -// Protocol definition for the query management requests. These -// requests do not require any response messages to be explicitly -// sent by workers. -// -// ATTENTION: each message sent to a worker must be preceeded by -// an int32 size (network-byte-ordered) word carrying a size -// of the message. -//////////////////////////////////////////////////////////////// - message QueryManagement { enum Operation { CANCEL_AFTER_RESTART = 1; // Cancel older queries before the specified query (excluding that one). diff --git a/src/wpublish/AddChunkGroupCommand.cc b/src/wpublish/AddChunkGroupCommand.cc deleted file mode 100644 index 6b7853652..000000000 --- a/src/wpublish/AddChunkGroupCommand.cc +++ /dev/null @@ -1,111 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * Copyright 2012-2018 AURA/LSST. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ - -// Class header -#include "wpublish/AddChunkGroupCommand.h" - -// System headers - -// Third-party headers -#include "XrdSsi/XrdSsiCluster.hh" - -// LSST headers -#include "lsst/log/Log.h" -#include "wbase/SendChannel.h" -#include "wpublish/ChunkInventory.h" -#include "xrdsvc/SsiProvider.h" -#include "xrdsvc/XrdName.h" - -extern XrdSsiProvider* XrdSsiProviderLookup; - -using namespace std; - -namespace { - -LOG_LOGGER _log = LOG_GET("lsst.qserv.wpublish.AddChunkGroupCommand"); - -} // namespace - -namespace lsst::qserv::wpublish { - -AddChunkGroupCommand::AddChunkGroupCommand(shared_ptr const& sendChannel, - shared_ptr const& chunkInventory, - mysql::MySqlConfig const& mySqlConfig, int chunk, - vector const& databases) - : wbase::WorkerCommand(sendChannel), - _chunkInventory(chunkInventory), - _mySqlConfig(mySqlConfig), - _chunk(chunk), - _databases(databases) {} - -void AddChunkGroupCommand::run() { - string const context = "AddChunkGroupCommand::" + string(__func__) + " "; - LOGS(_log, LOG_LVL_DEBUG, context); - - if (_databases.empty()) { - reportError( - "the list of database names in the group was found empty", - proto::WorkerCommandStatus::INVALID); - return; - } - - xrdsvc::SsiProviderServer* providerServer = - dynamic_cast(XrdSsiProviderLookup); - XrdSsiCluster* clusterManager = providerServer->GetClusterManager(); - - for (auto&& database : _databases) { - string const resource = "/chk/" + database + "/" + to_string(_chunk); - LOGS(_log, LOG_LVL_DEBUG, - context << " adding the chunk resource: " << resource - << " in DataContext=" << clusterManager->DataContext()); - try { - // Notify XRootD/cmsd and (depending on a mode) modify the provider's copy - // of the inventory. - clusterManager->Added(resource.c_str()); - if (clusterManager->DataContext()) { - providerServer->GetChunkInventory().add(database, _chunk); - } - // Notify QServ and update the database - _chunkInventory->add(database, _chunk, _mySqlConfig); - - } catch (InvalidParamError const& ex) { - reportError(ex.what(), proto::WorkerCommandStatus::INVALID); - return; - } catch (QueryError const& ex) { - reportError(ex.what()); - return; - } catch (exception const& ex) { - reportError("failed to add the chunk: " + string(ex.what())); - return; - } - } - proto::WorkerCommandChunkGroupR reply; - reply.mutable_status(); - _frameBuf.serialize(reply); - string str(_frameBuf.data(), _frameBuf.size()); - _sendChannel->sendStream(xrdsvc::StreamBuffer::createWithMove(str), true); - - LOGS(_log, LOG_LVL_DEBUG, context << "** SENT **"); -} - -} // namespace lsst::qserv::wpublish diff --git a/src/wpublish/AddChunkGroupCommand.h b/src/wpublish/AddChunkGroupCommand.h deleted file mode 100644 index fc97c682a..000000000 --- a/src/wpublish/AddChunkGroupCommand.h +++ /dev/null @@ -1,85 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * Copyright 2011-2018 LSST Corporation. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ -/// AddChunkGroupCommand.h -#ifndef LSST_QSERV_WPUBLISH_ADD_CHUNK_GROUP_COMMAND_H -#define LSST_QSERV_WPUBLISH_ADD_CHUNK_GROUP_COMMAND_H - -// System headers -#include -#include -#include - -// Qserv headers -#include "mysql/MySqlConfig.h" -#include "proto/worker.pb.h" -#include "wbase/WorkerCommand.h" - -// Forward declarations -namespace lsst::qserv { -namespace wbase { -class SendChannel; -} -namespace wpublish { -class ChunkInventory; -} -} // namespace lsst::qserv - -// This header declarations -namespace lsst::qserv::wpublish { - -/** - * Class AddChunkGroupCommand reloads a list of chunks from the database - */ -class AddChunkGroupCommand : public wbase::WorkerCommand { -public: - AddChunkGroupCommand& operator=(const AddChunkGroupCommand&) = delete; - AddChunkGroupCommand(const AddChunkGroupCommand&) = delete; - AddChunkGroupCommand() = delete; - - /** - * @param sendChannel communication channel for reporting results - * @param chunkInventory chunks known to the application - * @param mySqlConfig database connection parameters - * @param chunk chunk number - * @param databases names of databases in the group - */ - AddChunkGroupCommand(std::shared_ptr const& sendChannel, - std::shared_ptr const& chunkInventory, - mysql::MySqlConfig const& mySqlConfig, int chunk, - std::vector const& databases); - - virtual ~AddChunkGroupCommand() override = default; - -protected: - virtual void run() override; - -private: - std::shared_ptr _chunkInventory; - mysql::MySqlConfig _mySqlConfig; - int _chunk; - std::vector _databases; -}; - -} // namespace lsst::qserv::wpublish - -#endif // LSST_QSERV_WPUBLISH_ADD_CHUNK_GROUP_COMMAND_H \ No newline at end of file diff --git a/src/wpublish/CMakeLists.txt b/src/wpublish/CMakeLists.txt index da6317d0d..576b5787a 100644 --- a/src/wpublish/CMakeLists.txt +++ b/src/wpublish/CMakeLists.txt @@ -2,17 +2,8 @@ add_library(wpublish OBJECT) add_dependencies(wpublish proto) target_sources(wpublish PRIVATE - AddChunkGroupCommand.cc ChunkInventory.cc - ChunkListCommand.cc - GetChunkListCommand.cc - GetConfigCommand.cc - GetDbStatusCommand.cc - GetStatusCommand.cc QueriesAndChunks.cc - RemoveChunkGroupCommand.cc - SetChunkListCommand.cc - TestEchoCommand.cc ) target_include_directories(wpublish PRIVATE diff --git a/src/wpublish/ChunkInventory.cc b/src/wpublish/ChunkInventory.cc index 79b718066..eb112303a 100644 --- a/src/wpublish/ChunkInventory.cc +++ b/src/wpublish/ChunkInventory.cc @@ -140,8 +140,6 @@ class Validator : public lsst::qserv::ResourceUnit::Checker { switch (ru.unitType()) { case lsst::qserv::ResourceUnit::DBCHUNK: return chunkInventory.has(ru.db(), ru.chunk()); - case lsst::qserv::ResourceUnit::WORKER: - return chunkInventory.id() == ru.workerId(); case lsst::qserv::ResourceUnit::QUERY: return true; default: diff --git a/src/wpublish/ChunkListCommand.cc b/src/wpublish/ChunkListCommand.cc deleted file mode 100644 index 071135299..000000000 --- a/src/wpublish/ChunkListCommand.cc +++ /dev/null @@ -1,180 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * Copyright 2012-2018 AURA/LSST. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ - -// Class header -#include "wpublish/ChunkListCommand.h" - -// System headers -#include - -// Third-party headers -#include "XrdSsi/XrdSsiCluster.hh" - -// LSST headers -#include "lsst/log/Log.h" -#include "proto/worker.pb.h" -#include "wbase/SendChannel.h" -#include "wpublish/ChunkInventory.h" -#include "xrdsvc/SsiProvider.h" -#include "xrdsvc/XrdName.h" - -extern XrdSsiProvider* XrdSsiProviderLookup; - -using namespace std; - -namespace { - -LOG_LOGGER _log = LOG_GET("lsst.qserv.wpublish.ChunkListCommand"); - -/// Print the inventory status onto the logging stream -void dumpInventory(lsst::qserv::wpublish::ChunkInventory const& inventory, string const& context) { - ostringstream os; - inventory.dbgPrint(os); - LOGS(_log, LOG_LVL_TRACE, context << os.str()); -} - -} // namespace - -namespace lsst::qserv::wpublish { - -ChunkListCommand::ChunkListCommand(shared_ptr const& sendChannel, - shared_ptr const& chunkInventory, - mysql::MySqlConfig const& mySqlConfig, bool rebuild, bool reload) - : wbase::WorkerCommand(sendChannel), - _chunkInventory(chunkInventory), - _mySqlConfig(mySqlConfig), - _rebuild(rebuild), - _reload(reload) {} - -void ChunkListCommand::run() { - string const context = "ChunkListCommand::" + string(__func__) + " "; - LOGS(_log, LOG_LVL_DEBUG, context); - - proto::WorkerCommandUpdateChunkListR reply; - reply.mutable_status(); - - // Rebuild persistent list if requested - if (_rebuild) { - ChunkInventory newChunkInventory; - try { - xrdsvc::XrdName x; - newChunkInventory.rebuild(x.getName(), _mySqlConfig); - } catch (exception const& ex) { - reportError("database operation failed: " + - string(ex.what())); - return; - } - } - - // Rebuild the transient list and notify the caller if requested - if (_reload) { - // Load the new map from the database into a local variable - ChunkInventory newChunkInventory; - try { - xrdsvc::XrdName x; - newChunkInventory.init(x.getName(), _mySqlConfig); - } catch (exception const& ex) { - reportError("database operation failed: " + - string(ex.what())); - return; - } - ::dumpInventory(*_chunkInventory, context + "_chunkInventory: "); - ::dumpInventory(newChunkInventory, context + "newChunkInventory: "); - - // Compare two maps and worker identifiers to see which resources were - // were added or removed. Then Update the current map and notify XRootD - // accordingly. - ChunkInventory::ExistMap const removedChunks = *_chunkInventory - newChunkInventory; - ChunkInventory::ExistMap const addedChunks = newChunkInventory - *_chunkInventory; - - xrdsvc::SsiProviderServer* providerServer = - dynamic_cast(XrdSsiProviderLookup); - XrdSsiCluster* clusterManager = providerServer->GetClusterManager(); - - if (not removedChunks.empty()) { - for (auto&& entry : removedChunks) { - string const& database = entry.first; - for (int chunk : entry.second) { - string const resource = "/chk/" + database + "/" + to_string(chunk); - LOGS(_log, LOG_LVL_DEBUG, - context << "removing resource: " << resource - << " in DataContext=" << clusterManager->DataContext()); - try { - // Notify XRootD/cmsd and (depending on a mode) modify the provider's copy - // of the inventory. - clusterManager->Removed(resource.c_str()); - if (clusterManager->DataContext()) { - providerServer->GetChunkInventory().remove(database, chunk); - } - // Notify QServ - _chunkInventory->remove(database, chunk); - } catch (exception const& ex) { - reportError("failed to remove the chunk: " + - string(ex.what())); - return; - } - // Notify the caller of this service - proto::WorkerCommandChunk* ptr = reply.add_removed(); - ptr->set_db(database); - ptr->set_chunk(chunk); - } - } - } - if (not addedChunks.empty()) { - for (auto&& entry : addedChunks) { - string const& database = entry.first; - for (int chunk : entry.second) { - string const resource = "/chk/" + database + "/" + to_string(chunk); - LOGS(_log, LOG_LVL_DEBUG, - context + "adding resource: " << resource << " in DataContext=" - << clusterManager->DataContext()); - try { - // Notify XRootD/cmsd and (depending on a mode) modify the provider's copy - // of the inventory. - clusterManager->Added(resource.c_str()); - if (clusterManager->DataContext()) { - providerServer->GetChunkInventory().add(database, chunk); - } - // Notify QServ - _chunkInventory->add(database, chunk); - } catch (exception const& ex) { - reportError("failed to add the chunk: " + - string(ex.what())); - return; - } - // Notify the caller of this service - proto::WorkerCommandChunk* ptr = reply.add_added(); - ptr->set_db(database); - ptr->set_chunk(chunk); - } - } - } - } - _frameBuf.serialize(reply); - string str(_frameBuf.data(), _frameBuf.size()); - _sendChannel->sendStream(xrdsvc::StreamBuffer::createWithMove(str), true); - - LOGS(_log, LOG_LVL_DEBUG, context << "** SENT **"); -} - -} // namespace lsst::qserv::wpublish diff --git a/src/wpublish/ChunkListCommand.h b/src/wpublish/ChunkListCommand.h deleted file mode 100644 index 3cbe3cfb6..000000000 --- a/src/wpublish/ChunkListCommand.h +++ /dev/null @@ -1,126 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * Copyright 2011-2018 LSST Corporation. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ -#ifndef LSST_QSERV_WPUBLISH_CHUNK_LIST_COMMAND_H -#define LSST_QSERV_WPUBLISH_CHUNK_LIST_COMMAND_H - -// System headers -#include - -// Qserv headers -#include "wbase/WorkerCommand.h" -#include "mysql/MySqlConfig.h" - -// Forward declarations -namespace lsst::qserv::wpublish { -class ChunkInventory; -} // namespace lsst::qserv::wpublish - -// This header declarations -namespace lsst::qserv::wpublish { - -/** - * Class ChunkListCommand is the base class for operations with chunk lists - */ -class ChunkListCommand : public wbase::WorkerCommand { -public: - ChunkListCommand() = delete; - ChunkListCommand& operator=(ChunkListCommand const&) = delete; - ChunkListCommand(ChunkListCommand const&) = delete; - virtual ~ChunkListCommand() override = default; - -protected: - /** - * @param sendChannel communication channel for reporting results - * @param chunkInventory transient collection of available chunks to be reloaded (if requested) - * @param mySqlConfig database connection parameters - * @param rebuild rebuild the list from actual database tables - * @param reload reload the list in worker's memory - */ - ChunkListCommand(std::shared_ptr const& sendChannel, - std::shared_ptr const& chunkInventory, - mysql::MySqlConfig const& mySqlConfig, bool rebuild, bool reload); - - virtual void run() override; - -private: - // Parameters of the object - - std::shared_ptr _chunkInventory; - mysql::MySqlConfig const _mySqlConfig; - - bool _rebuild; - bool _reload; -}; - -/** - * Class ReloadChunkListCommand reloads a list of chunks from the database - */ -class ReloadChunkListCommand : public ChunkListCommand { -public: - // The default construction and copy semantics are prohibited - ReloadChunkListCommand() = delete; - ReloadChunkListCommand& operator=(ReloadChunkListCommand const&) = delete; - ReloadChunkListCommand(ReloadChunkListCommand const&) = delete; - - /** - * @param sendChannel communication channel for reporting results - * @param chunkInventory transient collection of available chunks to be reloaded (if requested) - * @param mySqlConfig database connection parameters - */ - ReloadChunkListCommand(std::shared_ptr const& sendChannel, - std::shared_ptr const& chunkInventory, - mysql::MySqlConfig const& mySqlConfig) - : ChunkListCommand(sendChannel, chunkInventory, mySqlConfig, false, true) {} - - ~ReloadChunkListCommand() override = default; -}; - -/** - * Class RebuildChunkListCommand rebuilds a persistent list of chunks and - * optionally (if requested) reloads a list of chunks from the database - * into a transient list. - */ -class RebuildChunkListCommand : public ChunkListCommand { -public: - // The default construction and copy semantics are prohibited - RebuildChunkListCommand() = delete; - RebuildChunkListCommand& operator=(RebuildChunkListCommand const&) = delete; - RebuildChunkListCommand(RebuildChunkListCommand const&) = delete; - - /** - * @param sendChannel communication channel for reporting results - * @param chunkInventory transient collection of available chunks to be reloaded (if requested) - * @param mySqlConfig database connection parameters - * @param reload reload the list in worker's memory - */ - RebuildChunkListCommand(std::shared_ptr const& sendChannel, - std::shared_ptr const& chunkInventory, - mysql::MySqlConfig const& mySqlConfig, bool reload) - : ChunkListCommand(sendChannel, chunkInventory, mySqlConfig, true, reload) {} - - ~RebuildChunkListCommand() override = default; -}; - -} // namespace lsst::qserv::wpublish - -#endif // LSST_QSERV_WPUBLISH_CHUNK_LIST_COMMAND_H \ No newline at end of file diff --git a/src/wpublish/GetChunkListCommand.cc b/src/wpublish/GetChunkListCommand.cc deleted file mode 100644 index 795e66b15..000000000 --- a/src/wpublish/GetChunkListCommand.cc +++ /dev/null @@ -1,76 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * Copyright 2012-2018 AURA/LSST. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ - -// Class header -#include "wpublish/GetChunkListCommand.h" - -// Qserv headers -#include "proto/worker.pb.h" -#include "wbase/SendChannel.h" -#include "wcontrol/ResourceMonitor.h" -#include "wpublish/ChunkInventory.h" - -// LSST headers -#include "lsst/log/Log.h" - -using namespace std; - -namespace { - -LOG_LOGGER _log = LOG_GET("lsst.qserv.wpublish.GetChunkListCommand"); - -} // anonymous namespace - -namespace lsst::qserv::wpublish { - -GetChunkListCommand::GetChunkListCommand(shared_ptr const& sendChannel, - shared_ptr const& chunkInventory, - shared_ptr const& resourceMonitor) - : wbase::WorkerCommand(sendChannel), - _chunkInventory(chunkInventory), - _resourceMonitor(resourceMonitor) {} - -void GetChunkListCommand::run() { - LOGS(_log, LOG_LVL_DEBUG, "GetChunkListCommand::" << __func__); - - proto::WorkerCommandGetChunkListR reply; - reply.mutable_status(); - ChunkInventory::ExistMap const existMap = _chunkInventory->existMap(); - for (auto&& entry : existMap) { - string const& db = entry.first; - - for (int chunk : entry.second) { - proto::WorkerCommandChunk* ptr = reply.add_chunks(); - ptr->set_db(db); - ptr->set_chunk(chunk); - ptr->set_use_count(_resourceMonitor->count(chunk, db)); - } - } - _frameBuf.serialize(reply); - string str(_frameBuf.data(), _frameBuf.size()); - _sendChannel->sendStream(xrdsvc::StreamBuffer::createWithMove(str), true); - - LOGS(_log, LOG_LVL_DEBUG, "GetChunkListCommand::" << __func__ << " ** SENT **"); -} - -} // namespace lsst::qserv::wpublish diff --git a/src/wpublish/GetChunkListCommand.h b/src/wpublish/GetChunkListCommand.h deleted file mode 100644 index 12ea23104..000000000 --- a/src/wpublish/GetChunkListCommand.h +++ /dev/null @@ -1,77 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * Copyright 2011-2018 LSST Corporation. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ -#ifndef LSST_QSERV_WPUBLISH_GET_CHUNK_LIST_COMMAND_H -#define LSST_QSERV_WPUBLISH_GET_CHUNK_LIST_COMMAND_H - -// System headers -#include - -// Qserv headers -#include "wbase/WorkerCommand.h" - -// Forward declarations -namespace lsst::qserv::wcontrol { -class ResourceMonitor; -} // namespace lsst::qserv::wcontrol - -namespace lsst::qserv::wpublish { -class ChunkInventory; -} // namespace lsst::qserv::wpublish - -// This header declarations -namespace lsst::qserv::wpublish { - -/** - * Class GetChunkListCommand returns a status of the chunk inventory - */ -class GetChunkListCommand : public wbase::WorkerCommand { -public: - // The default construction and copy semantics are prohibited - GetChunkListCommand() = delete; - GetChunkListCommand& operator=(GetChunkListCommand const&) = delete; - GetChunkListCommand(GetChunkListCommand const&) = delete; - - ~GetChunkListCommand() override = default; - - /** - * @param sendChannel communication channel for reporting results - * @param chunkInventory transient collection of available chunks to be reloaded (if requested) - * @param resourceMonitor XRootD resource monitor for finding which chunks are in use - */ - GetChunkListCommand(std::shared_ptr const& sendChannel, - std::shared_ptr const& chunkInventory, - std::shared_ptr const& resourceMonitor); - -protected: - void run() override; - -private: - // Parameters of the object - - std::shared_ptr _chunkInventory; - std::shared_ptr _resourceMonitor; -}; - -} // namespace lsst::qserv::wpublish - -#endif // LSST_QSERV_WPUBLISH_GET_CHUNK_LIST_COMMAND_H \ No newline at end of file diff --git a/src/wpublish/GetConfigCommand.cc b/src/wpublish/GetConfigCommand.cc deleted file mode 100644 index 8b683ae54..000000000 --- a/src/wpublish/GetConfigCommand.cc +++ /dev/null @@ -1,63 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ - -// Class header -#include "wpublish/GetConfigCommand.h" - -// Qserv headers -#include "proto/worker.pb.h" -#include "wbase/SendChannel.h" -#include "wconfig/WorkerConfig.h" - -// LSST headers -#include "lsst/log/Log.h" - -using namespace std; -using json = nlohmann::json; - -namespace { - -LOG_LOGGER _log = LOG_GET("lsst.qserv.wpublish.GetConfigCommand"); - -} // anonymous namespace - -namespace lsst::qserv::wpublish { - -GetConfigCommand::GetConfigCommand(shared_ptr const& sendChannel) - : wbase::WorkerCommand(sendChannel) {} - -void GetConfigCommand::run() { - string const context = "GetConfigCommand::" + string(__func__); - LOGS(_log, LOG_LVL_DEBUG, context); - - proto::WorkerCommandGetConfigR reply; - reply.mutable_status(); - reply.set_info(wconfig::WorkerConfig::instance()->toJson().dump()); - - _frameBuf.serialize(reply); - string str(_frameBuf.data(), _frameBuf.size()); - _sendChannel->sendStream(xrdsvc::StreamBuffer::createWithMove(str), true); - - LOGS(_log, LOG_LVL_DEBUG, context << " ** SENT **"); -} - -} // namespace lsst::qserv::wpublish diff --git a/src/wpublish/GetConfigCommand.h b/src/wpublish/GetConfigCommand.h deleted file mode 100644 index 994b17b5b..000000000 --- a/src/wpublish/GetConfigCommand.h +++ /dev/null @@ -1,61 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ -#ifndef LSST_QSERV_WPUBLISH_GET_CONFIG_COMMAND_H -#define LSST_QSERV_WPUBLISH_GET_CONFIG_COMMAND_H - -// System headers -#include - -// Qserv headers -#include "wbase/WorkerCommand.h" - -// Forward declarations -namespace lsst::qserv::wbase { -class SendChannel; -} // namespace lsst::qserv::wbase - -// This header declarations -namespace lsst::qserv::wpublish { - -/** - * Class GetConfigCommand returns configuration parameters the Qserv worker. - */ -class GetConfigCommand : public wbase::WorkerCommand { -public: - /** - * @param sendChannel The communication channel for reporting results. - */ - GetConfigCommand(std::shared_ptr const& sendChannel); - - GetConfigCommand() = delete; - GetConfigCommand& operator=(GetConfigCommand const&) = delete; - GetConfigCommand(GetConfigCommand const&) = delete; - - virtual ~GetConfigCommand() override = default; - -protected: - virtual void run() override; -}; - -} // namespace lsst::qserv::wpublish - -#endif // LSST_QSERV_WPUBLISH_GET_CONFIG_COMMAND_H diff --git a/src/wpublish/GetDbStatusCommand.cc b/src/wpublish/GetDbStatusCommand.cc deleted file mode 100644 index 7dd42a671..000000000 --- a/src/wpublish/GetDbStatusCommand.cc +++ /dev/null @@ -1,95 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ - -// Class header -#include "wpublish/GetDbStatusCommand.h" - -// System headers -#include -#include - -// Third party headers -#include "nlohmann/json.hpp" - -// Qserv headers -#include "mysql/MySqlUtils.h" -#include "proto/worker.pb.h" -#include "wbase/SendChannel.h" -#include "wconfig/WorkerConfig.h" -#include "wpublish/QueriesAndChunks.h" - -// LSST headers -#include "lsst/log/Log.h" - -using namespace std; -using json = nlohmann::json; - -namespace { - -LOG_LOGGER _log = LOG_GET("lsst.qserv.wpublish.GetDbStatusCommand"); - -} // anonymous namespace - -namespace lsst::qserv::wpublish { - -GetDbStatusCommand::GetDbStatusCommand(shared_ptr const& sendChannel, - shared_ptr const& queriesAndChunks) - : wbase::WorkerCommand(sendChannel), _queriesAndChunks(queriesAndChunks) {} - -void GetDbStatusCommand::run() { - string const context = "GetDbStatusCommand::" + string(__func__); - LOGS(_log, LOG_LVL_DEBUG, context); - - json result; - try { - bool const full = true; - result = mysql::MySqlUtils::processList(wconfig::WorkerConfig::instance()->getMySqlConfig(), full); - } catch (mysql::MySqlQueryError const& ex) { - LOGS(_log, LOG_LVL_ERROR, context << " " << ex.what()); - reportError(ex.what()); - return; - } - - // Amend the result with a map linking MySQL thread identifiers to the corresponding - // tasks that are being (or have been) processed by the worker. Note that only a subset - // of tasks is selected for the known MySQL threads. This prevents the monitoring - // system from pulling old tasks that may still keep records of the closed threads. - set activeMySqlThreadIds; - for (auto const& row : result["queries"]["rows"]) { - // The thread identifier is stored as a string at the very first element - // of the array. See mysql::MySqlUtils::processList for details. - activeMySqlThreadIds.insert(stoul(row[0].get())); - } - result["mysql_thread_to_task"] = _queriesAndChunks->mySqlThread2task(activeMySqlThreadIds); - - proto::WorkerCommandGetDbStatusR reply; - reply.mutable_status(); - reply.set_info(result.dump()); - - _frameBuf.serialize(reply); - string str(_frameBuf.data(), _frameBuf.size()); - _sendChannel->sendStream(xrdsvc::StreamBuffer::createWithMove(str), true); - - LOGS(_log, LOG_LVL_DEBUG, context << " ** SENT **"); -} - -} // namespace lsst::qserv::wpublish diff --git a/src/wpublish/GetDbStatusCommand.h b/src/wpublish/GetDbStatusCommand.h deleted file mode 100644 index 92cba286d..000000000 --- a/src/wpublish/GetDbStatusCommand.h +++ /dev/null @@ -1,71 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ -#ifndef LSST_QSERV_WPUBLISH_GET_DB_STATUS_COMMAND_H -#define LSST_QSERV_WPUBLISH_GET_DB_STATUS_COMMAND_H - -// System headers -#include - -// Qserv headers -#include "wbase/WorkerCommand.h" - -// Forward declarations -namespace lsst::qserv::wbase { -class SendChannel; -} // namespace lsst::qserv::wbase - -// Forward declarations -namespace lsst::qserv::wpublish { -class QueriesAndChunks; -} // namespace lsst::qserv::wpublish - -// This header declarations -namespace lsst::qserv::wpublish { - -/** - * Class GetDbStatusCommand returns various info on the status of the database - * service of the Qserv worker. - */ -class GetDbStatusCommand : public wbase::WorkerCommand { -public: - /** - * @param sendChannel The communication channel for reporting results. - */ - GetDbStatusCommand(std::shared_ptr const& sendChannel, - std::shared_ptr const& queriesAndChunks); - - GetDbStatusCommand() = delete; - GetDbStatusCommand& operator=(GetDbStatusCommand const&) = delete; - GetDbStatusCommand(GetDbStatusCommand const&) = delete; - - virtual ~GetDbStatusCommand() override = default; - -protected: - virtual void run() override; - -private: - std::shared_ptr const _queriesAndChunks; -}; - -} // namespace lsst::qserv::wpublish - -#endif // LSST_QSERV_WPUBLISH_GET_DB_STATUS_COMMAND_H diff --git a/src/wpublish/GetStatusCommand.cc b/src/wpublish/GetStatusCommand.cc deleted file mode 100644 index 2e2a698a1..000000000 --- a/src/wpublish/GetStatusCommand.cc +++ /dev/null @@ -1,75 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * Copyright 2012-2018 AURA/LSST. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ - -// Class header -#include "wpublish/GetStatusCommand.h" - -// Qserv headers -#include "proto/worker.pb.h" -#include "wbase/FileChannelShared.h" -#include "wbase/MsgProcessor.h" -#include "wbase/SendChannel.h" -#include "wcontrol/ResourceMonitor.h" - -// LSST headers -#include "lsst/log/Log.h" - -using namespace std; - -namespace { - -LOG_LOGGER _log = LOG_GET("lsst.qserv.wpublish.GetStatusCommand"); - -} // anonymous namespace - -namespace lsst::qserv::wpublish { - -GetStatusCommand::GetStatusCommand(shared_ptr const& sendChannel, - shared_ptr const& processor, - shared_ptr const& resourceMonitor, - wbase::TaskSelector const& taskSelector) - : wbase::WorkerCommand(sendChannel), - _processor(processor), - _resourceMonitor(resourceMonitor), - _taskSelector(taskSelector) {} - -void GetStatusCommand::run() { - LOGS(_log, LOG_LVL_DEBUG, "GetStatusCommand::" << __func__); - - nlohmann::json result; - result["processor"] = _processor->statusToJson(_taskSelector); - result["resources"] = _resourceMonitor->statusToJson(); - result["filesystem"] = wbase::FileChannelShared::statusToJson(); - - proto::WorkerCommandGetStatusR reply; - reply.mutable_status(); - reply.set_info(result.dump()); - - _frameBuf.serialize(reply); - string str(_frameBuf.data(), _frameBuf.size()); - _sendChannel->sendStream(xrdsvc::StreamBuffer::createWithMove(str), true); - - LOGS(_log, LOG_LVL_DEBUG, "GetStatusCommand::" << __func__ << " ** SENT **"); -} - -} // namespace lsst::qserv::wpublish diff --git a/src/wpublish/GetStatusCommand.h b/src/wpublish/GetStatusCommand.h deleted file mode 100644 index 69620f4d7..000000000 --- a/src/wpublish/GetStatusCommand.h +++ /dev/null @@ -1,82 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * Copyright 2011-2018 LSST Corporation. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ -#ifndef LSST_QSERV_WPUBLISH_GET_STATUS_COMMAND_H -#define LSST_QSERV_WPUBLISH_GET_STATUS_COMMAND_H - -// System headers -#include - -// Qserv headers -#include "wbase/TaskState.h" -#include "wbase/WorkerCommand.h" - -// Forward declarations -namespace lsst::qserv::wbase { -class MsgProcessor; -class SendChannel; -} // namespace lsst::qserv::wbase - -namespace lsst::qserv::wcontrol { -class ResourceMonitor; -} // namespace lsst::qserv::wcontrol - -// This header declarations -namespace lsst::qserv::wpublish { - -/** - * Class GetStatusCommand returns various info on the on-going status of - * a Qserv worker. - */ -class GetStatusCommand : public wbase::WorkerCommand { -public: - /** - * @param sendChannel The communication channel for reporting results. - * @param processor The message processor for extracting status info. - * @param resourceMonitor The XRootD resource monitor for finding which chunks are in use. - * @param taskSelector Task selection criterias. - */ - GetStatusCommand(std::shared_ptr const& sendChannel, - std::shared_ptr const& processor, - std::shared_ptr const& resourceMonitor, - wbase::TaskSelector const& taskSelector); - - GetStatusCommand() = delete; - GetStatusCommand& operator=(GetStatusCommand const&) = delete; - GetStatusCommand(GetStatusCommand const&) = delete; - - virtual ~GetStatusCommand() override = default; - -protected: - virtual void run() override; - -private: - // Parameters of the object - - std::shared_ptr const _processor; - std::shared_ptr const _resourceMonitor; - wbase::TaskSelector const _taskSelector; -}; - -} // namespace lsst::qserv::wpublish - -#endif // LSST_QSERV_WPUBLISH_GET_STATUS_COMMAND_H diff --git a/src/wpublish/RemoveChunkGroupCommand.cc b/src/wpublish/RemoveChunkGroupCommand.cc deleted file mode 100644 index 533243ee7..000000000 --- a/src/wpublish/RemoveChunkGroupCommand.cc +++ /dev/null @@ -1,138 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * Copyright 2012-2018 AURA/LSST. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ - -// Class header -#include "wpublish/RemoveChunkGroupCommand.h" - -// Third-party headers -#include "XrdSsi/XrdSsiCluster.hh" - -// Qserv headers -#include "wbase/SendChannel.h" -#include "wcontrol/ResourceMonitor.h" -#include "wpublish/ChunkInventory.h" -#include "xrdsvc/SsiProvider.h" -#include "xrdsvc/XrdName.h" - -// LSST headers -#include "lsst/log/Log.h" - -extern XrdSsiProvider* XrdSsiProviderLookup; - -using namespace std; - -namespace { - -LOG_LOGGER _log = LOG_GET("lsst.qserv.wpublish.RemoveChunkGroupCommand"); - -} // namespace - -namespace lsst::qserv::wpublish { - -RemoveChunkGroupCommand::RemoveChunkGroupCommand(shared_ptr const& sendChannel, - shared_ptr const& chunkInventory, - shared_ptr const& resourceMonitor, - mysql::MySqlConfig const& mySqlConfig, int chunk, - vector const& dbs, bool force) - : wbase::WorkerCommand(sendChannel), - _chunkInventory(chunkInventory), - _resourceMonitor(resourceMonitor), - _mySqlConfig(mySqlConfig), - _chunk(chunk), - _dbs(dbs), - _force(force) {} - -void RemoveChunkGroupCommand::run() { - string const context = "RemoveChunkGroupCommand::" + string(__func__) + " "; - LOGS(_log, LOG_LVL_DEBUG, context); - - if (_dbs.empty()) { - reportError( - "the list of database names in the group was found empty", - proto::WorkerCommandStatus::INVALID); - return; - } - - // Make sure none of the chunks in the group is not being used - // unless in the 'force' mode - if (!_force) { - if (_resourceMonitor->count(_chunk, _dbs)) { - reportError("some chunks of the group are in use", - proto::WorkerCommandStatus::IN_USE); - return; - } - } - - xrdsvc::SsiProviderServer* providerServer = - dynamic_cast(XrdSsiProviderLookup); - XrdSsiCluster* clusterManager = providerServer->GetClusterManager(); - - for (string const& db : _dbs) { - string const resource = "/chk/" + db + "/" + to_string(_chunk); - - LOGS(_log, LOG_LVL_DEBUG, - context << "removing the chunk resource: " << resource - << " in DataContext=" << clusterManager->DataContext()); - - try { - // Notify XRootD/cmsd and (depending on a mode) modify the provider's copy - // of the inventory. - clusterManager->Removed(resource.c_str()); - if (clusterManager->DataContext()) { - providerServer->GetChunkInventory().remove(db, _chunk); - } - - // Notify QServ and update the database - _chunkInventory->remove(db, _chunk, _mySqlConfig); - - } catch (InvalidParamError const& ex) { - reportError(ex.what(), proto::WorkerCommandStatus::INVALID); - return; - } catch (QueryError const& ex) { - reportError(ex.what()); - return; - } catch (exception const& ex) { - reportError("failed to remove the chunk: " + string(ex.what())); - return; - } - } - if (_resourceMonitor->count(_chunk, _dbs)) { - // Tell a caller that some of the associated resources are still - // in use by this worker even though they've been blocked from use for any - // further requests. It's up to a caller of this service to correctly - // interpret the effect of the operation based on a presence of the "force" - // flag in the request. - reportError("some chunks of the group are in use", - proto::WorkerCommandStatus::IN_USE); - return; - } - proto::WorkerCommandChunkGroupR reply; - reply.mutable_status(); - _frameBuf.serialize(reply); - string str(_frameBuf.data(), _frameBuf.size()); - _sendChannel->sendStream(xrdsvc::StreamBuffer::createWithMove(str), true); - - LOGS(_log, LOG_LVL_DEBUG, context << "** SENT **"); -} - -} // namespace lsst::qserv::wpublish diff --git a/src/wpublish/RemoveChunkGroupCommand.h b/src/wpublish/RemoveChunkGroupCommand.h deleted file mode 100644 index 958ef99e9..000000000 --- a/src/wpublish/RemoveChunkGroupCommand.h +++ /dev/null @@ -1,95 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * Copyright 2011-2018 LSST Corporation. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ -#ifndef LSST_QSERV_PUBLISH_REMOVE_CHUNK_GROUP_COMMAND_H -#define LSST_QSERV_PUBLISH_REMOVE_CHUNK_GROUP_COMMAND_H - -// System headers -#include -#include -#include - -// Qserv headers -#include "mysql/MySqlConfig.h" -#include "proto/worker.pb.h" -#include "wbase/WorkerCommand.h" - -// Forward declarations -namespace lsst::qserv::wbase { -class SendChannel; -} // namespace lsst::qserv::wbase - -namespace lsst::qserv::wcontrol { -class ResourceMonitor; -} // namespace lsst::qserv::wcontrol - -namespace lsst::qserv::wpublish { -class ChunkInventory; -} // namespace lsst::qserv::wpublish - -// This header declarations -namespace lsst::qserv::wpublish { - -/** - * Class RemoveChunkGroupCommand removes a group of chunks from XRootD - * and the worker's list of chunks. - */ -class RemoveChunkGroupCommand : public wbase::WorkerCommand { -public: - RemoveChunkGroupCommand() = delete; - RemoveChunkGroupCommand& operator=(const RemoveChunkGroupCommand&) = delete; - RemoveChunkGroupCommand(const RemoveChunkGroupCommand&) = delete; - - /** - * @param sendChannel communication channel for reporting results - * @param chunkInventory chunks known to the application - * @param resourceMonitor counters of resources which are being used - * @param mySqlConfig database connection parameters - * @param chunk chunk number - * @param dbs names of databases in the group - * @param force force chunk removal even if this chunk is in use - */ - RemoveChunkGroupCommand(std::shared_ptr const& sendChannel, - std::shared_ptr const& chunkInventory, - std::shared_ptr const& resourceMonitor, - mysql::MySqlConfig const& mySqlConfig, int chunk, - std::vector const& dbs, bool force); - - virtual ~RemoveChunkGroupCommand() override = default; - -protected: - virtual void run() override; - -private: - // Parameters of the object - - std::shared_ptr _chunkInventory; - std::shared_ptr _resourceMonitor; - mysql::MySqlConfig _mySqlConfig; - int _chunk; - std::vector _dbs; - bool _force; -}; - -} // namespace lsst::qserv::wpublish - -#endif // LSST_QSERV_PUBLISH_REMOVE_CHUNK_GROUP_COMMAND_H diff --git a/src/wpublish/SetChunkListCommand.cc b/src/wpublish/SetChunkListCommand.cc deleted file mode 100644 index 2a1061ea1..000000000 --- a/src/wpublish/SetChunkListCommand.cc +++ /dev/null @@ -1,221 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * Copyright 2012-2018 AURA/LSST. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ - -// Class header -#include "wpublish/SetChunkListCommand.h" - -// Third-party headers -#include "XrdSsi/XrdSsiCluster.hh" - -// Qserv headers -#include "wbase/SendChannel.h" -#include "wcontrol/ResourceMonitor.h" -#include "xrdsvc/SsiProvider.h" -#include "xrdsvc/XrdName.h" - -// LSST headers -#include "lsst/log/Log.h" - -extern XrdSsiProvider* XrdSsiProviderLookup; - -using namespace std; - -namespace { - -LOG_LOGGER _log = LOG_GET("lsst.qserv.wpublish.SetChunkListCommand"); - -} // anonymous namespace - -namespace lsst::qserv::wpublish { - -SetChunkListCommand::SetChunkListCommand(shared_ptr const& sendChannel, - shared_ptr const& chunkInventory, - shared_ptr const& resourceMonitor, - mysql::MySqlConfig const& mySqlConfig, - vector const& chunks, - vector const& databases, bool force) - : wbase::WorkerCommand(sendChannel), - _chunkInventory(chunkInventory), - _resourceMonitor(resourceMonitor), - _mySqlConfig(mySqlConfig), - _chunks(chunks), - _force(force) { - for (auto&& database : databases) { - _databases.insert(database); - } -} - -void SetChunkListCommand::_setChunks(proto::WorkerCommandSetChunkListR& reply, - ChunkInventory::ExistMap const& prevExistMap) { - for (auto const& entry : prevExistMap) { - string const& database = entry.first; - - // Exclude databases which are not in a scope of this command - if (0 == _databases.count(database)) continue; - - for (int chunk : entry.second) { - proto::WorkerCommandChunk* ptr = reply.add_chunks(); - ptr->set_db(database); - ptr->set_chunk(chunk); - ptr->set_use_count(_resourceMonitor->count(chunk, database)); - } - } -} - -void SetChunkListCommand::run() { - string const context = "SetChunkListCommand::" + string(__func__) + " "; - - LOGS(_log, LOG_LVL_DEBUG, context); - - // Store the current collection of chunks - ChunkInventory::ExistMap const prevExistMap = _chunkInventory->existMap(); - - // Build a temporary object representing a desired chunk list and - // compare it with the present one. - ChunkInventory::ExistMap newExistMap; - for (Chunk const& chunkEntry : _chunks) { - newExistMap[chunkEntry.database].insert(chunkEntry.chunk); - } - ChunkInventory const newChunkInventory(newExistMap, _chunkInventory->name(), _chunkInventory->id()); - - ChunkInventory::ExistMap const toBeRemovedExistMap = *_chunkInventory - newChunkInventory; - ChunkInventory::ExistMap const toBeAddedExistMap = newChunkInventory - *_chunkInventory; - - // Make sure none of the chunks in the 'to be removed' group is not being used - // unless in the 'force' mode - if (not _force) { - for (auto const& entry : toBeRemovedExistMap) { - string const& database = entry.first; - // Exclude databases which are not in a scope of this command - if (0 == _databases.count(database)) continue; - for (auto chunk : entry.second) { - if (_resourceMonitor->count(chunk, database) != 0) { - reportError( - "some chunks of the group are in use", proto::WorkerCommandStatus::IN_USE, - [&](auto& resp) { _setChunks(resp, prevExistMap); }); - return; - } - } - } - } - - // Begin making desired adjustments to the current inventory - - xrdsvc::SsiProviderServer* providerServer = - dynamic_cast(XrdSsiProviderLookup); - XrdSsiCluster* clusterManager = providerServer->GetClusterManager(); - - for (auto const& entry : toBeRemovedExistMap) { - string const& database = entry.first; - // Exclude databases which are not in a scope of this command - if (0 == _databases.count(database)) continue; - - for (auto chunk : entry.second) { - string const resource = "/chk/" + database + "/" + to_string(chunk); - - LOGS(_log, LOG_LVL_DEBUG, - context << "removing the chunk resource: " << resource - << " in DataContext=" << clusterManager->DataContext()); - - try { - // Notify QServ and update the database - _chunkInventory->remove(database, chunk, _mySqlConfig); - - // Notify XRootD/cmsd and (depending on a mode) modify the provider's copy - // of the inventory. - if (clusterManager->DataContext()) { - providerServer->GetChunkInventory().remove(database, chunk); - clusterManager->Removed(resource.c_str()); - } - } catch (InvalidParamError const& ex) { - reportError( - ex.what(), proto::WorkerCommandStatus::INVALID, - [&](auto& resp) { _setChunks(resp, prevExistMap); }); - return; - } catch (QueryError const& ex) { - reportError( - ex.what(), proto::WorkerCommandStatus::ERROR, - [&](auto& resp) { _setChunks(resp, prevExistMap); }); - return; - } catch (exception const& ex) { - reportError( - "failed to remove the chunk: " + string(ex.what()), proto::WorkerCommandStatus::ERROR, - [&](auto& resp) { _setChunks(resp, prevExistMap); }); - return; - } - } - } - for (auto const& entry : toBeAddedExistMap) { - string const& database = entry.first; - // Exclude databases which are not in a scope of this command - if (0 == _databases.count(database)) continue; - - for (auto chunk : entry.second) { - string const resource = "/chk/" + database + "/" + to_string(chunk); - - LOGS(_log, LOG_LVL_DEBUG, - context << "adding the chunk resource: " << resource - << " in DataContext=" << clusterManager->DataContext()); - - try { - // Notify QServ and update the database - _chunkInventory->add(database, chunk, _mySqlConfig); - - // Notify XRootD/cmsd and (depending on a mode) modify the provider's copy - // of the inventory. - if (clusterManager->DataContext()) { - providerServer->GetChunkInventory().add(database, chunk); - clusterManager->Added(resource.c_str()); - } - } catch (InvalidParamError const& ex) { - reportError( - ex.what(), proto::WorkerCommandStatus::INVALID, - [&](auto& resp) { _setChunks(resp, prevExistMap); }); - return; - } catch (QueryError const& ex) { - reportError( - ex.what(), proto::WorkerCommandStatus::ERROR, - [&](auto& resp) { _setChunks(resp, prevExistMap); }); - return; - } catch (exception const& ex) { - reportError( - "failed to add the chunk: " + string(ex.what()), proto::WorkerCommandStatus::ERROR, - [&](auto& resp) { _setChunks(resp, prevExistMap); }); - return; - } - } - } - - // Send back a reply - proto::WorkerCommandSetChunkListR reply; - reply.mutable_status(); - _setChunks(reply, prevExistMap); - - _frameBuf.serialize(reply); - string str(_frameBuf.data(), _frameBuf.size()); - _sendChannel->sendStream(xrdsvc::StreamBuffer::createWithMove(str), true); - - LOGS(_log, LOG_LVL_DEBUG, context << "** SENT **"); -} - -} // namespace lsst::qserv::wpublish diff --git a/src/wpublish/SetChunkListCommand.h b/src/wpublish/SetChunkListCommand.h deleted file mode 100644 index 581eec934..000000000 --- a/src/wpublish/SetChunkListCommand.h +++ /dev/null @@ -1,104 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * Copyright 2011-2018 LSST Corporation. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ -#ifndef LSST_QSERV_WPUBLISH_SET_CHUNK_LIST_COMMAND_H -#define LSST_QSERV_WPUBLISH_SET_CHUNK_LIST_COMMAND_H - -// System headers -#include -#include -#include -#include - -// Qserv headers -#include "mysql/MySqlConfig.h" -#include "proto/worker.pb.h" -#include "wbase/WorkerCommand.h" -#include "wpublish/ChunkInventory.h" - -// Forward declarations -namespace lsst::qserv::wbase { -class SendChannel; -} // namespace lsst::qserv::wbase - -namespace lsst::qserv::wcontrol { -class ResourceMonitor; -} // namespace lsst::qserv::wcontrol - -// This header declarations -namespace lsst::qserv::wpublish { - -/** - * Class SetChunkListCommand sets a new list of chunks - */ -class SetChunkListCommand : public wbase::WorkerCommand { -public: - /// An abstraction for a chunk to be set - struct Chunk { - std::string database; - unsigned int chunk; - }; - - SetChunkListCommand& operator=(const SetChunkListCommand&) = delete; - SetChunkListCommand(const SetChunkListCommand&) = delete; - SetChunkListCommand() = delete; - virtual ~SetChunkListCommand() override = default; - - /** - * @param sendChannel communication channel for reporting results - * @param chunkInventory chunks known to the application - * @param resourceMonitor counters of resources which are being used - * @param mySqlConfig database connection parameters - * @param chunks collection of chunks to replace the current one - * @param databases limit a scope of the operation to databases of this collection - * @param force force chunks removal even if chunks are in use - */ - SetChunkListCommand(std::shared_ptr const& sendChannel, - std::shared_ptr const& chunkInventory, - std::shared_ptr const& resourceMonitor, - mysql::MySqlConfig const& mySqlConfig, std::vector const& chunks, - std::vector const& databases, bool force); - -protected: - virtual void run() override; - -private: - /** - * Set the chunk list in the reply - * @param reply message to be initialized - * @param prevExistMap previous state of the ChunkList - */ - void _setChunks(proto::WorkerCommandSetChunkListR& reply, ChunkInventory::ExistMap const& prevExistMap); - - // Parameters of the object - - std::shared_ptr const _chunkInventory; - std::shared_ptr const _resourceMonitor; - mysql::MySqlConfig const _mySqlConfig; - std::vector const _chunks; - std::set _databases; - bool const _force; -}; - -} // namespace lsst::qserv::wpublish - -#endif // LSST_QSERV_WPUBLISH_SET_CHUNK_LIST_COMMAND_H diff --git a/src/wpublish/TestEchoCommand.cc b/src/wpublish/TestEchoCommand.cc deleted file mode 100644 index 665209cfb..000000000 --- a/src/wpublish/TestEchoCommand.cc +++ /dev/null @@ -1,69 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * Copyright 2012-2018 AURA/LSST. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ - -// Class header -#include "wpublish/TestEchoCommand.h" - -// System headers -#include - -// Third-party headers -#include "XrdSsi/XrdSsiCluster.hh" - -// Qserv headers -#include "proto/worker.pb.h" -#include "wbase/SendChannel.h" -#include "xrdsvc/SsiProvider.h" -#include "xrdsvc/XrdName.h" - -// LSST headers -#include "lsst/log/Log.h" - -using namespace std; - -namespace { - -LOG_LOGGER _log = LOG_GET("lsst.qserv.wpublish.TestEchoCommand"); - -} // anonymous namespace - -namespace lsst::qserv::wpublish { - -TestEchoCommand::TestEchoCommand(shared_ptr const& sendChannel, string const& value) - : wbase::WorkerCommand(sendChannel), _value(value) {} - -void TestEchoCommand::run() { - LOGS(_log, LOG_LVL_DEBUG, "TestEchoCommand::" << __func__); - - proto::WorkerCommandTestEchoR reply; - reply.mutable_status(); - reply.set_value(_value); - - _frameBuf.serialize(reply); - string str(_frameBuf.data(), _frameBuf.size()); - _sendChannel->sendStream(xrdsvc::StreamBuffer::createWithMove(str), true); - - LOGS(_log, LOG_LVL_DEBUG, "TestEchoCommand::" << __func__ << " ** SENT **"); -} - -} // namespace lsst::qserv::wpublish diff --git a/src/wpublish/TestEchoCommand.h b/src/wpublish/TestEchoCommand.h deleted file mode 100644 index 3cbb653a4..000000000 --- a/src/wpublish/TestEchoCommand.h +++ /dev/null @@ -1,68 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * Copyright 2011-2018 LSST Corporation. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ -#ifndef LSST_QSERV_WPUBLISH_TEST_ECHO_COMMAND_H -#define LSST_QSERV_WPUBLISH_TEST_ECHO_COMMAND_H - -// System headers -#include - -// Qserv headers -#include "wbase/WorkerCommand.h" - -// Forward declarations -namespace lsst::qserv::wbase { -class SendChannel; -} // namespace lsst::qserv::wbase - -// This header declarations -namespace lsst::qserv::wpublish { - -/** - * Class TestEchoCommand reloads a list of chunks from the database - */ -class TestEchoCommand : public wbase::WorkerCommand { -public: - // The default construction and copy semantics are prohibited - TestEchoCommand& operator=(TestEchoCommand const&) = delete; - TestEchoCommand(TestEchoCommand const&) = delete; - TestEchoCommand() = delete; - - /** - * @param sendChannel communication channel for reporting results - * @param value value to be send back to a client - */ - explicit TestEchoCommand(std::shared_ptr const& sendChannel, - std::string const& value); - - ~TestEchoCommand() override = default; - -protected: - void run() override; - -private: - std::string _value; -}; - -} // namespace lsst::qserv::wpublish - -#endif // LSST_QSERV_WPUBLISH_TEST_ECHO_COMMAND_H \ No newline at end of file diff --git a/src/xrdreq/CMakeLists.txt b/src/xrdreq/CMakeLists.txt index bfedd1e83..14974da04 100644 --- a/src/xrdreq/CMakeLists.txt +++ b/src/xrdreq/CMakeLists.txt @@ -2,17 +2,9 @@ add_library(xrdreq OBJECT) add_dependencies(xrdreq proto) target_sources(xrdreq PRIVATE - ChunkGroupQservRequest.cc - ChunkListQservRequest.cc - GetChunkListQservRequest.cc - GetConfigQservRequest.cc - GetDbStatusQservRequest.cc - GetStatusQservRequest.cc QservRequest.cc QueryManagementAction.cc QueryManagementRequest.cc - SetChunkListQservRequest.cc - TestEchoQservRequest.cc ) target_include_directories(xrdreq PRIVATE @@ -46,7 +38,6 @@ ENDFUNCTION() xrdreq_utils( qserv-query-management - qserv-worker-perf ) install( diff --git a/src/xrdreq/ChunkGroupQservRequest.cc b/src/xrdreq/ChunkGroupQservRequest.cc deleted file mode 100644 index 0ded683e6..000000000 --- a/src/xrdreq/ChunkGroupQservRequest.cc +++ /dev/null @@ -1,127 +0,0 @@ -/* - * LSST Data Management System - * Copyright 2018 LSST Corporation. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ - -// Class header -#include "xrdreq/ChunkGroupQservRequest.h" - -// System headers -#include - -// LSST headers -#include "lsst/log/Log.h" - -using namespace std; - -namespace { -LOG_LOGGER _log = LOG_GET("lsst.qserv.xrdreq.ChunkGroupQservRequest"); -} // namespace - -namespace lsst::qserv::xrdreq { - -ChunkGroupQservRequest::ChunkGroupQservRequest(bool add, unsigned int chunk, vector const& databases, - bool force, CallbackType onFinish) - : _add(add), _chunk(chunk), _databases(databases), _force(force), _onFinish(onFinish) { - LOGS(_log, LOG_LVL_TRACE, - "ChunkGroupQservRequest[" << (_add ? "add" : "remove") << "] ** CONSTRUCTED **"); -} - -ChunkGroupQservRequest::~ChunkGroupQservRequest() { - LOGS(_log, LOG_LVL_TRACE, "ChunkGroupQservRequest[" << (_add ? "add" : "remove") << "] ** DELETED **"); -} - -void ChunkGroupQservRequest::onRequest(proto::FrameBuffer& buf) { - proto::WorkerCommandH header; - header.set_command(_add ? proto::WorkerCommandH::ADD_CHUNK_GROUP - : proto::WorkerCommandH::REMOVE_CHUNK_GROUP); - buf.serialize(header); - - proto::WorkerCommandChunkGroupM message; - message.set_chunk(_chunk); - for (auto const& database : _databases) { - message.add_dbs(database); - } - message.set_force(_force); - buf.serialize(message); -} - -void ChunkGroupQservRequest::onResponse(proto::FrameBufferView& view) { - proto::WorkerCommandChunkGroupR reply; - view.parse(reply); - - LOGS(_log, LOG_LVL_TRACE, - "ChunkGroupQservRequest[" << (_add ? "add" : "remove") << "** SERVICE REPLY ** status: " - << proto::WorkerCommandStatus_Code_Name(reply.status().code())); - - if (nullptr != _onFinish) { - // Clearing the stored callback after finishing the up-stream notification - // has two purposes: - // - // 1. it guaranties (exactly) one time notification - // 2. it breaks the up-stream dependency on a caller object if a shared - // pointer to the object was mentioned as the lambda-function's closure - auto onFinish = move(_onFinish); - _onFinish = nullptr; - onFinish(reply.status().code(), reply.status().error()); - } -} - -void ChunkGroupQservRequest::onError(string const& error) { - if (nullptr != _onFinish) { - // Clearing the stored callback after finishing the up-stream notification - // has two purposes: - // - // 1. it guaranties (exactly) one time notification - // 2. it breaks the up-stream dependency on a caller object if a shared - // pointer to the object was mentioned as the lambda-function's closure - auto onFinish = move(_onFinish); - _onFinish = nullptr; - onFinish(proto::WorkerCommandStatus::ERROR, error); - } -} - -AddChunkGroupQservRequest::Ptr AddChunkGroupQservRequest::create(unsigned int chunk, - vector const& databases, - CallbackType onFinish) { - AddChunkGroupQservRequest::Ptr ptr(new AddChunkGroupQservRequest(chunk, databases, onFinish)); - ptr->setRefToSelf4keepAlive(ptr); - return ptr; -} - -AddChunkGroupQservRequest::AddChunkGroupQservRequest(unsigned int chunk, vector const& databases, - CallbackType onFinish) - : ChunkGroupQservRequest(true, chunk, databases, false, onFinish) {} - -RemoveChunkGroupQservRequest::Ptr RemoveChunkGroupQservRequest::create(unsigned int chunk, - vector const& databases, - bool force, CallbackType onFinish) { - RemoveChunkGroupQservRequest::Ptr ptr( - new RemoveChunkGroupQservRequest(chunk, databases, force, onFinish)); - ptr->setRefToSelf4keepAlive(ptr); - return ptr; -} - -RemoveChunkGroupQservRequest::RemoveChunkGroupQservRequest(unsigned int chunk, - vector const& databases, bool force, - CallbackType onFinish) - : ChunkGroupQservRequest(false, chunk, databases, force, onFinish) {} - -} // namespace lsst::qserv::xrdreq diff --git a/src/xrdreq/ChunkGroupQservRequest.h b/src/xrdreq/ChunkGroupQservRequest.h deleted file mode 100644 index d60b2bac8..000000000 --- a/src/xrdreq/ChunkGroupQservRequest.h +++ /dev/null @@ -1,165 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * Copyright 2011-2018 LSST Corporation. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ -/// ChunkGroupQservRequest.h -#ifndef LSST_QSERV_XRDREQ_CHUNK_GROUP_QSERV_REQUEST_H -#define LSST_QSERV_XRDREQ_CHUNK_GROUP_QSERV_REQUEST_H - -// System headers -#include -#include -#include - -// Qserv headers -#include "proto/worker.pb.h" -#include "xrdreq/QservRequest.h" - -// This header declarations -namespace lsst::qserv::xrdreq { - -/** - * Class ChunkGroupQservRequest implements a client-side request to - * the Qserv worker management services. - */ -class ChunkGroupQservRequest : public QservRequest { -public: - /// The callback function type to be used for notifications on - /// the operation completion. - using CallbackType = std::function; // error message (if failed) - - // Default construction and copy semantics is prohibited - ChunkGroupQservRequest() = delete; - ChunkGroupQservRequest(ChunkGroupQservRequest const&) = delete; - ChunkGroupQservRequest& operator=(ChunkGroupQservRequest const&) = delete; - - ~ChunkGroupQservRequest() override; - -protected: - /** - * @param add add a group if 'true', remove otherwise - * @param chunk chunk number - * @param databases names of databases in the group - * @param force force the proposed change even if the chunk is in use - * @param onFinish optional callback function to be called upon the completion - * (successful or not) of the request. - */ - ChunkGroupQservRequest(bool add, unsigned int chunk, std::vector const& databases, - bool force, CallbackType onFinish); - - void onRequest(proto::FrameBuffer& buf) override; - void onResponse(proto::FrameBufferView& view) override; - void onError(std::string const& error) override; - -private: - // Parameters of a request - - bool _add; - unsigned int _chunk; - std::vector _databases; - bool _force; - CallbackType _onFinish; -}; - -/** - * Class AddChunkGroupQservRequest implements a client-side request to - * the Qserv worker management services. - */ -class AddChunkGroupQservRequest : public ChunkGroupQservRequest { -public: - /// The pointer type for instances of the class - typedef std::shared_ptr Ptr; - - // Default construction and copy semantics is prohibited - AddChunkGroupQservRequest() = delete; - AddChunkGroupQservRequest(AddChunkGroupQservRequest const&) = delete; - AddChunkGroupQservRequest& operator=(AddChunkGroupQservRequest const&) = delete; - - ~AddChunkGroupQservRequest() override = default; - - /** - * Static factory method is needed to prevent issues with the lifespan - * and memory management of instances created otherwise (as values or via - * low-level pointers). - * - * @param chunk the chunk number - * @param databases names of databases in the group - * @param onFinish callback function to be called upon request completion - */ - static Ptr create(unsigned int chunk, std::vector const& databases, - CallbackType onFinish = nullptr); - -private: - /** - * @param chunk chunk number - * @param databases names of databases in the group - * @param onFinish optional callback function to be called upon the completion - * (successful or not) of the request. - */ - AddChunkGroupQservRequest(unsigned int chunk, std::vector const& databases, - CallbackType onFinish); -}; - -/** - * Class RemoveChunkGroupQservRequest implements a client-side request to - * the Qserv worker management services. - */ -class RemoveChunkGroupQservRequest : public ChunkGroupQservRequest { -public: - /// The pointer type for instances of the class - typedef std::shared_ptr Ptr; - - // Default construction and copy semantics is prohibited - RemoveChunkGroupQservRequest() = delete; - RemoveChunkGroupQservRequest(RemoveChunkGroupQservRequest const&) = delete; - RemoveChunkGroupQservRequest& operator=(RemoveChunkGroupQservRequest const&) = delete; - - ~RemoveChunkGroupQservRequest() override = default; - - /** - * Static factory method is needed to prevent issues with the lifespan - * and memory management of instances created otherwise (as values or via - * low-level pointers). - * - * @param chunk the chunk number - * @param databases names of databases in the group - * @param force force the proposed change even if the chunk is in use - * @param onFinish callback function to be called upon request completion - */ - static Ptr create(unsigned int chunk, std::vector const& databases, bool force, - CallbackType onFinish = nullptr); - -private: - /** - * @param chunk chunk number - * @param databases names of databases in the group - * @param force force the proposed change even if the chunk is in use - * @param onFinish optional callback function to be called upon the completion - * (successful or not) of the request. - */ - RemoveChunkGroupQservRequest(unsigned int chunk, std::vector const& databases, bool force, - CallbackType onFinish); -}; - -} // namespace lsst::qserv::xrdreq - -#endif // LSST_QSERV_XRDREQ_CHUNK_GROUP_QSERV_REQUEST_H diff --git a/src/xrdreq/ChunkListQservRequest.cc b/src/xrdreq/ChunkListQservRequest.cc deleted file mode 100644 index 839b9d25b..000000000 --- a/src/xrdreq/ChunkListQservRequest.cc +++ /dev/null @@ -1,138 +0,0 @@ -/* - * LSST Data Management System - * Copyright 2018 LSST Corporation. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ - -// Class header -#include "xrdreq/ChunkListQservRequest.h" - -// System headers -#include - -// Qserv headers -#include "lsst/log/Log.h" - -using namespace std; - -namespace { -LOG_LOGGER _log = LOG_GET("lsst.qserv.xrdreq.ChunkListQservRequest"); -} // namespace - -namespace lsst::qserv::xrdreq { - -ChunkListQservRequest::ChunkListQservRequest(bool rebuild, bool reload, CallbackType onFinish) - : _rebuild(rebuild), _reload(reload), _onFinish(onFinish) { - LOGS(_log, LOG_LVL_TRACE, "ChunkListQservRequest ** CONSTRUCTED **"); -} - -ChunkListQservRequest::~ChunkListQservRequest() { - LOGS(_log, LOG_LVL_TRACE, "ChunkListQservRequest ** DELETED **"); -} - -void ChunkListQservRequest::onRequest(proto::FrameBuffer& buf) { - proto::WorkerCommandH header; - header.set_command(proto::WorkerCommandH::UPDATE_CHUNK_LIST); - buf.serialize(header); - - proto::WorkerCommandUpdateChunkListM message; - message.set_rebuild(_rebuild); - message.set_reload(_reload); - buf.serialize(message); -} - -void ChunkListQservRequest::onResponse(proto::FrameBufferView& view) { - string const context = "ChunkListQservRequest "; - - proto::WorkerCommandUpdateChunkListR reply; - view.parse(reply); - - LOGS(_log, LOG_LVL_TRACE, - context << "** SERVICE REPLY ** status: " - << proto::WorkerCommandStatus_Code_Name(reply.status().code())); - - ChunkCollection added; - ChunkCollection removed; - - if (reply.status().code() == proto::WorkerCommandStatus::SUCCESS) { - int const numAdded = reply.added_size(); - for (int i = 0; i < numAdded; i++) { - proto::WorkerCommandChunk const& chunkEntry = reply.added(i); - Chunk chunk{chunkEntry.chunk(), chunkEntry.db()}; - added.push_back(chunk); - } - LOGS(_log, LOG_LVL_TRACE, context << "total chunks added: " << numAdded); - - int const numRemoved = reply.removed_size(); - for (int i = 0; i < numRemoved; i++) { - proto::WorkerCommandChunk const& chunkEntry = reply.removed(i); - Chunk chunk{chunkEntry.chunk(), chunkEntry.db()}; - removed.push_back(chunk); - } - LOGS(_log, LOG_LVL_TRACE, context << "total chunks removed: " << numRemoved); - } - if (nullptr != _onFinish) { - // Clearing the stored callback after finishing the up-stream notification - // has two purposes: - // - // 1. it guaranties (exactly) one time notification - // 2. it breaks the up-stream dependency on a caller object if a shared - // pointer to the object was mentioned as the lambda-function's closure - auto onFinish = move(_onFinish); - _onFinish = nullptr; - onFinish(reply.status().code(), reply.status().error(), added, removed); - } -} - -void ChunkListQservRequest::onError(string const& error) { - if (nullptr != _onFinish) { - // Clearing the stored callback after finishing the up-stream notification - // has two purposes: - // - // 1. it guaranties (exactly) one time notification - // 2. it breaks the up-stream dependency on a caller object if a shared - // pointer to the object was mentioned as the lambda-function's closure - auto onFinish = move(_onFinish); - _onFinish = nullptr; - onFinish(proto::WorkerCommandStatus::ERROR, error, ChunkCollection(), ChunkCollection()); - } -} - -ReloadChunkListQservRequest::Ptr ReloadChunkListQservRequest::create( - ChunkListQservRequest::CallbackType onFinish) { - ReloadChunkListQservRequest::Ptr ptr(new ReloadChunkListQservRequest(onFinish)); - ptr->setRefToSelf4keepAlive(ptr); - return ptr; -} - -ReloadChunkListQservRequest::ReloadChunkListQservRequest(ChunkListQservRequest::CallbackType onFinish) - : ChunkListQservRequest(false, true, onFinish) {} - -RebuildChunkListQservRequest::Ptr RebuildChunkListQservRequest::create( - bool reload, ChunkListQservRequest::CallbackType onFinish) { - RebuildChunkListQservRequest::Ptr ptr(new RebuildChunkListQservRequest(reload, onFinish)); - ptr->setRefToSelf4keepAlive(ptr); - return ptr; -} - -RebuildChunkListQservRequest::RebuildChunkListQservRequest(bool reload, - ChunkListQservRequest::CallbackType onFinish) - : ChunkListQservRequest(true, reload, onFinish) {} - -} // namespace lsst::qserv::xrdreq diff --git a/src/xrdreq/ChunkListQservRequest.h b/src/xrdreq/ChunkListQservRequest.h deleted file mode 100644 index 49cb908a8..000000000 --- a/src/xrdreq/ChunkListQservRequest.h +++ /dev/null @@ -1,162 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * Copyright 2011-2018 LSST Corporation. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ -#ifndef LSST_QSERV_XRDREQ_CHUNK_LIST_QSERV_REQUEST_H -#define LSST_QSERV_XRDREQ_CHUNK_LIST_QSERV_REQUEST_H - -// System headers -#include -#include -#include - -// Qserv headers -#include "proto/worker.pb.h" -#include "xrdreq/QservRequest.h" - -namespace lsst::qserv::xrdreq { - -/** - * Class ChunkListQservRequest the base class for client-side requests - * the Qserv worker services affecting chunk lists. - */ -class ChunkListQservRequest : public QservRequest { -public: - /// Struct Chunk a value type encapsulating a chunk number and the name - /// of a database - struct Chunk { - unsigned int chunk; - std::string database; - }; - - /// The ChunkCollection type represents a collection of chunks - using ChunkCollection = std::list; - - /// The callback function type to be used for notifications on - /// the operation completion. - using CallbackType = std::function; // chunks removed (if success) - - // Default construction and copy semantics are prohibited - ChunkListQservRequest() = delete; - ChunkListQservRequest(ChunkListQservRequest const&) = delete; - ChunkListQservRequest& operator=(ChunkListQservRequest const&) = delete; - - ~ChunkListQservRequest() override; - -protected: - /** - * @param rebuild rebuild the list from actual database tables - * @param reload reload the list in worker's memory - * @param onFinish optional callback function to be called upon the completion - * (successful or not) of the request. - */ - ChunkListQservRequest(bool rebuild, bool reload, CallbackType onFinish = nullptr); - - void onRequest(proto::FrameBuffer& buf) override; - - void onResponse(proto::FrameBufferView& view) override; - - void onError(std::string const& error) override; - -private: - // Parameters of the object - - bool _rebuild; - bool _reload; - CallbackType _onFinish; -}; - -/** - * Class ReloadChunkListQservRequest implements a client-side request to - * the Qserv worker management services. - */ -class ReloadChunkListQservRequest : public ChunkListQservRequest { -public: - /// The pointer type for instances of the class - typedef std::shared_ptr Ptr; - - /** - * Static factory method is needed to prevent issues with the lifespan - * and memory management of instances created otherwise (as values or via - * low-level pointers). - * - * @param onFinish optional callback function to be called upon the completion - * (successful or not) of the request. - */ - static Ptr create(CallbackType onFinish = nullptr); - - // Default construction and copy semantics are prohibited - ReloadChunkListQservRequest() = delete; - ReloadChunkListQservRequest(ReloadChunkListQservRequest const&) = delete; - ReloadChunkListQservRequest& operator=(ReloadChunkListQservRequest const&) = delete; - - ~ReloadChunkListQservRequest() override = default; - -protected: - /** - * @param onFinish optional callback function to be called upon the completion - * (successful or not) of the request. - */ - ReloadChunkListQservRequest(CallbackType onFinish); -}; - -/** - * Class RebuildChunkListQservRequest implements a client-side request to - * the Qserv worker management services. - */ -class RebuildChunkListQservRequest : public ChunkListQservRequest { -public: - /// The pointer type for instances of the class - typedef std::shared_ptr Ptr; - - /** - * Static factory method is needed to prevent issues with the lifespan - * and memory management of instances created otherwise (as values or via - * low-level pointers). - * - * @param reload reload the list in worker's memory - * @param onFinish optional callback function to be called upon the completion - * (successful or not) of the request. - */ - static Ptr create(bool reload, CallbackType onFinish = nullptr); - - // Default construction and copy semantics are prohibited - RebuildChunkListQservRequest() = delete; - RebuildChunkListQservRequest(RebuildChunkListQservRequest const&) = delete; - RebuildChunkListQservRequest& operator=(RebuildChunkListQservRequest const&) = delete; - - ~RebuildChunkListQservRequest() override = default; - -protected: - /** - * @param reload reload the list in worker's memory - * @param onFinish optional callback function to be called upon the completion - * (successful or not) of the request. - */ - RebuildChunkListQservRequest(bool reload, CallbackType onFinish); -}; - -} // namespace lsst::qserv::xrdreq - -#endif // LSST_QSERV_XRDREQ_CHUNK_LIST_QSERV_REQUEST_H diff --git a/src/xrdreq/GetChunkListQservRequest.cc b/src/xrdreq/GetChunkListQservRequest.cc deleted file mode 100644 index 2d152e03b..000000000 --- a/src/xrdreq/GetChunkListQservRequest.cc +++ /dev/null @@ -1,112 +0,0 @@ -/* - * LSST Data Management System - * Copyright 2018 LSST Corporation. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ - -// Class header -#include "xrdreq/GetChunkListQservRequest.h" - -// System headers -#include - -// LSST headers -#include "lsst/log/Log.h" - -using namespace std; - -namespace { -LOG_LOGGER _log = LOG_GET("lsst.qserv.xrdreq.GetChunkListQservRequest"); -} // namespace - -namespace lsst::qserv::xrdreq { - -GetChunkListQservRequest::Ptr GetChunkListQservRequest::create( - bool inUseOnly, GetChunkListQservRequest::CallbackType onFinish) { - GetChunkListQservRequest::Ptr ptr(new GetChunkListQservRequest(inUseOnly, onFinish)); - ptr->setRefToSelf4keepAlive(ptr); - return ptr; -} - -GetChunkListQservRequest::GetChunkListQservRequest(bool inUseOnly, - GetChunkListQservRequest::CallbackType onFinish) - : _inUseOnly(inUseOnly), _onFinish(onFinish) { - LOGS(_log, LOG_LVL_TRACE, "GetChunkListQservRequest ** CONSTRUCTED **"); -} - -GetChunkListQservRequest::~GetChunkListQservRequest() { - LOGS(_log, LOG_LVL_TRACE, "GetChunkListQservRequest ** DELETED **"); -} - -void GetChunkListQservRequest::onRequest(proto::FrameBuffer& buf) { - proto::WorkerCommandH header; - header.set_command(proto::WorkerCommandH::GET_CHUNK_LIST); - buf.serialize(header); -} - -void GetChunkListQservRequest::onResponse(proto::FrameBufferView& view) { - string const context = "GetChunkListQservRequest "; - - proto::WorkerCommandGetChunkListR reply; - view.parse(reply); - - LOGS(_log, LOG_LVL_TRACE, - context << "** SERVICE REPLY ** status: " - << proto::WorkerCommandStatus_Code_Name(reply.status().code())); - - ChunkCollection chunks; - - if (reply.status().code() == proto::WorkerCommandStatus::SUCCESS) { - int const num = reply.chunks_size(); - for (int i = 0; i < num; i++) { - proto::WorkerCommandChunk const& chunkEntry = reply.chunks(i); - if (_inUseOnly and not chunkEntry.use_count()) continue; - Chunk chunk{chunkEntry.chunk(), chunkEntry.db(), chunkEntry.use_count()}; - chunks.push_back(chunk); - } - LOGS(_log, LOG_LVL_TRACE, context << "total chunks: " << num); - } - if (nullptr != _onFinish) { - // Clearing the stored callback after finishing the up-stream notification - // has two purposes: - // - // 1. it guaranties (exactly) one time notification - // 2. it breaks the up-stream dependency on a caller object if a shared - // pointer to the object was mentioned as the lambda-function's closure - auto onFinish = move(_onFinish); - _onFinish = nullptr; - onFinish(reply.status().code(), reply.status().error(), chunks); - } -} - -void GetChunkListQservRequest::onError(string const& error) { - if (nullptr != _onFinish) { - // Clearing the stored callback after finishing the up-stream notification - // has two purposes: - // - // 1. it guaranties (exactly) one time notification - // 2. it breaks the up-stream dependency on a caller object if a shared - // pointer to the object was mentioned as the lambda-function's closure - auto onFinish = move(_onFinish); - _onFinish = nullptr; - onFinish(proto::WorkerCommandStatus::ERROR, error, ChunkCollection()); - } -} - -} // namespace lsst::qserv::xrdreq diff --git a/src/xrdreq/GetChunkListQservRequest.h b/src/xrdreq/GetChunkListQservRequest.h deleted file mode 100644 index b2ac23698..000000000 --- a/src/xrdreq/GetChunkListQservRequest.h +++ /dev/null @@ -1,105 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * Copyright 2011-2018 LSST Corporation. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ -#ifndef LSST_QSERV_XRDREQ_GET_CHUNK_LIST_QSERV_REQUEST_H -#define LSST_QSERV_XRDREQ_GET_CHUNK_LIST_QSERV_REQUEST_H - -// System headers -#include -#include -#include - -// Qserv headers -#include "proto/worker.pb.h" -#include "xrdreq/QservRequest.h" - -namespace lsst::qserv::xrdreq { - -/** - * Class GetChunkListQservRequest implements the client-side requests - * the Qserv worker services for a status of chunk lists. - */ -class GetChunkListQservRequest : public QservRequest { -public: - /// Struct Chunk a value type encapsulating a chunk number and the name - /// of a database - struct Chunk { - unsigned int chunk; - std::string database; - unsigned int use_count; - }; - - /// The ChunkCollection type refresens a collection of chunks - using ChunkCollection = std::list; - - /// The pointer type for instances of the class - typedef std::shared_ptr Ptr; - - /// The callback function type to be used for notifications on - /// the operation completion. - using CallbackType = std::function; // chunks (if success) - - /** - * Static factory method is needed to prevent issues with the lifespan - * and memory management of instances created otherwise (as values or via - * low-level pointers). - * - * @param inUseOnly only report chunks which are in use - * @param onFinish optional callback function to be called upon the completion - * (successful or not) of the request. - * @return smart pointer to the object of the class - */ - static Ptr create(bool inUseOnly, CallbackType onFinish = nullptr); - - // Default construction and copy semantics are prohibited - GetChunkListQservRequest() = delete; - GetChunkListQservRequest(GetChunkListQservRequest const&) = delete; - GetChunkListQservRequest& operator=(GetChunkListQservRequest const&) = delete; - - ~GetChunkListQservRequest() override; - -protected: - /** - * @param inUseOnly only report chunks which are in use - * @param onFinish optional callback function to be called upon the completion - * (successful or not) of the request. - */ - GetChunkListQservRequest(bool inUseOnly, CallbackType onFinish); - - void onRequest(proto::FrameBuffer& buf) override; - - void onResponse(proto::FrameBufferView& view) override; - - void onError(std::string const& error) override; - -private: - // Parameters of the object - - bool _inUseOnly; - CallbackType _onFinish; -}; - -} // namespace lsst::qserv::xrdreq - -#endif // LSST_QSERV_XRDREQ_GET_CHUNK_LIST_QSERV_REQUEST_H diff --git a/src/xrdreq/GetConfigQservRequest.cc b/src/xrdreq/GetConfigQservRequest.cc deleted file mode 100644 index 98896df0f..000000000 --- a/src/xrdreq/GetConfigQservRequest.cc +++ /dev/null @@ -1,93 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ - -// Class header -#include "xrdreq/GetConfigQservRequest.h" - -// LSST headers -#include "lsst/log/Log.h" - -using namespace std; - -namespace { - -LOG_LOGGER _log = LOG_GET("lsst.qserv.xrdreq.GetConfigQservRequest"); - -} // namespace - -namespace lsst::qserv::xrdreq { - -GetConfigQservRequest::Ptr GetConfigQservRequest::create(GetConfigQservRequest::CallbackType onFinish) { - GetConfigQservRequest::Ptr ptr(new GetConfigQservRequest(onFinish)); - ptr->setRefToSelf4keepAlive(ptr); - return ptr; -} - -GetConfigQservRequest::GetConfigQservRequest(GetConfigQservRequest::CallbackType onFinish) - : _onFinish(onFinish) { - LOGS(_log, LOG_LVL_TRACE, "GetConfigQservRequest ** CONSTRUCTED **"); -} - -GetConfigQservRequest::~GetConfigQservRequest() { - LOGS(_log, LOG_LVL_TRACE, "GetConfigQservRequest ** DELETED **"); -} - -void GetConfigQservRequest::onRequest(proto::FrameBuffer& buf) { - proto::WorkerCommandH header; - header.set_command(proto::WorkerCommandH::GET_CONFIG); - buf.serialize(header); -} - -void GetConfigQservRequest::onResponse(proto::FrameBufferView& view) { - proto::WorkerCommandGetDbStatusR reply; - view.parse(reply); - - if (nullptr != _onFinish) { - // Clearing the stored callback after finishing the up-stream notification - // has two purposes: - // - // 1. it guaranties (exactly) one time notification - // 2. it breaks the up-stream dependency on a caller object if a shared - // pointer to the object was mentioned as the lambda-function's closure - - auto onFinish = move(_onFinish); - _onFinish = nullptr; - onFinish(proto::WorkerCommandStatus::SUCCESS, string(), reply.info()); - } -} - -void GetConfigQservRequest::onError(string const& error) { - if (nullptr != _onFinish) { - // Clearing the stored callback after finishing the up-stream notification - // has two purposes: - // - // 1. it guaranties (exactly) one time notification - // 2. it breaks the up-stream dependency on a caller object if a shared - // pointer to the object was mentioned as the lambda-function's closure - - auto onFinish = move(_onFinish); - _onFinish = nullptr; - onFinish(proto::WorkerCommandStatus::ERROR, error, string()); - } -} - -} // namespace lsst::qserv::xrdreq diff --git a/src/xrdreq/GetConfigQservRequest.h b/src/xrdreq/GetConfigQservRequest.h deleted file mode 100644 index 6761e855b..000000000 --- a/src/xrdreq/GetConfigQservRequest.h +++ /dev/null @@ -1,87 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ -#ifndef LSST_QSERV_XRDREQ_GET_CONFIG_QSERV_REQUEST_H -#define LSST_QSERV_XRDREQ_GET_CONFIG_QSERV_REQUEST_H - -// System headers -#include -#include -#include -#include -#include - -// Qserv headers -#include "proto/worker.pb.h" -#include "xrdreq/QservRequest.h" - -namespace lsst::qserv::xrdreq { - -/** - * Class GetConfigQservRequest represents a request returning configuration - * parameters of the Qserv worker. - */ -class GetConfigQservRequest : public QservRequest { -public: - /// The pointer type for instances of the class - typedef std::shared_ptr Ptr; - - /// The callback function type to be used for notifications on - /// the operation completion. - using CallbackType = std::function; // worker info received (if success) - - /** - * Static factory method is needed to prevent issues with the lifespan - * and memory management of instances created otherwise (as values or via - * low-level pointers). - * - * @param onFinish (optional )callback function to be called upon the completion - * (successful or not) of the request. - * @see wbase::Task::Status - * @return the smart pointer to the object of the class - */ - static Ptr create(CallbackType onFinish = nullptr); - - GetConfigQservRequest() = delete; - GetConfigQservRequest(GetConfigQservRequest const&) = delete; - GetConfigQservRequest& operator=(GetConfigQservRequest const&) = delete; - - virtual ~GetConfigQservRequest() override; - -protected: - /// @see GetConfigQservRequest::create() - GetConfigQservRequest(CallbackType onFinish); - - virtual void onRequest(proto::FrameBuffer& buf) override; - virtual void onResponse(proto::FrameBufferView& view) override; - virtual void onError(std::string const& error) override; - -private: - // Parameters of the object - - CallbackType _onFinish; -}; - -} // namespace lsst::qserv::xrdreq - -#endif // LSST_QSERV_XRDREQ_GET_CONFIG_QSERV_REQUEST_H diff --git a/src/xrdreq/GetDbStatusQservRequest.cc b/src/xrdreq/GetDbStatusQservRequest.cc deleted file mode 100644 index 24e154594..000000000 --- a/src/xrdreq/GetDbStatusQservRequest.cc +++ /dev/null @@ -1,93 +0,0 @@ -/* - * LSST Data Management System - * Copyright 2018 LSST Corporation. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ - -// Class header -#include "xrdreq/GetDbStatusQservRequest.h" - -// LSST headers -#include "lsst/log/Log.h" - -using namespace std; - -namespace { - -LOG_LOGGER _log = LOG_GET("lsst.qserv.xrdreq.GetDbStatusQservRequest"); - -} // namespace - -namespace lsst::qserv::xrdreq { - -GetDbStatusQservRequest::Ptr GetDbStatusQservRequest::create(GetDbStatusQservRequest::CallbackType onFinish) { - GetDbStatusQservRequest::Ptr ptr(new GetDbStatusQservRequest(onFinish)); - ptr->setRefToSelf4keepAlive(ptr); - return ptr; -} - -GetDbStatusQservRequest::GetDbStatusQservRequest(GetDbStatusQservRequest::CallbackType onFinish) - : _onFinish(onFinish) { - LOGS(_log, LOG_LVL_TRACE, "GetDbStatusQservRequest ** CONSTRUCTED **"); -} - -GetDbStatusQservRequest::~GetDbStatusQservRequest() { - LOGS(_log, LOG_LVL_TRACE, "GetDbStatusQservRequest ** DELETED **"); -} - -void GetDbStatusQservRequest::onRequest(proto::FrameBuffer& buf) { - proto::WorkerCommandH header; - header.set_command(proto::WorkerCommandH::GET_DATABASE_STATUS); - buf.serialize(header); -} - -void GetDbStatusQservRequest::onResponse(proto::FrameBufferView& view) { - proto::WorkerCommandGetDbStatusR reply; - view.parse(reply); - - if (nullptr != _onFinish) { - // Clearing the stored callback after finishing the up-stream notification - // has two purposes: - // - // 1. it guaranties (exactly) one time notification - // 2. it breaks the up-stream dependency on a caller object if a shared - // pointer to the object was mentioned as the lambda-function's closure - - auto onFinish = move(_onFinish); - _onFinish = nullptr; - onFinish(proto::WorkerCommandStatus::SUCCESS, string(), reply.info()); - } -} - -void GetDbStatusQservRequest::onError(string const& error) { - if (nullptr != _onFinish) { - // Clearing the stored callback after finishing the up-stream notification - // has two purposes: - // - // 1. it guaranties (exactly) one time notification - // 2. it breaks the up-stream dependency on a caller object if a shared - // pointer to the object was mentioned as the lambda-function's closure - - auto onFinish = move(_onFinish); - _onFinish = nullptr; - onFinish(proto::WorkerCommandStatus::ERROR, error, string()); - } -} - -} // namespace lsst::qserv::xrdreq diff --git a/src/xrdreq/GetDbStatusQservRequest.h b/src/xrdreq/GetDbStatusQservRequest.h deleted file mode 100644 index be75c53a6..000000000 --- a/src/xrdreq/GetDbStatusQservRequest.h +++ /dev/null @@ -1,87 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ -#ifndef LSST_QSERV_XRDREQ_GET_DB_STATUS_QSERV_REQUEST_H -#define LSST_QSERV_XRDREQ_GET_DB_STATUS_QSERV_REQUEST_H - -// System headers -#include -#include -#include -#include -#include - -// Qserv headers -#include "proto/worker.pb.h" -#include "xrdreq/QservRequest.h" - -namespace lsst::qserv::xrdreq { - -/** - * Class GetDbStatusQservRequest represents a request returning various info - * on the status of the database service of the Qserv worker. - */ -class GetDbStatusQservRequest : public QservRequest { -public: - /// The pointer type for instances of the class - typedef std::shared_ptr Ptr; - - /// The callback function type to be used for notifications on - /// the operation completion. - using CallbackType = std::function; // worker info received (if success) - - /** - * Static factory method is needed to prevent issues with the lifespan - * and memory management of instances created otherwise (as values or via - * low-level pointers). - * - * @param onFinish (optional )callback function to be called upon the completion - * (successful or not) of the request. - * @see wbase::Task::Status - * @return the smart pointer to the object of the class - */ - static Ptr create(CallbackType onFinish = nullptr); - - GetDbStatusQservRequest() = delete; - GetDbStatusQservRequest(GetDbStatusQservRequest const&) = delete; - GetDbStatusQservRequest& operator=(GetDbStatusQservRequest const&) = delete; - - virtual ~GetDbStatusQservRequest() override; - -protected: - /// @see GetDbStatusQservRequest::create() - GetDbStatusQservRequest(CallbackType onFinish); - - virtual void onRequest(proto::FrameBuffer& buf) override; - virtual void onResponse(proto::FrameBufferView& view) override; - virtual void onError(std::string const& error) override; - -private: - // Parameters of the object - - CallbackType _onFinish; -}; - -} // namespace lsst::qserv::xrdreq - -#endif // LSST_QSERV_XRDREQ_GET_DB_STATUS_QSERV_REQUEST_H diff --git a/src/xrdreq/GetStatusQservRequest.cc b/src/xrdreq/GetStatusQservRequest.cc deleted file mode 100644 index 930a7beb4..000000000 --- a/src/xrdreq/GetStatusQservRequest.cc +++ /dev/null @@ -1,102 +0,0 @@ -/* - * LSST Data Management System - * Copyright 2018 LSST Corporation. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ - -// Class header -#include "xrdreq/GetStatusQservRequest.h" - -// LSST headers -#include "lsst/log/Log.h" - -using namespace std; - -namespace { -LOG_LOGGER _log = LOG_GET("lsst.qserv.xrdreq.GetStatusQservRequest"); -} // namespace - -namespace lsst::qserv::xrdreq { - -GetStatusQservRequest::Ptr GetStatusQservRequest::create(wbase::TaskSelector const& taskSelector, - GetStatusQservRequest::CallbackType onFinish) { - GetStatusQservRequest::Ptr ptr(new GetStatusQservRequest(taskSelector, onFinish)); - ptr->setRefToSelf4keepAlive(ptr); - return ptr; -} - -GetStatusQservRequest::GetStatusQservRequest(wbase::TaskSelector const& taskSelector, - GetStatusQservRequest::CallbackType onFinish) - : _taskSelector(taskSelector), _onFinish(onFinish) { - LOGS(_log, LOG_LVL_TRACE, "GetStatusQservRequest ** CONSTRUCTED **"); -} - -GetStatusQservRequest::~GetStatusQservRequest() { - LOGS(_log, LOG_LVL_TRACE, "GetStatusQservRequest ** DELETED **"); -} - -void GetStatusQservRequest::onRequest(proto::FrameBuffer& buf) { - proto::WorkerCommandH header; - header.set_command(proto::WorkerCommandH::GET_STATUS); - buf.serialize(header); - - proto::WorkerCommandGetStatusM message; - message.set_include_tasks(_taskSelector.includeTasks); - for (auto const& id : _taskSelector.queryIds) { - message.add_query_ids(id); - } - for (auto const& state : _taskSelector.taskStates) { - message.add_task_states(static_cast(state)); - } - message.set_max_tasks(_taskSelector.maxTasks); - buf.serialize(message); -} - -void GetStatusQservRequest::onResponse(proto::FrameBufferView& view) { - proto::WorkerCommandGetStatusR reply; - view.parse(reply); - - if (nullptr != _onFinish) { - // Clearing the stored callback after finishing the up-stream notification - // has two purposes: - // - // 1. it guaranties (exactly) one time notification - // 2. it breaks the up-stream dependency on a caller object if a shared - // pointer to the object was mentioned as the lambda-function's closure - auto onFinish = move(_onFinish); - _onFinish = nullptr; - onFinish(proto::WorkerCommandStatus::SUCCESS, string(), reply.info()); - } -} - -void GetStatusQservRequest::onError(string const& error) { - if (nullptr != _onFinish) { - // Clearing the stored callback after finishing the up-stream notification - // has two purposes: - // - // 1. it guaranties (exactly) one time notification - // 2. it breaks the up-stream dependency on a caller object if a shared - // pointer to the object was mentioned as the lambda-function's closure - auto onFinish = move(_onFinish); - _onFinish = nullptr; - onFinish(proto::WorkerCommandStatus::ERROR, error, string()); - } -} - -} // namespace lsst::qserv::xrdreq diff --git a/src/xrdreq/GetStatusQservRequest.h b/src/xrdreq/GetStatusQservRequest.h deleted file mode 100644 index c15561bad..000000000 --- a/src/xrdreq/GetStatusQservRequest.h +++ /dev/null @@ -1,92 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * Copyright 2011-2018 LSST Corporation. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ -#ifndef LSST_QSERV_XRDREQ_GET_STATUS_QSERV_REQUEST_H -#define LSST_QSERV_XRDREQ_GET_STATUS_QSERV_REQUEST_H - -// System headers -#include -#include -#include -#include -#include - -// Qserv headers -#include "proto/worker.pb.h" -#include "wbase/TaskState.h" -#include "xrdreq/QservRequest.h" - -namespace lsst::qserv::xrdreq { - -/** - * Class GetStatusQservRequest represents a request returning various info - * on the on-going status of a Qserv worker. - */ -class GetStatusQservRequest : public QservRequest { -public: - /// The pointer type for instances of the class - typedef std::shared_ptr Ptr; - - /// The callback function type to be used for notifications on - /// the operation completion. - using CallbackType = std::function; // worker info received (if success) - - /** - * Static factory method is needed to prevent issues with the lifespan - * and memory management of instances created otherwise (as values or via - * low-level pointers). - * - * @param taskSelector (optional) task selection criterias. - * @param onFinish (optional )callback function to be called upon the completion - * (successful or not) of the request. - * @see wbase::Task::Status - * @return the smart pointer to the object of the class - */ - static Ptr create(wbase::TaskSelector const& taskSelector = wbase::TaskSelector(), - CallbackType onFinish = nullptr); - - GetStatusQservRequest() = delete; - GetStatusQservRequest(GetStatusQservRequest const&) = delete; - GetStatusQservRequest& operator=(GetStatusQservRequest const&) = delete; - - virtual ~GetStatusQservRequest() override; - -protected: - /// @see GetStatusQservRequest::create() - GetStatusQservRequest(wbase::TaskSelector const& taskSelector, CallbackType onFinish); - - virtual void onRequest(proto::FrameBuffer& buf) override; - virtual void onResponse(proto::FrameBufferView& view) override; - virtual void onError(std::string const& error) override; - -private: - // Parameters of the object - - wbase::TaskSelector const _taskSelector; - CallbackType _onFinish; -}; - -} // namespace lsst::qserv::xrdreq - -#endif // LSST_QSERV_XRDREQ_GET_STATUS_QSERV_REQUEST_H diff --git a/src/xrdreq/SetChunkListQservRequest.cc b/src/xrdreq/SetChunkListQservRequest.cc deleted file mode 100644 index bb3e701a9..000000000 --- a/src/xrdreq/SetChunkListQservRequest.cc +++ /dev/null @@ -1,125 +0,0 @@ -/* - * LSST Data Management System - * Copyright 2018 LSST Corporation. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ - -// Class header -#include "xrdreq/SetChunkListQservRequest.h" - -// System headers -#include - -// LSST headers -#include "lsst/log/Log.h" - -using namespace std; - -namespace { -LOG_LOGGER _log = LOG_GET("lsst.qserv.xrdreq.SetChunkListQservRequest"); -} // namespace - -namespace lsst::qserv::xrdreq { - -SetChunkListQservRequest::Ptr SetChunkListQservRequest::create( - SetChunkListQservRequest::ChunkCollection const& chunks, vector const& databases, bool force, - SetChunkListQservRequest::CallbackType onFinish) { - SetChunkListQservRequest::Ptr ptr(new SetChunkListQservRequest(chunks, databases, force, onFinish)); - ptr->setRefToSelf4keepAlive(ptr); - return ptr; -} - -SetChunkListQservRequest::SetChunkListQservRequest(SetChunkListQservRequest::ChunkCollection const& chunks, - vector const& databases, bool force, - SetChunkListQservRequest::CallbackType onFinish) - : _chunks(chunks), _databases(databases), _force(force), _onFinish(onFinish) { - LOGS(_log, LOG_LVL_TRACE, "SetChunkListQservRequest ** CONSTRUCTED **"); -} - -SetChunkListQservRequest::~SetChunkListQservRequest() { - LOGS(_log, LOG_LVL_TRACE, "SetChunkListQservRequest ** DELETED **"); -} - -void SetChunkListQservRequest::onRequest(proto::FrameBuffer& buf) { - proto::WorkerCommandH header; - header.set_command(proto::WorkerCommandH::SET_CHUNK_LIST); - buf.serialize(header); - - proto::WorkerCommandSetChunkListM message; - for (auto const& chunkEntry : _chunks) { - proto::WorkerCommandChunk* ptr = message.add_chunks(); - ptr->set_db(chunkEntry.database); - ptr->set_chunk(chunkEntry.chunk); - } - for (auto&& database : _databases) { - message.add_databases(database); - } - message.set_force(_force); - buf.serialize(message); -} - -void SetChunkListQservRequest::onResponse(proto::FrameBufferView& view) { - static string const context = "SetChunkListQservRequest "; - - proto::WorkerCommandSetChunkListR reply; - view.parse(reply); - - LOGS(_log, LOG_LVL_TRACE, - context << "** SERVICE REPLY ** status: " - << proto::WorkerCommandStatus_Code_Name(reply.status().code())); - - ChunkCollection chunks; - - if (reply.status().code() == proto::WorkerCommandStatus::SUCCESS) { - int const num = reply.chunks_size(); - for (int i = 0; i < num; i++) { - proto::WorkerCommandChunk const& chunkEntry = reply.chunks(i); - Chunk chunk{chunkEntry.chunk(), chunkEntry.db(), chunkEntry.use_count()}; - chunks.push_back(chunk); - } - LOGS(_log, LOG_LVL_TRACE, context << "total chunks: " << num); - } - if (nullptr != _onFinish) { - // Clearing the stored callback after finishing the up-stream notification - // has two purposes: - // - // 1. it guaranties (exactly) one time notification - // 2. it breaks the up-stream dependency on a caller object if a shared - // pointer to the object was mentioned as the lambda-function's closure - auto onFinish = move(_onFinish); - _onFinish = nullptr; - onFinish(reply.status().code(), reply.status().error(), chunks); - } -} - -void SetChunkListQservRequest::onError(string const& error) { - if (nullptr != _onFinish) { - // Clearing the stored callback after finishing the up-stream notification - // has two purposes: - // - // 1. it guaranties (exactly) one time notification - // 2. it breaks the up-stream dependency on a caller object if a shared - // pointer to the object was mentioned as the lambda-function's closure - auto onFinish = move(_onFinish); - _onFinish = nullptr; - onFinish(proto::WorkerCommandStatus::ERROR, error, ChunkCollection()); - } -} - -} // namespace lsst::qserv::xrdreq diff --git a/src/xrdreq/SetChunkListQservRequest.h b/src/xrdreq/SetChunkListQservRequest.h deleted file mode 100644 index 4cf0d6e09..000000000 --- a/src/xrdreq/SetChunkListQservRequest.h +++ /dev/null @@ -1,111 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * Copyright 2011-2018 LSST Corporation. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ -#ifndef LSST_QSERV_XRDREQ_SET_CHUNK_LIST_QSERV_REQUEST_H -#define LSST_QSERV_XRDREQ_SET_CHUNK_LIST_QSERV_REQUEST_H - -// System headers -#include -#include -#include -#include - -// Qserv headers -#include "proto/worker.pb.h" -#include "xrdreq/QservRequest.h" - -namespace lsst::qserv::xrdreq { - -/** - * Class SetChunkListQservRequest implements the client-side requests - * the Qserv worker services for a status of chunk lists. - */ -class SetChunkListQservRequest : public QservRequest { -public: - /// Struct Chunk a value type encapsulating a chunk number and the name - /// of a database - struct Chunk { - unsigned int chunk; - std::string database; - unsigned int use_count; - }; - - /// The ChunkCollection type represents a collection of chunks - using ChunkCollection = std::list; - - /// The pointer type for instances of the class - typedef std::shared_ptr Ptr; - - /// The callback function type to be used for notifications on - /// the operation completion. - using CallbackType = std::function; // chunks (if success) - - /** - * Static factory method is needed to prevent issues with the lifespan - * and memory management of instances created otherwise (as values or via - * low-level pointers). - * - * ATTENTION: the 'use_count' field of structure Chunk is ignored by this - * class when used on its input. - * - * @param chunks collection of chunks to be transferred to the worker - * @param databases limit a scope of the operation to databases of this collection - * @param force force the proposed change even if the chunk is in use - * @param onFinish optional callback function to be called upon the completion - * (successful or not) of the request. - * @return smart pointer to the object of the class - */ - static Ptr create(ChunkCollection const& chunks, std::vector const& databases, - bool force = false, CallbackType onFinish = nullptr); - - // Default construction and copy semantics are prohibited - SetChunkListQservRequest() = delete; - SetChunkListQservRequest(SetChunkListQservRequest const&) = delete; - SetChunkListQservRequest& operator=(SetChunkListQservRequest const&) = delete; - - ~SetChunkListQservRequest() override; - -protected: - /// @see SetChunkListQservRequest::create()) - SetChunkListQservRequest(ChunkCollection const& chunks, std::vector const& databases, - bool force, CallbackType onFinish); - - void onRequest(proto::FrameBuffer& buf) override; - - void onResponse(proto::FrameBufferView& view) override; - - void onError(std::string const& error) override; - -private: - // Parameters of the object - - ChunkCollection const _chunks; - std::vector const _databases; - bool const _force; - CallbackType _onFinish; -}; - -} // namespace lsst::qserv::xrdreq - -#endif // LSST_QSERV_XRDREQ_SET_CHUNK_LIST_QSERV_REQUEST_H diff --git a/src/xrdreq/TestEchoQservRequest.cc b/src/xrdreq/TestEchoQservRequest.cc deleted file mode 100644 index 2864c2741..000000000 --- a/src/xrdreq/TestEchoQservRequest.cc +++ /dev/null @@ -1,101 +0,0 @@ -/* - * LSST Data Management System - * Copyright 2018 LSST Corporation. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ - -// Class header -#include "xrdreq/TestEchoQservRequest.h" - -// System headers -#include - -// LSST headers -#include "lsst/log/Log.h" - -using namespace std; - -namespace { -LOG_LOGGER _log = LOG_GET("lsst.qserv.xrdreq.TestEchoQservRequest"); -} // namespace - -namespace lsst::qserv::xrdreq { - -TestEchoQservRequest::Ptr TestEchoQservRequest::create(string const& value, - TestEchoQservRequest::CallbackType onFinish) { - TestEchoQservRequest::Ptr ptr(new TestEchoQservRequest(value, onFinish)); - ptr->setRefToSelf4keepAlive(ptr); - return ptr; -} - -TestEchoQservRequest::TestEchoQservRequest(string const& value, TestEchoQservRequest::CallbackType onFinish) - : _value(value), _onFinish(onFinish) { - LOGS(_log, LOG_LVL_TRACE, "TestEchoQservRequest ** CONSTRUCTED **"); -} - -TestEchoQservRequest::~TestEchoQservRequest() { - LOGS(_log, LOG_LVL_TRACE, "TestEchoQservRequest ** DELETED **"); -} - -void TestEchoQservRequest::onRequest(proto::FrameBuffer& buf) { - proto::WorkerCommandH header; - header.set_command(proto::WorkerCommandH::TEST_ECHO); - buf.serialize(header); - - proto::WorkerCommandTestEchoM echo; - echo.set_value(_value); - buf.serialize(echo); -} - -void TestEchoQservRequest::onResponse(proto::FrameBufferView& view) { - proto::WorkerCommandTestEchoR reply; - view.parse(reply); - - LOGS(_log, LOG_LVL_TRACE, - "TestEchoQservRequest ** SERVICE REPLY ** status: " - << proto::WorkerCommandStatus_Code_Name(reply.status().code())); - - if (nullptr != _onFinish) { - // Clearing the stored callback after finishing the up-stream notification - // has two purposes: - // - // 1. it guaranties (exactly) one time notification - // 2. it breaks the up-stream dependency on a caller object if a shared - // pointer to the object was mentioned as the lambda-function's closure - auto onFinish = move(_onFinish); - _onFinish = nullptr; - onFinish(reply.status().code(), reply.status().error(), _value, reply.value()); - } -} - -void TestEchoQservRequest::onError(string const& error) { - if (nullptr != _onFinish) { - // Clearing the stored callback after finishing the up-stream notification - // has two purposes: - // - // 1. it guaranties (exactly) one time notification - // 2. it breaks the up-stream dependency on a caller object if a shared - // pointer to the object was mentioned as the lambda-function's closure - auto onFinish = move(_onFinish); - _onFinish = nullptr; - onFinish(proto::WorkerCommandStatus::ERROR, error, _value, string()); - } -} - -} // namespace lsst::qserv::xrdreq diff --git a/src/xrdreq/TestEchoQservRequest.h b/src/xrdreq/TestEchoQservRequest.h deleted file mode 100644 index 089979989..000000000 --- a/src/xrdreq/TestEchoQservRequest.h +++ /dev/null @@ -1,96 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * Copyright 2011-2018 LSST Corporation. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ -#ifndef LSST_QSERV_XRDREQ_TEST_ECHO_QSERV_REQUEST_H -#define LSST_QSERV_XRDREQ_TEST_ECHO_QSERV_REQUEST_H - -// System headers -#include -#include -#include - -// Qserv headers -#include "proto/worker.pb.h" -#include "xrdreq/QservRequest.h" - -namespace lsst::qserv::xrdreq { - -/** - * Class TestEchoQservRequest represents a simple test request sending a string - * to the worker management service and expecting the same value back. - */ -class TestEchoQservRequest : public QservRequest { -public: - /// The pointer type for instances of the class - typedef std::shared_ptr Ptr; - - /// The callback function type to be used for notifications on - /// the operation completion. - using CallbackType = std::function; // value received (if success) - - /** - * Static factory method is needed to prevent issues with the lifespan - * and memory management of instances created otherwise (as values or via - * low-level pointers). - * - * @param value a value to be sent to the worker service - * @param onFinish optional callback function to be called upon the completion - * (successful or not) of the request. - * @return smart pointer to the object of the class - */ - static Ptr create(std::string const& value, CallbackType onFinish = nullptr); - - // Default construction and copy semantics is prohibited - TestEchoQservRequest() = delete; - TestEchoQservRequest(TestEchoQservRequest const&) = delete; - TestEchoQservRequest& operator=(TestEchoQservRequest const&) = delete; - - ~TestEchoQservRequest() override; - -protected: - /** - * Normal constructor - * - * @param value a value to be sent to the worker service - * @param onFinish function to be called upon the completion of a request - */ - TestEchoQservRequest(std::string const& value, CallbackType onFinish); - - void onRequest(proto::FrameBuffer& buf) override; - - void onResponse(proto::FrameBufferView& view) override; - - void onError(std::string const& error) override; - -private: - // Parameters of the object - - std::string _value; - CallbackType _onFinish; -}; - -} // namespace lsst::qserv::xrdreq - -#endif // LSST_QSERV_XRDREQ_TEST_ECHO_QSERV_REQUEST_H diff --git a/src/xrdreq/qserv-worker-notify.cc b/src/xrdreq/qserv-worker-notify.cc deleted file mode 100644 index c096085ce..000000000 --- a/src/xrdreq/qserv-worker-notify.cc +++ /dev/null @@ -1,340 +0,0 @@ -// System header -#include -#include -#include -#include -#include -#include -#include - -// Third party headers -#include "XrdSsi/XrdSsiProvider.hh" -#include "XrdSsi/XrdSsiService.hh" - -// Qserv headers -#include "global/intTypes.h" -#include "global/ResourceUnit.h" -#include "proto/worker.pb.h" -#include "util/BlockPost.h" -#include "util/CmdLineParser.h" -#include "xrdreq/ChunkGroupQservRequest.h" -#include "xrdreq/ChunkListQservRequest.h" -#include "xrdreq/GetChunkListQservRequest.h" -#include "xrdreq/GetStatusQservRequest.h" -#include "xrdreq/SetChunkListQservRequest.h" -#include "xrdreq/TestEchoQservRequest.h" - -/// This C++ symbol is provided by the SSI shared library -extern XrdSsiProvider* XrdSsiProviderClient; - -namespace global = lsst::qserv; -namespace proto = lsst::qserv::proto; -namespace util = lsst::qserv::util; -namespace xrdreq = lsst::qserv::xrdreq; - -using namespace std; - -namespace { - -// Command line parameters - -string operation; -string worker; -string inFileName; -unsigned int chunk; -vector dbs; -vector queryIds; -string value; -string serviceProviderLocation; -bool inUseOnly; -bool includeTasks; -bool reload; -bool force; -bool printReport; - -/** - * Read and parse a space/newline separated stream of pairs from the input - * file and fill replica entries into the collection. Each pair has - * the following format: - * - * : - * - * For example: - * - * LSST:123 LSST:124 LSST:23456 - * LSST:0 - * - * @param chunk - collection to be initialize - */ -void readInFile(xrdreq::SetChunkListQservRequest::ChunkCollection& chunks, vector& databases) { - chunks.clear(); - databases.clear(); - - set uniqueDatabaseNames; - - ifstream infile(inFileName); - if (not infile.good()) { - cerr << "failed to open file: " << inFileName << endl; - throw runtime_error("failed to open file: " + inFileName); - } - - string databaseAndChunk; - while (infile >> databaseAndChunk) { - if (databaseAndChunk.empty()) { - continue; - } - - string::size_type const pos = databaseAndChunk.rfind(':'); - if ((pos == string::npos) or (pos == 0) or (pos == databaseAndChunk.size() - 1)) { - throw runtime_error("failed to parse file: " + inFileName + - ", illegal :: pair: '" + databaseAndChunk + "'"); - } - unsigned long const chunk = stoul(databaseAndChunk.substr(pos + 1)); - string const database = databaseAndChunk.substr(0, pos); - chunks.emplace_back(xrdreq::SetChunkListQservRequest::Chunk{ - (unsigned int)chunk, database, 0 /* use_count (UNUSED) */ - }); - uniqueDatabaseNames.insert(database); - } - for (auto&& database : uniqueDatabaseNames) { - databases.push_back(database); - } -} - -int test() { - // Instantiate a request object - - atomic finished(false); - - shared_ptr request = nullptr; - - if ("GET_CHUNK_LIST" == operation) { - request = xrdreq::GetChunkListQservRequest::create( - inUseOnly, [&finished](proto::WorkerCommandStatus::Code code, string const& error, - xrdreq::GetChunkListQservRequest::ChunkCollection const& chunks) { - if (code != proto::WorkerCommandStatus::SUCCESS) { - cout << "code: " << proto::WorkerCommandStatus_Code_Name(code) << "\n" - << "error: " << error << endl; - } else { - cout << "# total chunks: " << chunks.size() << "\n" << endl; - if (chunks.size()) { - cout << " chunk | database | in use \n" - << "------------+----------------------------------+--------\n"; - for (auto const& entry : chunks) { - cout << " " << setw(10) << entry.chunk << " |" - << " " << setw(32) << entry.database << " |" - << " " << setw(6) << entry.use_count << " \n"; - } - cout << endl; - } - } - finished = true; - }); - - } else if ("SET_CHUNK_LIST" == operation) { - xrdreq::SetChunkListQservRequest::ChunkCollection chunks; - vector databases; - readInFile(chunks, databases); - - request = xrdreq::SetChunkListQservRequest::create( - chunks, databases, force, - [&finished](proto::WorkerCommandStatus::Code code, string const& error, - xrdreq::SetChunkListQservRequest::ChunkCollection const& chunks) { - if (code != proto::WorkerCommandStatus::SUCCESS) { - cout << "code: " << proto::WorkerCommandStatus_Code_Name(code) << "\n" - << "error: " << error << endl; - } else { - cout << "# total chunks: " << chunks.size() << "\n" << endl; - if (chunks.size()) { - cout << " chunk | database | in use \n" - << "------------+----------------------------------+--------\n"; - for (auto const& entry : chunks) { - cout << " " << setw(10) << entry.chunk << " |" - << " " << setw(32) << entry.database << " |" - << " " << setw(6) << entry.use_count << " \n"; - } - cout << endl; - } - } - finished = true; - }); - - } else if ("REBUILD_CHUNK_LIST" == operation) { - request = xrdreq::RebuildChunkListQservRequest::create( - reload, [&finished](proto::WorkerCommandStatus::Code code, string const& error, - xrdreq::ChunkListQservRequest::ChunkCollection const& added, - xrdreq::ChunkListQservRequest::ChunkCollection const& removed) { - if (code != proto::WorkerCommandStatus::SUCCESS) { - cout << "code: " << proto::WorkerCommandStatus_Code_Name(code) << "\n" - << "error: " << error << endl; - } else { - cout << "# chunks added: " << added.size() << "\n" - << "# chuks removed: " << removed.size() << endl; - } - finished = true; - }); - - } else if ("RELOAD_CHUNK_LIST" == operation) { - request = xrdreq::ReloadChunkListQservRequest::create( - [&finished](proto::WorkerCommandStatus::Code code, string const& error, - xrdreq::ChunkListQservRequest::ChunkCollection const& added, - xrdreq::ChunkListQservRequest::ChunkCollection const& removed) { - if (code != proto::WorkerCommandStatus::SUCCESS) { - cout << "code: " << proto::WorkerCommandStatus_Code_Name(code) << "\n" - << "error: " << error << endl; - } else { - cout << "# chunks added: " << added.size() << "\n" - << "# chuks removed: " << removed.size() << endl; - } - finished = true; - }); - - } else if ("ADD_CHUNK_GROUP" == operation) { - request = xrdreq::AddChunkGroupQservRequest::create( - chunk, dbs, [&finished](proto::WorkerCommandStatus::Code code, string const& error) { - if (code != proto::WorkerCommandStatus::SUCCESS) { - cout << "code: " << proto::WorkerCommandStatus_Code_Name(code) << "\n" - << "error: " << error << endl; - } - finished = true; - }); - - } else if ("REMOVE_CHUNK_GROUP" == operation) { - request = xrdreq::RemoveChunkGroupQservRequest::create( - chunk, dbs, force, [&finished](proto::WorkerCommandStatus::Code code, string const& error) { - if (code != proto::WorkerCommandStatus::SUCCESS) { - cout << "code: " << proto::WorkerCommandStatus_Code_Name(code) << "\n" - << "error: " << error << endl; - } - finished = true; - }); - - } else if ("TEST_ECHO" == operation) { - request = xrdreq::TestEchoQservRequest::create( - value, [&finished](proto::WorkerCommandStatus::Code code, string const& error, - string const& sent, string const& received) { - if (code != proto::WorkerCommandStatus::SUCCESS) { - cout << "code: " << proto::WorkerCommandStatus_Code_Name(code) << "\n" - << "error: " << error << endl; - } else { - cout << "value sent: " << sent << "\n" - << "value received: " << received << endl; - } - finished = true; - }); - - } else if ("GET_STATUS" == operation) { - request = xrdreq::GetStatusQservRequest::create( - includeTasks, queryIds, - [&finished](proto::WorkerCommandStatus::Code code, string const& error, string const& info) { - if (code != proto::WorkerCommandStatus::SUCCESS) { - cout << "code: " << proto::WorkerCommandStatus_Code_Name(code) << "\n" - << "error: " << error << endl; - } else { - cout << "worker info: " << info << endl; - } - finished = true; - }); - - } else { - return 1; - } - - // Connect to a service provider - XrdSsiErrInfo errInfo; - auto serviceProvider = XrdSsiProviderClient->GetService(errInfo, serviceProviderLocation); - if (nullptr == serviceProvider) { - cerr << "failed to contact service provider at: " << serviceProviderLocation - << ", error: " << errInfo.Get() << endl; - return 1; - } - cout << "connected to service provider at: " << serviceProviderLocation << endl; - - // Submit the request - XrdSsiResource resource(global::ResourceUnit::makeWorkerPath(worker)); - serviceProvider->ProcessRequest(*request, resource); - - // Block while the request is in progress - util::BlockPost blockPost(1000, 2000); - while (not finished) blockPost.wait(200); - - return 0; -} -} // namespace - -int main(int argc, const char* const argv[]) { - // Verify that the version of the library that we linked against is - // compatible with the version of the headers we compiled against. - - GOOGLE_PROTOBUF_VERIFY_VERSION; - - // Parse command line parameters - try { - util::CmdLineParser parser( - argc, argv, - "\n" - "Usage:\n" - " [ [ [...]]]\n" - " [--service=]\n" - " [--in-use-only]\n" - " [--include-tasks]\n" - " [--reload]\n" - " [--force>]\n" - " [--print-report]\n" - "\n" - "Supported operations and mandatory parameters:\n" - " GET_CHUNK_LIST \n" - " SET_CHUNK_LIST \n" - " REBUILD_CHUNK_LIST \n" - " RELOAD_CHUNK_LIST \n" - " ADD_CHUNK_GROUP [ [ ... ]]\n" - " REMOVE_CHUNK_GROUP [ [ ... ]]\n" - " TEST_ECHO \n" - " GET_STATUS [ [ ... ]]\n" - "\n" - "Flags an options:\n" - " --service= - location of a service provider (default: 'localhost:1094')\n" - " --in-use-only - used with GET_CHUNK_LIST to only report chunks which are in use.\n" - " Otherwise all chunks will be reported\n" - " --include-tasks - include detail info on the tasks\n" - " --reload - used with REBUILD_CHUNK_LIST to also reload the list into a " - "worker\n" - " --force - force operation in REMOVE_CHUNK_GROUP even for chunks in use\n" - " --print-report - print \n" - "\n" - "Parameters:\n" - " - unique identifier of a worker (example: 'worker-1')\n" - " - text file with space or newline separated pairs of :\n" - " - chunk number\n" - " - database name\n" - " - user query identifier\n" - " - arbitrary string\n"); - - ::operation = parser.parameterRestrictedBy( - 1, {"GET_CHUNK_LIST", "SET_CHUNK_LIST", "REBUILD_CHUNK_LIST", "RELOAD_CHUNK_LIST", - "ADD_CHUNK_GROUP", "REMOVE_CHUNK_GROUP", "TEST_ECHO", "GET_STATUS"}); - - ::worker = parser.parameter(2); - - if (parser.in(::operation, {"SET_CHUNK_LIST"})) { - ::inFileName = parser.parameter(3); - } else if (parser.in(::operation, {"ADD_CHUNK_GROUP", "REMOVE_CHUNK_GROUP"})) { - ::chunk = parser.parameter(3); - ::dbs = parser.parameters(4); - } else if (parser.in(::operation, {"TEST_ECHO"})) { - ::value = parser.parameter(3); - } else if (parser.in(::operation, {"GET_STATUS"})) { - ::queryIds = parser.parameters(2); - } - ::serviceProviderLocation = parser.option("service", "localhost:1094"); - ::inUseOnly = parser.flag("in-use-only"); - ::includeTasks = parser.flag("include-tasks"); - ::reload = parser.flag("reload"); - ::force = parser.flag("force"); - ::printReport = parser.flag("print-report"); - - } catch (exception const& ex) { - return 1; - } - return ::test(); -} diff --git a/src/xrdreq/qserv-worker-perf-chunks.cc b/src/xrdreq/qserv-worker-perf-chunks.cc deleted file mode 100644 index a48ffd46e..000000000 --- a/src/xrdreq/qserv-worker-perf-chunks.cc +++ /dev/null @@ -1,349 +0,0 @@ -/* - * LSST Data Management System - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ - -/** - * This application tests the performance of the XRootD/SSI protocol - * using Qserv workers as servers. The application also supports - * the multi-threaded option for initiating requests. - */ - -// System header -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// Third party headers -#include "XrdSsi/XrdSsiProvider.hh" -#include "XrdSsi/XrdSsiService.hh" - -// Qserv headers -#include "proto/worker.pb.h" -#include "util/BlockPost.h" -#include "util/CmdLineParser.h" -#include "util/File.h" -#include "xrdreq/TestEchoQservRequest.h" - -/// This C++ symbol is provided by the SSI shared library -extern XrdSsiProvider* XrdSsiProviderClient; - -namespace global = lsst::qserv; -namespace proto = lsst::qserv::proto; -namespace util = lsst::qserv::util; -namespace xrdreq = lsst::qserv::xrdreq; - -using namespace std; -using namespace std::placeholders; - -namespace { - -// Command line parameters - -string fileName; -unsigned int numRequests; -char value; -unsigned int valueSize; -string serviceProviderLocation; -unsigned int numResources; -bool resourceFirst; -unsigned int numThreads; -unsigned int flowControlLimit; -bool silent; -int xrootdCBThreadsMax; -int xrootdCBThreadsInit; -bool memoryCleanup; - -/** - * The synchronize counter, which is used to limit the number of the "in-flight" - * requests if the flow-control is enabled. The later is enabled if a value of - * the constructor's parameter 'maxRequestsAllowed' is not equal to 0. - */ -class Counter { -public: - typedef shared_ptr Ptr; - - static Ptr create(unsigned int maxRequestsAllowed) { return Ptr(new Counter(maxRequestsAllowed)); } - - Counter() = delete; - Counter(Counter const&) = delete; - Counter& operator=(Counter const&) = delete; - ~Counter() = default; - - void inc() { - if (_maxRequestsAllowed == 0) { - ++_counter; - } else { - unique_lock lock(_mtx); - _cv.wait(lock, [&]() { return _counter < _maxRequestsAllowed; }); - ++_counter; - } - } - - void dec() { - if (_maxRequestsAllowed == 0) { - --_counter; - } else { - unique_lock lock(_mtx); - --_counter; - lock.unlock(); - _cv.notify_one(); - } - } - - unsigned int counter() const { return _counter; } - -private: - explicit Counter(unsigned int maxRequestsAllowed) : _maxRequestsAllowed(maxRequestsAllowed) {} - - unsigned int _maxRequestsAllowed; - atomic _counter{0}; - std::mutex _mtx; - std::condition_variable _cv; -}; - -class RequestManager : public enable_shared_from_this { -public: - typedef shared_ptr Ptr; - typedef function - OnFinish; - - static Ptr create(Counter::Ptr const& numRequestsInFlight, XrdSsiService* serviceProvider, - string const& resourcePath, string const& payload, OnFinish const& onFinish, - bool memoryCleanup) { - return Ptr(new RequestManager(numRequestsInFlight, serviceProvider, resourcePath, payload, onFinish, - memoryCleanup)); - } - - RequestManager(RequestManager const&) = delete; - RequestManager& operator=(RequestManager const&) = delete; - - void start() { - _ptr = xrdreq::TestEchoQservRequest::create( - _payload, bind(&RequestManager::_finished, shared_from_this(), _1, _2, _3, _4)); - _numRequestsInFlight->inc(); - XrdSsiResource resource(_resourcePath); - _serviceProvider->ProcessRequest(*_ptr, resource); - } - -private: - void _finished(proto::WorkerCommandStatus::Code code, string const& error, string const& sent, - string const& received) { - _onFinish(code, error, sent, received); - _numRequestsInFlight->dec(); - if (_memoryCleanup) _ptr = nullptr; - } - - RequestManager(Counter::Ptr const& numRequestsInFlight, XrdSsiService* serviceProvider, - string const& resourcePath, string const& payload, OnFinish const& onFinish, - bool memoryCleanup) - : _numRequestsInFlight(numRequestsInFlight), - _serviceProvider(serviceProvider), - _resourcePath(resourcePath), - _payload(payload), - _onFinish(onFinish), - _memoryCleanup(memoryCleanup) {} - - Counter::Ptr const _numRequestsInFlight; - XrdSsiService* _serviceProvider; - string const _resourcePath; - string const _payload; - OnFinish const _onFinish; - bool const _memoryCleanup; - xrdreq::TestEchoQservRequest::Ptr _ptr; -}; - -int test() { - string const payload = string(valueSize, value); - - vector const resources = util::File::getLines(fileName, true); - if (not numResources or (resources.size() < numResources)) { - cerr << "error: specified number of resources not in the valid range: 1.." << numResources << endl; - return 1; - } - - // Configure threads at the XRootD/SSI client - XrdSsiProviderClient->SetCBThreads(xrootdCBThreadsMax, xrootdCBThreadsInit); - - // Connect to a service provider - XrdSsiErrInfo errInfo; - XrdSsiService* serviceProvider = XrdSsiProviderClient->GetService(errInfo, serviceProviderLocation); - if (nullptr == serviceProvider) { - cerr << "failed to contact service provider at: " << serviceProviderLocation - << ", error: " << errInfo.Get() << endl; - return 1; - } - cout << "connected to service provider at: " << serviceProviderLocation << endl; - - // Build a complete list of jobs (resource path names) to be processed - // in the specified order. Note that the actual (run time) ordering may - // be different if running this test in the multi-threaded mode. - vector jobs; - if (resourceFirst) { - for (unsigned int j = 0; j < numResources; ++j) { - auto&& resourcePath = resources[j]; - for (unsigned int i = 0; i < numRequests; ++i) { - jobs.push_back(resourcePath); - } - } - - } else { - for (unsigned int i = 0; i < numRequests; ++i) { - for (unsigned int j = 0; j < numResources; ++j) { - auto&& resourcePath = resources[j]; - jobs.push_back(resourcePath); - } - } - } - - // Make allocation of jobs to threads using the 'round-robin' method - vector> thread2jobs(numThreads); - for (unsigned int i = 0; i < jobs.size(); ++i) { - auto const threadIdx = i % numThreads; - auto&& resourcePath = jobs[i]; - thread2jobs[threadIdx].push_back(resourcePath); - } - - Counter::Ptr const numRequestsInFlight = Counter::create(flowControlLimit); - bool memoryCleanup = ::memoryCleanup; - - // Launch threads - vector> threads; - for (unsigned int tIdx = 0; tIdx < numThreads; ++tIdx) { - threads.emplace_back(new thread( - [&numRequestsInFlight, &payload, memoryCleanup](vector const& jobs, - XrdSsiService* serviceProvider) { - // Launch requests and keep them in the collection until they finish - // to prevent request being deleted while there are unhandled callbacks - // on request completion. - vector requests; - - for (auto&& resourcePath : jobs) { - auto const ptr = RequestManager::create( - numRequestsInFlight, serviceProvider, resourcePath, payload, - [](proto::WorkerCommandStatus::Code code, string const& error, - string const& sent, string const& received) { - if (not silent) { - if (status != proto::WorkerCommandStatus::SUCCESS) { - cout << "code: " << proto::WorkerCommandStatus_Code_Name(code) - << ", error: " << error << endl; - } else { - cout << "value sent: '" << sent << "', received: '" << received - << "'" << endl; - } - } - }, - memoryCleanup); - requests.push_back(ptr); - ptr->start(); - } - - // Block while at least one request is in progress - util::BlockPost blockPost(1000, 2000); - while (numRequestsInFlight->counter() != 0) { - blockPost.wait(200); - } - }, - thread2jobs[tIdx], serviceProvider)); - } - - // Wait before all threads will finish to avoid crashing the application - for (auto&& t : threads) { - t->join(); - } - return 0; -} -} // namespace - -int main(int argc, const char* const argv[]) { - // Verify that the version of the library that we linked against is - // compatible with the version of the headers we compiled against. - - GOOGLE_PROTOBUF_VERIFY_VERSION; - - // Parse command line parameters - try { - util::CmdLineParser parser( - argc, argv, - "\n" - "General syntax:\n" - " [flags, options...]\n" - "\n" - "Parameters:\n" - " - A file with resource paths (one resource per line).\n" - " - The number of requests per resource.\n" - "\n" - "Flags and options:\n" - " --service= - A location of a service provider (default: " - "'localhost:1094').\n" - " --num-resources= - The number of resources (default: 1, range: 1..*).\n" - " --resource-first - Iterate over resources, then over requests.\n" - " --num-threads= - The number of parallel threads (default: 1, range: 1..*).\n" - " --value= - An 8-bit unsigned decimal number to be used in composing " - "messages\n" - " sent to the services (default: 97 which is an ASCII code for " - "'a').\n" - " --value-size= - The number of times to replicate the when composing\n" - " payloads of requests sent to the worker services (default: " - "1).\n" - " --flow-control= - If the value is not 0 then it will turn on the flow control\n" - " for the requests processing. In this case a value of the " - "parameter\n" - " puts a cap on the maximum number of requests being processed " - "at each\n" - " moment of time (default: 0, range: 0..*).\n" - " --silent - Do not report any status or error messages, including\n" - " the ones sent via the LSST Logger API.\n" - " --xrootd-cdb-threads-max - The configuration parameter of the XRootD/SSI.\n" - " --xrootd-cdb-threads-init - The configuration parameter of the XRootD/SSI.\n" - " --memory-cleanup - Automatically delete XRootD/SSI requests as they finish\n" - " in order to clean the memory allocated by the requests.\n"); - - ::fileName = parser.parameter(1); - ::numRequests = parser.parameter(2); - - ::serviceProviderLocation = parser.option("service", "localhost:1094"); - ::numResources = parser.option("num-resources", 1); - ::resourceFirst = parser.flag("resource-first"); - ::numThreads = parser.option("num-threads", 1); - ::value = parser.option("value", 'a'); - ::valueSize = parser.option("value-size", 1); - ::flowControlLimit = parser.option("flow-control", 0); - ::silent = parser.flag("silent"); - ::xrootdCBThreadsMax = parser.option("xrootd-cdb-threads-max", 0); - ::xrootdCBThreadsInit = parser.option("xrootd-cdb-threads-init", 0); - ::memoryCleanup = parser.flag("memory-cleanup"); - - if (::valueSize == 0) throw invalid_argument("a value of option can't be 0"); - - } catch (exception const& ex) { - cerr << "error: " << ex.what() << endl; - return 1; - } - return ::test(); -} diff --git a/src/xrdreq/qserv-worker-perf.cc b/src/xrdreq/qserv-worker-perf.cc deleted file mode 100644 index 282f638a2..000000000 --- a/src/xrdreq/qserv-worker-perf.cc +++ /dev/null @@ -1,179 +0,0 @@ -// System header -#include -#include -#include -#include -#include -#include - -// Third party headers -#include "XrdSsi/XrdSsiProvider.hh" -#include "XrdSsi/XrdSsiService.hh" - -// Qserv headers -#include "global/ResourceUnit.h" -#include "proto/worker.pb.h" -#include "util/BlockPost.h" -#include "util/CmdLineParser.h" -#include "util/File.h" -#include "xrdreq/TestEchoQservRequest.h" - -/// This C++ symbol is provided by the SSI shared library -extern XrdSsiProvider* XrdSsiProviderClient; - -namespace global = lsst::qserv; -namespace proto = lsst::qserv::proto; -namespace util = lsst::qserv::util; -namespace xrdreq = lsst::qserv::xrdreq; - -using namespace std; - -namespace { - -// Command line parameters - -string fileName; -unsigned int numRequests; -string value; -string serviceProviderLocation; -unsigned int numWorkers; -bool workerFirst; -unsigned int cancelAfterMs; - -int test() { - vector const workers = util::File::getLines(fileName, true); - if (not numWorkers or (workers.size() < numWorkers)) { - cerr << "error: specified number of workers not in the valid range: 1.." << numWorkers << endl; - return 1; - } - - // Connect to a service provider - XrdSsiErrInfo errInfo; - auto serviceProvider = XrdSsiProviderClient->GetService(errInfo, serviceProviderLocation); - if (nullptr == serviceProvider) { - cerr << "failed to contact service provider at: " << serviceProviderLocation - << ", error: " << errInfo.Get() << endl; - return 1; - } - cout << "connected to service provider at: " << serviceProviderLocation << endl; - - // Store request pointers here to prevent them deleted too early - vector requests; - - atomic finished(0); - - if (workerFirst) { - for (unsigned int j = 0; j < numWorkers; ++j) { - string const& worker = workers[j]; - - for (unsigned int i = 0; i < numRequests; ++i) { - auto request = xrdreq::TestEchoQservRequest::create( - value, [&finished](proto::WorkerCommandStatus::Code code, string const& error, - string const& sent, string const& received) { - if (code != proto::WorkerCommandStatus::SUCCESS) { - cout << "code: " << proto::WorkerCommandStatus_Code_Name(code) << "\n" - << "error: " << error << endl; - } else { - cout << "value sent: " << sent << "\n" - << "value received: " << received << endl; - } - finished--; - }); - requests.push_back(request); - - // Submit the request - finished++; - XrdSsiResource resource(global::ResourceUnit::makeWorkerPath(worker)); - serviceProvider->ProcessRequest(*request, resource); - } - } - - } else { - for (unsigned int i = 0; i < numRequests; ++i) { - for (unsigned int j = 0; j < numWorkers; ++j) { - string const& worker = workers[j]; - auto request = xrdreq::TestEchoQservRequest::create( - value, [&finished](proto::WorkerCommandStatus::Code code, string const& error, - string const& sent, string const& received) { - if (code != proto::WorkerCommandStatus::SUCCESS) { - cout << "code: " << proto::WorkerCommandStatus_Code_Name(code) << "\n" - << "error: " << error << endl; - } else { - cout << "value sent: " << sent << "\n" - << "value received: " << received << endl; - } - finished--; - }); - requests.push_back(request); - - // Submit the request - finished++; - XrdSsiResource resource(global::ResourceUnit::makeWorkerPath(worker)); - serviceProvider->ProcessRequest(*request, resource); - } - } - } - if (cancelAfterMs == 0) { - // Block while at least one request is in progress - util::BlockPost blockPost(1000, 2000); - while (finished) { - blockPost.wait(200); - } - } else { - // Request cancellation timeout is used to test the correctness of - // the XRootD/SSI implementation under heavy loads. - util::BlockPost blockPost(cancelAfterMs, cancelAfterMs + 1); - blockPost.wait(); - for (auto&& request : requests) { - request->cancel(); - } - } - return 0; -} -} // namespace - -int main(int argc, const char* const argv[]) { - // Verify that the version of the library that we linked against is - // compatible with the version of the headers we compiled against. - - GOOGLE_PROTOBUF_VERIFY_VERSION; - - // Parse command line parameters - try { - util::CmdLineParser parser( - argc, argv, - "\n" - "Usage:\n" - " \n" - " [--service=]\n" - " [--num-workers=]\n" - " [--worker-first]\b" - " [--cancel-after=]\n" - "\n" - "Flags an options:\n" - " --service= - location of a service provider (default: 'localhost:1094')\n" - " --num-workers= - the number of workers (default: 1, range: 1..10)\n" - " --worker-first - iterate over workers, then over requests\n" - " --cancel-after= \n" - " - the number of milliseconds to wait before cancelling\n" - " all requests (default 0 means no cancellation)\n" - "\n" - "Parameters:\n" - " - a file with worker identifiers (one worker per line)\n" - " - the number of requests per worker\n" - " - arbitrary string\n"); - - ::fileName = parser.parameter(1); - ::numRequests = parser.parameter(2); - ::value = parser.parameter(3); - - ::serviceProviderLocation = parser.option("service", "localhost:1094"); - ::numWorkers = parser.option("num-workers", 1); - ::workerFirst = parser.flag("worker-first"); - ::cancelAfterMs = parser.option("cancel-after", 0); - - } catch (exception const& ex) { - return 1; - } - return ::test(); -} diff --git a/src/xrdreq/qserv-worker-status.cc b/src/xrdreq/qserv-worker-status.cc deleted file mode 100644 index d84e1387d..000000000 --- a/src/xrdreq/qserv-worker-status.cc +++ /dev/null @@ -1,181 +0,0 @@ -// System header -#include -#include -#include -#include -#include - -// Third party headers -#include "XrdSsi/XrdSsiProvider.hh" -#include "XrdSsi/XrdSsiService.hh" - -// Qserv headers -#include "global/intTypes.h" -#include "global/ResourceUnit.h" -#include "proto/worker.pb.h" -#include "util/BlockPost.h" -#include "util/CmdLineParser.h" -#include "util/File.h" -#include "xrdreq/GetStatusQservRequest.h" - -/// This C++ symbol is provided by the SSI shared library -extern XrdSsiProvider* XrdSsiProviderClient; - -namespace global = lsst::qserv; -namespace proto = lsst::qserv::proto; -namespace util = lsst::qserv::util; -namespace xrdreq = lsst::qserv::xrdreq; - -using namespace std; - -namespace { - -// Command line parameters - -string fileName; -unsigned int numRequests; -string serviceProviderLocation; -unsigned int numWorkers; -vector queryIds; -bool workerFirst; -unsigned int cancelAfterMs; -bool includeTasks; - -int test() { - vector const workers = util::File::getLines(fileName, true); - if (not numWorkers or (workers.size() < numWorkers)) { - cerr << "error: specified number of workers not in the valid range: 1.." << numWorkers << endl; - return 1; - } - - // Connect to a service provider - XrdSsiErrInfo errInfo; - auto serviceProvider = XrdSsiProviderClient->GetService(errInfo, serviceProviderLocation); - if (nullptr == serviceProvider) { - cerr << "failed to contact service provider at: " << serviceProviderLocation - << ", error: " << errInfo.Get() << endl; - return 1; - } - cout << "connected to service provider at: " << serviceProviderLocation << endl; - - // Store request pointers here to prevent them deleted too early - vector requests; - - atomic finished(0); - - if (workerFirst) { - for (unsigned int j = 0; j < numWorkers; ++j) { - string const& worker = workers[j]; - - for (unsigned int i = 0; i < numRequests; ++i) { - auto request = xrdreq::GetStatusQservRequest::create( - includeTasks, queryIds, - [&finished](proto::WorkerCommandStatus::Code code, string const& error, - string const& info) { - if (code != proto::WorkerCommandStatu::SUCCESS) { - cout << "code: " << proto::WorkerCommandStatus_Code_Name(code) << "\n" - << "error: " << error << endl; - } else { - cout << "info: " << info << endl; - } - finished--; - }); - requests.push_back(request); - - // Submit the request - finished++; - XrdSsiResource resource(global::ResourceUnit::makeWorkerPath(worker)); - serviceProvider->ProcessRequest(*request, resource); - } - } - - } else { - for (unsigned int i = 0; i < numRequests; ++i) { - for (unsigned int j = 0; j < numWorkers; ++j) { - string const& worker = workers[j]; - auto request = xrdreq::GetStatusQservRequest::create( - [&finished](proto::WorkerCommandStatus::Code code, string const& error, - string const& info) { - if (code != proto::WorkerCommandStatu::SUCCESS) { - cout << "code: " << proto::WorkerCommandStatus_Code_Name(code) << "\n" - << "error: " << error << endl; - } else { - cout << "info: " << info << endl; - } - finished--; - }); - requests.push_back(request); - - // Submit the request - finished++; - XrdSsiResource resource(global::ResourceUnit::makeWorkerPath(worker)); - serviceProvider->ProcessRequest(*request, resource); - } - } - } - if (cancelAfterMs == 0) { - // Block while at least one request is in progress - util::BlockPost blockPost(1000, 2000); - while (finished) { - blockPost.wait(200); - } - } else { - // Request cancellation timeout is used to test the correctness of - // the XRootD/SSI implementation under heavy loads. - util::BlockPost blockPost(cancelAfterMs, cancelAfterMs + 1); - blockPost.wait(); - for (auto&& request : requests) { - request->cancel(); - } - } - return 0; -} -} // namespace - -int main(int argc, const char* const argv[]) { - // Verify that the version of the library that we linked against is - // compatible with the version of the headers we compiled against. - - GOOGLE_PROTOBUF_VERIFY_VERSION; - - // Parse command line parameters - try { - util::CmdLineParser parser( - argc, argv, - "\n" - "Usage:\n" - " [ [ ... ]]\n" - " [--service=]\n" - " [--num-workers=]\n" - " [--worker-first]\b" - " [--include-tasks]\n" - " [--cancel-after=]\n" - "\n" - "Flags an options:\n" - " --service= - location of a service provider (default: 'localhost:1094')\n" - " --num-workers= - the number of workers (default: 1, range: 1..10)\n" - " --worker-first - iterate over workers, then over requests\n" - " --include-tasks - include detail info on the tasks\n" - " --cancel-after= \n" - " - the number of milliseconds to wait before cancelling\n" - " all requests (default 0 means no cancellation)\n" - "\n" - "Parameters:\n" - " - a file with worker identifiers (one worker per line)\n" - " - the number of requests per worker\n" - " - user query identifier\n"); - - ::fileName = parser.parameter(1); - ::numRequests = parser.parameter(2); - ::queryIds = parser.parameters(3); - - ::serviceProviderLocation = parser.option("service", "localhost:1094"); - ::numWorkers = parser.option("num-workers", 1); - ::workerFirst = parser.flag("worker-first"); - ::cancelAfterMs = parser.option("cancel-after", 0); - - } catch (exception const& ex) { - return 1; - } - return ::test(); -} diff --git a/src/xrdsvc/SsiProvider.cc b/src/xrdsvc/SsiProvider.cc index 9aae72f20..f7a068411 100644 --- a/src/xrdsvc/SsiProvider.cc +++ b/src/xrdsvc/SsiProvider.cc @@ -160,19 +160,6 @@ XrdSsiProvider::rStat SsiProviderServer::QueryResource(char const* rName, char c // Tell the caller we do not have the chunk. LOGS(_log, LOG_LVL_DEBUG, "SsiProvider Query " << rName << " absent"); return notPresent; - - } else if (ru.unitType() == ResourceUnit::WORKER) { - // Extract the worker name and alidate it against the one which is - // provided through the inventory - if (not _chunkInventory.id().empty() and _chunkInventory.id() == ru.workerId()) { - LOGS(_log, LOG_LVL_DEBUG, "SsiProvider Query " << rName << " present"); - return isPresent; - } - - // Tell the caller we don't recognize this worker - LOGS(_log, LOG_LVL_DEBUG, "SsiProvider Query " << rName << " absent"); - return notPresent; - } else if (ru.unitType() == ResourceUnit::QUERY) { return isPresent; } @@ -201,12 +188,6 @@ void SsiProviderServer::ResourceAdded(const char* rName) { _chunkInventory.add(ru.db(), ru.chunk()); LOGS(_log, LOG_LVL_DEBUG, "SsiProvider ResourceAdded " << rName); return; - - } else if (ru.unitType() == ResourceUnit::WORKER) { - // Replace the unique identifier of the worker with the new one - _chunkInventory.resetId(ru.workerId()); - LOGS(_log, LOG_LVL_DEBUG, "SsiProvider ResourceAdded " << rName); - return; } LOGS(_log, LOG_LVL_DEBUG, "SsiProvider ResourceAdded " << rName << " invalid"); } @@ -221,13 +202,6 @@ void SsiProviderServer::ResourceRemoved(const char* rName) { _chunkInventory.remove(ru.db(), ru.chunk()); LOGS(_log, LOG_LVL_DEBUG, "SsiProvider ResourceRemoved " << rName); return; - - } else if (ru.unitType() == ResourceUnit::WORKER) { - // Clear the unique identifier of the worker to prevent any incoming - // requests to the worker - _chunkInventory.resetId(""); - LOGS(_log, LOG_LVL_DEBUG, "SsiProvider ResourceRemoved " << rName); - return; } LOGS(_log, LOG_LVL_DEBUG, "SsiProvider ResourceRemoved " << rName << " invalid"); } diff --git a/src/xrdsvc/SsiRequest.cc b/src/xrdsvc/SsiRequest.cc index c2a74d600..3913e6f3b 100644 --- a/src/xrdsvc/SsiRequest.cc +++ b/src/xrdsvc/SsiRequest.cc @@ -51,16 +51,7 @@ #include "wconfig/WorkerConfig.h" #include "wcontrol/Foreman.h" #include "wcontrol/ResourceMonitor.h" -#include "wpublish/AddChunkGroupCommand.h" #include "wpublish/ChunkInventory.h" -#include "wpublish/ChunkListCommand.h" -#include "wpublish/GetChunkListCommand.h" -#include "wpublish/GetConfigCommand.h" -#include "wpublish/GetDbStatusCommand.h" -#include "wpublish/GetStatusCommand.h" -#include "wpublish/RemoveChunkGroupCommand.h" -#include "wpublish/SetChunkListCommand.h" -#include "wpublish/TestEchoCommand.h" #include "xrdsvc/ChannelStream.h" namespace proto = lsst::qserv::proto; @@ -70,21 +61,6 @@ namespace { LOG_LOGGER _log = LOG_GET("lsst.qserv.xrdsvc.SsiRequest"); -/** - * Translate the Protobuf message into the task selector. - */ -wbase::TaskSelector proto2taskSelector(proto::WorkerCommandGetStatusM const& message) { - wbase::TaskSelector selector; - selector.includeTasks = message.include_tasks(); - for (int i = 0, num = message.query_ids_size(); i < num; ++i) { - selector.queryIds.push_back(message.query_ids(i)); - } - for (int i = 0, num = message.task_states_size(); i < num; ++i) { - selector.taskStates.push_back(static_cast(message.task_states(i))); - } - selector.maxTasks = message.max_tasks(); - return selector; -} } // namespace namespace lsst::qserv::xrdsvc { @@ -215,25 +191,6 @@ void SsiRequest::execute(XrdSsiRequest& req) { "Enqueued TaskMsg for " << ru << " in " << t.getElapsed() << " seconds"); break; } - case ResourceUnit::WORKER: { - LOGS(_log, LOG_LVL_DEBUG, "Parsing WorkerCommand for resource=" << _resourceName); - - wbase::WorkerCommand::Ptr const command = parseWorkerCommand(sendChannel, reqData, reqSize); - if (not command) return; - - // The buffer must be released before submitting commands for - // further processing. - ReleaseRequestBuffer(); - _foreman->processCommand(command); // Queues the command to be run later. - - LOGS(_log, LOG_LVL_DEBUG, "Enqueued WorkerCommand for resource=" << _resourceName); - ++countLimiter; - if (countLimiter % 500 == 0) { - LOGS(_log, LOG_LVL_DEBUG, "Forcing instance count to the log"); - util::InstanceCount ic("ForcingPrint_LDB"); // LockupDB - } - break; - } case ResourceUnit::QUERY: { LOGS(_log, LOG_LVL_DEBUG, "Parsing request details for resource=" << _resourceName); proto::QueryManagement request; @@ -300,114 +257,6 @@ void SsiRequest::execute(XrdSsiRequest& req) { // to actually do something once everything is actually setup. } -wbase::WorkerCommand::Ptr SsiRequest::parseWorkerCommand( - std::shared_ptr const& sendChannel, char const* reqData, int reqSize) { - wbase::WorkerCommand::Ptr command; - try { - // reqData has the entire request, so we can unpack it without waiting for - // more data. - proto::FrameBufferView view(reqData, reqSize); - - proto::WorkerCommandH header; - view.parse(header); - - LOGS(_log, LOG_LVL_DEBUG, - "WorkerCommandH: command=" << proto::WorkerCommandH_Command_Name(header.command()) - << " resource=" << _resourceName); - - switch (header.command()) { - case proto::WorkerCommandH::TEST_ECHO: { - proto::WorkerCommandTestEchoM echo; - view.parse(echo); - - command = std::make_shared(sendChannel, echo.value()); - break; - } - case proto::WorkerCommandH::ADD_CHUNK_GROUP: - case proto::WorkerCommandH::REMOVE_CHUNK_GROUP: { - proto::WorkerCommandChunkGroupM group; - view.parse(group); - - std::vector dbs; - for (int i = 0, num = group.dbs_size(); i < num; ++i) dbs.push_back(group.dbs(i)); - - int const chunk = group.chunk(); - bool const force = group.force(); - - if (header.command() == proto::WorkerCommandH::ADD_CHUNK_GROUP) - command = std::make_shared( - sendChannel, _foreman->chunkInventory(), _foreman->mySqlConfig(), chunk, dbs); - else - command = std::make_shared( - sendChannel, _foreman->chunkInventory(), _foreman->resourceMonitor(), - _foreman->mySqlConfig(), chunk, dbs, force); - break; - } - case proto::WorkerCommandH::UPDATE_CHUNK_LIST: { - proto::WorkerCommandUpdateChunkListM message; - view.parse(message); - - if (message.rebuild()) - command = std::make_shared( - sendChannel, _foreman->chunkInventory(), _foreman->mySqlConfig(), - message.reload()); - else - command = std::make_shared( - sendChannel, _foreman->chunkInventory(), _foreman->mySqlConfig()); - break; - } - case proto::WorkerCommandH::GET_CHUNK_LIST: { - command = std::make_shared( - sendChannel, _foreman->chunkInventory(), _foreman->resourceMonitor()); - break; - } - case proto::WorkerCommandH::SET_CHUNK_LIST: { - proto::WorkerCommandSetChunkListM message; - view.parse(message); - - std::vector chunks; - for (int i = 0, num = message.chunks_size(); i < num; ++i) { - chunks.push_back(wpublish::SetChunkListCommand::Chunk{message.chunks(i).db(), - message.chunks(i).chunk()}); - } - std::vector databases; - for (int i = 0, num = message.databases_size(); i < num; ++i) { - databases.push_back(message.databases(i)); - } - bool const force = message.force(); - command = std::make_shared( - sendChannel, _foreman->chunkInventory(), _foreman->resourceMonitor(), - _foreman->mySqlConfig(), chunks, databases, force); - break; - } - case proto::WorkerCommandH::GET_STATUS: { - proto::WorkerCommandGetStatusM message; - view.parse(message); - command = std::make_shared( - sendChannel, _foreman, _foreman->resourceMonitor(), ::proto2taskSelector(message)); - break; - } - case proto::WorkerCommandH::GET_DATABASE_STATUS: { - command = std::make_shared(sendChannel, - _foreman->queriesAndChunks()); - break; - } - case proto::WorkerCommandH::GET_CONFIG: { - command = std::make_shared(sendChannel); - break; - } - default: - reportError("Unsupported command " + proto::WorkerCommandH_Command_Name(header.command()) + - " found in WorkerCommandH on worker resource=" + _resourceName); - break; - } - - } catch (proto::FrameBufferError const& ex) { - reportError("Failed to decode a worker management command, error: " + std::string(ex.what())); - } - return command; -} - /// Called by SSI to free resources. void SsiRequest::Finished(XrdSsiRequest& req, XrdSsiRespInfo const& rinfo, bool cancel) { // Step 8 util::HoldTrack::Mark markA(ERR_LOC, "SsiRequest::Finished start"); diff --git a/src/xrdsvc/SsiRequest.h b/src/xrdsvc/SsiRequest.h index 8fa371af6..68cbe176e 100644 --- a/src/xrdsvc/SsiRequest.h +++ b/src/xrdsvc/SsiRequest.h @@ -35,7 +35,6 @@ // Qserv headers #include "global/ResourceUnit.h" #include "mysql/MySqlConfig.h" -#include "wbase/WorkerCommand.h" #include "xrdsvc/StreamBuffer.h" // Forward declarations @@ -107,18 +106,6 @@ class SsiRequest : public XrdSsiResponder, public std::enable_shared_from_this const& sendChannel, - char const* reqData, int reqSize); - private: ValidatorPtr _validator; ///< validates request against what's available std::shared_ptr const _foreman; ///< actual msg processor