From eafd82d2ad17eed133265d0b899e3156a1063883 Mon Sep 17 00:00:00 2001 From: dentiny Date: Fri, 31 Jan 2025 00:33:11 +0000 Subject: [PATCH 1/2] performance job id Signed-off-by: dentiny --- src/ray/raylet/worker_pool.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/ray/raylet/worker_pool.cc b/src/ray/raylet/worker_pool.cc index 117d0a506615f..72b116f329c40 100644 --- a/src/ray/raylet/worker_pool.cc +++ b/src/ray/raylet/worker_pool.cc @@ -253,6 +253,7 @@ void WorkerPool::RemoveWorkerProcess(State &state, state.worker_processes.erase(proc_startup_token); } +// Intuition is, job id only decides which parameters to use for the executable, reduce a few of them. std::pair, ProcessEnvironment> WorkerPool::BuildProcessCommandArgs(const Language &language, rpc::JobConfig *job_config, @@ -357,6 +358,8 @@ WorkerPool::BuildProcessCommandArgs(const Language &language, worker_command_args.push_back("--worker-launch-time-ms=" + std::to_string(current_sys_time_ms())); worker_command_args.push_back("--node-id=" + node_id_.Hex()); + // worker_command_args.push_back("--runtime-env-hash=" + + // std::to_string(runtime_env_hash)); worker_command_args.push_back("--runtime-env-hash=" + std::to_string(runtime_env_hash)); } else if (language == Language::CPP) { @@ -368,8 +371,8 @@ WorkerPool::BuildProcessCommandArgs(const Language &language, if (serialized_runtime_env_context != "{}" && !serialized_runtime_env_context.empty()) { worker_command_args.push_back("--language=" + Language_Name(language)); - worker_command_args.push_back("--serialized-runtime-env-context=" + - serialized_runtime_env_context); + // worker_command_args.push_back("--serialized-runtime-env-context=" + + // serialized_runtime_env_context); } else if (language == Language::PYTHON && worker_command_args.size() >= 2 && worker_command_args[1].find(kSetupWorkerFilename) != std::string::npos) { // Check that the arg really is the path to the setup worker before erasing it, to From 02bb28da3b849e8c97f6c4d69b8780131167d4b0 Mon Sep 17 00:00:00 2001 From: dentiny Date: Fri, 31 Jan 2025 01:19:17 +0000 Subject: [PATCH 2/2] guard with flag Signed-off-by: dentiny --- src/ray/raylet/worker_pool.cc | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/ray/raylet/worker_pool.cc b/src/ray/raylet/worker_pool.cc index 72b116f329c40..f350b610ca683 100644 --- a/src/ray/raylet/worker_pool.cc +++ b/src/ray/raylet/worker_pool.cc @@ -253,7 +253,8 @@ void WorkerPool::RemoveWorkerProcess(State &state, state.worker_processes.erase(proc_startup_token); } -// Intuition is, job id only decides which parameters to use for the executable, reduce a few of them. +// Intuition is, job id only decides which parameters to use for the executable, reduce a +// few of them. std::pair, ProcessEnvironment> WorkerPool::BuildProcessCommandArgs(const Language &language, rpc::JobConfig *job_config, @@ -358,8 +359,10 @@ WorkerPool::BuildProcessCommandArgs(const Language &language, worker_command_args.push_back("--worker-launch-time-ms=" + std::to_string(current_sys_time_ms())); worker_command_args.push_back("--node-id=" + node_id_.Hex()); - // worker_command_args.push_back("--runtime-env-hash=" + - // std::to_string(runtime_env_hash)); + if (!is_prefetch) { + worker_command_args.push_back("--runtime-env-hash=" + + std::to_string(runtime_env_hash)); + } worker_command_args.push_back("--runtime-env-hash=" + std::to_string(runtime_env_hash)); } else if (language == Language::CPP) { @@ -371,8 +374,10 @@ WorkerPool::BuildProcessCommandArgs(const Language &language, if (serialized_runtime_env_context != "{}" && !serialized_runtime_env_context.empty()) { worker_command_args.push_back("--language=" + Language_Name(language)); - // worker_command_args.push_back("--serialized-runtime-env-context=" + - // serialized_runtime_env_context); + if (!is_prefetch) { + worker_command_args.push_back("--serialized-runtime-env-context=" + + serialized_runtime_env_context); + } } else if (language == Language::PYTHON && worker_command_args.size() >= 2 && worker_command_args[1].find(kSetupWorkerFilename) != std::string::npos) { // Check that the arg really is the path to the setup worker before erasing it, to @@ -387,6 +392,8 @@ WorkerPool::BuildProcessCommandArgs(const Language &language, worker_command_args.push_back("--ray-debugger-external"); } + is_prefetch = false; + ProcessEnvironment env; if (!IsIOWorkerType(worker_type)) { // We pass the job ID to worker processes via an environment variable, so we don't @@ -1485,6 +1492,8 @@ void WorkerPool::PrestartWorkers(const TaskSpecification &task_spec, } } +thread_local bool is_prefetch = false; + void WorkerPool::PrestartWorkersInternal(const TaskSpecification &task_spec, int64_t num_needed) { RAY_LOG(DEBUG) << "PrestartWorkers " << num_needed; @@ -1510,6 +1519,7 @@ void WorkerPool::PrestartWorkersInternal(const TaskSpecification &task_spec, << setup_error_message; return; } + is_prefetch = true; PopWorkerStatus status; StartWorkerProcess(task_spec.GetLanguage(), rpc::WorkerType::WORKER,