[QNN-EP] - Tie HTP power config id lifetime to session (#26457)

### Description

- Remove PerThreadContext (Only used for id management)
- Create ManagedHtpPowerConfigId to manage destruction of id
- Only create one htp power config id per session (previously was one
per thread)



### Motivation and Context
One session can potentially be used on multiple threads for execution,
and there is a set max number of htp power config ids that can be used
at any given time. If enough sessions and enough threads are created,
then the maximum number of power config ids can be reached easily (see
ticket).

Additionally, all power configurations are available on a per-session
basis. Therefore, there is no reason to have more than one power config
id per session.

Removal of PerThreadContext is due to the fact that it was only used to
contain and destroy the power config ids on thread termination. As such,
there is no more need for PerThreadContext.

---------

Co-authored-by: quic_calvnguy <quic_calvnguy@quic_inc.com>

Author: quic-calvnguy

onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc

@@ -1555,6 +1555,61 @@ Status QnnBackendManager::SetRpcPowerConfigs(uint32_t htp_power_config_client_id
   return Status::OK();
 }
 
+Status QnnBackendManager::SetPerThreadHtpPowerConfigs(const std::thread::id& thread_id, bool pre_run) {
+  PerThreadHtpPowerConfigs_t htp_power_configs;
+  if (!GetPerThreadHtpPowerConfigMapping(thread_id, htp_power_configs)) {
+    return Status::OK();
+  }
+
+  auto htp_power_config_id = htp_power_configs.power_config_id;
+  if (pre_run) {
+    if (htp_power_configs.pre_run_perf_mode.has_value()) {
+      ORT_RETURN_IF_ERROR(SetHtpPowerConfig(htp_power_config_id, *htp_power_configs.pre_run_perf_mode));
+    }
+
+    if (htp_power_configs.rpc_configs.has_value()) {
+      ORT_RETURN_IF_ERROR(SetRpcPowerConfigs(htp_power_config_id,
+                                             htp_power_configs.rpc_configs->rpc_control_latency,
+                                             htp_power_configs.rpc_configs->rpc_polling_time));
+    }
+  } else if (htp_power_configs.post_run_perf_mode.has_value()) {
+    ORT_RETURN_IF_ERROR(SetHtpPowerConfig(htp_power_config_id, *htp_power_configs.post_run_perf_mode));
+  }
+
+  return Status::OK();
+}
+
+Status QnnBackendManager::AddPerThreadHtpPowerConfigMapping(const std::thread::id& thread_id,
+                                                            const PerThreadHtpPowerConfigs_t& htp_power_configs) {
+  std::lock_guard<std::mutex> lock(per_thread_power_configs_mutex_);
+
+  auto res = per_thread_power_configs_.find(thread_id);
+  ORT_RETURN_IF(res != per_thread_power_configs_.end(), "Trying to set HtpPowerConfigs for thread id ", thread_id,
+                " but one already exists!");
+
+  per_thread_power_configs_.emplace(thread_id, std::move(htp_power_configs));
+
+  return Status::OK();
+}
+
+bool QnnBackendManager::GetPerThreadHtpPowerConfigMapping(const std::thread::id& thread_id,
+                                                          PerThreadHtpPowerConfigs_t& htp_power_configs) {
+  std::lock_guard<std::mutex> lock(per_thread_power_configs_mutex_);
+
+  auto it = per_thread_power_configs_.find(thread_id);
+  if (it == per_thread_power_configs_.end()) {
+    return false;
+  }
+
+  htp_power_configs = it->second;
+  return true;
+}
+
+void QnnBackendManager::RemovePerThreadHtpPowerConfigMapping(const std::thread::id& thread_id) {
+  std::lock_guard<std::mutex> lock(per_thread_power_configs_mutex_);
+  per_thread_power_configs_.erase(thread_id);
+}
+
 Status QnnBackendManager::DestroyHTPPowerConfigID(uint32_t htp_power_config_id) {
   QnnDevice_Infrastructure_t qnn_device_infra = nullptr;
   auto status = qnn_interface_.deviceGetInfrastructure(&qnn_device_infra);

onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h

@@ -15,6 +15,7 @@
 #include <mutex>
 #include <string>
 #include <string_view>
+#include <thread>
 #include <unordered_map>
 #include <vector>
 
@@ -172,6 +173,13 @@ class QnnBackendManager : public std::enable_shared_from_this<QnnBackendManager>
                             uint32_t rpc_control_latency,
                             uint32_t rpc_polling_time);
 
+  Status SetPerThreadHtpPowerConfigs(const std::thread::id& thread_id, bool pre_run);
+
+  Status AddPerThreadHtpPowerConfigMapping(const std::thread::id& thread_id,
+                                           const PerThreadHtpPowerConfigs_t& htp_power_configs);
+
+  void RemovePerThreadHtpPowerConfigMapping(const std::thread::id& thread_id);
+
   const QNN_INTERFACE_VER_TYPE& GetQnnInterface() { return qnn_interface_; }
 
   const Qnn_ContextHandle_t& GetQnnContext(int index = 0) {
@@ -334,6 +342,9 @@ class QnnBackendManager : public std::enable_shared_from_this<QnnBackendManager>
   // Transfers ownership of `context_handle` (i.e., responsibility of freeing it) to this instance
   Status AddQnnContextHandle(Qnn_ContextHandle_t context_handle);
 
+  bool GetPerThreadHtpPowerConfigMapping(const std::thread::id& thread_id,
+                                         PerThreadHtpPowerConfigs_t& htp_power_configs);
+
  private:
   // assume Qnn_ContextHandle_t is a pointer and able to be wrapped with std::unique_ptr
   static_assert(std::is_pointer_v<Qnn_ContextHandle_t>);
@@ -464,6 +475,10 @@ class QnnBackendManager : public std::enable_shared_from_this<QnnBackendManager>
   uint32_t soc_model_ = QNN_SOC_MODEL_UNKNOWN;
   const std::vector<OpPackage> op_packages_;
   bool skip_qnn_version_check_ = false;
+
+  // Mapping of thread id to on-run-start/end power configs
+  std::mutex per_thread_power_configs_mutex_;
+  std::unordered_map<std::thread::id, PerThreadHtpPowerConfigs_t> per_thread_power_configs_;
 };
 
 }  // namespace qnn

onnxruntime/core/providers/qnn/builder/qnn_def.h

@@ -71,6 +71,19 @@ enum class HtpPerformanceMode : uint8_t {
   kHtpExtremePowerSaver,
 };
 
+typedef struct RpcPowerConfigs {
+  uint32_t rpc_control_latency = 0;
+  uint32_t rpc_polling_time = 0;
+} RpcPowerConfigs_t;
+
+typedef struct PerThreadHtpPowerConfigs {
+  std::optional<HtpPerformanceMode> pre_run_perf_mode;
+  std::optional<HtpPerformanceMode> post_run_perf_mode;
+  std::optional<RpcPowerConfigs_t> rpc_configs;
+
+  uint32_t power_config_id = 0;
+} PerThreadHtpPowerConfigs_t;
+
 enum class ContextPriority : uint8_t {
   LOW = 0,
   NORMAL,

onnxruntime/core/providers/qnn/builder/qnn_model.cc

@@ -6,6 +6,7 @@
 #include <iostream>
 #include <fstream>
 #include <gsl/gsl>
+#include <thread>
 #include "QnnOpDef.h"
 
 #include "core/providers/qnn/builder/op_builder_factory.h"
@@ -346,8 +347,11 @@ Status QnnModel::ExecuteGraph(const Ort::KernelContext& context,
       profiling_info.start_time = qnn::utils::GetTimeStampInUs();
     }
 #endif
-
     auto profile_backend_handle = qnn_backend_manager_->GetQnnProfileHandle();
+
+    auto thread_id = std::this_thread::get_id();
+    ORT_RETURN_IF_ERROR(qnn_backend_manager_->SetPerThreadHtpPowerConfigs(thread_id, true));
+
     execute_status = qnn_interface.graphExecute(graph_info_->Graph(),
                                                 qnn_inputs.data(),
                                                 static_cast<uint32_t>(qnn_inputs.size()),
@@ -363,6 +367,8 @@ Status QnnModel::ExecuteGraph(const Ort::KernelContext& context,
     }
 #endif
 
+    ORT_RETURN_IF_ERROR(qnn_backend_manager_->SetPerThreadHtpPowerConfigs(thread_id, false));
+
     // NOTE: This function returns immediately when profiling is disabled.
     // Extracting profiling data can be expensive, but it is typically only enabled for debugging purposes
     // and not in production. We can improve synchronization for event profiling if it becomes an issue.