Refactor js_generic to use common DynamicJSEndpointRegistry hierarchy, and expose as an extensible public header (#6710)

Author: eddyashton

CHANGELOG.md

@@ -13,6 +13,14 @@ and this project adheres Fto [Semantic Versioning](http://semver.org/spec/v2.0.0
 
 - The function `ccf::get_js_plugins()` and associated FFI plugin system for JS is deprecated. Similar functionality should now be implemented through a `js::Extension` returned from `DynamicJSEndpointRegistry::get_extensions()`.
 
+## [6.0.0-dev11]
+
+[6.0.0-dev11]: https://github.com/microsoft/CCF/releases/tag/6.0.0-dev11
+
+### Added
+
+- Applications can now extend `js_generic` (ie - a JS app where JS endpoints are edited by governance transactions), from the public header `ccf/js/samples/governance_driven_registry.h`. The API for existing JS-programmability apps using `DynamicJSEndpointRegistry` should be unaffected.
+
 ## [6.0.0-dev10]
 
 [6.0.0-dev10]: https://github.com/microsoft/CCF/releases/tag/6.0.0-dev10

CMakeLists.txt

@@ -416,45 +416,9 @@ set(CCF_NETWORK_TEST_ARGS
     ${TEST_LOGGING_LEVEL} --worker-threads ${WORKER_THREADS}
 )
 
-set(JS_GENERIC_SOURCES ${CCF_DIR}/src/apps/js_generic/js_generic_base.cpp)
-if(COMPILE_TARGET STREQUAL "snp")
-  add_library(js_generic_base.snp STATIC ${JS_GENERIC_SOURCES})
-  add_san(js_generic_base.snp)
-  add_warning_checks(js_generic_base.snp)
-  target_link_libraries(js_generic_base.snp PUBLIC ccf.snp)
-  target_compile_options(js_generic_base.snp PRIVATE ${COMPILE_LIBCXX})
-  target_compile_definitions(
-    js_generic_base.snp PUBLIC INSIDE_ENCLAVE VIRTUAL_ENCLAVE
-                               _LIBCPP_HAS_THREAD_API_PTHREAD PLATFORM_SNP
-  )
-  set_property(TARGET js_generic_base.snp PROPERTY POSITION_INDEPENDENT_CODE ON)
-  install(
-    TARGETS js_generic_base.snp
-    EXPORT ccf
-    DESTINATION lib
-  )
-elseif(COMPILE_TARGET STREQUAL "virtual")
-  add_library(js_generic_base.virtual STATIC ${JS_GENERIC_SOURCES})
-  add_san(js_generic_base.virtual)
-  add_warning_checks(js_generic_base.virtual)
-  target_link_libraries(js_generic_base.virtual PUBLIC ccf.virtual)
-  target_compile_options(js_generic_base.virtual PRIVATE ${COMPILE_LIBCXX})
-  set_property(
-    TARGET js_generic_base.virtual PROPERTY POSITION_INDEPENDENT_CODE ON
-  )
-  install(
-    TARGETS js_generic_base.virtual
-    EXPORT ccf
-    DESTINATION lib
-  )
-endif()
 # SNIPPET_START: JS generic application
 add_ccf_app(
-  js_generic
-  SRCS ${CCF_DIR}/src/apps/js_generic/js_generic.cpp
-  LINK_LIBS_ENCLAVE js_generic_base.enclave
-  LINK_LIBS_VIRTUAL js_generic_base.virtual
-  LINK_LIBS_SNP js_generic_base.snp INSTALL_LIBS ON
+  js_generic SRCS ${CCF_DIR}/src/apps/js_generic/js_generic.cpp INSTALL_LIBS ON
 )
 # SNIPPET_END: JS generic application
 

include/ccf/js/registry.h

@@ -19,10 +19,13 @@
 
 namespace ccf::js
 {
+  static constexpr auto default_js_registry_kv_prefix =
+    "public:custom_endpoints";
+
   struct CustomJSEndpoint : public ccf::endpoints::Endpoint
   {};
 
-  // By subclassing DynamicJSEndpointRegistry, an application gains the
+  // By subclassing BaseDynamicJSEndpointRegistry, an application gains the
   // ability to execute custom JavaScript endpoints, and exposes the ability to
   // install them via install_custom_endpoints(). The JavaScript code for these
   // endpoints is stored in the internal KV store under a namespace configured
@@ -31,31 +34,14 @@ namespace ccf::js
   // proposal in governance, and the payload format is currently identical,
   // except the controlling logic resides in the application space.
   //
-  // Known limitations:
-  //
-  // No auditability yet, COSE Sign1 auth is recommended, but the signature is
-  // not stored.
-  // No support for historical endpoints yet.
-  // No support for import from external modules.
-  //
   // Additional functionality compared to set_js_app:
-  //
-  // The KV namespace can be private, to keep the application confidential if
+  // - The KV namespace can be private, to keep the application confidential if
   // desired.
-  class DynamicJSEndpointRegistry : public ccf::UserEndpointRegistry
+  class BaseDynamicJSEndpointRegistry : public ccf::UserEndpointRegistry
   {
   private:
     std::shared_ptr<ccf::js::AbstractInterpreterCache> interpreter_cache =
       nullptr;
-    std::string modules_map;
-    std::string metadata_map;
-    std::string interpreter_flush_map;
-    std::string modules_quickjs_version_map;
-    std::string modules_quickjs_bytecode_map;
-    std::string runtime_options_map;
-    std::string recent_actions_map;
-    std::string audit_input_map;
-    std::string audit_info_map;
 
     ccf::js::NamespaceRestriction namespace_restriction;
 
@@ -75,10 +61,18 @@ namespace ccf::js
       ccf::endpoints::CommandEndpointContext& endpoint_ctx,
       const ccf::TxID& tx_id);
 
+  protected:
+    std::string modules_map;
+    std::string metadata_map;
+    std::string interpreter_flush_map;
+    std::string modules_quickjs_version_map;
+    std::string modules_quickjs_bytecode_map;
+    std::string runtime_options_map;
+
   public:
-    DynamicJSEndpointRegistry(
+    BaseDynamicJSEndpointRegistry(
       ccf::AbstractNodeContext& context,
-      const std::string& kv_prefix = "public:custom_endpoints");
+      const std::string& kv_prefix = default_js_registry_kv_prefix);
 
     /**
      * Call this to populate the KV with JS endpoint definitions, so they can
@@ -133,29 +127,6 @@ namespace ccf::js
      */
     ccf::ApiResult get_js_runtime_options_v1(
       ccf::JSRuntimeOptions& options, ccf::kv::ReadOnlyTx& tx);
-
-    /**
-     * Record action details by storing them in KV maps using a common format,
-     * for the purposes of offline audit using the ledger.
-     */
-    ccf::ApiResult record_action_for_audit_v1(
-      ccf::kv::Tx& tx,
-      ccf::ActionFormat format,
-      const std::string& user_id,
-      const std::string& action_name,
-      const std::vector<uint8_t>& action_body);
-
-    /**
-     * Check an action is not being replayed, by looking it up
-     * in the history of recent actions. To place an upper bound on the history
-     * size, an authenticated timestamp (@p created_at) is required.
-     */
-    ccf::ApiResult check_action_not_replayed_v1(
-      ccf::kv::Tx& tx,
-      uint64_t created_at,
-      const std::span<const uint8_t> action,
-      ccf::InvalidArgsReason& reason);
-
     /// \defgroup Overrides for base EndpointRegistry functions, looking up JS
     /// endpoints before delegating to base implementation.
     ///@{
@@ -172,6 +143,9 @@ namespace ccf::js
       const ccf::TxID& tx_id) override;
 
     void build_api(nlohmann::json& document, ccf::kv::ReadOnlyTx& tx) override;
+
+    std::set<RESTVerb> get_allowed_verbs(
+      ccf::kv::Tx&, const ccf::RpcContext& rpc_ctx) override;
     ///@}
 
     virtual ccf::js::extensions::Extensions get_extensions(
@@ -180,4 +154,48 @@ namespace ccf::js
       return {};
     };
   };
+
+  // Extends BaseDynamicJSEndpointRegistry with methods for making actions
+  // auditable and preventing replay. These should be used if apps are not
+  // deployed through governance, to ensure that app-modification is safely and
+  // clearly tracked in the ledger history
+  class DynamicJSEndpointRegistry : public BaseDynamicJSEndpointRegistry
+  {
+  protected:
+    std::string recent_actions_map;
+    std::string audit_input_map;
+    std::string audit_info_map;
+
+  public:
+    DynamicJSEndpointRegistry(
+      ccf::AbstractNodeContext& context,
+      const std::string& kv_prefix = default_js_registry_kv_prefix) :
+      BaseDynamicJSEndpointRegistry(context, kv_prefix),
+      recent_actions_map(fmt::format("{}.recent_actions", kv_prefix)),
+      audit_input_map(fmt::format("{}.audit.input", kv_prefix)),
+      audit_info_map(fmt::format("{}.audit.info", kv_prefix))
+    {}
+
+    /**
+     * Record action details by storing them in KV maps using a common format,
+     * for the purposes of offline audit using the ledger.
+     */
+    ccf::ApiResult record_action_for_audit_v1(
+      ccf::kv::Tx& tx,
+      ccf::ActionFormat format,
+      const std::string& user_id,
+      const std::string& action_name,
+      const std::vector<uint8_t>& action_body);
+
+    /**
+     * Check an action is not being replayed, by looking it up
+     * in the history of recent actions. To place an upper bound on the history
+     * size, an authenticated timestamp (@p created_at) is required.
+     */
+    ccf::ApiResult check_action_not_replayed_v1(
+      ccf::kv::Tx& tx,
+      uint64_t created_at,
+      const std::span<const uint8_t> action,
+      ccf::InvalidArgsReason& reason);
+  };
 }

include/ccf/js/samples/governance_driven_registry.h

@@ -0,0 +1,36 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the Apache 2.0 License.
+#pragma once
+
+#include "ccf/js/registry.h"
+#include "ccf/service/tables/jsengine.h"
+#include "ccf/service/tables/modules.h"
+
+namespace ccf::js
+{
+  // This sample extends the generic BaseDynamicJSEndpointRegistry to read JS
+  // endpoints (code, metadata, options) from governance tables. Specifically,
+  // tables populated by actions in the default sample CCF constitution
+  // (set_js_app). This can be sub-classed to modify the dispatch or execution
+  // behaviour, or to provide further JS extension APIs via get_extensions().
+  //
+  // An application running this registry with no further extensions is shipped
+  // with the CCF releases as `js_generic`.
+  class GovernanceDrivenJSRegistry
+    : public ccf::js::BaseDynamicJSEndpointRegistry
+  {
+  public:
+    GovernanceDrivenJSRegistry(AbstractNodeContext& context) :
+      // Note: We do not pass a kv_prefix here, instead we explicitly, manually
+      // construct each map name to match previously used values
+      ccf::js::BaseDynamicJSEndpointRegistry(context)
+    {
+      modules_map = ccf::Tables::MODULES;
+      metadata_map = ccf::endpoints::Tables::ENDPOINTS;
+      interpreter_flush_map = ccf::Tables::INTERPRETER_FLUSH;
+      modules_quickjs_version_map = ccf::Tables::MODULES_QUICKJS_VERSION;
+      modules_quickjs_bytecode_map = ccf::Tables::MODULES_QUICKJS_BYTECODE;
+      runtime_options_map = ccf::Tables::JSENGINE;
+    }
+  };
+} // namespace ccf::js

src/apps/js_generic/js_generic.cpp

@@ -1,13 +1,15 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the Apache 2.0 License.
+
 #include "ccf/app_interface.h"
-#include "js_generic_base.h"
+#include "ccf/js/samples/governance_driven_registry.h"
 
 namespace ccf
 {
   std::unique_ptr<ccf::endpoints::EndpointRegistry> make_user_endpoints(
     ccf::AbstractNodeContext& context)
   {
-    return make_user_endpoints_impl(context);
+    return std::make_unique<ccf::js::GovernanceDrivenJSRegistry>(context);
   }
+
 } // namespace ccf