diff --git a/.prototools b/.prototools index bbd26d490..fd4c9c10a 100644 --- a/.prototools +++ b/.prototools @@ -1,4 +1,5 @@ [plugins] +# moon-test = "file://./crates/cli/tests/fixtures/moon-schema.yaml" moon-test = "https://raw.githubusercontent.com/moonrepo/moon/master/proto-plugin.toml" wasm-test = "file://./plugins/target/wasm32-wasi/debug/proto_wasm_test.wasm" diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ef43b855..959c18698 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,13 @@ - [Node](https://github.com/moonrepo/tools/blob/master/tools/node/CHANGELOG.md) - [Python](https://github.com/moonrepo/tools/blob/master/tools/python/CHANGELOG.md) - [Rust](https://github.com/moonrepo/tools/blob/master/tools/rust/CHANGELOG.md) -- [TOML schema](https://github.com/moonrepo/tools/blob/master/tools/internal-schema/CHANGELOG.md) +- [Schema (TOML, JSON, YAML)](https://github.com/moonrepo/tools/blob/master/tools/internal-schema/CHANGELOG.md) + +## Unreleased + +#### 🚀 Updates + +- Added support for JSON and YAML based configurations for non-WASM schema based plugins. This is an alternative to TOML, but supports all the same settings. ## 0.41.5 @@ -28,7 +34,7 @@ #### 🧩 Plugins -- Updated `internal_schema_plugin` (TOML) to v0.15.1. +- Updated `schema_tool` (TOML) to v0.15.1. - Added `{versionMajor}`, `{versionMajorMinor}`, `{versionYear}`, `{versionYearMonth}`, `{versionPrerelease}`, and `{versionBuild}` tokens. #### ⚙️ Internal diff --git a/Cargo.lock b/Cargo.lock index f1cefb109..819cfc0b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2345,6 +2345,7 @@ name = "proto_core" version = "0.41.5" dependencies = [ "clap", + "convert_case", "indexmap", "miette", "minisign-verify", @@ -2423,7 +2424,6 @@ dependencies = [ "serde", "serde_json", "starbase_sandbox", - "toml", "warpgate", ] @@ -3025,6 +3025,19 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "sha2" version = "0.10.8" @@ -3250,9 +3263,11 @@ dependencies = [ "fs4", "json-strip-comments", "miette", + "regex", "reqwest", "serde", "serde_json", + "serde_yaml", "starbase_styles", "thiserror", "tokio", @@ -3731,6 +3746,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.9.0" diff --git a/crates/cli/src/commands/plugin/search.rs b/crates/cli/src/commands/plugin/search.rs index 216cb6aca..3ebc78696 100644 --- a/crates/cli/src/commands/plugin/search.rs +++ b/crates/cli/src/commands/plugin/search.rs @@ -71,8 +71,10 @@ pub async fn search(session: ProtoSession, args: SearchPluginArgs) -> AppResult PluginAuthor::Object(author) => &author.name, }), Cell::new(match plugin.format { + PluginFormat::Json => "JSON", PluginFormat::Toml => "TOML", PluginFormat::Wasm => "WASM", + PluginFormat::Yaml => "YAML", }), Cell::new(&plugin.description), Cell::new(plugin.locator.to_string()) diff --git a/crates/cli/tests/fixtures/moon-schema.json b/crates/cli/tests/fixtures/moon-schema.json new file mode 100644 index 000000000..6bbc71bc3 --- /dev/null +++ b/crates/cli/tests/fixtures/moon-schema.json @@ -0,0 +1,23 @@ +{ + "bin": "moon-test", + "name": "moon-test", + "type": "cli", + "platform": { + "linux": { + "downloadFile": "moon-{arch}-unknown-linux-{libc}" + }, + "macos": { + "downloadFile": "moon-{arch}-apple-darwin" + }, + "windows": { + "downloadFile": "moon-{arch}-pc-windows-msvc.exe" + } + }, + "install": { + "downloadUrl": "https://github.com/moonrepo/moon/releases/download/v{version}/{download_file}", + "unpack": false + }, + "resolve": { + "gitUrl": "https://github.com/moonrepo/moon" + } +} diff --git a/crates/cli/tests/fixtures/moon-schema.yaml b/crates/cli/tests/fixtures/moon-schema.yaml new file mode 100644 index 000000000..fc5185ed4 --- /dev/null +++ b/crates/cli/tests/fixtures/moon-schema.yaml @@ -0,0 +1,18 @@ +bin: "moon-test" +name: "moon-test" +type: "cli" + +platform: + linux: + downloadFile: "moon-{arch}-unknown-linux-{libc}" + macos: + downloadFile: "moon-{arch}-apple-darwin" + windows: + downloadFile: "moon-{arch}-pc-windows-msvc.exe" + +install: + downloadUrl: "https://github.com/moonrepo/moon/releases/download/v{version}/{download_file}" + unpack: false + +resolve: + gitUrl: "https://github.com/moonrepo/moon" diff --git a/crates/cli/tests/plugins_test.rs b/crates/cli/tests/plugins_test.rs index 66856868b..909875af6 100644 --- a/crates/cli/tests/plugins_test.rs +++ b/crates/cli/tests/plugins_test.rs @@ -57,7 +57,7 @@ mod plugins { use super::*; #[tokio::test(flavor = "multi_thread")] - async fn downloads_and_installs_plugin_from_file() { + async fn downloads_and_installs_toml_plugin_from_file() { run_tests(|env| { let root_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); @@ -73,6 +73,40 @@ mod plugins { .await; } + #[tokio::test(flavor = "multi_thread")] + async fn downloads_and_installs_json_plugin_from_file() { + run_tests(|env| { + let root_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + + load_tool_from_locator( + Id::raw("moon"), + env.to_owned(), + PluginLocator::File(Box::new(FileLocator { + file: "./tests/fixtures/moon-schema.json".into(), + path: Some(root_dir.join("./tests/fixtures/moon-schema.json")), + })), + ) + }) + .await; + } + + #[tokio::test(flavor = "multi_thread")] + async fn downloads_and_installs_yaml_plugin_from_file() { + run_tests(|env| { + let root_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + + load_tool_from_locator( + Id::raw("moon"), + env.to_owned(), + PluginLocator::File(Box::new(FileLocator { + file: "./tests/fixtures/moon-schema.yaml".into(), + path: Some(root_dir.join("./tests/fixtures/moon-schema.yaml")), + })), + ) + }) + .await; + } + #[tokio::test] #[should_panic(expected = "does not exist")] async fn errors_for_missing_file() { @@ -333,7 +367,33 @@ mod plugins { #[test] fn supports_toml_schema() { - let sandbox = create_empty_proto_sandbox_with_tools(); + let sandbox = create_empty_proto_sandbox_with_tools("toml"); + + sandbox + .run_bin(|cmd| { + cmd.arg("install").arg("moon-test"); + }) + .success(); + + // Doesn't create shims + } + + #[test] + fn supports_json_schema() { + let sandbox = create_empty_proto_sandbox_with_tools("json"); + + sandbox + .run_bin(|cmd| { + cmd.arg("install").arg("moon-test"); + }) + .success(); + + // Doesn't create shims + } + + #[test] + fn supports_yaml_schema() { + let sandbox = create_empty_proto_sandbox_with_tools("yaml"); sandbox .run_bin(|cmd| { diff --git a/crates/cli/tests/utils.rs b/crates/cli/tests/utils.rs index 7ccb83a26..837e338f0 100644 --- a/crates/cli/tests/utils.rs +++ b/crates/cli/tests/utils.rs @@ -59,17 +59,23 @@ pub fn create_empty_proto_sandbox() -> ProtoSandbox { ProtoSandbox::new(starbase_sandbox::create_empty_sandbox()) } -pub fn create_empty_proto_sandbox_with_tools() -> ProtoSandbox { +pub fn create_empty_proto_sandbox_with_tools(ext: &str) -> ProtoSandbox { let sandbox = create_empty_proto_sandbox(); + let schema_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("./tests/fixtures") + .join(format!("moon-schema.{ext}")); sandbox.create_file( ".prototools", - r#" + format!( + r#" moon-test = "1.0.0" [plugins] -moon-test = "https://raw.githubusercontent.com/moonrepo/moon/master/proto-plugin.toml" +moon-test = "file://{}" "#, + schema_path.to_string_lossy().replace("\\", "/") + ), ); sandbox diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index ca0ba0b4d..55bd3282c 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -19,6 +19,7 @@ warpgate = { version = "0.18.0", path = "../warpgate", features = [ "schematic", ] } clap = { workspace = true, optional = true } +convert_case = "0.6.0" indexmap = { workspace = true } miette = { workspace = true } minisign-verify = "0.2.2" @@ -41,7 +42,7 @@ sha2 = { workspace = true } shell-words = { workspace = true } starbase_archive = { workspace = true } starbase_styles = { workspace = true } -starbase_utils = { workspace = true, features = ["fs-lock"] } +starbase_utils = { workspace = true, features = ["fs-lock", "yaml"] } thiserror = { workspace = true } tracing = { workspace = true } url = { version = "2.5.2", features = ["serde"] } diff --git a/crates/core/src/registry/data.rs b/crates/core/src/registry/data.rs index 18afb3925..e3ea08715 100644 --- a/crates/core/src/registry/data.rs +++ b/crates/core/src/registry/data.rs @@ -7,8 +7,10 @@ use warpgate::{Id, PluginLocator}; #[derive(Debug, Deserialize, Serialize, Schematic)] #[serde(rename_all = "lowercase")] pub enum PluginFormat { + Json, Toml, Wasm, + Yaml, } /// Information about a person. diff --git a/crates/core/src/tool_loader.rs b/crates/core/src/tool_loader.rs index 48b6b304f..3ab4fecfb 100644 --- a/crates/core/src/tool_loader.rs +++ b/crates/core/src/tool_loader.rs @@ -2,9 +2,10 @@ use crate::error::ProtoError; use crate::proto::ProtoEnvironment; use crate::proto_config::SCHEMA_PLUGIN_KEY; use crate::tool::Tool; -use starbase_utils::{json, toml}; +use convert_case::{Case, Casing}; +use starbase_utils::{json, toml, yaml}; use std::fmt::Debug; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use tracing::{debug, instrument, trace}; use warpgate::{inject_default_manifest_config, Id, PluginLocator, PluginManifest, Wasm}; @@ -82,6 +83,61 @@ pub async fn load_schema_plugin_with_proto( .await } +pub fn load_schema_config(plugin_path: &Path) -> miette::Result { + let mut is_toml = false; + let mut schema: json::JsonValue = match plugin_path + .extension() + .and_then(|ext| ext.to_str()) + .unwrap() + { + "toml" => { + is_toml = true; + toml::read_file(plugin_path)? + } + "json" | "jsonc" => json::read_file(plugin_path)?, + "yaml" | "yml" => yaml::read_file(plugin_path)?, + _ => unimplemented!(), + }; + + // Convert object keys to kebab-case since the original + // configuration format was based on TOML + fn convert_config(config: &mut json::JsonValue, is_toml: bool) { + match config { + json::JsonValue::Array(array) => { + for item in array { + convert_config(item, is_toml); + } + } + json::JsonValue::Object(object) => { + let mut map = json::JsonMap::default(); + + for (key, value) in object.iter_mut() { + convert_config(value, is_toml); + + map.insert( + if is_toml { + key.to_owned() + } else { + key.from_case(Case::Camel).to_case(Case::Kebab) + }, + value.to_owned(), + ); + } + + // serde_json doesn't allow mutating keys in place, + // so we need to rebuild the entire map... + object.clear(); + object.extend(map); + } + _ => {} + } + } + + convert_config(&mut schema, is_toml); + + Ok(schema) +} + #[instrument(name = "load_tool", skip(proto))] pub async fn load_tool_from_locator( id: impl AsRef + Debug, @@ -93,31 +149,34 @@ pub async fn load_tool_from_locator( let locator = locator.as_ref(); let plugin_path = proto.get_plugin_loader()?.load_plugin(id, locator).await?; + let plugin_ext = plugin_path + .extension() + .and_then(|ext| ext.to_str()) + .map(|ext| ext.to_owned()); - // If a TOML plugin, we need to load the WASM plugin for it, - // wrap it, and modify the plugin manifest. - let mut manifest = if plugin_path.extension().is_some_and(|ext| ext == "toml") { - debug!(source = ?plugin_path, "Loading TOML plugin"); - - let mut manifest = Tool::create_plugin_manifest( - proto, - Wasm::file(load_schema_plugin_with_proto(proto).await?), - )?; + let mut manifest = match plugin_ext.as_deref() { + Some("wasm") => { + debug!(source = ?plugin_path, "Loading WASM plugin"); - // Convert TOML to JSON - let schema: json::JsonValue = toml::read_file(plugin_path)?; - let schema = json::format(&schema, false)?; + Tool::create_plugin_manifest(proto, Wasm::file(plugin_path))? + } + Some("toml" | "json" | "jsonc" | "yaml" | "yml") => { + debug!(format = plugin_ext, source = ?plugin_path, "Loading non-WASM plugin"); - trace!(schema = %schema, "Storing schema settings"); + let mut manifest = Tool::create_plugin_manifest( + proto, + Wasm::file(load_schema_plugin_with_proto(proto).await?), + )?; - manifest.config.insert("proto_schema".to_string(), schema); - manifest + let schema = json::format(&load_schema_config(&plugin_path)?, false)?; - // Otherwise, just use the WASM plugin as is - } else { - debug!(source = ?plugin_path, "Loading WASM plugin"); + trace!(schema = %schema, "Storing schema settings"); - Tool::create_plugin_manifest(proto, Wasm::file(plugin_path))? + manifest.config.insert("proto_schema".to_string(), schema); + manifest + } + // This case is handled by warpgate when loading the plugin + _ => unimplemented!(), }; inject_default_manifest_config(id, &proto.home, &mut manifest)?; diff --git a/crates/pdk-test-utils/Cargo.toml b/crates/pdk-test-utils/Cargo.toml index 1a52a1b33..b9ab39c27 100644 --- a/crates/pdk-test-utils/Cargo.toml +++ b/crates/pdk-test-utils/Cargo.toml @@ -14,8 +14,7 @@ warpgate = { version = "0.18.0", path = "../warpgate" } serde = { workspace = true } serde_json = { workspace = true } starbase_sandbox = { workspace = true } -toml = { version = "0.8.19", optional = true } [features] default = [] -schema = ["dep:toml"] +schema = [] diff --git a/crates/pdk-test-utils/src/config_builder.rs b/crates/pdk-test-utils/src/config_builder.rs index ab6c9325d..bf3e14606 100644 --- a/crates/pdk-test-utils/src/config_builder.rs +++ b/crates/pdk-test-utils/src/config_builder.rs @@ -2,12 +2,12 @@ use serde::Serialize; use warpgate::test_utils::ConfigBuilder; pub trait ProtoConfigBuilder { - fn toml_schema(&mut self, schema: serde_json::Value) -> &mut Self; + fn schema_config(&mut self, schema: serde_json::Value) -> &mut Self; fn tool_config(&mut self, config: impl Serialize) -> &mut Self; } impl ProtoConfigBuilder for ConfigBuilder { - fn toml_schema(&mut self, schema: serde_json::Value) -> &mut Self { + fn schema_config(&mut self, schema: serde_json::Value) -> &mut Self { self.insert("proto_schema", schema) } diff --git a/crates/pdk-test-utils/src/sandbox.rs b/crates/pdk-test-utils/src/sandbox.rs index 9ce2038c1..7244b9454 100644 --- a/crates/pdk-test-utils/src/sandbox.rs +++ b/crates/pdk-test-utils/src/sandbox.rs @@ -125,10 +125,7 @@ impl ProtoWasmSandbox { { use crate::config_builder::ProtoConfigBuilder; - let schema = fs::read_to_string(&schema_path).unwrap(); - let schema: serde_json::Value = toml::from_str(&schema).unwrap(); - - config.toml_schema(schema); + config.schema_config(proto_core::load_schema_config(&schema_path).unwrap()); } }) .await diff --git a/plugins/Cargo.lock b/plugins/Cargo.lock index 4c2126902..cf6e92064 100644 --- a/plugins/Cargo.lock +++ b/plugins/Cargo.lock @@ -2003,6 +2003,7 @@ dependencies = [ name = "proto_core" version = "0.41.5" dependencies = [ + "convert_case", "indexmap", "miette", "minisign-verify", @@ -2656,6 +2657,19 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "sha2" version = "0.10.8" @@ -2826,9 +2840,11 @@ dependencies = [ "fs4", "json-strip-comments", "miette", + "regex", "reqwest", "serde", "serde_json", + "serde_yaml", "starbase_styles", "thiserror", "tokio", @@ -3219,6 +3235,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.9.0"