Merge pull request #331 from tsloughter/schema-url

Support for Schema URLs on the Resource and Tracer (Instrumentation Library)

Author: tsloughter

CHANGELOG.md

@@ -25,10 +25,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   - `OTEL_SPAN_LINK_COUNT_LIMIT`: Limit number of Links on a Span.
   - `OTEL_EVENT_ATTRIBUTE_COUNT_LIMIT`: Limit on number of Attributes on an Event.
   - `OTEL_LINK_ATTRIBUTE_COUNT_LIMIT`: Limit on number of Attributes on a Link.
-     
+- Support for a Schema URL in a Resource and in the Instrumentation Library
+  information of a Tracer. For more information about Schema URLs see the
+  specification for [Resources](https://github.com/open-telemetry/opentelemetry-specification/blob/v1.8.0/specification/resource/sdk.md)
+  and [getting a Tracer](https://github.com/open-telemetry/opentelemetry-specification/blob/v1.8.0/specification/trace/api.md).
+- `OTEL_CREATE_APPLICATION_TRACERS` is a new environment variable,
+  `create_application_tracers` is the application environment key, for disabling
+  the automatic creation of Tracers for every Application at boot. The old keys,
+  `OTEL_REGISTER_LOADED_APPLICATIONS` and `register_loaded_applications`, will
+  continue to work as well.
+
 ### Removed
 
 - The `sampler` option to `start_span` and `with_span` was removed.
+- `register_tracer` has been removed and now `get_tracer` will cache the Tracer
+  after creation if needed.
 
 ## [API 1.0.0-rc.3.2] - 2021-10-13
 

apps/opentelemetry/include/otel_resource.hrl

@@ -1,3 +1,6 @@
+%% The schema url for telemetry resources
+-define(SCHEMA_URL, <<"https://opentelemetry.io/schemas/1.8.0">>).
+
 %% if any `service' attribute is defined then
 %% `service.name' and `service.instance.id' are required
 

apps/opentelemetry/src/opentelemetry_app.erl

@@ -38,7 +38,7 @@ start(_StartType, _StartArgs) ->
     %% must be done after the supervisor starts so that otel_tracer_server is running
     %% TODO: make this work with release upgrades. Currently if an application's version
     %% changes the version in the tracer will not be updated.
-    register_loaded_application_tracers(Config),
+    create_loaded_application_tracers(Config),
 
     SupResult.
 
@@ -59,10 +59,10 @@ setup_text_map_propagators(#{text_map_propagators := List}) ->
     CompositePropagator = otel_propagator_text_map_composite:create(List),
     opentelemetry:set_text_map_propagator(CompositePropagator).
 
-register_loaded_application_tracers(#{register_loaded_applications := true}) ->
+create_loaded_application_tracers(#{create_application_tracers := true}) ->
     %% TODO: filter out OTP apps that will not have any instrumentation
     LoadedApplications = application:loaded_applications(),
-    opentelemetry:register_applications(LoadedApplications),
+    opentelemetry:create_application_tracers(LoadedApplications),
     ok;
-register_loaded_application_tracers(_) ->
+create_loaded_application_tracers(_) ->
     ok.

apps/opentelemetry/src/otel_configuration.erl

@@ -28,7 +28,8 @@
 %% required configuration
 %% using a map instead of a record because there can be more values
 -type t() :: #{log_level := atom(),
-               register_loaded_applications := boolean(),
+               register_loaded_applications := boolean() | undefined,
+               create_application_tracers := boolean() | undefined,
                id_generator := module(),
                deny_list := [atom()],
                resource_detectors => [module()],
@@ -51,7 +52,8 @@
 -spec new() -> t().
 new() ->
     #{log_level => info,
-      register_loaded_applications => true,
+      register_loaded_applications => undefined,
+      create_application_tracers => undefined,
       id_generator => otel_id_generator,
       deny_list => [],
       resource_detectors => [otel_resource_env_var,
@@ -93,7 +95,26 @@ span_limits(AppEnv, ConfigMap) ->
 
 -spec general(list(), t()) -> t().
 general(AppEnv, ConfigMap) ->
-    merge_list_with_environment(config_mappings(general_sdk), AppEnv, ConfigMap).
+    Config = merge_list_with_environment(config_mappings(general_sdk), AppEnv, ConfigMap),
+
+    %% merge the old `register_loaded_applications' with the new config key
+    %% `create_application_tracers' that has replaced it
+    Config1 = maps:update_with(create_application_tracers,
+                               fun(undefined) ->
+                                       %% `create_application_tracers' isn't set so update
+                                       %% with the `register_loaded_applications' value
+                                       %% or `true' if it too isn't set
+                                       case maps:get(register_loaded_applications, Config) of
+                                           undefined ->
+                                               true;
+                                           Bool ->
+                                               Bool
+                                       end;
+                                  (Bool) ->
+                                       Bool
+                               end, Config),
+
+    Config1.
 
 -spec sweeper(list(), t()) -> t().
 sweeper(AppEnv, ConfigMap=#{sweeper := DefaultSweeperConfig}) ->
@@ -231,13 +252,17 @@ report_cb(#{source := transform,
 
 config_mappings(general_sdk) ->
     [{"OTEL_LOG_LEVEL", log_level, existing_atom},
+
+     %% `register_loaded_applications' is kept for backwards compatibility
      {"OTEL_REGISTER_LOADED_APPLICATIONS", register_loaded_applications, boolean},
+     {"OTEL_CREATE_APPLICATION_TRACERS", create_application_tracers, boolean},
+
      {"OTEL_ID_GENERATOR", id_generator, existing_atom},
-     {"OTEL_DENY_LIST", deny_list, list},
+     {"OTEL_DENY_LIST", deny_list, existing_atom_list},
      {"OTEL_PROPAGATORS", text_map_propagators, propagators},
      {"OTEL_TRACES_EXPORTER", traces_exporter, exporter},
      {"OTEL_METRICS_EXPORTER", metrics_exporter, exporter},
-     {"OTEL_RESOURCE_DETECTORS", resource_detectors, kvlist_value},
+     {"OTEL_RESOURCE_DETECTORS", resource_detectors, existing_atom_list},
      {"OTEL_RESOURCE_DETECTOR_TIMEOUT", resource_detector_timeout, integer}];
 config_mappings(otel_batch_processor) ->
     [{"OTEL_BSP_SCHEDULE_DELAY_MILLIS", scheduled_delay_ms, integer},
@@ -269,6 +294,19 @@ config_mappings(_) ->
 
 transform(_, undefined) ->
     undefined;
+transform(existing_atom_list, [A | _]=List) when is_atom(A) ->
+    List;
+transform(existing_atom_list, String) when is_list(String) ->
+    List = string:split(String, ",", all),
+    lists:filtermap(fun(A) ->
+                            try transform(existing_atom, string:trim(A)) of
+                                Value ->
+                                    {true, Value}
+                            catch
+                                _:_ ->
+                                    false
+                            end
+                    end, List);
 transform(exporter, "otlp") ->
     {opentelemetry_exporter, #{}};
 transform(exporter, "jaeger") ->
@@ -333,7 +371,7 @@ transform(sampler, {"parentbased_traceidratio", Probability}) ->
 transform(sampler, Value) ->
     Value;
 transform(key_value_list, Value) when is_list(Value) ->
-    case io_lib:printable_list(Value) of
+    case io_lib:printable_unicode_list(Value) of
         true ->
             Pairs = string:split(Value, ",", all),
             lists:filtermap(fun(Pair) ->

apps/opentelemetry/src/otel_resource.erl

@@ -19,23 +19,33 @@
 -module(otel_resource).
 
 -export([create/1,
+         create/2,
          merge/2,
+         schema_url/1,
          attributes/1]).
 
--type key() :: unicode:latin1_chardata().
--type value() :: unicode:latin1_chardata() | integer() | float() | boolean().
--type resource() :: {otel_resource, otel_attributes:t()}.
--opaque t() :: resource().
+-include("otel_resource.hrl").
 
--export_type([t/0]).
+-type key() :: unicode:latin1_binary().
+%% values allowed in attributes of a resource are limited
+-type value() :: unicode:latin1_binary() | integer() | float() | boolean().
+-type schema_url() :: uri_string:uri_string().
 
 -define(MAX_LENGTH, 255).
 
-%% verifies each key and value and drops any that don't pass verification
+-record(resource, {schema_url :: schema_url() | undefined,
+                   attributes :: otel_attributes:t()}).
+-type t() :: #resource{}.
+
 -spec create(#{key() => value()} | [{key(), value()}]) -> t().
-create(Map) when is_map(Map) ->
-    create(maps:to_list(Map));
-create(List) when is_list(List) ->
+create(Attributes) ->
+    create(Attributes, undefined).
+
+%% verifies each key and value and drops any that don't pass verification
+-spec create(#{key() => value()} | [{key(), value()}], schema_url() | undefined) -> t().
+create(Map, SchemaUrl) when is_map(Map) ->
+    create(maps:to_list(Map), SchemaUrl);
+create(List, SchemaUrl) when is_list(List) ->
     List1 = lists:filtermap(fun({K, V}) ->
                                     %% TODO: log an info or debug message when dropping?
                                     case check_key(K) andalso check_value(V) of
@@ -45,20 +55,43 @@ create(List) when is_list(List) ->
                                             false
                                     end
                             end, lists:ukeysort(1, List)),
-    {otel_resource, otel_attributes:new(List1, 128, 255)}.
+    #resource{schema_url=SchemaUrl,
+              attributes=otel_attributes:new(List1, 128, 255)}.
+
+-spec schema_url(t()) -> schema_url() | undefined.
+schema_url(#resource{schema_url=Schema}) ->
+    Schema.
 
 -spec attributes(t()) -> otel_attributes:t().
-attributes({otel_resource, Attributes}) ->
+attributes(#resource{attributes=Attributes}) ->
     Attributes.
 
-%% In case of collision the updating, first argument, resource takes precedence.
+%% in case of collision the updating, first argument, resource takes precedence.
 -spec merge(t(), t()) -> t().
-merge({otel_resource, NewAttributes}, {otel_resource, CurrentAttributes}) ->
+merge(#resource{schema_url=NewSchemaUrl,
+                attributes=NewAttributes}, Current=#resource{schema_url=CurrentSchemaUrl,
+                                                             attributes=CurrentAttributes}) ->
+    SchameUrl = merge_schema_url(NewSchemaUrl, CurrentSchemaUrl),
     NewMap = otel_attributes:map(NewAttributes),
-    {otel_resource, otel_attributes:set(NewMap, CurrentAttributes)}.
+    Current#resource{schema_url=SchameUrl,
+                     attributes=otel_attributes:set(NewMap, CurrentAttributes)}.
 
 %%
 
+%% when merging resources the schemas are checked to verify they match.
+%% if they do match then the schema is set to `undefined' and the attributes are kept.
+merge_schema_url(SchemaUrl, undefined) ->
+    SchemaUrl;
+merge_schema_url(undefined, SchemaUrl) ->
+    SchemaUrl;
+merge_schema_url(NewSchemaUrl, CurrentSchemaUrl) ->
+    merge_schema_url_(uri_string:normalize(NewSchemaUrl), uri_string:normalize(CurrentSchemaUrl)).
+
+merge_schema_url_(SchemaUrl, SchemaUrl) ->
+    SchemaUrl;
+merge_schema_url_(_, _) ->
+    undefined.
+
 %% all resource strings, key or value, must be latin1 with length less than 255
 check_string(S) ->
     string:length(S) =< ?MAX_LENGTH.

apps/opentelemetry/src/otel_resource_app_env.erl

@@ -50,22 +50,27 @@ parse(_) ->
 parse_values(Key, Values) when is_map(Values) ->
     parse_values(Key, maps:to_list(Values));
 parse_values(Key, Values) when is_list(Values) ->
-    lists:flatmap(fun({SubKey, Value=[{_,_}|_]}) ->
-                          %% list of tuples means we have more subkeys
-                          parse_values([Key, ".", to_string(SubKey)], Value);
-                     ({SubKey, Value}) when is_map(Value) ->
-                          %% map value means we have more subkeys
-                          parse_values([Key, ".", to_string(SubKey)], Value);
-                     ({SubKey, Value})->
-                          [{unicode:characters_to_list([Key, ".", to_string(SubKey)]), Value}]
-                  end, Values);
+    case io_lib:printable_unicode_list(Values) of
+        true ->
+            [{unicode:characters_to_binary(Key), unicode:characters_to_binary(Values)}];
+        false ->
+            lists:flatmap(fun({SubKey, Value=[{_,_}|_]}) ->
+                                  %% list of tuples means we have more subkeys
+                                  parse_values([Key, ".", to_string(SubKey)], Value);
+                             ({SubKey, Value}) when is_map(Value) ->
+                                  %% map value means we have more subkeys
+                                  parse_values([Key, ".", to_string(SubKey)], Value);
+                             ({SubKey, Value})->
+                                  [{unicode:characters_to_binary([Key, ".", to_string(SubKey)]), Value}]
+                          end, Values)
+    end;
 parse_values(Key, Value) ->
-    [{unicode:characters_to_list(Key), Value}].
+    [{unicode:characters_to_binary(Key), Value}].
 
--spec to_string(atom() | binary() | list()) -> list().
+-spec to_string(atom() | binary() | list()) -> binary().
 to_string(K) when is_atom(K) ->
-    atom_to_list(K);
-to_string(K) when is_binary(K) ->
-    binary_to_list(K);
+    atom_to_binary(K, utf8);
+to_string(K) when is_list(K) ->
+    list_to_binary(K);
 to_string(K) ->
     K.

apps/opentelemetry/src/otel_resource_detector.erl

@@ -166,9 +166,9 @@ required_attributes(Resource) ->
 process_attributes() ->
     OtpVsn = otp_vsn(),
     ErtsVsn = erts_vsn(),
-    [{?PROCESS_RUNTIME_NAME, emulator()},
-     {?PROCESS_RUNTIME_VERSION, ErtsVsn},
-     {?PROCESS_RUNTIME_DESCRIPTION, runtime_description(OtpVsn, ErtsVsn)}].
+    [{?PROCESS_RUNTIME_NAME, unicode:characters_to_binary(emulator())},
+     {?PROCESS_RUNTIME_VERSION, unicode:characters_to_binary(ErtsVsn)},
+     {?PROCESS_RUNTIME_DESCRIPTION, unicode:characters_to_binary(runtime_description(OtpVsn, ErtsVsn))}].
 
 runtime_description(OtpVsn, ErtsVsn) ->
     io_lib:format("Erlang/OTP ~s erts-~s", [OtpVsn, ErtsVsn]).
@@ -185,7 +185,7 @@ emulator() ->
 prog_name() ->
     %% RELEASE_PROG is set by mix and rebar3 release scripts
     %% PROGNAME is an OS variable set by `erl' and rebar3 release scripts
-    os_or_default("RELEASE_PROG", os_or_default("PROGNAME", "erl")).
+    unicode:characters_to_binary(os_or_default("RELEASE_PROG", os_or_default("PROGNAME", <<"erl">>))).
 
 os_or_default(EnvVar, Default) ->
     case os:getenv(EnvVar) of
@@ -232,7 +232,9 @@ add_service_name(Resource, ProgName) ->
         ServiceName ->
             %% service.name resource first to override any other service.name
             %% attribute that could be set in the resource
-            otel_resource:merge(otel_resource:create([{?SERVICE_NAME, ServiceName}]), Resource)
+            ServiceNameResource = otel_resource:create([{?SERVICE_NAME,
+                                                         unicode:characters_to_binary(ServiceName)}]),
+            otel_resource:merge(ServiceNameResource, Resource)
     end.
 
 service_release_name(ProgName) ->
@@ -244,5 +246,5 @@ service_release_name(ProgName) ->
                                       _ -> [{?SERVICE_VERSION, RelVsn}]
                                   end]);
         _ ->
-            otel_resource:create([{?SERVICE_NAME, "unknown_service:" ++ ProgName}])
+            otel_resource:create([{?SERVICE_NAME, <<"unknown_service:", ProgName/binary>>}])
     end.

apps/opentelemetry/src/otel_tracer_default.erl

@@ -20,8 +20,7 @@
 -behaviour(otel_tracer).
 
 -export([start_span/4,
-         with_span/5
-        ]).
+         with_span/5]).
 
 -include_lib("opentelemetry_api/include/opentelemetry.hrl").
 -include("otel_tracer.hrl").
@@ -53,4 +52,3 @@ with_span(Ctx, Tracer, SpanName, Opts, Fun) ->
         _ = otel_span_ets:end_span(SpanCtx),
         otel_ctx:detach(Ctx)
     end.
-

apps/opentelemetry/src/otel_tracer_server.erl

@@ -93,24 +93,22 @@ init(#{id_generator := IdGeneratorModule,
 
 handle_call(resource, _From, State=#state{resource=Resource}) ->
     {reply, Resource, State};
-handle_call({get_tracer, InstrumentationLibrary}, _From, State=#state{shared_tracer=Tracer,
-                                                                      deny_list=_DenyList}) ->
-    {reply, {Tracer#tracer.module,
-             Tracer#tracer{instrumentation_library=InstrumentationLibrary}}, State};
-handle_call({register_tracer, Name, Vsn}, _From, State=#state{shared_tracer=Tracer,
-                                                              deny_list=DenyList}) ->
+handle_call({get_tracer, Name, Vsn, SchemaUrl}, _From, State=#state{shared_tracer=Tracer,
+                                                                    deny_list=DenyList}) ->
     %% TODO: support semver constraints in denylist
     case proplists:is_defined(Name, DenyList) of
         true ->
-            _ = opentelemetry:set_tracer(Name, {otel_tracer_noop, []}),
-            {reply, ok, State};
+            {reply, {otel_tracer_noop, []}, State};
         false ->
-            InstrumentationLibrary = opentelemetry:instrumentation_library(Name, Vsn),
+            InstrumentationLibrary = opentelemetry:instrumentation_library(Name, Vsn, SchemaUrl),
             TracerTuple = {Tracer#tracer.module,
                            Tracer#tracer{instrumentation_library=InstrumentationLibrary}},
-            _ = opentelemetry:set_tracer(Name, TracerTuple),
-            {reply, ok, State}
+            {reply, TracerTuple, State}
     end;
+handle_call({get_tracer, InstrumentationLibrary}, _From, State=#state{shared_tracer=Tracer,
+                                                                      deny_list=_DenyList}) ->
+    {reply, {Tracer#tracer.module,
+             Tracer#tracer{instrumentation_library=InstrumentationLibrary}}, State};
 handle_call(force_flush, _From, State=#state{processors=Processors}) ->
     Reply = lists:foldl(fun(Processor, Result) ->
                                 case force_flush_(Processor) of