diff --git a/lib/datadog/appsec/api_security/endpoint_collection/rails_routes_serializer.rb b/lib/datadog/appsec/api_security/endpoint_collection/rails_routes_serializer.rb new file mode 100644 index 00000000000..67e5fb8f8f0 --- /dev/null +++ b/lib/datadog/appsec/api_security/endpoint_collection/rails_routes_serializer.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Datadog + module AppSec + module APISecurity + module EndpointCollection + # This module serializes Rails Journey Router routes. + class RailsRoutesSerializer + FORMAT_SUFFIX = "(.:format)" + + def initialize(routes) + @routes = routes + end + + def to_enum + Enumerator.new do |yielder| + @routes.each do |route| + next unless route.dispatcher? + + yielder.yield serialize_route(route) + end + end + end + + private + + def serialize_route(route) + method = route.verb.empty? ? "*" : route.verb + path = route.path.spec.to_s.delete_suffix(FORMAT_SUFFIX) + + { + type: "REST", + resource_name: "#{method} #{path}", + operation_name: "http.request", + method: method, + path: path + } + end + end + end + end + end +end diff --git a/lib/datadog/appsec/configuration/settings.rb b/lib/datadog/appsec/configuration/settings.rb index 571ef2f1005..b22f172eb39 100644 --- a/lib/datadog/appsec/configuration/settings.rb +++ b/lib/datadog/appsec/configuration/settings.rb @@ -352,6 +352,15 @@ def self.add_settings!(base) o.default true end + settings :endpoint_collection do + # Enables reporting of application routes at application start via telemetry + option :enabled do |o| + o.type :bool, nilable: true + o.env 'DD_API_SECURITY_ENDPOINT_COLLECTION_ENABLED' + o.default false + end + end + # NOTE: Unfortunately, we have to go with Float due to other libs # setup, even tho we don't plan to support sub-second delays. # diff --git a/lib/datadog/appsec/contrib/rails/patcher.rb b/lib/datadog/appsec/contrib/rails/patcher.rb index ea73fb26981..63947c96994 100644 --- a/lib/datadog/appsec/contrib/rails/patcher.rb +++ b/lib/datadog/appsec/contrib/rails/patcher.rb @@ -10,6 +10,7 @@ require_relative 'gateway/request' require_relative 'patches/render_to_body_patch' require_relative 'patches/process_action_patch' +require_relative '../../api_security/endpoint_collection/rails_routes_serializer' require_relative '../../../tracing/contrib/rack/middlewares' @@ -20,6 +21,7 @@ module Rails # Patcher for AppSec on Rails module Patcher GUARD_ACTION_CONTROLLER_ONCE_PER_APP = Hash.new { |h, key| h[key] = Datadog::Core::Utils::OnlyOnce.new } + GUARD_ROUTES_REPORTING_ONCE_PER_APP = Hash.new { |h, key| h[key] = Datadog::Core::Utils::OnlyOnce.new } BEFORE_INITIALIZE_ONLY_ONCE_PER_APP = Hash.new { |h, key| h[key] = Datadog::Core::Utils::OnlyOnce.new } AFTER_INITIALIZE_ONLY_ONCE_PER_APP = Hash.new { |h, key| h[key] = Datadog::Core::Utils::OnlyOnce.new } @@ -38,6 +40,7 @@ def patch patch_before_initialize patch_after_initialize patch_action_controller + subscribe_to_routes_loaded Patcher.instance_variable_set(:@patched, true) end @@ -128,6 +131,31 @@ def patch_action_controller GUARD_ACTION_CONTROLLER_ONCE_PER_APP[self].run do ::ActionController::Base.prepend(Patches::RenderToBodyPatch) end + + # Rails 7.1 adds `after_routes_loaded` hook + if Datadog::AppSec::Contrib::Rails::Patcher.target_version < Gem::Version.new('7.1') + Datadog::AppSec::Contrib::Rails::Patcher.report_routes_via_telemetry(::Rails.application.routes.routes) + end + end + end + + def subscribe_to_routes_loaded + ::ActiveSupport.on_load(:after_routes_loaded) do |app| + Datadog::AppSec::Contrib::Rails::Patcher.report_routes_via_telemetry(app.routes.routes) + end + end + + def report_routes_via_telemetry(routes) + # We do not support Rails 4.x for Endpoint Collection, + # mainly because the Route#verb was a Regexp before Rails 5.0 + return if target_version < Gem::Version.new('5.0') + + return unless Datadog.configuration.appsec.api_security.endpoint_collection.enabled + + GUARD_ROUTES_REPORTING_ONCE_PER_APP[self].run do + AppSec.telemetry.app_endpoints_loaded( + APISecurity::EndpointCollection::RailsRoutesSerializer.new(routes).to_enum + ) end end diff --git a/lib/datadog/core/configuration/supported_configurations.rb b/lib/datadog/core/configuration/supported_configurations.rb index 054847a86c7..a2159507761 100644 --- a/lib/datadog/core/configuration/supported_configurations.rb +++ b/lib/datadog/core/configuration/supported_configurations.rb @@ -10,6 +10,7 @@ module Configuration {"DD_AGENT_HOST" => {version: ["A"]}, "DD_API_KEY" => {version: ["A"]}, "DD_API_SECURITY_ENABLED" => {version: ["A"]}, + "DD_API_SECURITY_ENDPOINT_COLLECTION_ENABLED" => {version: ["A"]}, "DD_API_SECURITY_REQUEST_SAMPLE_RATE" => {version: ["A"]}, "DD_API_SECURITY_SAMPLE_DELAY" => {version: ["A"]}, "DD_APM_TRACING_ENABLED" => {version: ["A"]}, diff --git a/sig/datadog/appsec/api_security/endpoint_collection/rails_routes_serializer.rbs b/sig/datadog/appsec/api_security/endpoint_collection/rails_routes_serializer.rbs new file mode 100644 index 00000000000..5fffbf903e5 --- /dev/null +++ b/sig/datadog/appsec/api_security/endpoint_collection/rails_routes_serializer.rbs @@ -0,0 +1,33 @@ +module Datadog + module AppSec + module APISecurity + module EndpointCollection + interface _Route + def dispatcher?: () -> bool + def verb: () -> String + def path: () -> _RoutePath + end + + interface _RoutePath + def spec: () -> _RouteSpec + end + + interface _RouteSpec + def to_s: () -> String + end + + class RailsRoutesSerializer + FORMAT_SUFFIX: String + + @routes: Array[_Route] + + def initialize: (Array[_Route] routes) -> void + + def to_enum: () -> Enumerator[Core::Telemetry::Event::AppEndpointsLoaded::endpoint] + + def serialize_route: (_Route route) -> Core::Telemetry::Event::AppEndpointsLoaded::endpoint + end + end + end + end +end diff --git a/sig/datadog/appsec/contrib/rails/patcher.rbs b/sig/datadog/appsec/contrib/rails/patcher.rbs index 095d037d1a2..9d93945aa74 100644 --- a/sig/datadog/appsec/contrib/rails/patcher.rbs +++ b/sig/datadog/appsec/contrib/rails/patcher.rbs @@ -27,6 +27,10 @@ module Datadog def self?.after_initialize: (untyped app) -> untyped + def self.subscribe_to_routes_loaded: () -> void + + def self.report_routes_via_telemetry: () -> void + def self?.setup_security: () -> untyped end end diff --git a/spec/datadog/appsec/api_security/endpoint_collection/rails_routes_serializer_spec.rb b/spec/datadog/appsec/api_security/endpoint_collection/rails_routes_serializer_spec.rb new file mode 100644 index 00000000000..a9cbdd7169c --- /dev/null +++ b/spec/datadog/appsec/api_security/endpoint_collection/rails_routes_serializer_spec.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'datadog/appsec/api_security/endpoint_collection/rails_routes_serializer' + +RSpec.describe Datadog::AppSec::APISecurity::EndpointCollection::RailsRoutesSerializer do + describe '#to_enum' do + it 'returns an Enumerator' do + expect(described_class.new([]).to_enum).to be_a(Enumerator) + end + + it 'correctly serializes routes' do + routes = described_class.new([ + build_route_double(method: 'GET', path: '/events') + ]).to_enum + + expect(routes.count).to eq(1) + + aggregate_failures 'serialized attributes' do + expect(routes.first.fetch(:type)).to eq('REST') + expect(routes.first.fetch(:resource_name)).to eq('GET /events') + expect(routes.first.fetch(:operation_name)).to eq('http.request') + expect(routes.first.fetch(:method)).to eq('GET') + expect(routes.first.fetch(:path)).to eq('/events') + end + end + + it 'removes rails format suffix from the path' do + routes = described_class.new([ + build_route_double(method: 'GET', path: '/events(.:format)') + ]).to_enum + + aggregate_failures 'path attributes' do + expect(routes.first.fetch(:resource_name)).to eq('GET /events') + expect(routes.first.fetch(:path)).to eq('/events') + end + end + + it 'sets method to * for wildcard routes' do + routes = described_class.new([ + build_route_double(method: '*', path: '/') + ]).to_enum + + aggregate_failures 'path attributes' do + expect(routes.first.fetch(:resource_name)).to eq('* /') + expect(routes.first.fetch(:method)).to eq('*') + end + end + + it 'skips non-dispatcher routes for now' do + routes = described_class.new([ + build_route_double(method: nil, path: 'admin', is_dispatcher: false) + ]).to_enum + + expect(routes.to_a).to be_empty + end + end + + def build_route_double(method:, path:, is_dispatcher: true) + instance_double( + 'ActionDispatch::Journey::Route', + dispatcher?: is_dispatcher, + verb: method, + path: instance_double( + 'ActionDispatch::Journey::Path::Pattern', + spec: path + ) + ) + end +end diff --git a/spec/datadog/appsec/configuration/settings_spec.rb b/spec/datadog/appsec/configuration/settings_spec.rb index 04a68713cc9..f1ffb5f399e 100644 --- a/spec/datadog/appsec/configuration/settings_spec.rb +++ b/spec/datadog/appsec/configuration/settings_spec.rb @@ -1015,6 +1015,44 @@ def patcher end end end + + describe 'endpoint_collection' do + describe '#enabled' do + context 'when DD_API_SECURITY_ENDPOINT_COLLECTION_ENABLED is undefined' do + around do |example| + ClimateControl.modify('DD_API_SECURITY_ENDPOINT_COLLECTION_ENABLED' => nil) { example.run } + end + + it { expect(settings.appsec.api_security.endpoint_collection.enabled).to eq(false) } + end + + context 'when DD_API_SECURITY_ENDPOINT_COLLECTION_ENABLED is set to true' do + around do |example| + ClimateControl.modify('DD_API_SECURITY_ENDPOINT_COLLECTION_ENABLED' => 'true') { example.run } + end + + it { expect(settings.appsec.api_security.endpoint_collection.enabled).to eq(true) } + end + + context 'when DD_API_SECURITY_ENDPOINT_COLLECTION_ENABLED is set to false' do + around do |example| + ClimateControl.modify('DD_API_SECURITY_ENDPOINT_COLLECTION_ENABLED' => 'false') { example.run } + end + + it { expect(settings.appsec.api_security.endpoint_collection.enabled).to eq(false) } + end + end + + describe '#enabled=' do + [true, false].each do |value| + context "when given #{value}" do + before { settings.appsec.api_security.endpoint_collection.enabled = value } + + it { expect(settings.appsec.api_security.endpoint_collection.enabled).to eq(value) } + end + end + end + end end describe 'sca' do diff --git a/spec/datadog/appsec/integration/contrib/rails/endpoint_collection_spec.rb b/spec/datadog/appsec/integration/contrib/rails/endpoint_collection_spec.rb new file mode 100644 index 00000000000..6837f365e20 --- /dev/null +++ b/spec/datadog/appsec/integration/contrib/rails/endpoint_collection_spec.rb @@ -0,0 +1,296 @@ +# frozen_string_literal: true + +require 'datadog/tracing/contrib/support/spec_helper' +require 'datadog/appsec/spec_helper' +require 'rack/test' + +require 'action_controller/railtie' +require 'active_record' +require 'sqlite3' +require 'devise' + +RSpec.describe 'Rails Endpoint Collection' do + include Rack::Test::Methods + include Warden::Test::Helpers + + before do + # TODO: Remove Devise + # We do not need Devise for this spec, but since Devise pollutes the global namespace, + # it leads to this test being flaky, since it is run in the same process as other + # integration tests that use Devise. + + # NOTE: By doing this we are emulating the initial load of the devise rails + # engine for every test case. It will install the required middleware. + # WARNING: This is a hack! + Devise.send(:remove_const, :Engine) + + load File.join(Gem.loaded_specs['devise'].full_gem_path, 'lib/devise/rails.rb') + + Devise.warden_config = Warden::Config.new + Devise.class_variable_set(:@@warden_configured, nil) # rubocop:disable Style/ClassVars + Devise.configure_warden! + + Devise.setup do |config| + config.secret_key = 'test-secret-key' + + require 'devise/orm/active_record' + + config.sign_out_via = :delete + config.responder.error_status = :unprocessable_entity + config.responder.redirect_status = :see_other + config.sign_out_all_scopes = false + config.parent_controller = 'TestApplicationController' + config.paranoid = true + config.stretches = 1 + config.password_length = 6..8 + config.http_authenticatable = true + end + + # app/models + stub_const('User', Class.new(ActiveRecord::Base)).tap do |klass| + klass.establish_connection({adapter: 'sqlite3', database: ':memory:'}) + klass.connection.create_table 'users', force: :cascade do |t| + t.string :username, null: false + t.string :email, default: '', null: false + t.string :encrypted_password, default: '', null: false + end + + klass.class_eval do + devise :database_authenticatable, :registerable, :validatable + end + + # prevent internal sql requests from showing up + klass.count + end + + stub_const('TestApplicationController', Class.new(ActionController::Base)).class_eval do + end + + # NOTE: Unfortunately, can't figure out why devise receives 3 times `finalize!` + # of the RouteSet patch, hence it's bypassed with below hack. + # The order of hacks matters! + allow(Devise).to receive(:regenerate_helpers!) + + app = Class.new(Rails::Application) do + config.root = __dir__ + config.secret_key_base = 'test-secret-key-base' + config.action_dispatch.show_exceptions = :rescuable + config.hosts.clear + config.eager_load = false + config.consider_all_requests_local = true + # NOTE: For debugging replace with $stdout + config.logger = Rails.logger = Logger.new(StringIO.new) + + config.file_watcher = Class.new(ActiveSupport::FileUpdateChecker) do + def initialize(files, dirs = {}, &block) + dirs = dirs.delete('') if dirs.include?('') + + super + end + end + end + + stub_const('RailsTest::Application', app) + + Datadog.configure do |c| + c.tracing.enabled = true + + c.appsec.enabled = true + c.appsec.api_security.endpoint_collection.enabled = true + c.appsec.instrument :rails + c.appsec.instrument :devise + + c.remote.enabled = false + end + + allow(Datadog::AppSec.telemetry).to receive(:app_endpoints_loaded) + + # app.initialize! + app.routes.draw do + resources :products + + resources :users, only: %i[index show] do + resources :photos, only: %i[index create destroy] + end + + get '/photos(/:id)', to: 'photos#display' + + root to: 'home#show' + + get '/job-queue', to: 'job_queue#index', constraints: {subdomain: 'tech-stuff'} + + namespace :admin do + get '/stats', to: 'statistics#index' + + post '/sign-in', to: 'sessions#create' + delete '/sign-out', to: 'sessions#destroy' + end + + match '/search', to: 'search#index', via: :all + end + + allow(Rails).to receive(:application).and_return(app) + + allow_any_instance_of(Datadog::Tracing::Transport::HTTP::Client).to receive(:send_request) + allow_any_instance_of(Datadog::Tracing::Transport::Traces::Transport).to receive(:native_events_supported?) + end + + after do + clear_traces! + + Datadog.configuration.reset! + Datadog.registry[:rack].reset_configuration! + + Datadog::AppSec::RateLimiter.reset! + Datadog::AppSec::APISecurity::Sampler.reset! + end + + it 'reports routes via telemetry' do + ActiveSupport.run_load_hooks(:after_routes_loaded, Rails.application) + + expect(Datadog::AppSec.telemetry).to have_received(:app_endpoints_loaded) do |arg| + expect(arg.to_a).to contain_exactly( + { + type: 'REST', + resource_name: 'GET /', + operation_name: 'http.request', + method: 'GET', + path: '/' + }, + { + type: 'REST', + resource_name: 'GET /products', + operation_name: 'http.request', + method: 'GET', + path: '/products' + }, + { + type: 'REST', + resource_name: 'GET /products/new', + operation_name: 'http.request', + method: 'GET', + path: '/products/new' + }, + { + type: 'REST', + resource_name: 'POST /products', + operation_name: 'http.request', + method: 'POST', + path: '/products' + }, + { + type: 'REST', + resource_name: 'GET /products/:id', + operation_name: 'http.request', + method: 'GET', + path: '/products/:id' + }, + { + type: 'REST', + resource_name: 'GET /products/:id/edit', + operation_name: 'http.request', + method: 'GET', + path: '/products/:id/edit' + }, + { + type: 'REST', + resource_name: 'PATCH /products/:id', + operation_name: 'http.request', + method: 'PATCH', + path: '/products/:id' + }, + { + type: 'REST', + resource_name: 'PUT /products/:id', + operation_name: 'http.request', + method: 'PUT', + path: '/products/:id' + }, + { + type: 'REST', + resource_name: 'DELETE /products/:id', + operation_name: 'http.request', + method: 'DELETE', + path: '/products/:id' + }, + { + type: 'REST', + resource_name: 'GET /users', + operation_name: 'http.request', + method: 'GET', + path: '/users' + }, + { + type: 'REST', + resource_name: 'GET /users/:id', + operation_name: 'http.request', + method: 'GET', + path: '/users/:id' + }, + { + type: 'REST', + resource_name: 'GET /users/:user_id/photos', + operation_name: 'http.request', + method: 'GET', + path: '/users/:user_id/photos' + }, + { + type: 'REST', + resource_name: 'POST /users/:user_id/photos', + operation_name: 'http.request', + method: 'POST', + path: '/users/:user_id/photos' + }, + { + type: 'REST', + resource_name: 'DELETE /users/:user_id/photos/:id', + operation_name: 'http.request', + method: 'DELETE', + path: '/users/:user_id/photos/:id' + }, + { + type: 'REST', + resource_name: 'GET /photos(/:id)', + operation_name: 'http.request', + method: 'GET', + path: '/photos(/:id)' + }, + { + type: 'REST', + resource_name: 'GET /admin/stats', + operation_name: 'http.request', + method: 'GET', + path: '/admin/stats' + }, + { + type: 'REST', + resource_name: 'POST /admin/sign-in', + operation_name: 'http.request', + method: 'POST', + path: '/admin/sign-in' + }, + { + type: 'REST', + resource_name: 'DELETE /admin/sign-out', + operation_name: 'http.request', + method: 'DELETE', + path: '/admin/sign-out' + }, + { + type: 'REST', + resource_name: 'GET /job-queue', + operation_name: 'http.request', + method: 'GET', + path: '/job-queue' + }, + { + type: 'REST', + resource_name: '* /search', + operation_name: 'http.request', + method: '*', + path: '/search' + } + ) + end + end +end diff --git a/supported-configurations.json b/supported-configurations.json index fa5503ea345..25d1205804a 100644 --- a/supported-configurations.json +++ b/supported-configurations.json @@ -10,6 +10,9 @@ "DD_API_SECURITY_ENABLED": { "version": ["A"] }, + "DD_API_SECURITY_ENDPOINT_COLLECTION_ENABLED": { + "version": ["A"] + }, "DD_API_SECURITY_REQUEST_SAMPLE_RATE": { "version": ["A"] },