From ee605962a2f8ed75cc797891a287a536ef4a0c61 Mon Sep 17 00:00:00 2001 From: Gustavo Caso Date: Thu, 28 Sep 2023 10:48:07 +0200 Subject: [PATCH] add defaults scanners to the WAF ruleset --- lib/datadog/appsec/assets.rb | 4 + .../appsec/assets/waf_rules/processors.json | 7 ++ .../appsec/assets/waf_rules/scanners.json | 114 ++++++++++++++++++ lib/datadog/appsec/processor/rule_merger.rb | 10 +- sig/datadog/appsec/assets.rbs | 2 + sig/datadog/appsec/processor/rule_merger.rbs | 4 +- .../appsec/processor/rule_merger_spec.rb | 14 +++ spec/datadog/appsec/processor_spec.rb | 15 ++- 8 files changed, 167 insertions(+), 3 deletions(-) create mode 100644 lib/datadog/appsec/assets/waf_rules/scanners.json diff --git a/lib/datadog/appsec/assets.rb b/lib/datadog/appsec/assets.rb index b923e9a3b65..9e77544252f 100644 --- a/lib/datadog/appsec/assets.rb +++ b/lib/datadog/appsec/assets.rb @@ -14,6 +14,10 @@ def waf_processors read('waf_rules/processors.json') end + def waf_scanners + read('waf_rules/scanners.json') + end + def blocked(format: :html) (@blocked ||= {})[format] ||= read("blocked.#{format}") end diff --git a/lib/datadog/appsec/assets/waf_rules/processors.json b/lib/datadog/appsec/assets/waf_rules/processors.json index 1668a96453f..a765b3c6ec7 100644 --- a/lib/datadog/appsec/assets/waf_rules/processors.json +++ b/lib/datadog/appsec/assets/waf_rules/processors.json @@ -77,6 +77,13 @@ ], "output": "_dd.appsec.s.res.body" } + ], + "scanners": [ + { + "tags": { + "category": "pii" + } + } ] }, "evaluate": false, diff --git a/lib/datadog/appsec/assets/waf_rules/scanners.json b/lib/datadog/appsec/assets/waf_rules/scanners.json new file mode 100644 index 00000000000..930dffe8a4d --- /dev/null +++ b/lib/datadog/appsec/assets/waf_rules/scanners.json @@ -0,0 +1,114 @@ +[ + { + "id": "d962f7ddb3f55041e39195a60ff79d4814a7c331", + "name": "US Passport Scanner", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "passport", + "options": { + "case_sensitive": false, + "min_length": 8 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b[0-9A-Z]{9}\\b|\\b[0-9]{6}[A-Z][0-9]{2}\\b", + "options": { + "case_sensitive": false, + "min_length": 8 + } + } + }, + "tags": { + "type": "passport_number", + "category": "pii" + } + }, + { + "id": "ac6d683cbac77f6e399a14990793dd8fd0fca333", + "name": "US Vehicle Identification Number Scanner", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "vehicle[_\\s-]*identification[_\\s-]*number|vin", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b[A-HJ-NPR-Z0-9]{17}\\b", + "options": { + "case_sensitive": false, + "min_length": 17 + } + } + }, + "tags": { + "type": "vin", + "category": "pii" + } + }, + { + "id": "de0899e0cbaaa812bb624cf04c912071012f616d", + "name": "UK National Insurance Number Scanner", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "national[\\s_]?(?:insurance(?:\\s+number)?)?|NIN|NI[\\s_]?number|insurance[\\s_]?number", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b[A-Z]{2}\\d{6}[A-Z]?\\b", + "options": { + "case_sensitive": false, + "min_length": 8 + } + } + }, + "tags": { + "type": "uk_nin", + "category": "pii" + } + }, + { + "id": "450239afc250a19799b6c03dc0e16fd6a4b2a1af", + "name": "Canadian Social Insurance Number Scanner", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "social[\\s_]?(?:insurance(?:\\s+number)?)?|SIN|Canadian[\\s_]?(?:social[\\s_]?(?:insurance)?|insurance[\\s_]?number)?", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b\\d{3}-\\d{3}-\\d{3}\\b", + "options": { + "case_sensitive": false, + "min_length": 11 + } + } + }, + "tags": { + "type": "canadian_sin", + "category": "pii" + } + } +] diff --git a/lib/datadog/appsec/processor/rule_merger.rb b/lib/datadog/appsec/processor/rule_merger.rb index afe094abcbb..1396ecf718c 100644 --- a/lib/datadog/appsec/processor/rule_merger.rb +++ b/lib/datadog/appsec/processor/rule_merger.rb @@ -25,10 +25,17 @@ def initialize(version1, version2) [] end + DEFAULT_WAF_SCANNERS = begin + JSON.parse(Datadog::AppSec::Assets.waf_scanners) + rescue StandardError => e + Datadog.logger.error { "libddwaf rulemerger failed to parse default waf scanners. Error: #{e.inspect}" } + [] + end + class << self def merge( rules:, data: [], overrides: [], exclusions: [], custom_rules: [], - processors: DEFAULT_WAF_PROCESSORS + processors: DEFAULT_WAF_PROCESSORS, scanners: DEFAULT_WAF_SCANNERS ) combined_rules = combine_rules(rules) @@ -42,6 +49,7 @@ def merge( combined_rules['exclusions'] = combined_exclusions if combined_exclusions combined_rules['custom_rules'] = combined_custom_rules if combined_custom_rules combined_rules['processors'] = processors + combined_rules['scanners'] = scanners combined_rules end diff --git a/sig/datadog/appsec/assets.rbs b/sig/datadog/appsec/assets.rbs index baa503c7851..dd013d7ff7c 100644 --- a/sig/datadog/appsec/assets.rbs +++ b/sig/datadog/appsec/assets.rbs @@ -7,6 +7,8 @@ module Datadog def self?.waf_processors: () -> ::String + def self?.waf_scanners: () -> ::String + def self?.blocked: (?format: ::Symbol) -> ::String def self?.path: () -> ::Pathname diff --git a/sig/datadog/appsec/processor/rule_merger.rbs b/sig/datadog/appsec/processor/rule_merger.rbs index 841fbfb3a69..979b45260e6 100644 --- a/sig/datadog/appsec/processor/rule_merger.rbs +++ b/sig/datadog/appsec/processor/rule_merger.rbs @@ -12,10 +12,12 @@ module Datadog type exclusions = ::Array[::Hash[::String, untyped]] type custom_rules = ::Array[::Hash[::String, untyped]] type processors = ::Array[::Hash[::String, untyped]] + type scanners = ::Array[::Hash[::String, untyped]] DEFAULT_WAF_PROCESSORS: processors + DEFAULT_WAF_SCANNERS: processors - def self.merge: (rules: ::Array[rules], ?data: ::Array[data], ?overrides: ::Array[overrides], ?exclusions: ::Array[exclusions], ?custom_rules: ::Array[custom_rules], ?processors: processors) -> rules + def self.merge: (rules: ::Array[rules], ?data: ::Array[data], ?overrides: ::Array[overrides], ?exclusions: ::Array[exclusions], ?custom_rules: ::Array[custom_rules], ?processors: processors, ?scanners: scanners) -> rules private diff --git a/spec/datadog/appsec/processor/rule_merger_spec.rb b/spec/datadog/appsec/processor/rule_merger_spec.rb index a1049f18aa7..f89c37226cd 100644 --- a/spec/datadog/appsec/processor/rule_merger_spec.rb +++ b/spec/datadog/appsec/processor/rule_merger_spec.rb @@ -698,4 +698,18 @@ expect(result).to include('processors' => 'hello') end end + + context 'scanners' do + it 'merges default scanners' do + result = described_class.merge(rules: rules) + expect(result).to include('rules' => rules[0]['rules']) + expect(result).to include('scanners' => described_class::DEFAULT_WAF_SCANNERS) + end + + it 'merges the provided processors' do + result = described_class.merge(rules: rules, scanners: 'hello') + expect(result).to include('rules' => rules[0]['rules']) + expect(result).to include('scanners' => 'hello') + end + end end diff --git a/spec/datadog/appsec/processor_spec.rb b/spec/datadog/appsec/processor_spec.rb index 77e0f018482..04c838d4dd1 100644 --- a/spec/datadog/appsec/processor_spec.rb +++ b/spec/datadog/appsec/processor_spec.rb @@ -127,7 +127,10 @@ end RSpec.describe Datadog::AppSec::Processor::Context do - let(:ruleset) { Datadog::AppSec::Processor::RuleLoader.load_rules(ruleset: :recommended) } + let(:ruleset) do + rules = Datadog::AppSec::Processor::RuleLoader.load_rules(ruleset: :recommended) + Datadog::AppSec::Processor::RuleMerger.merge(rules: [rules]) + end let(:input_safe) { { 'server.request.headers.no_cookies' => { 'user-agent' => 'Ruby' } } } let(:input_sqli) { { 'server.request.query' => { 'q' => '1 OR 1;' } } } @@ -320,6 +323,16 @@ expect(context.extract_schema).to eq dummy_result end + + it 'returns schema extraction information' do + input = { 'server.request.query' => { 'vin' => '4Y1SL65848Z411439' } } + context.run(input, timeout) + + results = context.extract_schema + derivatives = results.derivatives + expect(derivatives).to_not be_empty + expect(derivatives['_dd.appsec.s.req.query']).to eq([{ 'vin' => [8, { 'category' => 'pii', 'type' => 'vin' }] }]) + end end context 'when extrct_schema? returns false' do