[APPSEC-10967] add defaults scanners to the WAF ruleset

lib/datadog/appsec/assets.rb

@@ -14,6 +14,10 @@ def waf_processors
         read('waf_rules/processors.json')
       end
 
+      def waf_scanners
+        read('waf_rules/scanners.json')
+      end
+
       def blocked(format: :html)
         (@blocked ||= {})[format] ||= read("blocked.#{format}")
       end

lib/datadog/appsec/assets/waf_rules/processors.json

@@ -77,6 +77,13 @@
           ],
           "output": "_dd.appsec.s.res.body"
         }
+      ],
+      "scanners": [
+        {
+          "tags": {
+            "category": "pii"
+          }
+        }
       ]
     },
     "evaluate": false,

lib/datadog/appsec/assets/waf_rules/scanners.json

@@ -0,0 +1,114 @@
+[
+  {
+    "id": "d962f7ddb3f55041e39195a60ff79d4814a7c331",
+    "name": "US Passport Scanner",
+    "key": {
+      "operator": "match_regex",
+      "parameters": {
+        "regex": "passport",
+        "options": {
+          "case_sensitive": false,
+          "min_length": 8
+        }
+      }
+    },
+    "value": {
+      "operator": "match_regex",
+      "parameters": {
+        "regex": "\\b[0-9A-Z]{9}\\b|\\b[0-9]{6}[A-Z][0-9]{2}\\b",
+        "options": {
+          "case_sensitive": false,
+          "min_length": 8
+        }
+      }
+    },
+    "tags": {
+      "type": "passport_number",
+      "category": "pii"
+    }
+  },
+  {
+    "id": "ac6d683cbac77f6e399a14990793dd8fd0fca333",
+    "name": "US Vehicle Identification Number Scanner",
+    "key": {
+      "operator": "match_regex",
+      "parameters": {
+        "regex": "vehicle[_\\s-]*identification[_\\s-]*number|vin",
+        "options": {
+          "case_sensitive": false,
+          "min_length": 3
+        }
+      }
+    },
+    "value": {
+      "operator": "match_regex",
+      "parameters": {
+        "regex": "\\b[A-HJ-NPR-Z0-9]{17}\\b",
+        "options": {
+          "case_sensitive": false,
+          "min_length": 17
+        }
+      }
+    },
+    "tags": {
+      "type": "vin",
+      "category": "pii"
+    }
+  },
+  {
+    "id": "de0899e0cbaaa812bb624cf04c912071012f616d",
+    "name": "UK National Insurance Number Scanner",
+    "key": {
+      "operator": "match_regex",
+      "parameters": {
+        "regex": "national[\\s_]?(?:insurance(?:\\s+number)?)?|NIN|NI[\\s_]?number|insurance[\\s_]?number",
+        "options": {
+          "case_sensitive": false,
+          "min_length": 3
+        }
+      }
+    },
+    "value": {
+      "operator": "match_regex",
+      "parameters": {
+        "regex": "\\b[A-Z]{2}\\d{6}[A-Z]?\\b",
+        "options": {
+          "case_sensitive": false,
+          "min_length": 8
+        }
+      }
+    },
+    "tags": {
+      "type": "uk_nin",
+      "category": "pii"
+    }
+  },
+  {
+    "id": "450239afc250a19799b6c03dc0e16fd6a4b2a1af",
+    "name": "Canadian Social Insurance Number Scanner",
+    "key": {
+      "operator": "match_regex",
+      "parameters": {
+        "regex": "social[\\s_]?(?:insurance(?:\\s+number)?)?|SIN|Canadian[\\s_]?(?:social[\\s_]?(?:insurance)?|insurance[\\s_]?number)?",
+        "options": {
+          "case_sensitive": false,
+          "min_length": 3
+        }
+      }
+    },
+    "value": {
+      "operator": "match_regex",
+      "parameters": {
+        "regex": "\\b\\d{3}-\\d{3}-\\d{3}\\b",
+        "options": {
+          "case_sensitive": false,
+          "min_length": 11
+        }
+      }
+    },
+    "tags": {
+      "type": "canadian_sin",
+      "category": "pii"
+    }
+  }
+]

lib/datadog/appsec/processor/rule_merger.rb

@@ -25,10 +25,17 @@ def initialize(version1, version2)
           []
         end
 
+        DEFAULT_WAF_SCANNERS = begin
+          JSON.parse(Datadog::AppSec::Assets.waf_scanners)
+        rescue StandardError => e
+          Datadog.logger.error { "libddwaf rulemerger failed to parse default waf scanners. Error: #{e.inspect}" }
+          []
+        end
+
         class << self
           def merge(
             rules:, data: [], overrides: [], exclusions: [], custom_rules: [],
-            processors: DEFAULT_WAF_PROCESSORS
+            processors: DEFAULT_WAF_PROCESSORS, scanners: DEFAULT_WAF_SCANNERS
           )
             combined_rules = combine_rules(rules)
 
@@ -42,6 +49,7 @@ def merge(
             combined_rules['exclusions'] = combined_exclusions if combined_exclusions
             combined_rules['custom_rules'] = combined_custom_rules if combined_custom_rules
             combined_rules['processors'] = processors
+            combined_rules['scanners'] = scanners
             combined_rules
           end
 

sig/datadog/appsec/assets.rbs

@@ -7,6 +7,8 @@ module Datadog
 
       def self?.waf_processors: () -> ::String
 
+      def self?.waf_scanners: () -> ::String
+
       def self?.blocked: (?format: ::Symbol) -> ::String
 
       def self?.path: () -> ::Pathname

sig/datadog/appsec/processor/rule_merger.rbs

@@ -12,10 +12,12 @@ module Datadog
         type exclusions = ::Array[::Hash[::String, untyped]]
         type custom_rules = ::Array[::Hash[::String, untyped]]
         type processors = ::Array[::Hash[::String, untyped]]
+        type scanners = ::Array[::Hash[::String, untyped]]
 
         DEFAULT_WAF_PROCESSORS: processors
+        DEFAULT_WAF_SCANNERS: processors
 
-        def self.merge: (rules: ::Array[rules], ?data: ::Array[data], ?overrides: ::Array[overrides], ?exclusions: ::Array[exclusions], ?custom_rules: ::Array[custom_rules], ?processors: processors) -> rules
+        def self.merge: (rules: ::Array[rules], ?data: ::Array[data], ?overrides: ::Array[overrides], ?exclusions: ::Array[exclusions], ?custom_rules: ::Array[custom_rules], ?processors: processors, ?scanners: scanners) -> rules
 
         private
 

spec/datadog/appsec/processor/rule_merger_spec.rb

@@ -698,4 +698,18 @@
       expect(result).to include('processors' => 'hello')
     end
   end
+
+  context 'scanners' do
+    it 'merges default scanners' do
+      result = described_class.merge(rules: rules)
+      expect(result).to include('rules' => rules[0]['rules'])
+      expect(result).to include('scanners' => described_class::DEFAULT_WAF_SCANNERS)
+    end
+
+    it 'merges the provided processors' do
+      result = described_class.merge(rules: rules, scanners: 'hello')
+      expect(result).to include('rules' => rules[0]['rules'])
+      expect(result).to include('scanners' => 'hello')
+    end
+  end
 end

spec/datadog/appsec/processor_spec.rb

@@ -127,7 +127,10 @@
 end
 
 RSpec.describe Datadog::AppSec::Processor::Context do
-  let(:ruleset) { Datadog::AppSec::Processor::RuleLoader.load_rules(ruleset: :recommended) }
+  let(:ruleset) do
+    rules = Datadog::AppSec::Processor::RuleLoader.load_rules(ruleset: :recommended)
+    Datadog::AppSec::Processor::RuleMerger.merge(rules: [rules])
+  end
 
   let(:input_safe) { { 'server.request.headers.no_cookies' => { 'user-agent' => 'Ruby' } } }
   let(:input_sqli) { { 'server.request.query' => { 'q' => '1 OR 1;' } } }
@@ -320,6 +323,16 @@
 
         expect(context.extract_schema).to eq dummy_result
       end
+
+      it 'returns schema extraction information' do
+        input = { 'server.request.query' => { 'vin' => '4Y1SL65848Z411439' } }
+        context.run(input, timeout)
+
+        results = context.extract_schema
+        derivatives = results.derivatives
+        expect(derivatives).to_not be_empty
+        expect(derivatives['_dd.appsec.s.req.query']).to eq([{ 'vin' => [8, { 'category' => 'pii', 'type' => 'vin' }] }])
+      end
     end
 
     context 'when extrct_schema? returns false' do

spec/datadog/appsec/remote_spec.rb

@@ -160,7 +160,8 @@
               'transformers' => [],
               'on_match' => ['block']
             }],
-            'processors' => Datadog::AppSec::Processor::RuleMerger::DEFAULT_WAF_PROCESSORS
+            'processors' => Datadog::AppSec::Processor::RuleMerger::DEFAULT_WAF_PROCESSORS,
+            'scanners' => Datadog::AppSec::Processor::RuleMerger::DEFAULT_WAF_SCANNERS,
           }
 
           expect(Datadog::AppSec).to receive(:reconfigure).with(ruleset: expected_ruleset, actions: [])