Merge pull request #1580 from mururu/fix-1553

Mask all secret parameters in worker section, fix #1553

lib/fluent/engine.rb

@@ -43,6 +43,8 @@ def initialize
       @suppress_config_dump = false
 
       @system_config = SystemConfig.new
+
+      @dry_run_mode = false
     end
 
     MAINLOOP_SLEEP_INTERVAL = 0.3
@@ -54,6 +56,8 @@ def initialize
     attr_reader :matches, :sources
     attr_reader :system_config
 
+    attr_accessor :dry_run_mode
+
     def init(system_config)
       @system_config = system_config
 
@@ -157,7 +161,7 @@ def configure(conf)
       $log.enable_event(true) if @log_event_router
 
       unless @suppress_config_dump
-        $log.info :worker0, "using configuration file: #{conf.to_s.rstrip}"
+        $log.info :supervisor, "using configuration file: #{conf.to_s.rstrip}"
       end
     end
 

lib/fluent/root_agent.rb

@@ -76,6 +76,9 @@ def configure(conf)
           raise ConfigError, "worker id #{target_worker_id} specified by <worker> directive is not allowed. Available worker id is between 0 and #{(Fluent::Engine.system_config.workers - 1)}"
         end
 
+        ## On dry_run mode, all worker sections have to be configured on supervisor (recognized as worker_id = 0).
+        target_worker_id = 0 if Fluent::Engine.dry_run_mode
+
         e.elements.each do |elem|
           unless ['source', 'match', 'filter', 'label'].include?(elem.name)
             raise ConfigError, "<worker> section cannot have <#{elem.name}> directive"

lib/fluent/supervisor.rb

@@ -457,7 +457,7 @@ def run_supervisor
         end
       end
 
-      dry_run if @dry_run
+      dry_run_cmd if @dry_run
       supervise
     end
 
@@ -518,24 +518,42 @@ def create_socket_manager
       ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
     end
 
-    def dry_run
+    def dry_run_cmd
       $log.info "starting fluentd-#{Fluent::VERSION} as dry run mode"
-      change_privilege
-      init_engine
-      run_configure
+      @system_config.suppress_config_dump = true
+      dry_run
       exit 0
     rescue => e
       $log.error "dry run failed: #{e}"
       exit 1
     end
 
+    ## Set Engine's dry_run_mode true to override all target_id of worker sections
+    def dry_run
+      begin
+        Fluent::Engine.dry_run_mode = true
+        change_privilege
+        init_engine
+        run_configure
+      rescue Fluent::ConfigError => e
+        $log.error "config error", file: @config_path, error: e
+        $log.debug_backtrace
+        exit!(1)
+      ensure
+        Fluent::Engine.dry_run_mode = false
+      end
+    end
+
     def show_plugin_config
       name, type = @show_plugin_config.split(":") # input:tail
       $log.info "Use fluent-plugin-config-format --format=txt #{name} #{type}"
       exit 0
     end
 
     def supervise
+      # Make dumpable conf, which is set corresponding_proxies for all elements in all worker sections
+      dry_run
+
       Process.setproctitle("supervisor:#{@process_name}") if @process_name
       $log.info "starting fluentd-#{Fluent::VERSION}", pid: Process.pid
 

test/command/test_fluentd.rb

@@ -442,8 +442,7 @@ def assert_fluentd_fails_to_start(cmdline, *pattern_list, timeout: 10)
 
       assert_fluentd_fails_to_start(
         create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
-        "error_class=SyntaxError",
-        "in_buggy.rb:5: syntax error, unexpected end-of-input, expecting keyword_end",
+        "in_buggy.rb:5: syntax error, unexpected end-of-input, expecting keyword_end (SyntaxError)"
       )
     end
   end
@@ -821,4 +820,48 @@ def write(chunk)
       )
     end
   end
+
+  sub_test_case 'config dump' do
+    test 'all secret parameters in worker section is sealed' do
+      script =  <<-EOC
+require 'fluent/plugin/input'
+module Fluent::Plugin
+  class FakeInput < Input
+    Fluent::Plugin.register_input('fake', self)
+    config_param :secret, :string, secret: true
+  end
+end
+EOC
+      plugin_path = create_plugin_file('in_fake.rb', script)
+
+      conf = <<CONF
+<system>
+  workers 2
+</system>
+<worker 0>
+  <source>
+    @type fake
+    secret secret0
+  </source>
+  <match>
+    @type null
+  </match>
+</worker>
+<worker 1>
+  <source>
+    @type fake
+    secret secret1
+  </source>
+  <match>
+    @type null
+  </match>
+</worker>
+CONF
+      conf_path = create_conf_file('secret_in_worker.conf', conf)
+      assert File.exist?(conf_path)
+
+      assert_log_matches(create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
+                         "secret xxxxxx", patterns_not_match: ["secret secret0", "secret secret1"])
+    end
+  end
 end