Merge pull request #509 from rodjek/pdk-985

(PDK-985) Split validation targets into chunks of 1000

lib/pdk/cli/exec_group.rb

@@ -6,44 +6,58 @@
 module PDK
   module CLI
     class ExecGroup
-      attr_reader :commands
-
       def initialize(message, opts = {})
         @options = opts.merge(PDK::CLI::Util.spinner_opts_for_platform)
 
         if PDK::CLI::Util.interactive?
-          @multi_spinner = TTY::Spinner::Multi.new("[:spinner] #{message}", @options)
-          @multi_spinner.auto_spin
+          @spinner = if parallel?
+                       TTY::Spinner::Multi.new("[:spinner] #{message}", @options)
+                     else
+                       TTY::Spinner.new("[:spinner] #{message}", @options)
+                     end
+          @spinner.auto_spin
         end
 
-        @threads = []
+        @threads_or_procs = []
         @exit_codes = []
       end
 
-      def register
+      def parallel?
+        @options[:parallel].nil? ? true : @options[:parallel]
+      end
+
+      def register(&block)
         raise PDK::CLI::FatalError, 'No block registered' unless block_given?
 
-        @threads << Thread.new do
-          GettextSetup.initialize(File.absolute_path('../../../locales', File.dirname(__FILE__)))
-          GettextSetup.negotiate_locale!(GettextSetup.candidate_locales)
-          @exit_codes << yield
-        end
+        @threads_or_procs << if parallel?
+                               Thread.new do
+                                 GettextSetup.initialize(File.absolute_path('../../../locales', File.dirname(__FILE__)))
+                                 GettextSetup.negotiate_locale!(GettextSetup.candidate_locales)
+                                 @exit_codes << yield
+                               end
+                             else
+                               block
+                             end
       end
 
       def add_spinner(message, opts = {})
         return unless PDK::CLI::Util.interactive?
-        @multi_spinner.register("[:spinner] #{message}", @options.merge(opts).merge(PDK::CLI::Util.spinner_opts_for_platform))
+        @spinner.register("[:spinner] #{message}", @options.merge(opts).merge(PDK::CLI::Util.spinner_opts_for_platform))
       end
 
       def exit_code
-        @threads.each(&:join)
+        if parallel?
+          @threads_or_procs.each(&:join)
+        else
+          @exit_codes = @threads_or_procs.map(&:call)
+        end
 
         exit_code = @exit_codes.max
 
-        if exit_code.zero? && @multi_spinner
-          @multi_spinner.success
-        elsif @multi_spinner
-          @multi_spinner.error
+        if exit_code.zero? && @spinner
+          @spinner.success
+        elsif @spinner
+          @spinner.error
         end
 
         exit_code

lib/pdk/validate/base_validator.rb

@@ -115,7 +115,14 @@ def self.invoke(report, options = {})
         # targets array in another array. This is so we can loop through the
         # invokes with the same logic, regardless of which invoke style is
         # needed.
-        targets = (self::INVOKE_STYLE == :per_target) ? targets.combination(1).to_a : Array[targets]
+        #
+        if self::INVOKE_STYLE == :per_target
+          targets = targets.combination(1).to_a
+        else
+          targets = targets.each_slice(1000).to_a
+          options[:split_exec] = PDK::CLI::ExecGroup.new(spinner_text(targets), parallel: false)
+        end
+
         exit_codes = []
 
         targets.each do |invokation_targets|
@@ -124,22 +131,32 @@ def self.invoke(report, options = {})
 
           command = PDK::CLI::Exec::Command.new(*cmd_argv).tap do |c|
             c.context = :module
-            exec_group = options[:exec_group]
-            if exec_group
-              sub_spinner = exec_group.add_spinner(spinner_text(invokation_targets))
-              c.register_spinner(sub_spinner)
-            else
-              c.add_spinner(spinner_text(invokation_targets))
+            unless options[:split_exec]
+              exec_group = options[:exec_group]
+              if exec_group
+                sub_spinner = exec_group.add_spinner(spinner_text(invokation_targets))
+                c.register_spinner(sub_spinner)
+              else
+                c.add_spinner(spinner_text(invokation_targets))
+              end
             end
           end
 
-          result = command.execute!
-          exit_codes << result[:exit_code]
+          if options[:split_exec]
+            options[:split_exec].register do
+              result = command.execute!
+              parse_output(report, result, invokation_targets)
+              result[:exit_code]
+            end
+          else
+            result = command.execute!
+            exit_codes << result[:exit_code]
 
-          parse_output(report, result, invokation_targets)
+            parse_output(report, result, invokation_targets)
+          end
         end
 
-        exit_codes.max
+        options.key?(:split_exec) ? options[:split_exec].exit_code : exit_codes.max
       end
     end
   end

spec/unit/pdk/validate/base_validator_spec.rb

@@ -9,6 +9,43 @@
     end
   end
 
+  describe '.invoke' do
+    before(:each) do
+      allow(described_class).to receive(:parse_targets).and_return([(1..1001).map(&:to_s), [], []])
+      allow(described_class).to receive(:cmd).and_return('dummy_cmd')
+      allow(PDK::Util::Bundler).to receive(:ensure_binstubs!).with('dummy_cmd')
+      allow(described_class).to receive(:parse_output)
+    end
+
+    let(:dummy_exec) do
+      instance_double(PDK::CLI::Exec::Command, :context= => nil, :execute! => { exit_code: 0 })
+    end
+
+    after(:each) do
+      described_class.invoke(instance_double(PDK::Report))
+    end
+
+    context 'when validating less than 1000 targets' do
+      before(:each) do
+        allow(described_class).to receive(:parse_targets).and_return([(1..999).map(&:to_s), [], []])
+      end
+
+      it 'executes the validator once' do
+        expect(PDK::CLI::Exec::Command).to receive(:new).and_return(dummy_exec).once
+      end
+    end
+
+    context 'when validating more than 1000 targets' do
+      before(:each) do
+        allow(described_class).to receive(:parse_targets).and_return([(1..1001).map(&:to_s), [], []])
+      end
+
+      it 'executes the validator for each block of up to 1000 targets' do
+        expect(PDK::CLI::Exec::Command).to receive(:new).and_return(dummy_exec).twice
+      end
+    end
+  end
+
   describe '.parse_targets' do
     subject(:target_files) { described_class.parse_targets(targets: targets) }