diff --git a/CHANGELOG.md b/CHANGELOG.md index 9089c7b9..7183db11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,12 +10,24 @@ The format is based on [Keep a Changelog], and this project adheres to ## [Unreleased] +[Unreleased]: https://github.com/envato/stack_master/compare/v2.13.4...HEAD + +## [2.14.0] - 2024-02-05 + ### Added +- Allow the use of [commander](https://github.com/commander-rb/commander) + major version 5 ([#375]). + - Test on Ruby 3.3 in the CI build ([#376]). -[Unreleased]: https://github.com/envato/stack_master/compare/v2.13.4...HEAD +- Introduce `user_data_file`, `user_data_file_as_lines`, and `include_file` + convenience methods to the YAML ERB template compiler ([#377]). + +[2.13.4]: https://github.com/envato/stack_master/compare/v2.13.4...v2.14.0 +[#375]: https://github.com/envato/stack_master/pull/375 [#376]: https://github.com/envato/stack_master/pull/376 +[#377]: https://github.com/envato/stack_master/pull/377 ## [2.13.4] - 2023-08-02 @@ -23,7 +35,7 @@ The format is based on [Keep a Changelog], and this project adheres to - Resolve SparkleFormation template error caused by `SortedSet` class being removed from the `set` library in Ruby 3 ([#374]). -[2.13.3]: https://github.com/envato/stack_master/compare/v2.13.3...v2.13.4 +[2.13.4]: https://github.com/envato/stack_master/compare/v2.13.3...v2.13.4 [#374]: https://github.com/envato/stack_master/pull/374 ## [2.13.3] - 2023-02-01 diff --git a/lib/stack_master.rb b/lib/stack_master.rb index f52904e5..bdc16996 100644 --- a/lib/stack_master.rb +++ b/lib/stack_master.rb @@ -45,6 +45,8 @@ module StackMaster autoload :StackDefinition, 'stack_master/stack_definition' autoload :TemplateCompiler, 'stack_master/template_compiler' autoload :Identity, 'stack_master/identity' + autoload :CloudFormationInterpolatingEruby, 'stack_master/cloudformation_interpolating_eruby' + autoload :CloudFormationTemplateEruby, 'stack_master/cloudformation_template_eruby' autoload :StackDiffer, 'stack_master/stack_differ' autoload :Validator, 'stack_master/validator' diff --git a/lib/stack_master/cloudformation_interpolating_eruby.rb b/lib/stack_master/cloudformation_interpolating_eruby.rb new file mode 100644 index 00000000..852e7363 --- /dev/null +++ b/lib/stack_master/cloudformation_interpolating_eruby.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +require 'erubis' + +module StackMaster + # This class is a modified version of `Erubis::Eruby`. It allows using + # `<%= %>` ERB expressions to interpolate values into a source string. We use + # this capability to enrich user data scripts with data and parameters pulled + # from the AWS CloudFormation service. The evaluation produces an array of + # objects ready for use in a CloudFormation `Fn::Join` intrinsic function. + class CloudFormationInterpolatingEruby < Erubis::Eruby + include Erubis::ArrayEnhancer + + # Load a template from a file at the specified path and evaluate it. + def self.evaluate_file(source_path, context = Erubis::Context.new) + template_contents = File.read(source_path) + eruby = new(template_contents) + eruby.filename = source_path + eruby.evaluate(context) + end + + # @return [Array] The result of evaluating the source: an array of strings + # from the source intermindled with Hash objects from the ERB + # expressions. To be included in a CloudFormation template, this + # value needs to be used in a CloudFormation `Fn::Join` intrinsic + # function. + # @see Erubis::Eruby#evaluate + # @example + # CloudFormationInterpolatingEruby.new("my_variable=<%= { 'Ref' => 'Param1' } %>;").evaluate + # #=> ['my_variable=', { 'Ref' => 'Param1' }, ';'] + def evaluate(_context = Erubis::Context.new) + format_lines_for_cloudformation(super) + end + + # @see Erubis::Eruby#add_expr + def add_expr(src, code, indicator) + if indicator == '=' + src << " #{@bufvar} << (" << code << ');' + else + super + end + end + + private + + # Split up long strings containing multiple lines. One string per line in the + # CloudFormation array makes the compiled template and diffs more readable. + def format_lines_for_cloudformation(source) + source.flat_map do |lines| + lines = lines.to_s if lines.is_a?(Symbol) + next(lines) unless lines.is_a?(String) + + newlines = Array.new(lines.count("\n"), "\n") + newlines = lines.split("\n").map { |line| "#{line}#{newlines.pop}" } + newlines.insert(0, "\n") if lines.start_with?("\n") + newlines + end + end + end +end diff --git a/lib/stack_master/cloudformation_template_eruby.rb b/lib/stack_master/cloudformation_template_eruby.rb new file mode 100644 index 00000000..50e51161 --- /dev/null +++ b/lib/stack_master/cloudformation_template_eruby.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require 'erubis' +require 'json' + +module StackMaster + # This class is a modified version of `Erubis::Eruby`. It provides extra + # helper methods to ease the dynamic creation of CloudFormation templates + # with ERB. These helper methods are available within `<%= %>` expressions. + class CloudFormationTemplateEruby < Erubis::Eruby + # Adds the contents of an EC2 userdata script to the CloudFormation + # template. Allows using the ERB `<%= %>` expressions within the user data + # script to interpolate CloudFormation values. + def user_data_file(filepath) + JSON.pretty_generate({ 'Fn::Base64' => { 'Fn::Join' => ['', user_data_file_as_lines(filepath)] } }) + end + + # Evaluate the ERB template at the specified filepath and return the result + # as an array of lines. Allows using ERB `<%= %>` expressions to interpolate + # CloudFormation objects into the result. + def user_data_file_as_lines(filepath) + StackMaster::CloudFormationInterpolatingEruby.evaluate_file(filepath, self) + end + + # Add the contents of another file into the CloudFormation template as a + # string. ERB `<%= %>` expressions within the referenced file are not + # evaluated. + def include_file(filepath) + JSON.pretty_generate(File.read(filepath)) + end + end +end diff --git a/lib/stack_master/sparkle_formation/template_file.rb b/lib/stack_master/sparkle_formation/template_file.rb index 1ca35cb7..f3ff7853 100644 --- a/lib/stack_master/sparkle_formation/template_file.rb +++ b/lib/stack_master/sparkle_formation/template_file.rb @@ -5,19 +5,6 @@ module StackMaster module SparkleFormation TemplateFileNotFound = ::Class.new(StandardError) - class SfEruby < Erubis::Eruby - include Erubis::ArrayEnhancer - - def add_expr(src, code, indicator) - case indicator - when '=' - src << " #{@bufvar} << (" << code << ');' - else - super - end - end - end - class TemplateContext < AttributeStruct include ::SparkleFormation::SparkleAttribute include ::SparkleFormation::SparkleAttribute::Aws @@ -49,47 +36,12 @@ def render(file_name, vars = {}) end end - # Splits up long strings with multiple lines in them to multiple strings - # in the CF array. Makes the compiled template and diffs more readable. - class CloudFormationLineFormatter - def self.format(template) - new(template).format - end - - def initialize(template) - @template = template - end - - def format - @template.flat_map do |lines| - lines = lines.to_s if Symbol === lines - if String === lines - newlines = [] - lines.count("\n").times do - newlines << "\n" - end - newlines = lines.split("\n").map do |line| - "#{line}#{newlines.pop}" - end - if lines.start_with?("\n") - newlines.insert(0, "\n") - end - newlines - else - lines - end - end - end - end - module Template def self.render(prefix, file_name, vars) file_path = File.join(::SparkleFormation.sparkle_path, prefix, file_name) - template = File.read(file_path) template_context = TemplateContext.build(vars, prefix) - compiled_template = SfEruby.new(template).evaluate(template_context) - CloudFormationLineFormatter.format(compiled_template) - rescue Errno::ENOENT => e + CloudFormationInterpolatingEruby.evaluate_file(file_path, template_context) + rescue Errno::ENOENT Kernel.raise TemplateFileNotFound, "Could not find template file at path: #{file_path}" end end diff --git a/lib/stack_master/template_compilers/yaml_erb.rb b/lib/stack_master/template_compilers/yaml_erb.rb index 79c0df14..57bed748 100644 --- a/lib/stack_master/template_compilers/yaml_erb.rb +++ b/lib/stack_master/template_compilers/yaml_erb.rb @@ -3,13 +3,12 @@ module StackMaster::TemplateCompilers class YamlErb def self.require_dependencies - require 'erubis' require 'yaml' end def self.compile(template_dir, template, compile_time_parameters, _compiler_options = {}) template_file_path = File.join(template_dir, template) - template = Erubis::Eruby.new(File.read(template_file_path)) + template = StackMaster::CloudFormationTemplateEruby.new(File.read(template_file_path)) template.filename = template_file_path template.result(params: compile_time_parameters) diff --git a/lib/stack_master/version.rb b/lib/stack_master/version.rb index df4e8397..a7d2dc04 100644 --- a/lib/stack_master/version.rb +++ b/lib/stack_master/version.rb @@ -1,3 +1,3 @@ module StackMaster - VERSION = "2.13.4" + VERSION = "2.14.0" end diff --git a/spec/fixtures/templates/erb/user_data.sh.erb b/spec/fixtures/templates/erb/user_data.sh.erb new file mode 100644 index 00000000..a19cca81 --- /dev/null +++ b/spec/fixtures/templates/erb/user_data.sh.erb @@ -0,0 +1,5 @@ +#!/bin/bash + +echo 'Hello, World!' +REGION=<%= { 'Ref' => 'AWS::Region' } %> +echo $REGION diff --git a/spec/fixtures/templates/erb/user_data.yml.erb b/spec/fixtures/templates/erb/user_data.yml.erb new file mode 100644 index 00000000..35f783c0 --- /dev/null +++ b/spec/fixtures/templates/erb/user_data.yml.erb @@ -0,0 +1,7 @@ +Description: A test case for storing the userdata script in a dedicated file + +Resources: + LaunchConfig: + Type: 'AWS::AutoScaling::LaunchConfiguration' + Properties: + UserData: <%= user_data_file(File.join(__dir__, 'user_data.sh.erb')) %> diff --git a/spec/stack_master/cloudformation_interpolating_eruby_spec.rb b/spec/stack_master/cloudformation_interpolating_eruby_spec.rb new file mode 100644 index 00000000..a73028f7 --- /dev/null +++ b/spec/stack_master/cloudformation_interpolating_eruby_spec.rb @@ -0,0 +1,62 @@ +RSpec.describe(StackMaster::CloudFormationInterpolatingEruby) do + describe('#evaluate') do + subject(:evaluate) { described_class.new(user_data).evaluate } + + context('given a simple user data script') do + let(:user_data) { <<~SHELL } + #!/bin/bash + + REGION=ap-southeast-2 + echo $REGION + SHELL + + it 'returns an array of lines' do + expect(evaluate).to eq([ + "#!/bin/bash\n", + "\n", + "REGION=ap-southeast-2\n", + "echo $REGION\n", + ]) + end + end + + context('given a user data script referring parameters') do + let(:user_data) { <<~SHELL } + #!/bin/bash + <%= { 'Ref' => 'Param1' } %> <%= { 'Ref' => 'Param2' } %> + SHELL + + it 'includes CloudFormation objects in the array' do + expect(evaluate).to eq([ + "#!/bin/bash\n", + { 'Ref' => 'Param1' }, + ' ', + { 'Ref' => 'Param2' }, + "\n", + ]) + end + end + end + + describe('.evaluate_file') do + subject(:evaluate_file) { described_class.evaluate_file('my/userdata.sh') } + + context('given a simple user data script file') do + before { allow(File).to receive(:read).with('my/userdata.sh').and_return(<<~SHELL) } + #!/bin/bash + + REGION=ap-southeast-2 + echo $REGION + SHELL + + it 'returns an array of lines' do + expect(evaluate_file).to eq([ + "#!/bin/bash\n", + "\n", + "REGION=ap-southeast-2\n", + "echo $REGION\n", + ]) + end + end + end +end diff --git a/spec/stack_master/cloudformation_template_eruby_spec.rb b/spec/stack_master/cloudformation_template_eruby_spec.rb new file mode 100644 index 00000000..86ac349a --- /dev/null +++ b/spec/stack_master/cloudformation_template_eruby_spec.rb @@ -0,0 +1,124 @@ +RSpec.describe(StackMaster::CloudFormationTemplateEruby) do + subject(:evaluate) do + eruby = described_class.new(template) + eruby.evaluate(eruby) + end + + describe('.user_data_file') do + context('given a template that loads a simple user data script file') do + let(:template) { <<~YAML} + Resources: + LaunchConfig: + Type: 'AWS::AutoScaling::LaunchConfiguration' + Properties: + UserData: <%= user_data_file('my/userdata.sh') %> + YAML + + before do + allow(File).to receive(:read).with('my/userdata.sh').and_return(<<~SHELL) + #!/bin/bash + + REGION=ap-southeast-2 + echo $REGION + SHELL + end + + it 'embeds the script in the evaluated CFN template' do + expect(evaluate).to eq(<<~YAML) + Resources: + LaunchConfig: + Type: 'AWS::AutoScaling::LaunchConfiguration' + Properties: + UserData: { + "Fn::Base64": { + "Fn::Join": [ + "", + [ + "#!/bin/bash\\n", + "\\n", + "REGION=ap-southeast-2\\n", + "echo $REGION\\n" + ] + ] + } + } + YAML + end + end + + context('given a template that loads a user data script file that includes another file') do + let(:template) { <<~YAML} + Resources: + LaunchConfig: + Type: 'AWS::AutoScaling::LaunchConfiguration' + Properties: + UserData: <%= user_data_file('my/userdata.sh') %> + YAML + + before do + allow(File).to receive(:read).with('my/userdata.sh').and_return(<<~SHELL) + #!/bin/bash + echo 'Hello from userdata.sh' + <%= user_data_file_as_lines('my/other.sh') %> + SHELL + allow(File).to receive(:read).with('my/other.sh').and_return(<<~SHELL) + echo 'Hello from other.sh' + SHELL + end + + it 'embeds the script in the evaluated CFN template' do + expect(evaluate).to eq(<<~YAML) + Resources: + LaunchConfig: + Type: 'AWS::AutoScaling::LaunchConfiguration' + Properties: + UserData: { + "Fn::Base64": { + "Fn::Join": [ + "", + [ + "#!/bin/bash\\n", + "echo 'Hello from userdata.sh'\\n", + "echo 'Hello from other.sh'\\n", + "\\n" + ] + ] + } + } + YAML + end + end + end + + describe('.include_file') do + context('given a template that loads a lambda script') do + let(:template) { <<~YAML} + Resources: + Function: + Type: 'AWS::Lambda::Function' + Properties: + Code: + ZipFile: <%= include_file('my/lambda.sh') %> + YAML + + before do + allow(File).to receive(:read).with('my/lambda.sh').and_return(<<~SHELL) + #!/bin/bash + + echo 'Hello, world!' + SHELL + end + + it 'embeds the script in the evaluated CFN template' do + expect(evaluate).to eq(<<~YAML) + Resources: + Function: + Type: 'AWS::Lambda::Function' + Properties: + Code: + ZipFile: "#!/bin/bash\\n\\necho 'Hello, world!'\\n" + YAML + end + end + end +end diff --git a/spec/stack_master/template_compilers/yaml_erb_spec.rb b/spec/stack_master/template_compilers/yaml_erb_spec.rb index cc45f4b3..afcebedb 100644 --- a/spec/stack_master/template_compilers/yaml_erb_spec.rb +++ b/spec/stack_master/template_compilers/yaml_erb_spec.rb @@ -4,18 +4,28 @@ before(:all) { described_class.require_dependencies } describe '.compile' do - let(:compile_time_parameters) { { 'SubnetCidrs' => ['10.0.0.0/28:ap-southeast-2', '10.0.2.0/28:ap-southeast-1'] } } - - def compile - described_class.compile(stack_definition.template_dir, stack_definition.template, compile_time_parameters) + subject(:compile) do + described_class.compile( + stack_definition.template_dir, + stack_definition.template, + compile_time_parameters + ) end context 'a YAML template using a loop over compile time parameters' do - let(:stack_definition) { StackMaster::StackDefinition.new(template_dir: 'spec/fixtures/templates/erb', - template: 'compile_time_parameters_loop.yml.erb') } + let(:stack_definition) do + StackMaster::StackDefinition.new( + template_dir: 'spec/fixtures/templates/erb', + template: 'compile_time_parameters_loop.yml.erb' + ) + end + + let(:compile_time_parameters) do + { 'SubnetCidrs' => ['10.0.0.0/28:ap-southeast-2', '10.0.2.0/28:ap-southeast-1'] } + end it 'renders the expected output' do - expect(compile).to eq <<~EOEXPECTED + expect(compile).to eq(<<~YAML) --- Description: "A test case for generating subnet resources in a loop" Parameters: @@ -39,7 +49,47 @@ def compile VpcId: !Ref Vpc CidrBlock: 10.0.2.0/28 AvailabilityZone: ap-southeast-1 - EOEXPECTED + YAML + end + end + + context 'a YAML template using loading a userdata script from an external file' do + let(:stack_definition) do + StackMaster::StackDefinition.new( + template_dir: 'spec/fixtures/templates/erb', + template: 'user_data.yml.erb' + ) + end + + let(:compile_time_parameters) { {} } + + it 'renders the expected output' do + expect(compile).to eq(<<~YAML) + Description: A test case for storing the userdata script in a dedicated file + + Resources: + LaunchConfig: + Type: 'AWS::AutoScaling::LaunchConfiguration' + Properties: + UserData: { + "Fn::Base64": { + "Fn::Join": [ + "", + [ + "#!/bin/bash\\n", + "\\n", + "echo 'Hello, World!'\\n", + "REGION=", + { + "Ref": "AWS::Region" + }, + "\\n", + "echo $REGION\\n" + ] + ] + } + } + YAML end end end