diff --git a/Gemfile b/Gemfile index 744c0b5..7bb5356 100644 --- a/Gemfile +++ b/Gemfile @@ -6,6 +6,7 @@ git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } gem 'bundler', '~> 1.16' gem 'haml' +gem 'kramdown' group :development, :test do gem 'rake', '~> 12.3' @@ -14,10 +15,6 @@ group :development, :test do gem 'rubocop' end -group :development do - gem 'commonmarker' -end - group :test do gem 'simplecov', require: false end diff --git a/Gemfile.lock b/Gemfile.lock index c60a37e..ea4dedd 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -2,23 +2,21 @@ GEM remote: https://rubygems.org/ specs: ast (2.4.0) - commonmarker (0.17.9) - ruby-enum (~> 0.5) - concurrent-ruby (1.0.5) diff-lcs (1.3) docile (1.3.0) haml (5.0.4) temple (>= 0.8.0) tilt - i18n (1.0.1) - concurrent-ruby (~> 1.0) json (2.1.0) + kramdown (2.3.1) + rexml parallel (1.12.1) parser (2.5.1.0) ast (~> 2.4.0) powerpack (0.1.1) rainbow (3.0.0) rake (12.3.3) + rexml (3.2.4) rspec (3.7.0) rspec-core (~> 3.7.0) rspec-expectations (~> 3.7.0) @@ -41,8 +39,6 @@ GEM rainbow (>= 2.2.2, < 4.0) ruby-progressbar (~> 1.7) unicode-display_width (~> 1.0, >= 1.0.1) - ruby-enum (0.7.2) - i18n ruby-progressbar (1.9.0) simplecov (0.16.1) docile (~> 1.1) @@ -58,8 +54,8 @@ PLATFORMS DEPENDENCIES bundler (~> 1.16) - commonmarker haml + kramdown rake (~> 12.3) rspec (~> 3.0) rspec_junit_formatter diff --git a/bin/run-dev b/bin/run-dev index e16fabe..10b40de 100755 --- a/bin/run-dev +++ b/bin/run-dev @@ -5,7 +5,6 @@ require 'bundler/setup' require 'pathname' $LOAD_PATH.unshift(Pathname.new(__FILE__).dirname.dirname.join('lib').to_s) -require 'commonmarker' require 'yaml' require 'test_summary_buildkite_plugin' @@ -20,8 +19,7 @@ module TestSummaryBuildkitePlugin log(args, stdin: stdin) if args.first == 'annotate' context = args[2] - content = CommonMarker.render_html(stdin) - html = HamlRender.render('test_layout', content: content) + html = Utils.standalone_markdown(stdin) FileUtils.mkdir_p('tmp') out = Pathname.new('tmp').join(context + '.html') out.open('w') { |file| file.write(html) } diff --git a/lib/test_summary_buildkite_plugin.rb b/lib/test_summary_buildkite_plugin.rb index 752a85e..f4d1056 100644 --- a/lib/test_summary_buildkite_plugin.rb +++ b/lib/test_summary_buildkite_plugin.rb @@ -3,11 +3,13 @@ require 'json' require 'test_summary_buildkite_plugin/agent' +require 'test_summary_buildkite_plugin/error_handler' require 'test_summary_buildkite_plugin/failure' require 'test_summary_buildkite_plugin/formatter' require 'test_summary_buildkite_plugin/haml_render' require 'test_summary_buildkite_plugin/input' -require 'test_summary_buildkite_plugin/runner' +require 'test_summary_buildkite_plugin/main' +require 'test_summary_buildkite_plugin/processor' require 'test_summary_buildkite_plugin/tap' require 'test_summary_buildkite_plugin/truncater' require 'test_summary_buildkite_plugin/utils' @@ -18,6 +20,6 @@ def self.run plugins = JSON.parse(ENV.fetch('BUILDKITE_PLUGINS'), symbolize_names: true) # plugins is an array of hashes, keyed by # options = plugins.find { |k, _| k.to_s.include?('test-summary') }.values.first - Runner.new(options).run + Main.new(options).run end end diff --git a/lib/test_summary_buildkite_plugin/error_handler.rb b/lib/test_summary_buildkite_plugin/error_handler.rb new file mode 100644 index 0000000..98f6abb --- /dev/null +++ b/lib/test_summary_buildkite_plugin/error_handler.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module TestSummaryBuildkitePlugin + module ErrorHandler + def handle_error(err, diagnostics = nil) + if fail_on_error + raise err + else + Utils.log_error(err, diagnostics) + end + end + end +end diff --git a/lib/test_summary_buildkite_plugin/formatter.rb b/lib/test_summary_buildkite_plugin/formatter.rb index 984fdeb..4c8cdc9 100644 --- a/lib/test_summary_buildkite_plugin/formatter.rb +++ b/lib/test_summary_buildkite_plugin/formatter.rb @@ -4,51 +4,55 @@ module TestSummaryBuildkitePlugin class Formatter - def self.create(**options) + def self.create(input:, output_path:, options:) options[:type] ||= 'details' type = options[:type].to_sym raise "Unknown type: #{type}" unless TYPES.key?(type) - TYPES[type].new(options) + TYPES[type].new(input: input, output_path: output_path, options: options) end class Base - attr_reader :options + attr_reader :input, :output_path, :options - def initialize(options = {}) + def initialize(input:, output_path:, options:) + @input = input + @output_path = output_path @options = options || {} end - def markdown(input) + def markdown(truncate) return nil if input.failures.count.zero? - [heading(input), input_markdown(input), footer(input)].compact.join("\n\n") + [heading(truncate), input_markdown(truncate), footer].compact.join("\n\n") end protected - def input_markdown(input) - if show_first.negative? || show_first >= include_failures(input).count - failures_markdown(include_failures(input)) + def input_markdown(truncate) + failures = include_failures(truncate) + + if show_first.negative? || show_first >= failures.count + failures_markdown(failures, truncate) elsif show_first.zero? - details('Show failures', failures_markdown(include_failures(input))) + details('Show failures', failures_markdown(failures, truncate)) else - failures_markdown(include_failures(input)[0...show_first]) + - details('Show additional failures', failures_markdown(include_failures(input)[show_first..-1])) + failures_markdown(failures[0...show_first], false) + + details('Show additional failures', failures_markdown(failures[show_first..-1], truncate)) end end - def failures_markdown(failures) - render_template('failures', failures: failures) + def failures_markdown(failures, truncate) + render_template('failures', failures: failures, output_path: truncate ? output_path : nil) end - def heading(input) + def heading(truncate) count = input.failures.count - show_count = include_failures(input).count + show_count = include_failures(truncate).count s = "##### #{input.label}: #{count} failure#{'s' unless count == 1}" s += "\n\n_Including first #{show_count} failures_" if show_count < count s end - def footer(input) + def footer job_ids = input.failures.map(&:job_id).uniq.reject(&:nil?) render_template('footer', job_ids: job_ids) end @@ -65,11 +69,7 @@ def type options[:type] || 'details' end - def truncate - options[:truncate] - end - - def include_failures(input) + def include_failures(truncate) if truncate input.failures[0...truncate] else diff --git a/lib/test_summary_buildkite_plugin/input.rb b/lib/test_summary_buildkite_plugin/input.rb index b92ea73..038eee9 100644 --- a/lib/test_summary_buildkite_plugin/input.rb +++ b/lib/test_summary_buildkite_plugin/input.rb @@ -16,6 +16,8 @@ def self.create(type:, **options) end class Base + include ErrorHandler + attr_reader :label, :artifact_path, :options def initialize(label:, artifact_path:, **options) @@ -77,14 +79,6 @@ def job_id_regex DEFAULT_JOB_ID_REGEX end end - - def handle_error(err) - if fail_on_error - raise err - else - Utils.log_error(err) - end - end end class OneLine < Base diff --git a/lib/test_summary_buildkite_plugin/runner.rb b/lib/test_summary_buildkite_plugin/main.rb similarity index 58% rename from lib/test_summary_buildkite_plugin/runner.rb rename to lib/test_summary_buildkite_plugin/main.rb index dbac504..70f9c4f 100644 --- a/lib/test_summary_buildkite_plugin/runner.rb +++ b/lib/test_summary_buildkite_plugin/main.rb @@ -1,8 +1,9 @@ # frozen_string_literal: true module TestSummaryBuildkitePlugin - class Runner + class Main MAX_MARKDOWN_SIZE = 100_000 + OUTPUT_PATH = 'test-summary.html' attr_reader :options @@ -11,23 +12,37 @@ def initialize(options) end def run - markdown = Truncater.new( + processor = Processor.new( + formatter_options: formatter, max_size: MAX_MARKDOWN_SIZE, + output_path: OUTPUT_PATH, inputs: inputs, - formatter_opts: options[:formatter], fail_on_error: fail_on_error - ).markdown - if markdown.nil? || markdown.empty? + ) + + if processor.truncated_markdown.nil? || processor.truncated_markdown.empty? puts('No errors found! 🎉') else - annotate(markdown) + upload_artifact(processor.inputs_markdown) + annotate(processor.truncated_markdown) end end + private + + def upload_artifact(markdown) + File.write(OUTPUT_PATH, Utils.standalone_markdown(markdown)) + Agent.run('artifact', 'upload', OUTPUT_PATH) + end + def annotate(markdown) Agent.run('annotate', '--context', context, '--style', style, stdin: markdown) end + def formatter + options[:formatter] || {} + end + def inputs @inputs ||= options[:inputs].map { |opts| Input.create(opts.merge(fail_on_error: fail_on_error)) } end diff --git a/lib/test_summary_buildkite_plugin/processor.rb b/lib/test_summary_buildkite_plugin/processor.rb new file mode 100644 index 0000000..949ac44 --- /dev/null +++ b/lib/test_summary_buildkite_plugin/processor.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module TestSummaryBuildkitePlugin + class Processor + include ErrorHandler + + attr_reader :formatter_options, :max_size, :output_path, :inputs, :fail_on_error + + def initialize(formatter_options:, max_size:, output_path:, inputs:, fail_on_error:) + @formatter_options = formatter_options + @max_size = max_size + @output_path = output_path + @inputs = inputs + @fail_on_error = fail_on_error + @_formatters = {} + end + + def truncated_markdown + @truncated_markdown ||= begin + truncater = Truncater.new( + max_size: max_size, + max_truncate: inputs.map(&:failures).map(&:count).max + ) do |truncate| + inputs_markdown(truncate) + end + + truncater.markdown + rescue StandardError => e + handle_error(e, diagnostics) + HamlRender.render('truncater_exception', {}) + end + end + + def inputs_markdown(truncate = nil) + inputs.map { |input| input_markdown(input, truncate) }.compact.join("\n\n") + end + + private + + def input_markdown(input, truncate) + formatter(input).markdown(truncate) + rescue StandardError => e + handle_error(e) + end + + def formatter(input) + @_formatters[input] ||= Formatter.create(input: input, output_path: output_path, options: formatter_options) + end + + def diagnostics + { + formatter: formatter_options, + inputs: inputs.map do |input| + { + type: input.class, + failure_count: input.failures.count, + markdown_bytesize: input_markdown(input, nil)&.bytesize + } + end + } + end + end +end diff --git a/lib/test_summary_buildkite_plugin/truncater.rb b/lib/test_summary_buildkite_plugin/truncater.rb index 9b41578..9c5fa33 100644 --- a/lib/test_summary_buildkite_plugin/truncater.rb +++ b/lib/test_summary_buildkite_plugin/truncater.rb @@ -2,19 +2,17 @@ module TestSummaryBuildkitePlugin class Truncater - attr_reader :max_size, :inputs, :formatter_opts, :fail_on_error + attr_reader :max_size, :max_truncate - def initialize(max_size:, inputs:, formatter_opts: {}, fail_on_error: false) + def initialize(max_size:, max_truncate:, &blk) @max_size = max_size - @inputs = inputs - @formatter_opts = formatter_opts || {} - @fail_on_error = fail_on_error - @_input_markdown = {} - @_formatter = {} + @max_truncate = max_truncate + @blk = blk + @_truncations = {} end def markdown - requested = markdown_with_truncation(nil) + requested = with_truncation(nil) if requested.empty? || requested.bytesize < max_size # we can use it as-is, no need to truncate return requested @@ -26,65 +24,23 @@ def markdown # The block must return false for every value before the result # and true for the result and every value after best_truncate = (0..max_truncate).to_a.reverse.bsearch do |truncate| - puts "Test truncating to #{truncate}: bytesize=#{markdown_with_truncation(truncate).bytesize}" - markdown_with_truncation(truncate).bytesize <= max_size + puts "Test truncating to #{truncate}: bytesize=#{with_truncation(truncate).bytesize}" + with_truncation(truncate).bytesize <= max_size end if best_truncate.nil? # If we end up here, we failed to find a valid truncation value # ASAICT this should never happen but if it does, something is very wrong # so ask the user to let us know - return bug_report_message + return nil end puts "Optimal truncation: #{best_truncate}" - markdown_with_truncation(best_truncate) + with_truncation(best_truncate) end private - def formatter(truncate) - @_formatter[truncate] ||= Formatter.create(formatter_opts.merge(truncate: truncate)) - end - - def input_markdown(input, truncate = nil) - @_input_markdown[[input, truncate]] ||= formatter(truncate).markdown(input) - rescue StandardError => e - if fail_on_error - raise - else - Utils.log_error(e) - nil - end - end - - def markdown_with_truncation(truncate) - inputs.map { |input| input_markdown(input, truncate) }.compact.join("\n\n") - end - - def max_truncate - @max_truncate ||= inputs.map(&:failures).map(&:count).max - end - - def bug_report_message - puts - puts 'Optimization failed 😱' - puts 'Please report this to https://github.com/bugcrowd/test-summary-buildkite-plugin/issues' - puts 'with the test log above and the details below.' - puts JSON.pretty_generate(diagnostics) - HamlRender.render('truncater_exception', {}) - end - - def diagnostics - { - max_size: max_size, - formatter: formatter_opts, - inputs: inputs.map do |input| - { - type: input.class, - failure_count: input.failures.count, - markdown_bytesize: input_markdown(input, nil)&.bytesize - } - end - } + def with_truncation(truncate) + @_truncations[truncate] ||= @blk.call(truncate) end end end diff --git a/lib/test_summary_buildkite_plugin/utils.rb b/lib/test_summary_buildkite_plugin/utils.rb index 143126f..83746de 100644 --- a/lib/test_summary_buildkite_plugin/utils.rb +++ b/lib/test_summary_buildkite_plugin/utils.rb @@ -1,11 +1,20 @@ # frozen_string_literal: true +require 'kramdown' + module TestSummaryBuildkitePlugin module Utils - def log_error(err) + def self.log_error(err, diagnostics = nil) puts "#{err.class}: #{err.message}\n\n#{err.backtrace.join("\n")}" + puts + puts 'Please report this to https://github.com/bugcrowd/test-summary-buildkite-plugin/issues' + puts 'with the log above and the details below, if present.' + puts JSON.pretty_generate(diagnostics) unless diagnostics.nil? end - module_function :log_error + def self.standalone_markdown(markdown) + content = Kramdown::Document.new(markdown).to_html + HamlRender.render('standalone_layout', content: content) + end end end diff --git a/spec/test_summary_buildkite_plugin/formatter_spec.rb b/spec/test_summary_buildkite_plugin/formatter_spec.rb index a4d4b31..7a37c4f 100644 --- a/spec/test_summary_buildkite_plugin/formatter_spec.rb +++ b/spec/test_summary_buildkite_plugin/formatter_spec.rb @@ -7,9 +7,10 @@ let(:truncate) { nil } let(:input) { double(TestSummaryBuildkitePlugin::Input::Base, label: 'animals') } let(:failures) { [] } - let(:options) { { type: type, show_first: show_first, truncate: truncate } } + let(:options) { { type: type, show_first: show_first } } + let(:formatter) { described_class.create(input: input, output_path: 'output_path', options: options) } - subject(:markdown) { described_class.create(options).markdown(input) } + subject(:markdown) { formatter.markdown(truncate) } before do allow(input).to receive(:failures).and_return(failures) @@ -72,6 +73,10 @@ expect(markdown).to include('See all failures') + end end context 'without truncation' do @@ -245,7 +258,7 @@ end describe 'with no formatter options' do - subject(:markdown) { described_class.create.markdown(input) } + let(:options) { {} } let(:failures) do [TestSummaryBuildkitePlugin::Failure::Structured.new( summary: 'ponies are awesome', diff --git a/spec/test_summary_buildkite_plugin/runner_spec.rb b/spec/test_summary_buildkite_plugin/main_spec.rb similarity index 90% rename from spec/test_summary_buildkite_plugin/runner_spec.rb rename to spec/test_summary_buildkite_plugin/main_spec.rb index 8f15aa7..fe75153 100644 --- a/spec/test_summary_buildkite_plugin/runner_spec.rb +++ b/spec/test_summary_buildkite_plugin/main_spec.rb @@ -2,11 +2,11 @@ require 'spec_helper' -RSpec.describe TestSummaryBuildkitePlugin::Runner do +RSpec.describe TestSummaryBuildkitePlugin::Main do let(:params) { { inputs: inputs } } - let(:runner) { described_class.new(params) } + let(:main) { described_class.new(params) } - subject(:run) { runner.run } + subject(:run) { main.run } context 'with no failures' do let(:inputs) do diff --git a/spec/test_summary_buildkite_plugin/processor_spec.rb b/spec/test_summary_buildkite_plugin/processor_spec.rb new file mode 100644 index 0000000..0d809e4 --- /dev/null +++ b/spec/test_summary_buildkite_plugin/processor_spec.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe TestSummaryBuildkitePlugin::Processor do + let(:processor) do + described_class.new( + formatter_options: {}, + max_size: 50_000, + output_path: 'foo', + inputs: inputs, + fail_on_error: fail_on_error + ) + end + + context 'formatter raises exceptions' do + let(:formatter1) { spy } + let(:formatter2) { spy } + + let(:input1) { double(TestSummaryBuildkitePlugin::Input::Base) } + let(:input2) { double(TestSummaryBuildkitePlugin::Input::Base) } + let(:inputs) { [input1, input2] } + + before do + allow(processor).to receive(:formatter).with(input1).and_return(formatter1) + allow(formatter1).to receive(:markdown).with(nil).and_raise('life sucks') + allow(processor).to receive(:formatter).with(input2).and_return(formatter2) + allow(formatter2).to receive(:markdown).with(nil).and_return('awesome markdown') + allow(input1).to receive(:failures).and_return([]) + allow(input2).to receive(:failures).and_return([]) + end + + context 'without fail_on_error' do + let(:fail_on_error) { false } + + it 'continues' do + expect(processor.truncated_markdown).to include('awesome markdown') + end + + it 'logs the error' do + expect { processor.truncated_markdown }.to output(/life sucks/).to_stdout + end + end + + context 'with fail_on_error' do + let(:fail_on_error) { true } + + it 'raises error' do + expect { processor.truncated_markdown }.to raise_error('life sucks') + end + end + end +end diff --git a/spec/test_summary_buildkite_plugin/truncater_spec.rb b/spec/test_summary_buildkite_plugin/truncater_spec.rb index e2139ba..0714623 100644 --- a/spec/test_summary_buildkite_plugin/truncater_spec.rb +++ b/spec/test_summary_buildkite_plugin/truncater_spec.rb @@ -3,31 +3,13 @@ require 'spec_helper' RSpec.describe TestSummaryBuildkitePlugin::Truncater do - let(:max_size) { 5000 } - let(:input1) { double(TestSummaryBuildkitePlugin::Input::Base, label: 'animals') } - let(:input2) { double(TestSummaryBuildkitePlugin::Input::Base, label: 'cars') } - let(:inputs) { [input1, input2] } - let(:failures1) do - %w[dog cat pony horse unicorn].map { |x| TestSummaryBuildkitePlugin::Failure::Unstructured.new(x) } - end - let(:failures2) do - %w[toyota honda holden ford mazda volkswagen].map { |x| TestSummaryBuildkitePlugin::Failure::Unstructured.new(x) } - end - let(:formatter_opts) { nil } - let(:fail_on_error) { false } - let(:options) { { max_size: max_size, inputs: inputs, formatter_opts: formatter_opts, fail_on_error: fail_on_error } } - let(:truncater) { described_class.new(options) } + let(:max_size) { 400 } + let(:truncater) { described_class.new(max_size: max_size, max_truncate: 200, &blk) } subject(:truncated) { truncater.markdown } - before do - allow(input1).to receive(:failures).and_return(failures1) - allow(input2).to receive(:failures).and_return(failures2) - end - - context 'when no failures' do - let(:failures1) { [] } - let(:failures2) { [] } + context 'when empty string' do + let(:blk) { Proc.new { '' } } # rubocop:disable Style/Proc it 'returns nothing' do is_expected.to be_empty @@ -35,61 +17,30 @@ end context 'when below max size' do - let(:max_size) { 50_000 } + let(:blk) { Proc.new { 'foo' } } # rubocop:disable Style/Proc it 'returns all failures' do - is_expected.to include('dog', 'cat', 'pony', 'horse', 'unicorn') - is_expected.to include('toyota', 'honda', 'holden', 'ford', 'mazda', 'volkswagen') + is_expected.to eq('foo') end end context 'when above max size' do - let(:max_size) { 400 } + let(:blk) { Proc.new { |truncate| 'foo' * (truncate || max_size) } } # rubocop:disable Style/Proc it 'returns something below max size' do expect(truncated.bytesize).to be <= max_size end it 'optimally truncates' do - is_expected.to include('Including first 4') + is_expected.to eq('foo' * 133) # 399 characters end end context 'when optimization fails' do - before do - allow(truncater).to receive(:markdown_with_truncation).and_return('a' * (max_size + 1)) - end - - it 'shows a helpful error' do - is_expected.to include('ANNOTATION ERROR') - end - end - - context 'formatter raises exceptions' do - let(:formatter) { spy } - - before do - allow(truncater).to receive(:formatter).and_return(formatter) - allow(formatter).to receive(:markdown).with(input1).and_raise('life sucks') - allow(formatter).to receive(:markdown).with(input2).and_return('awesome markdown') - end - - context 'without fail_on_error' do - it 'continues' do - expect(truncated).to include('awesome markdown') - end - - it 'logs the error' do - expect { truncated }.to output(/life sucks/).to_stdout - end - end - - context 'with fail_on_error' do - let(:fail_on_error) { true } + let(:blk) { Proc.new { 'foo' * max_size } } # rubocop:disable Style/Proc - it 'raises error' do - expect { truncated }.to raise_error('life sucks') - end + it 'returns nil' do + is_expected.to be_nil end end end diff --git a/templates/details/failures.html.haml b/templates/details/failures.html.haml index ad1d4cc..31a5c27 100644 --- a/templates/details/failures.html.haml +++ b/templates/details/failures.html.haml @@ -27,3 +27,7 @@ - else %code = failure.summary + +- if output_path + %a{href: "artifact://#{output_path}"} + See all failures diff --git a/templates/test_layout.html.haml b/templates/standalone_layout.html.haml similarity index 100% rename from templates/test_layout.html.haml rename to templates/standalone_layout.html.haml diff --git a/templates/summary/failures.html.haml b/templates/summary/failures.html.haml index 092cf94..349a206 100644 --- a/templates/summary/failures.html.haml +++ b/templates/summary/failures.html.haml @@ -1,3 +1,7 @@ %pre %code = failures.map(&:summary).join("\n") + +- if output_path + %a{href: "artifact://#{output_path}"} + See all failures