Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rename parameter names with the rule of other params #1412

Merged
merged 1 commit into from
Jan 10, 2017
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions lib/fluent/plugin/buffer.rb
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,10 @@ class BufferChunkOverflowError < BufferError; end # A record size is larger than

# If user specify this value and (chunk_size * queue_length) is smaller than total_size,
# then total_size is automatically configured to that value
config_param :queue_length_limit, :integer, default: nil
config_param :queue_limit_length, :integer, default: nil

# optional new limitations
config_param :chunk_records_limit, :integer, default: nil
config_param :chunk_limit_records, :integer, default: nil

# if chunk size (or records) is 95% or more after #write, then that chunk will be enqueued
config_param :chunk_full_threshold, :float, default: DEFAULT_CHUNK_FULL_THRESHOLD
Expand All @@ -73,8 +73,8 @@ def initialize

@chunk_limit_size = nil
@total_limit_size = nil
@queue_length_limit = nil
@chunk_records_limit = nil
@queue_limit_length = nil
@chunk_limit_records = nil

@stage = {} #=> Hash (metadata -> chunk) : not flushed yet
@queue = [] #=> Array (chunks) : already flushed (not written)
Expand All @@ -92,8 +92,8 @@ def persistent?
def configure(conf)
super

unless @queue_length_limit.nil?
@total_limit_size = @chunk_limit_size * @queue_length_limit
unless @queue_limit_length.nil?
@total_limit_size = @chunk_limit_size * @queue_limit_length
end
end

Expand Down Expand Up @@ -142,7 +142,7 @@ def storable?

## TODO: for back pressure feature
# def used?(ratio)
# @total_size_limit * ratio > @stage_size + @queue_size
# @total_limit_size * ratio > @stage_size + @queue_size
# end

def resume
Expand Down Expand Up @@ -433,11 +433,11 @@ def clear_queue!
end

def chunk_size_over?(chunk)
chunk.bytesize > @chunk_limit_size || (@chunk_records_limit && chunk.size > @chunk_records_limit)
chunk.bytesize > @chunk_limit_size || (@chunk_limit_records && chunk.size > @chunk_limit_records)
end

def chunk_size_full?(chunk)
chunk.bytesize >= @chunk_limit_size * @chunk_full_threshold || (@chunk_records_limit && chunk.size >= @chunk_records_limit * @chunk_full_threshold)
chunk.bytesize >= @chunk_limit_size * @chunk_full_threshold || (@chunk_limit_records && chunk.size >= @chunk_limit_records * @chunk_full_threshold)
end

class ShouldRetry < StandardError; end
Expand Down
2 changes: 1 addition & 1 deletion lib/fluent/plugin_helper/compat_parameters.rb
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ module CompatParameters
"retry_limit" => "retry_max_times",
"max_retry_wait" => "retry_max_interval",
"buffer_chunk_limit" => "chunk_limit_size",
"buffer_queue_limit" => "queue_length_limit",
"buffer_queue_limit" => "queue_limit_length",
"buffer_queue_full_action" => "overflow_action",
"flush_at_shutdown" => "flush_at_shutdown",
}
Expand Down
18 changes: 9 additions & 9 deletions test/plugin/test_buffer.rb
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ def create_chunk_es(metadata, es)
end

test 'chunk records limit is ignored in default' do
assert_nil @p.chunk_records_limit
assert_nil @p.chunk_limit_records
end

test '#storable? checks total size of staged and enqueued(includes dequeued chunks) against total_limit_size' do
Expand Down Expand Up @@ -1113,9 +1113,9 @@ def create_chunk_es(metadata, es)
end
end

sub_test_case 'with configuration includes chunk_records_limit' do
sub_test_case 'with configuration includes chunk_limit_records' do
setup do
@p = create_buffer({"chunk_limit_size" => 1024, "total_limit_size" => 10240, "chunk_records_limit" => 6})
@p = create_buffer({"chunk_limit_size" => 1024, "total_limit_size" => 10240, "chunk_limit_records" => 6})
@dm0 = dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil)
@dm1 = dm1 = create_metadata(Time.parse('2016-04-11 16:10:00 +0000').to_i, nil, nil)
@dm2 = dm2 = create_metadata(Time.parse('2016-04-11 16:20:00 +0000').to_i, nil, nil)
Expand All @@ -1139,7 +1139,7 @@ def create_chunk_es(metadata, es)
end

test '#chunk_size_over? returns true if too many records exists in a chunk even if its bytes is less than limit' do
assert_equal 6, @p.chunk_records_limit
assert_equal 6, @p.chunk_limit_records

m = create_metadata(Time.parse('2016-04-11 16:40:00 +0000').to_i)

Expand All @@ -1155,7 +1155,7 @@ def create_chunk_es(metadata, es)
end

test '#chunk_size_full? returns true if enough many records exists in a chunk even if its bytes is less than limit' do
assert_equal 6, @p.chunk_records_limit
assert_equal 6, @p.chunk_limit_records

m = create_metadata(Time.parse('2016-04-11 16:40:00 +0000').to_i)

Expand All @@ -1171,9 +1171,9 @@ def create_chunk_es(metadata, es)
end
end

sub_test_case 'with configuration includes queue_length_limit' do
sub_test_case 'with configuration includes queue_limit_length' do
setup do
@p = create_buffer({"chunk_limit_size" => 1024, "total_limit_size" => 10240, "queue_length_limit" => 5})
@p = create_buffer({"chunk_limit_size" => 1024, "total_limit_size" => 10240, "queue_limit_length" => 5})
@dm0 = dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil)
@dm1 = dm1 = create_metadata(Time.parse('2016-04-11 16:10:00 +0000').to_i, nil, nil)
@dm2 = dm2 = create_metadata(Time.parse('2016-04-11 16:20:00 +0000').to_i, nil, nil)
Expand All @@ -1196,9 +1196,9 @@ def create_chunk_es(metadata, es)
@p.start
end

test '#configure will overwrite standard configuration if queue_length_limit' do
test '#configure will overwrite standard configuration if queue_limit_length' do
assert_equal 1024, @p.chunk_limit_size
assert_equal 5, @p.queue_length_limit
assert_equal 5, @p.queue_limit_length
assert_equal (1024*5), @p.total_limit_size
end
end
Expand Down
2 changes: 1 addition & 1 deletion test/plugin_helper/test_compat_parameters.rb
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ def write(chunk); end # dummy
assert @i.buffer_config.flush_at_shutdown

assert_equal 8*1024*1024, @i.buffer.chunk_limit_size
assert_equal 1024, @i.buffer.queue_length_limit
assert_equal 1024, @i.buffer.queue_limit_length
end
end

Expand Down