67 lines
1.6 KiB
Ruby
67 lines
1.6 KiB
Ruby
|
require 'bundler'
|
||
|
|
require 'rdoc/task'
|
||
|
|
require 'rake/testtask'
|
||
|
|
|
||
|
|
task :default => :test
|
||
|
|
|
||
|
|
Bundler::GemHelper.install_tasks
|
||
|
|
|
||
|
|
RDoc::Task.new do |task|
|
||
|
|
task.rdoc_dir = 'doc'
|
||
|
|
task.title = 'BufferedTokenizer'
|
||
|
|
task.rdoc_files.include('lib/**/*.rb')
|
||
|
|
end
|
||
|
|
|
||
|
|
Rake::TestTask.new :test do |t|
|
||
|
|
t.libs << 'lib'
|
||
|
|
t.test_files = FileList['test/**/*.rb']
|
||
|
|
end
|
||
|
|
|
||
|
|
desc "Benchmark the current implementation"
|
||
|
|
task :bench do
|
||
|
|
require 'benchmark'
|
||
|
|
require File.expand_path('lib/buftok', File.dirname(__FILE__))
|
||
|
|
|
||
|
|
n = 50000
|
||
|
|
delimiter = "\n\n"
|
||
|
|
|
||
|
|
frequency1 = 1000
|
||
|
|
puts "generating #{n} strings, with #{delimiter.inspect} every #{frequency1} strings..."
|
||
|
|
data1 = (0...n).map do |i|
|
||
|
|
(((i % frequency1 == 1) ? "\n" : "") +
|
||
|
|
("s" * i) +
|
||
|
|
((i % frequency1 == 0) ? "\n" : "")).freeze
|
||
|
|
end
|
||
|
|
|
||
|
|
frequency2 = 10
|
||
|
|
puts "generating #{n} strings, with #{delimiter.inspect} every #{frequency2} strings..."
|
||
|
|
data2 = (0...n).map do |i|
|
||
|
|
(((i % frequency2 == 1) ? "\n" : "") +
|
||
|
|
("s" * i) +
|
||
|
|
((i % frequency2 == 0) ? "\n" : "")).freeze
|
||
|
|
end
|
||
|
|
|
||
|
|
Benchmark.bmbm do |x|
|
||
|
|
x.report("1 char, freq: #{frequency1}") do
|
||
|
|
bt1 = BufferedTokenizer.new
|
||
|
|
n.times { |i| bt1.extract(data1[i]) }
|
||
|
|
end
|
||
|
|
|
||
|
|
x.report("2 char, freq: #{frequency1}") do
|
||
|
|
bt2 = BufferedTokenizer.new(delimiter)
|
||
|
|
n.times { |i| bt2.extract(data1[i]) }
|
||
|
|
end
|
||
|
|
|
||
|
|
x.report("1 char, freq: #{frequency2}") do
|
||
|
|
bt3 = BufferedTokenizer.new
|
||
|
|
n.times { |i| bt3.extract(data2[i]) }
|
||
|
|
end
|
||
|
|
|
||
|
|
x.report("2 char, freq: #{frequency2}") do
|
||
|
|
bt4 = BufferedTokenizer.new(delimiter)
|
||
|
|
n.times { |i| bt4.extract(data2[i]) }
|
||
|
|
end
|
||
|
|
|
||
|
|
end
|
||
|
|
end
|