innertube-1.1.0/0000755000004100000410000000000012576036132013541 5ustar www-datawww-datainnertube-1.1.0/innertube.gemspec0000644000004100000410000000153512576036132017105 0ustar www-datawww-data$:.push File.expand_path("../lib", __FILE__) require 'innertube/version' Gem::Specification.new do |gem| gem.name = 'innertube' gem.version = Innertube::VERSION gem.summary = "A thread-safe resource pool, originally borne in riak-client (Ripple)." gem.description = "Because everyone needs their own pool library." gem.email = [ "sean@basho.com", "aphyr@aphyr.com" ] gem.homepage = "http://github.com/basho/innertube" gem.authors = ["Sean Cribbs", "Kyle Kingsbury"] gem.add_development_dependency 'rspec', '~> 2.10.0' # Files ignores = File.read(".gitignore").split(/\r?\n/).reject{ |f| f =~ /^(#.+|\s*)$/ }.map {|f| Dir[f] }.flatten gem.files = (Dir['**/*','.gitignore'] - ignores).reject {|f| !File.file?(f) } gem.test_files = (Dir['spec/**/*','.gitignore'] - ignores).reject {|f| !File.file?(f) } gem.require_paths = ['lib'] end innertube-1.1.0/Gemfile0000644000004100000410000000004612576036132015034 0ustar www-datawww-datasource "http://rubygems.org" gemspec innertube-1.1.0/spec/0000755000004100000410000000000012576036132014473 5ustar www-datawww-datainnertube-1.1.0/spec/spec_helper.rb0000644000004100000410000000070712576036132017315 0ustar www-datawww-data$: << File.expand_path('../../lib', __FILE__) $: << File.expand_path('..', __FILE__) require 'rubygems' require 'rspec' require 'innertube' require 'support/verbose_formatter' require 'support/timeout' RSpec.configure do |config| config.mock_with :rspec config.filter_run :focus => true config.run_all_when_everything_filtered = true if defined?(::Java) seed = Time.now.utc config.seed = seed else config.order = :random end end innertube-1.1.0/spec/innertube_spec.rb0000644000004100000410000002517712576036132020041 0ustar www-datawww-datarequire 'spec_helper' require 'thread' require 'thwait' describe Innertube::Pool do def wait_all(threads) message "Waiting on #{threads.size} threads: " ThreadsWait.all_waits(*threads) do |t| message "<#{threads.index(t) + 1}> " end message "\n" end let(:pool_members) { pool.instance_variable_get(:@pool) } let(:pool) { described_class.new(lambda { [0] }, lambda { |x| }) } it 'yields a new object when the pool is empty' do pool.take do |x| x.should == [0] end end it 'retains a single object for serial access' do n = 100 n.times do |i| pool.take do |x| x.should == [i] x[0] += 1 end end pool.size.should == 1 end it 'should be re-entrant' do n = 10 n.times do |i| pool.take do |x| x.replace [1] pool.take do |y| y.replace [2] pool.take do |z| z.replace [3] pool.take do |t| t.replace [4] end end end end end pool_members.map { |e| e.object.first }.sort.should == [1,2,3,4] end it 'should be fillable with existing resources' do pool.fill(["Apple", "Banana", "Kiwi"]) pool_members.size.should == 3 pool.take do |x| x.should eq('Apple') pool.take do |y| y.should eq('Banana') pool.take do |z| z.should eq('Kiwi') end end end end it 'should unlock when exceptions are raised' do begin pool.take do |x| x << 1 pool.take do |y| x << 2 y << 3 raise end end rescue end pool_members.should be_all {|e| not e.owner } pool_members.map { |e| e.object }.should =~ [[0,1,2],[0,3]] end context 'when BadResource is raised' do let(:pool) do described_class.new(lambda { mock('resource').tap {|m| m.should_receive(:close) } }, lambda { |res| res.close }) end it 'should remove the member from the pool' do lambda do pool.take do |x| raise Innertube::Pool::BadResource end end.should raise_error(Innertube::Pool::BadResource) pool_members.size.should == 0 end end context 'threaded access' do let!(:pool) { described_class.new(lambda { [] }, lambda { |x| }) } it 'should allocate n objects for n concurrent operations' do # n threads concurrently allocate and sign objects from the pool n = 10 readyq = Queue.new finishq = Queue.new threads = (0...n).map do Thread.new do pool.take do |x| readyq << 1 x << Thread.current finishq.pop end end end # Give the go-ahead to all threads n.times { readyq.pop } # Let all threads finish n.times { finishq << 1 } # Wait for completion ThreadsWait.all_waits(*threads) # Should have taken exactly n objects to do this pool_members.size.should == n # And each one should be signed exactly once pool_members.map do |e| e.object.size.should == 1 e.object.first end.should =~ threads end it 'take with filter and default' do n = 10 pool = described_class.new(lambda { [] }, lambda { |x| }) # Allocate several elements of the pool q = Queue.new finishq = Queue.new threads = (0...n).map do |i| Thread.new do pool.take do |a| q << 1 a << i finishq.pop end end end # Wait for all threads to have acquired an element n.times { q.pop } # Let all threads finish n.times { finishq << 1 } # Wait for completion # threads.each {|t| t.join } ThreadsWait.all_waits(*threads) # Get and delete existing even elements got = [] (n / 2).times do begin pool.take( :filter => lambda { |x| x.first.even? }, :default => [:default] ) do |x| got << x.first raise Innertube::Pool::BadResource end rescue Innertube::Pool::BadResource end end got.should =~ (0...n).select(&:even?) # This time, no even elements exist, so we should get the default. pool.take(:filter => lambda { |x| x.first.even? }, :default => :default) do |x| x.should == :default end end it 'iterates over a snapshot of all connections, even ones in use' do started = Queue.new n = 30 threads = (0..n).map do Thread.new do psleep = 0.75 * rand # up to 50ms sleep pool.take do |a| started << 1 a << rand sleep psleep end end end n.times { started.pop } touched = [] pool.each {|e| touched << e } wait_all threads touched.should be_all {|item| pool_members.any? {|e| e.object == item } } end context 'clearing the pool' do let(:pool) do described_class.new(lambda { mock('connection').tap {|m| m.should_receive(:teardown) }}, lambda { |b| b.teardown }) end it 'should remove all elements' do n = 10 q, fq = Queue.new, Queue.new # Allocate several elements of the pool threads = (0...n).map do |i| Thread.new do pool.take do |a| q << i sleep(rand * 0.5) message "W<#{i}> " fq.pop message "X<#{i}> " end end end # Wait for all threads to have acquired an element n.times { message "S<#{q.pop}> " } # Start a thread to push stuff onto the finish queue, allowing # the worker threads to exit pusher = Thread.new do n.times do |i| message "R<#{i}> " fq << 1 sleep(rand * 0.1) end end # Clear the pool while threads still have elements checked out message "S " pool.clear message "X " # Wait for threads to complete wait_all(threads + [pusher]) pool_members.should be_empty end end context 'conditionally deleting members' do let(:pool) { described_class.new( lambda { [] }, lambda { |x| } ) } it 'should remove them from the pool' do n = 10 # Allocate several elements of the pool q = Queue.new threads = (0...n).map do |i| Thread.new do pool.take do |a| message "S<#{i}> " a << i q << i Thread.pass end end end # Wait for all threads to have acquired an element n.times { message "X<#{q.pop}> " } # Delete odd elements pool.delete_if do |x| x.first.odd? end # Verify odds are gone. pool_members.all? do |x| x.object.first.even? end.should == true # Wait for threads wait_all threads end end it 'iteration race-condition regression', :timeout => 60 do # This simulates a race-condition where the condition variable # waited on by the iterator until an element is released might # be signaled before the iterator begins waiting, thus dropping # the signal and sending the iterator into an infinite wait. # First we pick a largish random thread count, and split it into # threads that release before the iterator starts (split) and # ones that release while the iterator is busy (rest). n = rand(250) split = rand(n) rest = n - split message "[#{n}:#{split}] " # We use two queues to signal between the main thread and the # workers, and a queue to communicate with the iterator thread sq, fq, iq = Queue.new, Queue.new, Queue.new # Start all the worker threads threads = (0...n).map do |i| Thread.new do pool.take do |e| # Signal to the main thread that we're inside the take sq << i+1 # Block waiting on the main thread. When reactivated, log # the exit of the thread fq.pop message "X<#{i+1}> " sq << Thread.current end end end # Wait for all workers to start up, log their startup to the console n.times { message "S<#{sq.pop}> " } message "[all started] " # Now signal for the first group to continue finished = [] split.times { fq << 1; finished << sq.pop } wait_all finished message "[first group #{split}] " # Start the iterator thread iterator = Thread.new do Thread.current[:wait] = true pool.each do |e| # Block in the first iteration so the other workers can exit # while the iterator is not waiting on the condition variable if Thread.current[:wait] sq << 'i' iq.pop Thread.current[:wait] = false end # Make sure we've touched every element of the pool by # modifying every entry. e << 1 end message "X " end # Wait on the iterator thread to start message "S<#{sq.pop}> " # Now signal the remaining workers to finish, and wait on all # workers to exit (even ones that exited in the first pass) finished.clear rest.times { fq << 1; finished << sq.pop } wait_all(finished) message "[second group #{rest}] " # Now signal the iterator to continue, and wait for it to exit iq << 1 wait_all([ iterator ]) # Finally, verify that all elements of the pool were touched by # the iterator pool_members.each {|e| e.object.size.should == 1 } end it 'stress test', :timeout => 60 do n = rand(400) passes = rand(20) rounds = rand(200) breaker = rand message "[#{n}t:#{rounds}r:#{passes}p:#{'%0.5f' % breaker}b] " threads = (0...n).map do Thread.new do rounds.times do |i| pool.take do |a| a.should == [] a << Thread.current a.should == [Thread.current] # Pass and check passes.times do Thread.pass # Nobody else should get ahold of this while I'm idle a.should == [Thread.current] break if rand > breaker end a.delete Thread.current message "." end end end end wait_all threads end end end innertube-1.1.0/spec/support/0000755000004100000410000000000012576036132016207 5ustar www-datawww-datainnertube-1.1.0/spec/support/timeout.rb0000644000004100000410000000047512576036132020230 0ustar www-datawww-datarequire 'timeout' RSpec.configure do |config| config.include Timeout config.around(:each) do |example| time = example.metadata[:timeout] || 30 begin timeout(time, Timeout::Error) do example.run end rescue Timeout::Error => e example.send :set_exception, e end end end innertube-1.1.0/spec/support/verbose_formatter.rb0000644000004100000410000000451212576036132022266 0ustar www-datawww-datarequire 'rspec/core/formatters/base_text_formatter' class VerboseFormatter < RSpec::Core::Formatters::BaseTextFormatter attr_reader :column, :current_indentation def initialize(*args) super @mutex = Mutex.new @column = @current_indentation = 0 end def start(count) super output.puts output.puts "Running suite with seed #{RSpec.configuration.seed}\n" output.puts end def example_group_started(example_group) super # $stderr.puts example_group.metadata.inspect output.puts "#{padding}#{example_group.metadata[:example_group][:description_args].first}" indent! end def example_group_finished(example_group) super outdent! end def example_started(example) output.puts "#{padding}#{example.description}:" indent! end def message(m) @mutex.synchronize do messages = m.split(/\r?\n/).reject {|s| s.empty? } messages.each do |message| if column + message.length > max_columns output.puts @column = current_indentation end if at_left_margin? output.print "#{padding}#{message}" else output.print message end @column += message.length end end end def example_passed(example) super print_example_result green("PASS") end def example_failed(example) super print_example_result red("FAIL") end def example_pending(example) super print_example_result yellow("PENDING: #{example.metadata[:execution_result][:pending_message]}") end private def print_example_result(text) output.puts unless at_left_margin? output.puts "#{padding}#{text}" output.puts outdent! end def at_left_margin? column == current_indentation end def max_columns @max_columns ||= ENV.include?('COLUMNS') ? ENV['COLUMNS'].to_i : 72 end def indent_width 2 end def padding ' ' * current_indentation end def indent! @current_indentation += indent_width @column = @current_indentation end def outdent! @current_indentation -= indent_width @column = @current_indentation end end module ExposeFormatter def message(string) RSpec.configuration.formatters.first.message(string) end end RSpec.configure do |config| config.include ExposeFormatter config.add_formatter VerboseFormatter end innertube-1.1.0/lib/0000755000004100000410000000000012576036132014307 5ustar www-datawww-datainnertube-1.1.0/lib/innertube.rb0000644000004100000410000001325012576036132016630 0ustar www-datawww-datarequire 'thread' require 'set' # Innertube is a re-entrant thread-safe resource pool that was # extracted from the Riak Ruby Client # (https://github.com/basho/riak-ruby-client). # @see Pool module Innertube # A re-entrant thread-safe resource pool that generates new resources on # demand. # @private class Pool # Raised when a taken element should be deleted from the pool. class BadResource < RuntimeError; end # An element of the pool. Comprises an object with an owning # thread. Not usually needed by user code, and should not be # modified outside the {Pool}'s lock. class Element attr_reader :object, :owner # Creates a pool element # @param [Object] object the resource to wrap into the pool element def initialize(object) @object = object @owner = nil end # Claims this element of the pool for the current Thread. # Do not call this manually, it is only used from inside the pool. def lock @owner = Thread.current end # @return [true,false] Is this element locked/claimed? def locked? !unlocked? end # Releases this element of the pool from the current Thread. def unlock @owner = nil end # @return [true,false] Is this element available for use? def unlocked? owner.nil? end end # Creates a new resource pool. # @param [Proc, #call] open a callable which allocates a new object for the # pool # @param [Proc, #call] close a callable which is called with an # object before it is freed. def initialize(open, close) @open = open @close = close @lock = Mutex.new @iterator = Mutex.new @element_released = ConditionVariable.new @pool = Set.new end # Populate the pool with existing, open resources. # @param [Array] An array of resources. def fill(resources) @lock.synchronize do resources.each do |r| @pool << Element.new(r) end end end # On each element of the pool, calls close(element) and removes it. # @private def clear each_element do |e| delete_element e end end alias :close :clear # Deletes an element of the pool. Calls the close callback on its object. # Not intended for external use. # @param [Element] e the element to remove from the pool def delete_element(e) @close.call(e.object) @lock.synchronize do @pool.delete e end end private :delete_element # Locks each element in turn and closes/deletes elements for which the # object passes the block. # @yield [object] a block that should determine whether an element # should be deleted from the pool # @yieldparam [Object] object the resource def delete_if raise ArgumentError, "block required" unless block_given? each_element do |e| if yield e.object delete_element e end end end # Acquire an element of the pool. Yields the object. If all # elements are claimed, it will create another one. # @yield [resource] a block that will perform some action with the # element of the pool # @yieldparam [Object] resource a resource managed by the pool. # Locked for the duration of the block # @param [Proc, #call] :filter a callable which receives objects and has # the opportunity to reject each in turn. # @param [Object] :default if no resources are available, use this object # instead of calling #open. # @private def take(opts = {}) raise ArgumentError, "block required" unless block_given? result = nil element = nil opts[:filter] ||= proc {|_| true } @lock.synchronize do element = @pool.find { |e| e.unlocked? && opts[:filter].call(e.object) } unless element # No objects were acceptable resource = opts[:default] || @open.call element = Element.new(resource) @pool << element end element.lock end begin result = yield element.object rescue BadResource delete_element element raise ensure # Unlock if element element.unlock @element_released.signal end end result end alias >> take # Iterate over a snapshot of the pool. Yielded objects are locked # for the duration of the block. This may block the current thread # until elements in the snapshot are released by other threads. # @yield [element] a block that will do something with each # element in the pool # @yieldparam [Element] element the current element in the # iteration def each_element targets = @pool.to_a unlocked = [] @iterator.synchronize do until targets.empty? @lock.synchronize do @element_released.wait(@iterator) if targets.all? {|e| e.locked? } unlocked, targets = targets.partition {|e| e.unlocked? } unlocked.each {|e| e.lock } end unlocked.each do |e| begin yield e ensure e.unlock end end end end end # As each_element, but yields objects, not wrapper elements. # @yield [resource] a block that will do something with each # resource in the pool # @yieldparam [Object] resource the current resource in the # iteration def each each_element do |e| yield e.object end end # @return [Integer] the number of the resources in the pool def size @lock.synchronize { @pool.size } end end end innertube-1.1.0/lib/innertube/0000755000004100000410000000000012576036132016302 5ustar www-datawww-datainnertube-1.1.0/lib/innertube/version.rb0000644000004100000410000000005112576036132020310 0ustar www-datawww-datamodule Innertube VERSION = "1.1.0" end innertube-1.1.0/metadata.yml0000644000004100000410000000336512576036132016053 0ustar www-datawww-data--- !ruby/object:Gem::Specification name: innertube version: !ruby/object:Gem::Version version: 1.1.0 prerelease: platform: ruby authors: - Sean Cribbs - Kyle Kingsbury autorequire: bindir: bin cert_chain: [] date: 2013-07-29 00:00:00.000000000 Z dependencies: - !ruby/object:Gem::Dependency name: rspec requirement: !ruby/object:Gem::Requirement none: false requirements: - - ~> - !ruby/object:Gem::Version version: 2.10.0 type: :development prerelease: false version_requirements: !ruby/object:Gem::Requirement none: false requirements: - - ~> - !ruby/object:Gem::Version version: 2.10.0 description: Because everyone needs their own pool library. email: - sean@basho.com - aphyr@aphyr.com executables: [] extensions: [] extra_rdoc_files: [] files: - Gemfile - innertube.gemspec - lib/innertube/version.rb - lib/innertube.rb - LICENSE - README.md - spec/innertube_spec.rb - spec/spec_helper.rb - spec/support/timeout.rb - spec/support/verbose_formatter.rb - .gitignore homepage: http://github.com/basho/innertube licenses: [] post_install_message: rdoc_options: [] require_paths: - lib required_ruby_version: !ruby/object:Gem::Requirement none: false requirements: - - ! '>=' - !ruby/object:Gem::Version version: '0' required_rubygems_version: !ruby/object:Gem::Requirement none: false requirements: - - ! '>=' - !ruby/object:Gem::Version version: '0' requirements: [] rubyforge_project: rubygems_version: 1.8.23 signing_key: specification_version: 3 summary: A thread-safe resource pool, originally borne in riak-client (Ripple). test_files: - spec/innertube_spec.rb - spec/spec_helper.rb - spec/support/timeout.rb - spec/support/verbose_formatter.rb - .gitignore has_rdoc: innertube-1.1.0/.gitignore0000644000004100000410000000027312576036132015533 0ustar www-datawww-data## MAC OS .DS_Store ## TEXTMATE *.tmproj tmtags ## EMACS *~ \#* .\#* ## VIM *.swp ## PROJECT::GENERAL coverage rdoc pkg ## PROJECT::SPECIFIC .bundle Gemfile.lock **/bin *.rbc .rvmrc innertube-1.1.0/LICENSE0000644000004100000410000000127512576036132014553 0ustar www-datawww-dataCopyright 2011-2012 Sean Cribbs, Kyle Kingsbury and Basho Technologies, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. All of the files in this project are under the project-wide license unless they are otherwise marked. innertube-1.1.0/README.md0000644000004100000410000000335012576036132015021 0ustar www-datawww-data# Innertube Innertube is a thread-safe, re-entrant resource pool, extracted from the [Riak Ruby Client](/basho/riak-ruby-client), where it was used to pool connections to [Riak](/basho/riak). It is free to use and modify, licensed under the Apache 2.0 License. ## Example ```ruby # ------------------------------------------------------- # Basics # ------------------------------------------------------- # Create a pool with open/close callables pool = Innertube::Pool.new(proc { Connection.new }, proc {|c| c.disconnect }) # Optionally, fill the pool with existing resources pool.fill([conn1, conn2, conn3]) # Grab a connection from the pool, returns the same value # as the block pool.take {|conn| conn.ping } # => true # Raise the BadResource exception if the resource is no # longer good pool.take do |conn| raise Innertube::Pool::BadResource unless conn.connected? conn.ping end # Innertube helps your code be re-entrant! Take more resources # while you have one checked out. pool.take do |conn| conn.stream_tweets do |tweet| pool.take {|conn2| conn2.increment :tweets } end end # ------------------------------------------------------- # Iterations: These are slow because they have guarantees # about visiting all current elements of the pool. # ------------------------------------------------------- # Do something with every connection in the pool pool.each {|conn| puts conn.get_stats } # Expunge some expired connections from the pool pool.delete_if {|conn| conn.idle_time > 5 } ``` ## Credits The pool was originally implemented by [Kyle Kingsbury](/aphyr) and extracted by [Sean Cribbs](/seancribbs), when bugged about it by [Pat Allan](/freelancing-god) at [EuRuKo 2012](http://www.euruko2012.org/).