Improved Postgresql compatibility and failover support

.travis.yml

@@ -1,9 +1,6 @@
 language: ruby
 
 rvm:
-  - 1.8.7
-  - 1.9.2
-  - 1.9.3
   - 2.0.0
   - 2.1.1
   - jruby
@@ -28,19 +25,4 @@ before_script:
 
 before_install:
   - gem update --system 2.1.11
-  - gem --version
-
-matrix:
-  exclude:
-    - rvm: 1.8.7
-      gemfile: gemfiles/ar40.gemfile
-    - rvm: 1.9.2
-      gemfile: gemfiles/ar40.gemfile
-    - rvm: 1.8.7
-      gemfile: gemfiles/ar41.gemfile
-    - rvm: 1.9.2
-      gemfile: gemfiles/ar41.gemfile
-    - rvm: 1.8.7
-      gemfile: gemfiles/ar42.gemfile
-    - rvm: 1.9.2
-      gemfile: gemfiles/ar42.gemfile
+  - gem --version

lib/active_record/connection_adapters/makara_abstract_adapter.rb

@@ -16,13 +16,18 @@ class ErrorHandler < ::Makara::ErrorHandler
 
 
         CONNECTION_MATCHERS = [
-          /(closed|lost|no|terminating|terminated)\s?([^\s]+)?\sconnection/i,
-          /gone away/i,
-          /connection[^:]+refused/i,
-          /could not connect/i,
-          /can\'t connect/i,
-          /cannot connect/i,
-          /connection[^:]+closed/i
+          /(closed|lost|no|terminating|terminated)\s?([^\s]+)?\sconnection/,
+          /gone away/,
+          /connection[^:]+refused/,
+          /could not connect/,
+          /can\'t connect/,
+          /cannot connect/,
+          /connection[^:]+closed/,
+          /can\'t get socket descriptor/,
+          /connection to [a-z0-9.]+:[0-9]+ refused/,
+          /timeout expired/,
+          /could not translate host name/,
+          /the database system is (starting|shutting)/
         ].map(&:freeze).freeze
 
 
@@ -99,8 +104,8 @@ def custom_error_message?(connection, message)
       end
 
 
-      hijack_method :execute, :select_rows, :exec_query
-      send_to_all :connect, :disconnect!, :reconnect!, :verify!, :clear_cache!, :reset!
+      hijack_method :execute, :select_rows, :exec_query, :transaction
+      send_to_all :connect, :reconnect!, :verify!, :clear_cache!, :reset!
 
       SQL_MASTER_MATCHERS           = [/\A\s*select.+for update\Z/i, /select.+lock in share mode\Z/i].map(&:freeze).freeze
       SQL_SLAVE_MATCHERS            = [/\A\s*select\s/i].map(&:freeze).freeze
@@ -140,16 +145,18 @@ def initialize(config)
       def appropriate_connection(method_name, args)
         if needed_by_all?(method_name, args)
 
-          # slave pool must run first.
-          @slave_pool.provide_each do |con|
-            hijacked do
-              yield con
+          handling_an_all_execution(method_name) do
+            # slave pool must run first.
+            @slave_pool.provide_each do |con|
+              hijacked do
+                yield con
+              end
             end
-          end
 
-          @master_pool.provide_each do |con|
-            hijacked do
-              yield con
+            @master_pool.provide_each do |con|
+              hijacked do
+                yield con
+              end
             end
           end
 

lib/makara/connection_wrapper.rb

@@ -1,4 +1,3 @@
-require 'delegate'
 require 'active_support/core_ext/hash/keys'
 
 # Makara::ConnectionWrapper wraps the instance of an underlying connection.
@@ -7,15 +6,23 @@
 # Makara::Proxy.
 
 module Makara
-  class ConnectionWrapper < ::SimpleDelegator
+  class ConnectionWrapper
 
-    def initialize(proxy, connection, config)
-      super(connection)
+    attr_accessor :initial_error, :config
+
+    # invalid queries caused by connections switching that needs to be replaced
+    SQL_REPLACE = {"SET client_min_messages TO ''".freeze => "SET client_min_messages TO 'warning'".freeze}.freeze
 
+    def initialize(proxy, connection, config)
       @config = config.symbolize_keys
+      @connection = connection
       @proxy  = proxy
 
-      _makara_decorate_connection(connection)
+      if connection.nil?
+        _makara_blacklist!
+      else
+        _makara_decorate_connection(connection)
+      end
     end
 
     # the weight of the current node
@@ -35,6 +42,8 @@ def _makara_blacklisted?
 
     # blacklist this node for @config[:blacklist_duration] seconds
     def _makara_blacklist!
+      @connection.disconnect! if @connection
+      @connection = nil
       @blacklisted_until = Time.now.to_i + @config[:blacklist_duration]
     end
 
@@ -48,20 +57,55 @@ def _makara_custom_error_matchers
       @custom_error_matchers ||= (@config[:connection_error_matchers] || [])
     end
 
+    def _makara_connected?
+      _makara_connection.present?
+    rescue Makara::Errors::BlacklistConnection
+      false
+    end
+
+    def _makara_connection
+      current = @connection
+
+      if current
+        current
+      else # blacklisted connection or initial error
+        new_connection = @proxy.graceful_connection_for(@config)
+
+        # Already wrapped because of initial failure
+        if new_connection.is_a?(Makara::ConnectionWrapper)
+          _makara_blacklist!
+          raise Makara::Errors::BlacklistConnection.new(self, new_connection.initial_error)
+        else
+          @connection = new_connection
+          _makara_decorate_connection(new_connection)
+          new_connection
+        end
+      end
+    end
+
+    def execute(*args)
+      SQL_REPLACE.each do |find, replace|
+        if args[0] == find
+          args[0] = replace
+        end
+      end
+
+      _makara_connection.execute(*args)
+    end
+
     # we want to forward all private methods, since we could have kicked out from a private scenario
     def method_missing(m, *args, &block)
-      target = __getobj__
-      begin
-        target.respond_to?(m, true) ? target.__send__(m, *args, &block) : super(m, *args, &block)
-      ensure
-        $@.delete_if {|t| %r"\A#{Regexp.quote(__FILE__)}:#{__LINE__-2}:"o =~ t} if $@
+      if _makara_connection.respond_to?(m)
+        _makara_connection.public_send(m, *args, &block)
+      else # probably private method
+        _makara_connection.__send__(m, *args, &block)
       end
     end
 
 
     class_eval <<-RUBY_EVAL, __FILE__, __LINE__ + 1
       def respond_to#{RUBY_VERSION.to_s =~ /^1.8/ ? nil : '_missing'}?(m, include_private = false)
-        __getobj__.respond_to?(m, true)
+        _makara_connection.respond_to?(m, true)
       end
     RUBY_EVAL
 

lib/makara/errors/blacklist_connection.rb

@@ -2,7 +2,10 @@ module Makara
   module Errors
     class BlacklistConnection < ::StandardError
 
+      attr_reader :original_error
+
       def initialize(connection, error)
+        @original_error = error
         super "[Makara/#{connection._makara_name}] #{error.message}"
       end
 

lib/makara/pool.rb

@@ -9,7 +9,7 @@ class Pool
 
     # there are cases when we understand the pool is busted and we essentially want to skip
     # all execution
-    attr_writer :disabled
+    attr_accessor :disabled
     attr_reader :blacklist_errors
     attr_reader :role
     attr_reader :connections
@@ -38,7 +38,15 @@ def add(config)
       config[:name] ||= "#{@role}/#{@connections.length + 1}"
 
       connection = yield
-      wrapper = Makara::ConnectionWrapper.new(@proxy, connection, config)
+
+      # already wrapped because of initial error
+      if connection.is_a?(Makara::ConnectionWrapper)
+        connection.config = config # to add :name
+        wrapper = connection
+      else
+        wrapper = Makara::ConnectionWrapper.new(@proxy, connection, config)
+      end
+
 
       # the weight results in N references to the connection, not N connections
       wrapper._makara_weight.times{ @connections << wrapper }
@@ -54,11 +62,6 @@ def add(config)
       wrapper
     end
 
-    def any
-      @connections.first
-    end
-
-
     # send this method to all available nodes
     def send_to_all(method, *args)
       ret = nil
@@ -98,10 +101,12 @@ def provide(allow_stickiness = true)
         value
 
       # if we've made any connections within this pool, we should report the blackout.
-      else
+      elsif connection_made?
         err = Makara::Errors::AllConnectionsBlacklisted.new(self, @blacklist_errors)
         @blacklist_errors = []
         raise err
+      else
+        raise Makara::Errors::NoConnectionsAvailable.new(@role) unless @disabled
       end
 
     # when a connection causes a blacklist error within the provided block, we blacklist it then retry
@@ -116,6 +121,11 @@ def provide(allow_stickiness = true)
     protected
 
 
+    # have we connected to any of the underlying connections.
+    def connection_made?
+      @connections.any?(&:_makara_connected?)
+    end
+
 
     # Get the next non-blacklisted connection. If the proxy is setup
     # to be sticky, provide back the current connection assuming it is