(POOLER-129) Allow setting weights for backends

CHANGELOG.md

@@ -17,6 +17,7 @@ git logs & PR history.
 ### Added
 - Re-write check\_pool in pool\_manager to improve readability
 - Add a docker-compose file for testing vmpooler
+- Add capability to weight backends when an alias spans multiple backends (POOLER-129)
 
 # [0.2.0](/~https://github.com/puppetlabs/vmpooler/compare/0.1.0...0.2.0)
 

Gemfile

@@ -1,6 +1,7 @@
 source ENV['GEM_SOURCE'] || 'https://rubygems.org'
 
 gem 'json', '>= 1.8'
+gem 'pickup', '~> 0.0.11'
 gem 'puma', '~> 3.11'
 gem 'rack', '~> 2.0'
 gem 'rake', '~> 12.3'

lib/vmpooler.rb

@@ -1,15 +1,16 @@
 module Vmpooler
   require 'date'
   require 'json'
-  require 'open-uri'
   require 'net/ldap'
+  require 'open-uri'
+  require 'pickup'
   require 'rbvmomi'
   require 'redis'
+  require 'set'
   require 'sinatra/base'
   require 'time'
   require 'timeout'
   require 'yaml'
-  require 'set'
 
   %w[api graphite logger pool_manager statsd dummy_statsd generic_connection_pool].each do |lib|
     require "vmpooler/#{lib}"

lib/vmpooler/api/v1.rb

@@ -37,15 +37,38 @@ def need_token!
     end
 
     def fetch_single_vm(template)
-      vm = backend.spop('vmpooler__ready__' + template)
-      return [vm, template] if vm
-
+      template_backends = [template]
       aliases = Vmpooler::API.settings.config[:alias]
-      if aliases && aliased_template = aliases[template]
-        vm = backend.spop('vmpooler__ready__' + aliased_template)
-        return [vm, aliased_template] if vm
+      if aliases
+        template_backends << aliases[template] if aliases[template]
+
+        pool_index = pool_index(pools)
+        weighted_pools = {}
+        template_backends.each do |t|
+          next unless pool_index.key? t
+          index = pool_index[t]
+          clone_target = pools[index]['clone_target'] || config['clone_target']
+          next unless config.key?('backend_weight')
+          weight = config['backend_weight'][clone_target]
+          if weight
+            weighted_pools[t] = weight
+          end
+        end
+
+        if weighted_pools.count == template_backends.count
+          pickup = Pickup.new(weighted_pools)
+          selection = pickup.pick
+          template_backends.delete(selection)
+          template_backends.unshift(selection)
+        else
+          template_backends = template_backends.sample(template_backends.count)
+        end
       end
 
+      template_backends.each do |t|
+        vm = backend.spop('vmpooler__ready__' + t)
+        return [vm, t] if vm
+      end
       [nil, nil]
     end
 

lib/vmpooler/pool_manager.rb

@@ -188,8 +188,10 @@ def has_mismatched_hostname?(vm, pool, provider)
 
       # Check if the hostname has magically changed from underneath Pooler
       vm_hash = provider.get_vm(pool['name'], vm)
+      return unless vm_hash.is_a? Hash
       hostname = vm_hash['hostname']
 
+      return if hostname.nil?
       return if hostname.empty?
       return if hostname == vm
       $redis.smove('vmpooler__ready__' + pool['name'], 'vmpooler__completed__' + pool['name'], vm)
@@ -870,7 +872,7 @@ def check_ready_pool_vms(pool_name, provider, pool_check_response, inventory, po
         if inventory[vm]
           begin
             pool_check_response[:checked_ready_vms] += 1
-            check_ready_vm(vm, pool_name, pool_ttl, provider)
+            check_ready_vm(vm, pool_name, pool_ttl || 0, provider)
           rescue => err
             $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating ready VMs: #{err}")
           end

lib/vmpooler/providers/vsphere.rb

@@ -428,7 +428,7 @@ def destroy_vm(pool_name, vm_name)
 
         def vm_ready?(_pool_name, vm_name)
           begin
-            open_socket(vm_name)
+            open_socket(vm_name, global_config[:config]['domain'])
           rescue => _err
             return false
           end

spec/unit/pool_manager_spec.rb

@@ -2974,9 +2974,7 @@
         # mock response from create_inventory
         {vm => 1}
       }
-      let(:big_lifetime) {
-        2000
-      }
+      let(:big_lifetime) { 2000 }
       before(:each) do
         allow(subject).to receive(:check_ready_vm)
         create_ready_vm(pool,vm,token)
@@ -2992,7 +2990,7 @@
         expect(subject).to receive(:check_ready_vm).and_raise(RuntimeError,'MockError')
         expect(logger).to receive(:log).with('d', "[!] [#{pool}] _check_pool failed with an error while evaluating ready VMs: MockError")
 
-        subject.check_ready_pool_vms(pool, provider, pool_check_response, inventory)
+        subject.check_ready_pool_vms(pool, provider, pool_check_response, inventory, big_lifetime)
       end
 
       it 'should use the pool TTL if set' do
@@ -3511,12 +3509,8 @@
       end
 
       it 'captures #create_inventory errors correctly' do
-        allow(subject).to receive(:create_inventory).and_raise(
-          RuntimeError,'Mock Error'
-        )
-        expect {
-          subject._check_pool(pool_object, provider)
-        }.to_not raise_error(RuntimeError, /Mock Error/)
+        allow(subject).to receive(:create_inventory).and_raise(RuntimeError,'Mock Error')
+        subject._check_pool(pool_object, provider)
       end
 
       it 'should return early if an error occurs' do

spec/unit/providers/vsphere_spec.rb

@@ -925,9 +925,10 @@
   end
 
   describe '#vm_ready?' do
+    let(:domain) { nil }
     context 'When a VM is ready' do
       before(:each) do
-        expect(subject).to receive(:open_socket).with(vmname)
+        expect(subject).to receive(:open_socket).with(vmname, domain)
       end
 
       it 'should return true' do
@@ -939,7 +940,7 @@
       # TODO not sure how to handle a VM that is passed in but
       # not located in the pool.  Is that ready or not?
       before(:each) do
-        expect(subject).to receive(:open_socket).with(vmname)
+        expect(subject).to receive(:open_socket).with(vmname, domain)
       end
 
       it 'should return true' do

vmpooler.gemspec

@@ -17,6 +17,7 @@ Gem::Specification.new do |s|
   s.bindir        = 'bin'
   s.executables   = 'vmpooler'
   s.require_paths = ["lib"]
+  s.add_dependency 'pickup', '~> 0.0.11'
   s.add_dependency 'puma', '~> 3.11'
   s.add_dependency 'rack', '~> 2.0'
   s.add_dependency 'rake', '~> 12.3'

vmpooler.yaml.example

@@ -477,6 +477,14 @@
 #     Expects a boolean value
 #     (optional; default: false)
 #
+#   - backend_weight
+#     A hash of clone_target values with weights assigned to allow selecting VMs by alias with selection probability
+#     This setting is only used when there is a pool title and matching alias that both have values set in backend weight.
+#     When both conditions are met then the next VM is selected by probability using backend weight. When weight is not set
+#     in this configuration then distribution of load is random.
+#     Expects a hash value
+#     (optional)
+#
 # Example:
 
 :config:
@@ -493,6 +501,9 @@
   domain: 'example.com'
   prefix: 'poolvm-'
   experimental_features: true
+  backend_weight:
+    'backend1': 60
+    'backend2': 40
 
 # :pools:
 #