Merge pull request #291 from mattkirby/improved_docs

(doc) Document config via environment

README.md

@@ -14,17 +14,20 @@ At [Puppet, Inc.](http://puppet.com) we run acceptance tests on thousands of dis
 
 ### Prerequisites
 
-vmpooler requires the following Ruby gems be installed:
+vmpooler is available as a gem
 
-- [json](http://rubygems.org/gems/json)
-- [rbvmomi](http://rubygems.org/gems/rbvmomi)
-- [redis](http://rubygems.org/gems/redis)
-- [sinatra](http://rubygems.org/gems/sinatra)
+To use the gem `gem install vmpooler`
 
-It also requires that a [Redis](http://redis.io/) server exists somewhere, as this is the datastore used for vmpooler's inventory and queueing services.
+### Dependencies
+
+Vmpooler requires a [Redis](http://redis.io/) server. This is the datastore used for vmpooler's inventory and queueing services.
 
 ### Configuration
 
+Configuration for vmpooler may be provided via environment variables, or a configuration file.
+
+Please see this [configuration](docs/configuration.md) document for more details about configuring vmpooler via environment variables.
+
 The following YAML configuration sets up two pools, `debian-7-i386` and `debian-7-x86_64`, which contain 5 running VMs each:
 
 ```
@@ -62,59 +65,27 @@ See the provided YAML configuration example, [vmpooler.yaml.example](vmpooler.ya
 
 ### Running via Docker
 
-A [Dockerfile](Dockerfile) is included in this repository to allow running vmpooler inside a Docker container.  A `vmpooler.yaml` configuration file can be embedded in the current working directory, or specified inline in a `VMPOOLER_CONFIG` environment variable.  To build and run:
+A [Dockerfile](Dockerfile) is included in this repository to allow running vmpooler inside a Docker container. A configuration file can be used via volume mapping, and specifying the destination as the configuration file via environment variables, or the application can be configured with environment variables alone. The Dockerfile provides an entrypoint so you may choose whether to run API, or manager services. The default behavior will run both. To build and run:
 
 ```
 docker build -t vmpooler . && docker run -e VMPOOLER_CONFIG -p 80:4567 -it vmpooler
 ```
 
-### Running Docker inside Vagrant
-
-A [Vagrantfile](Vagrantfile) is also included in this repository so that you dont have to run Docker on your local computer.
-To use it run:
+To run only the API and dashboard
 
 ```
-vagrant up
-vagrant ssh
-docker run -p 8080:4567 -v /vagrant/vmpooler.yaml.example:/var/lib/vmpooler/vmpooler.yaml -it --rm --name pooler vmpooler
+docker run -p 80:4567 -it vmpooler api
 ```
 
-To run vmpooler with the example dummy provider you can replace the above docker command with this:
+To run only the manager component
 
 ```
-docker run -e VMPOOLER_DEBUG=true -p 8080:4567 -v /vagrant/vmpooler.yaml.dummy-example:/var/lib/vmpooler/vmpooler.yaml -e VMPOOLER_LOG='/var/log/vmpooler/vmpooler.log' -it --rm --name pooler vmpooler
+docker run -it vmpooler manager
 ```
 
-Either variation will allow you to access the dashboard from [localhost:8080](http://localhost:8080/).
-
-### Running directly in Vagrant
-
-You can also run vmpooler directly in the Vagrant box. To do so run this:
-
-```
-vagrant up
-vagrant ssh
-cd /vagrant
-
-# Do this if using the dummy provider
-export VMPOOLER_DEBUG=true
-cp vmpooler.yaml.dummy-example vmpooler.yaml
-
-# vmpooler needs a redis server.
-sudo yum -y install redis
-sudo systemctl start redis
-
-# Optional: Choose your ruby version or use jruby
-# ruby 2.4.x is used by default
-rvm list
-rvm use jruby-9.1.7.0
-
-gem install bundler
-bundle install
-bundle exec ruby vmpooler
-```
+### Running Docker inside Vagrant
 
-When run this way you can access vmpooler from your local computer via [localhost:4567](http://localhost:4567/).
+A vagrantfile is included in this repository. Please see [vagrant instructions](docs/vagrant.md) for details.
 
 ## API and Dashboard
 

docs/configuration.md

@@ -0,0 +1,196 @@
+# Configuring vmpooler
+
+Vmpooler configuration can be provided via file or environment variable. Some configuration elements are unique to API or manager components. This document aims to describe options for providing vmpooler configuration via environment variables.
+
+# Table of contents
+1. [Global options](#global)
+2. [Manager specific options](#manager)
+3. [API specific options](#API)
+
+## Global options <a name="global"></a>
+
+These options affect manager and API applications.
+
+### VMPOOLER\_CONFIG
+
+Provide the entire configuration as a blob of yaml. Individual parameters passed via environment variable will override any setting in this blob, or a configuration file passed in.
+
+### VMPOOLER\_CONFIG\_FILE
+
+Path to a the file to use when loading the vmpooler configuration. This is only evaluated if `VMPOOLER_CONFIG` has not been specified.
+
+### DOMAIN\_NAME
+
+If set, returns a top-level 'domain' JSON key in POST requests
+(optional)
+
+### REDIS\_SERVER
+
+The redis server to use for vmpooler.
+(optional; default: 'localhost')
+
+### REDIS\_PORT
+
+The port to use when connecting to redis.
+(optional)
+
+### REDIS\_PASSWORD
+
+The password to use when connecting to redis.
+(optional)
+
+### REDIS\_DATA\_TTL
+
+How long to keep data when set to expire in hours.
+(optional; default: 168)
+
+### STATSD\_SERVER
+
+The FQDN hostname of the statsd daemon.
+(optional)
+
+### STATSD\_PREFIX
+
+The prefix to use while storing statsd data.
+(optional; default: 'vmpooler')
+
+### STATSD\_PORT
+
+The UDP port to communicate with the statsd daemon.
+(optional; default: 8125)
+
+### GRAPHITE\_SERVER
+
+The FQDN hostname of the Graphite server.
+(optional)
+
+### GRAPHITE\_PREFIX
+
+The prefix to use while storing Graphite data.
+(optional; default: 'vmpooler')
+
+### GRAPHITE\_PORT
+
+The TCP port to communicate with the graphite server.
+(optional; default: 2003)
+
+## Manager options <a name="manager"></a>
+
+### TASK\_LIMIT
+
+The number of concurrent VM creation tasks to perform. Only affects manager.
+(optional; default: 10)
+
+### MIGRATION\_LIMIT
+
+When set to any value greater than 0 enable VM migration at checkout.
+When enabled this capability will evaluate a VM for migration to a different host when it is requested in an effort to maintain a more even distribution of load across compute resources. The number of migrations in progress at any one time are constrained by this value.
+(optional)
+
+### VM\_CHECKTIME
+
+How often (in minutes) to check the sanity of VMs in 'ready' queues.
+(optional; default: 15)
+
+### VM\_LIFETIME
+
+How long (in hours) to keep VMs in 'running' queues before destroying.
+(optional; default: 24)
+
+### VM\_LIFETIME\_AUTH
+
+Same as `vm_lifetime`, but applied if a valid authentication token is
+included during the request.
+(required)
+
+### VM\_PREFIX
+
+If set, prefixes all created VMs with this string. This should include a separator.
+(optional; default: '')
+
+### LOGFILE
+
+The file to use for logging manager operations.
+(optional; default: '/var/log/vmpooler.log')
+
+### CLONE\_TARGET
+
+The target cluster VMs are cloned into (host with least VMs chosen)
+(optional; default: same cluster/host as origin template)
+
+### TIMEOUT
+
+How long (in minutes) before marking a clone as 'failed' and retrying.
+(optional; default: 15)
+
+### MAX\_TRIES
+
+Set the max number of times a connection should retry in VM providers. This optional setting allows a user to dial in retry limits to suit your environment.
+(optional; default: 3)
+
+### RETRY\_FACTOR
+
+When retrying, each attempt sleeps for the try count * retry_factor.
+Increase this number to lengthen the delay between retry attempts.
+This is particularly useful for instances with a large number of pools
+to prevent a thundering herd when retrying connections.
+(optional; default: 10)
+
+### CREATE\_FOLDERS
+
+Create the pool folder specified in the pool configuration.
+Note: this will only create the last folder when it does not exist. It will not create any parent folders.
+(optional; default: false)
+
+### CREATE\_TEMPLATE\_DELTA\_DISKS
+
+Create backing delta disks for the specified templates to support creating linked clones.
+(optional; default: false)
+
+### PURGE\_UNCONFIGURED\_FOLDERS
+
+Enable purging of VMs and folders detected within the base folder path that are not configured for the provider
+Only a single layer of folders and their child VMs are evaluated from detected base folder paths
+A base folder path for 'vmpooler/redhat-7' would be 'vmpooler'
+When enabled in the global configuration then purging is enabled for all providers
+Expects a boolean value
+(optional; default: false)
+
+## API options <a name="API"></a>
+
+### AUTH\_PROVIDER
+
+The provider to use for authentication.
+(optional)
+
+### LDAP\_HOST
+
+The FQDN hostname of the LDAP server.
+(optional)
+
+### LDAP\_PORT
+
+The port used to connect to the LDAP service.
+(optional; default: 389)
+
+### LDAP\_BASE
+
+The base DN used for LDAP searches.
+This can be a string providing a single DN. For multiple DNs please specify the DNs as an array in a configuration file.
+(optional)
+
+### LDAP\_USER\_OBJECT
+
+The LDAP object-type used to designate a user object.
+(optional)
+
+### SITE\_NAME
+
+The name of your deployment.
+(optional; default: 'vmpooler')
+
+### EXPERIMENTAL\_FEATURES
+
+Enable experimental API capabilities such as changing pool template and size without application restart
+Expects a boolean value
+(optional; default: false)

docs/vagrant.md

@@ -0,0 +1,45 @@
+A [Vagrantfile](Vagrantfile) is also included in this repository so that you dont have to run Docker on your local computer.
+To use it run:
+
+```
+vagrant up
+vagrant ssh
+docker run -p 8080:4567 -v /vagrant/vmpooler.yaml.example:/var/lib/vmpooler/vmpooler.yaml -it --rm --name pooler vmpooler
+```
+
+To run vmpooler with the example dummy provider you can replace the above docker command with this:
+
+```
+docker run -e VMPOOLER_DEBUG=true -p 8080:4567 -v /vagrant/vmpooler.yaml.dummy-example:/var/lib/vmpooler/vmpooler.yaml -e VMPOOLER_LOG='/var/log/vmpooler/vmpooler.log' -it --rm --name pooler vmpooler
+```
+
+Either variation will allow you to access the dashboard from [localhost:8080](http://localhost:8080/).
+
+### Running directly in Vagrant
+
+You can also run vmpooler directly in the Vagrant box. To do so run this:
+
+```
+vagrant up
+vagrant ssh
+cd /vagrant
+
+# Do this if using the dummy provider
+export VMPOOLER_DEBUG=true
+cp vmpooler.yaml.dummy-example vmpooler.yaml
+
+# vmpooler needs a redis server.
+sudo yum -y install redis
+sudo systemctl start redis
+
+# Optional: Choose your ruby version or use jruby
+# ruby 2.4.x is used by default
+rvm list
+rvm use jruby-9.1.7.0
+
+gem install bundler
+bundle install
+bundle exec ruby vmpooler
+```
+
+When run this way you can access vmpooler from your local computer via [localhost:4567](http://localhost:4567/).

lib/vmpooler.rb

@@ -59,12 +59,12 @@ def self.config(filepath = 'vmpooler.yaml')
     parsed_config[:config]['clone_target'] = ENV['CLONE_TARGET'] if ENV['CLONE_TARGET']
     parsed_config[:config]['timeout'] = ENV['TIMEOUT'] if ENV['TIMEOUT']
     parsed_config[:config]['vm_lifetime_auth'] = ENV['VM_LIFETIME_AUTH'] if ENV['VM_LIFETIME_AUTH']
-    parsed_config[:config]['ssh_key'] = ENV['SSH_KEY'] if ENV['SSH_KEY']
     parsed_config[:config]['max_tries'] = ENV['MAX_TRIES'] if ENV['MAX_TRIES']
     parsed_config[:config]['retry_factor'] = ENV['RETRY_FACTOR'] if ENV['RETRY_FACTOR']
     parsed_config[:config]['create_folders'] = ENV['CREATE_FOLDERS'] if ENV['CREATE_FOLDERS']
     parsed_config[:config]['create_template_delta_disks'] = ENV['CREATE_TEMPLATE_DELTA_DISKS'] if ENV['CREATE_TEMPLATE_DELTA_DISKS']
     parsed_config[:config]['experimental_features'] = ENV['EXPERIMENTAL_FEATURES'] if ENV['EXPERIMENTAL_FEATURES']
+    parsed_config[:config]['purge_unconfigured_folders'] = ENV['PURGE_UNCONFIGURED_FOLDERS'] if ENV['PURGE_UNCONFIGURED_FOLDERS']
 
     parsed_config[:redis] = parsed_config[:redis] || {}
     parsed_config[:redis]['server'] = ENV['REDIS_SERVER'] || parsed_config[:redis]['server'] || 'localhost'
@@ -79,12 +79,14 @@ def self.config(filepath = 'vmpooler.yaml')
 
     parsed_config[:graphite] = parsed_config[:graphite] || {} if ENV['GRAPHITE_SERVER']
     parsed_config[:graphite]['server'] = ENV['GRAPHITE_SERVER'] if ENV['GRAPHITE_SERVER']
+    parsed_config[:graphite]['prefix'] = ENV['GRAPHITE_PREFIX'] if ENV['GRAPHITE_PREFIX']
+    parsed_config[:graphite]['port'] = ENV['GRAPHITE_PORT'] if ENV['GRAPHITE_PORT']
 
     parsed_config[:auth] = parsed_config[:auth] || {} if ENV['AUTH_PROVIDER']
     if parsed_config.has_key? :auth
       parsed_config[:auth]['provider'] = ENV['AUTH_PROVIDER'] if ENV['AUTH_PROVIDER']
       parsed_config[:auth][:ldap] = parsed_config[:auth][:ldap] || {} if parsed_config[:auth]['provider'] == 'ldap'
-      parsed_config[:auth][:ldap]['server'] = ENV['LDAP_SERVER'] if ENV['LDAP_SERVER']
+      parsed_config[:auth][:ldap]['host'] = ENV['LDAP_HOST'] if ENV['LDAP_HOST']
       parsed_config[:auth][:ldap]['port'] = ENV['LDAP_PORT'] if ENV['LDAP_PORT']
       parsed_config[:auth][:ldap]['base'] = ENV['LDAP_BASE'] if ENV['LDAP_BASE']
       parsed_config[:auth][:ldap]['user_object'] = ENV['LDAP_USER_OBJECT'] if ENV['LDAP_USER_OBJECT']

vmpooler.yaml.example

@@ -448,6 +448,10 @@
 #     Note: this will only create the last folder when it does not exist. It will not create any parent folders
 #     (optional; default: false)
 #
+#   - create_template_delta_disks (Only affects vSphere Provider)
+#     Create backing delta disks for the specified templates to support creating linked clones.
+#     (optional; default: false)
+#
 #   - host_selection_max_age (Only affects vSphere Provider)
 #     The maximum age of the provider_hosts list in seconds
 #     The list is repopulated when the difference between now and the last time