Skip to content

Commit

Permalink
pubsys: change "parameter overrides" to "conditional parameters"
Browse files Browse the repository at this point in the history
Rather than template directories with override files for specific variants and
architectures, this uses a single TOML file for all parameters with (optional)
keys restricting parameters to specific variants and arches.

This model is clearer, based on early feedback, and also allows users to
specify parameters that are conditional to a variant *and* an architecture,
whereas the override files were more like "or" matches.
  • Loading branch information
tjkirch committed Aug 24, 2020
1 parent 2f40216 commit 7a65061
Show file tree
Hide file tree
Showing 5 changed files with 73 additions and 65 deletions.
13 changes: 7 additions & 6 deletions Makefile.toml
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,11 @@ PUBLISH_KEY = "default"
# AMIs. (You can also specify PUBLISH_ROOT_VOLUME_SIZE to override the root
# volume size; by default it's the image size, rounded up.)
PUBLISH_DATA_VOLUME_SIZE = "20"
# This can be overridden with -e to change the path to the directory containing
# SSM parameter template files. These files determine the parameter names and
# values that will be published to SSM when you run `cargo make ssm`.
PUBLISH_SSM_TEMPLATES_PATH = "${BUILDSYS_ROOT_DIR}/tools/pubsys/policies/ssm"
# This can be overridden with -e to change the path to the file containing SSM
# parameter templates. This file determines the parameter names and values
# that will be published to SSM when you run `cargo make ssm`. See
# tools/pubsys/policies/ssm/README.md for more information.
PUBLISH_SSM_TEMPLATES_PATH = "${BUILDSYS_ROOT_DIR}/tools/pubsys/policies/ssm/defaults.toml"

# You can also set PUBLISH_REGIONS to override the list of regions from
# Infra.toml for AMI and SSM commands; it's a comma-separated list like
Expand Down Expand Up @@ -507,7 +508,7 @@ pubsys \
--arch "${BUILDSYS_ARCH}" \
--variant "${BUILDSYS_VARIANT}" \
--version "${BUILDSYS_VERSION_FULL}" \
--template-dir "${PUBLISH_SSM_TEMPLATES_PATH}" \
--template-path "${PUBLISH_SSM_TEMPLATES_PATH}" \
\
${PUBLISH_REGIONS:+--regions "${PUBLISH_REGIONS}"} \
${ALLOW_CLOBBER:+--allow-clobber}
Expand Down Expand Up @@ -539,7 +540,7 @@ pubsys \
--variant "${BUILDSYS_VARIANT}" \
--source "${source}" \
--target "${target}" \
--template-dir "${PUBLISH_SSM_TEMPLATES_PATH}" \
--template-path "${PUBLISH_SSM_TEMPLATES_PATH}" \
\
${PUBLISH_REGIONS:+--regions "${PUBLISH_REGIONS}"}
'''
Expand Down
18 changes: 13 additions & 5 deletions tools/pubsys/policies/ssm/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,18 @@ The available variables include:
* `image_version`, for example "0.5.0-e0ddf1b"
* `region`, for example "us-west-2"

# Overrides
# Conditional parameters

You can also add or override parameters that are specific to `variant` or `arch`.
To do so, create a directory named "variant" or "arch" inside parameters directory, and create a file named after the specific variant or arch for which you want overrides.
You can also list parameters that only apply to specific variants or architectures.
To do so, add `variant` or `arch` keys (or both) to your parameter definition.
The parameter will only be populated if the current `variant` or `arch` matches one of the values in the list.
(If both `variant` and `arch` are listed, the build must match an entry from both lists.)

For example, to add extra parameters just for the "aarch64" architecture, create `arch/aarch64.toml`.
Inside you can put the same types of `[[parameter]]` declarations that you see in `defaults.toml`, but they'll only be applied for `aarch64` builds.
For example, to add an extra parameter that's only set for "aarch64" builds of the "aws-ecs-1" variant:
```
[[parameter]]
arch = ["aarch64"]
variant = ["aws-ecs-1"]
name = "/a/special/aarch64/ecs/parameter"
value = "{image_name}"
```
16 changes: 12 additions & 4 deletions tools/pubsys/src/aws/promote_ssm/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,9 @@ pub(crate) struct PromoteArgs {
#[structopt(long, use_delimiter = true)]
regions: Vec<String>,

/// Directory holding the parameter template files
/// File holding the parameter templates
#[structopt(long)]
template_dir: PathBuf,
template_path: PathBuf,
}

/// Common entrypoint from main()
Expand Down Expand Up @@ -100,14 +100,22 @@ pub(crate) async fn run(args: &Args, promote_args: &PromoteArgs) -> Result<()> {

info!(
"Parsing SSM parameter templates from {}",
promote_args.template_dir.display()
promote_args.template_path.display()
);
// Doesn't matter which build context we use to find template files because version isn't used
// in their naming
let template_parameters =
template::get_parameters(&promote_args.template_dir, &source_build_context)
template::get_parameters(&promote_args.template_path, &source_build_context)
.context(error::FindTemplates)?;

if template_parameters.parameters.is_empty() {
info!(
"No parameters for this arch/variant in {}",
promote_args.template_path.display()
);
return Ok(());
}

// Render parameter names into maps of {template string => rendered value}. We need the
// template strings so we can associate source parameters with target parameters that came
// from the same template, so we know what to copy.
Expand Down
16 changes: 12 additions & 4 deletions tools/pubsys/src/aws/ssm/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,9 @@ pub(crate) struct SsmArgs {
#[structopt(long, use_delimiter = true)]
regions: Vec<String>,

/// Directory holding the parameter template files
/// File holding the parameter templates
#[structopt(long)]
template_dir: PathBuf,
template_path: PathBuf,

/// Allows overwrite of existing parameters
#[structopt(long)]
Expand Down Expand Up @@ -96,11 +96,19 @@ pub(crate) async fn run(args: &Args, ssm_args: &SsmArgs) -> Result<()> {

info!(
"Parsing SSM parameter templates from {}",
ssm_args.template_dir.display()
ssm_args.template_path.display()
);
let template_parameters = template::get_parameters(&ssm_args.template_dir, &build_context)
let template_parameters = template::get_parameters(&ssm_args.template_path, &build_context)
.context(error::FindTemplates)?;

if template_parameters.parameters.is_empty() {
info!(
"No parameters for this arch/variant in {}",
ssm_args.template_path.display()
);
return Ok(());
}

let new_parameters =
template::render_parameters(template_parameters, amis, ssm_prefix, &build_context)
.context(error::RenderTemplates)?;
Expand Down
75 changes: 29 additions & 46 deletions tools/pubsys/src/aws/ssm/template.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
use super::{BuildContext, SsmKey, SsmParameters};
use crate::aws::ami::Image;
use log::{info, trace};
use log::trace;
use rusoto_core::Region;
use serde::{Deserialize, Serialize};
use snafu::{ensure, ResultExt};
Expand All @@ -17,6 +17,12 @@ use tinytemplate::TinyTemplate;
pub(crate) struct TemplateParameter {
pub(crate) name: String,
pub(crate) value: String,

// User can say parameters only apply to these variants/arches
#[serde(default, rename = "variant")]
pub(crate) variants: Vec<String>,
#[serde(default, rename = "arch")]
pub(crate) arches: Vec<String>,
}

/// Represents a set of SSM parameters, in a format that allows for clear definition of
Expand All @@ -28,60 +34,39 @@ pub(crate) struct TemplateParameters {
pub(crate) parameters: Vec<TemplateParameter>,
}

impl TemplateParameters {
fn extend(&mut self, other: Self) {
self.parameters.extend(other.parameters)
}
}

/// Finds and deserializes template parameters from the template directory, taking into account
/// overrides requested by the user
/// Deserializes template parameters from the template file, taking into account conditional
/// parameters that may or may not apply based on our build context.
pub(crate) fn get_parameters(
template_dir: &Path,
template_path: &Path,
build_context: &BuildContext<'_>,
) -> Result<TemplateParameters> {
let defaults_path = template_dir.join("defaults.toml");
let defaults_str = fs::read_to_string(&defaults_path).context(error::File {
let templates_str = fs::read_to_string(&template_path).context(error::File {
op: "read",
path: &defaults_path,
path: &template_path,
})?;
let mut template_parameters: TemplateParameters =
toml::from_str(&defaults_str).context(error::InvalidToml {
path: &defaults_path,
toml::from_str(&templates_str).context(error::InvalidToml {
path: &template_path,
})?;
trace!("Parsed default templates: {:#?}", template_parameters);

// Allow the user to add/override parameters specific to variant or arch. Because these are
// added after the defaults, they will take precedence. (It doesn't make sense to override
// based on the version argument.)
let mut context = HashMap::new();
context.insert("variant", build_context.variant);
context.insert("arch", build_context.arch);
for (key, value) in context {
let override_path = template_dir.join(key).join(format!("{}.toml", value));
if override_path.exists() {
info!(
"Parsing SSM parameter overrides from {}",
override_path.display()
);
let template_str = fs::read_to_string(&override_path).context(error::File {
op: "read",
path: &override_path,
})?;
let override_parameters: TemplateParameters =
toml::from_str(&template_str).context(error::InvalidToml {
path: &override_path,
})?;
trace!("Parsed override templates: {:#?}", override_parameters);
template_parameters.extend(override_parameters);
}
}
trace!("Parsed templates: {:#?}", template_parameters);

// You shouldn't point to an empty file, but if all the entries are removed by
// conditionals below, we allow that and just don't set any parameters.
ensure!(
!template_parameters.parameters.is_empty(),
error::NoTemplates { path: template_dir }
error::NoTemplates {
path: template_path
}
);

let variant = build_context.variant.to_string();
let arch = build_context.arch.to_string();
template_parameters.parameters.retain(|p| {
(p.variants.is_empty() || p.variants.contains(&variant))
&& (p.arches.is_empty() || p.arches.contains(&arch))
});
trace!("Templates after conditionals: {:#?}", template_parameters);

Ok(template_parameters)
}

Expand Down Expand Up @@ -201,9 +186,7 @@ mod error {
},

#[snafu(display("Found no parameter templates in {}", path.display()))]
NoTemplates {
path: PathBuf,
},
NoTemplates { path: PathBuf },

#[snafu(display("Error rendering template from '{}': {}", template, source))]
RenderTemplate {
Expand Down

0 comments on commit 7a65061

Please sign in to comment.