Add cargo make ami-public and ami-private targets

Makefile.toml

@@ -388,6 +388,9 @@ case "${BUILDSYS_ARCH,,}" in
       arch="${BUILDSYS_ARCH}" ;;
 esac
 
+ami_output="${BUILDSYS_OUTPUT_DIR}/${BUILDSYS_NAME_FULL}-amis.json"
+ami_output_latest="${BUILDSYS_OUTPUT_DIR}/latest/${BUILDSYS_NAME_VARIANT}-amis.json"
+
 pubsys \
    --infra-config-path "${PUBLISH_INFRA_CONFIG_PATH}" \
    \
@@ -401,8 +404,71 @@ pubsys \
    --arch "${arch}" \
    --name "${PUBLISH_AMI_NAME}" \
    --description "${PUBLISH_AMI_DESCRIPTION:-${PUBLISH_AMI_NAME}}" \
+   \
+   --ami-output "${ami_output}" \
+   \
    ${NO_PROGRESS:+--no-progress} \
    ${PUBLISH_REGIONS:+--regions "${PUBLISH_REGIONS}"}
+
+ln -snf "../${ami_output##*/}" "${ami_output_latest}"
+'''
+]
+
+[tasks.ami-public]
+# Rather than depend on "build", which currently rebuilds images each run, we
+# depend on publish-tools and check for the input file below to save time.
+# This does mean that `cargo make ami` must be run before `cargo make ami-public`.
+dependencies = ["publish-tools"]
+script_runner = "bash"
+script = [
+'''
+set -e
+
+export PATH="${BUILDSYS_TOOLS_DIR}/bin:${PATH}"
+
+ami_input="${BUILDSYS_OUTPUT_DIR}/${BUILDSYS_NAME_FULL}-amis.json"
+if [ ! -s "${ami_input}" ]; then
+   echo "AMI input file doesn't exist for the current version/commit - ${BUILDSYS_VERSION_FULL} - please run 'cargo make ami'" >&2
+   exit 1
+fi
+
+pubsys \
+   --infra-config-path "${PUBLISH_INFRA_CONFIG_PATH}" \
+   \
+   publish-ami \
+   --make-public \
+   \
+   --ami-input "${ami_input}" \
+   ${PUBLISH_REGIONS:+--regions "${PUBLISH_REGIONS}"}
+'''
+]
+
+[tasks.ami-private]
+# Rather than depend on "build", which currently rebuilds images each run, we
+# depend on publish-tools and check for the input file below to save time.
+# This does mean that `cargo make ami` must be run before `cargo make ami-private`.
+dependencies = ["publish-tools"]
+script_runner = "bash"
+script = [
+'''
+set -e
+
+export PATH="${BUILDSYS_TOOLS_DIR}/bin:${PATH}"
+
+ami_input="${BUILDSYS_OUTPUT_DIR}/${BUILDSYS_NAME_FULL}-amis.json"
+if [ ! -s "${ami_input}" ]; then
+   echo "AMI input file doesn't exist for the current version/commit - ${BUILDSYS_VERSION_FULL} - please run 'cargo make ami'" >&2
+   exit 1
+fi
+
+pubsys \
+   --infra-config-path "${PUBLISH_INFRA_CONFIG_PATH}" \
+   \
+   publish-ami \
+   --make-private \
+   \
+   --ami-input "${ami_input}" \
+   ${PUBLISH_REGIONS:+--regions "${PUBLISH_REGIONS}"}
 '''
 ]
 

tools/pubsys/src/aws/ami/mod.rs

@@ -5,8 +5,8 @@ mod register;
 mod snapshot;
 mod wait;
 
-use crate::aws::client::build_client;
-use crate::config::{AwsConfig, InfraConfig};
+use crate::aws::{client::build_client, region_from_string};
+use crate::config::InfraConfig;
 use crate::Args;
 use futures::future::{join, lazy, ready, FutureExt};
 use futures::stream::{self, StreamExt};
@@ -17,6 +17,7 @@ use rusoto_ebs::EbsClient;
 use rusoto_ec2::{CopyImageError, CopyImageRequest, CopyImageResult, Ec2, Ec2Client};
 use snafu::{ensure, OptionExt, ResultExt};
 use std::collections::{HashMap, VecDeque};
+use std::fs::File;
 use std::path::PathBuf;
 use structopt::StructOpt;
 use wait::wait_for_ami;
@@ -60,10 +61,31 @@ pub(crate) struct AmiArgs {
     /// Regions where you want the AMI, the first will be used as the base for copying
     #[structopt(long, use_delimiter = true)]
     regions: Vec<String>,
+
+    /// If specified, save created regional AMI IDs in JSON at this path.
+    #[structopt(long)]
+    ami_output: Option<PathBuf>,
 }
 
 /// Common entrypoint from main()
 pub(crate) async fn run(args: &Args, ami_args: &AmiArgs) -> Result<()> {
+    match _run(args, ami_args).await {
+        Ok(ami_ids) => {
+            // Write the AMI IDs to file if requested
+            if let Some(ref path) = ami_args.ami_output {
+                let file = File::create(path).context(error::FileCreate { path })?;
+                serde_json::to_writer_pretty(file, &ami_ids).context(error::Serialize { path })?;
+                info!("Wrote AMI data to {}", path.display());
+            }
+            Ok(())
+        }
+        Err(e) => Err(e),
+    }
+}
+
+async fn _run(args: &Args, ami_args: &AmiArgs) -> Result<HashMap<String, String>> {
+    let mut ami_ids = HashMap::new();
+
     info!(
         "Using infra config from path: {}",
         args.infra_config_path.display()
@@ -80,7 +102,7 @@ pub(crate) async fn run(args: &Args, ami_args: &AmiArgs) -> Result<()> {
         aws.regions.clone()
     }
     .into_iter()
-    .map(|name| region_from_string(&name, &aws))
+    .map(|name| region_from_string(&name, &aws).context(error::ParseRegion))
     .collect::<Result<VecDeque<Region>>>()?;
 
     // We register in this base region first, then copy from there to any other regions.
@@ -138,9 +160,11 @@ pub(crate) async fn run(args: &Args, ami_args: &AmiArgs) -> Result<()> {
         (new_id, false)
     };
 
+    ami_ids.insert(base_region.name().to_string(), image_id.clone());
+
     // If we don't need to copy AMIs, we're done.
     if regions.is_empty() {
-        return Ok(());
+        return Ok(ami_ids);
     }
 
     // Wait for AMI to be available so it can be copied
@@ -187,6 +211,7 @@ pub(crate) async fn run(args: &Args, ami_args: &AmiArgs) -> Result<()> {
                 region.name(),
                 id
             );
+            ami_ids.insert(region.name().to_string(), id.clone());
             continue;
         }
         let request = CopyImageRequest {
@@ -214,7 +239,7 @@ pub(crate) async fn run(args: &Args, ami_args: &AmiArgs) -> Result<()> {
 
     // If all target regions already have the AMI, we're done.
     if copy_requests.is_empty() {
-        return Ok(());
+        return Ok(ami_ids);
     }
 
     // Start requests; they return almost immediately and the copying work is done by the service
@@ -234,12 +259,24 @@ pub(crate) async fn run(args: &Args, ami_args: &AmiArgs) -> Result<()> {
     let mut saw_error = false;
     for (region, copy_response) in copy_responses {
         match copy_response {
-            Ok(success) => info!(
-                "Registered AMI '{}' in region {}: {}",
-                ami_args.name,
-                region.name(),
-                success.image_id.unwrap_or_else(|| "<missing>".to_string())
-            ),
+            Ok(success) => {
+                if let Some(image_id) = success.image_id {
+                    info!(
+                        "Registered AMI '{}' in {}: {}",
+                        ami_args.name,
+                        region.name(),
+                        image_id,
+                    );
+                    ami_ids.insert(region.name().to_string(), image_id);
+                } else {
+                    saw_error = true;
+                    error!(
+                        "Registered AMI '{}' in {} but didn't receive an AMI ID!",
+                        ami_args.name,
+                        region.name(),
+                    );
+                }
+            }
             Err(e) => {
                 saw_error = true;
                 error!("Copy to {} failed: {}", region.name(), e);
@@ -249,25 +286,13 @@ pub(crate) async fn run(args: &Args, ami_args: &AmiArgs) -> Result<()> {
 
     ensure!(!saw_error, error::AmiCopy);
 
-    Ok(())
-}
-
-/// Builds a Region from the given region name, and uses the custom endpoint from the AWS config,
-/// if specified in aws.region.REGION.endpoint.
-fn region_from_string(name: &str, aws: &AwsConfig) -> Result<Region> {
-    let maybe_endpoint = aws.region.get(name).and_then(|r| r.endpoint.clone());
-    Ok(match maybe_endpoint {
-        Some(endpoint) => Region::Custom {
-            name: name.to_string(),
-            endpoint,
-        },
-        None => name.parse().context(error::ParseRegion { name })?,
-    })
+    Ok(ami_ids)
 }
 
 mod error {
     use crate::aws::{self, ami};
     use snafu::Snafu;
+    use std::path::PathBuf;
 
     #[derive(Debug, Snafu)]
     #[snafu(visibility = "pub(super)")]
@@ -283,7 +308,15 @@ mod error {
         },
 
         #[snafu(display("Error reading config: {}", source))]
-        Config { source: crate::config::Error },
+        Config {
+            source: crate::config::Error,
+        },
+
+        #[snafu(display("Failed to create file '{}': {}", path.display(), source))]
+        FileCreate {
+            path: PathBuf,
+            source: std::io::Error,
+        },
 
         #[snafu(display("Error getting AMI ID for {} {} in {}: {}", arch, name, region, source))]
         GetAmiId {
@@ -294,12 +327,12 @@ mod error {
         },
 
         #[snafu(display("Infra.toml is missing {}", missing))]
-        MissingConfig { missing: String },
+        MissingConfig {
+            missing: String,
+        },
 
-        #[snafu(display("Failed to parse region '{}': {}", name, source))]
         ParseRegion {
-            name: String,
-            source: rusoto_signature::region::ParseRegionError,
+            source: crate::aws::Error,
         },
 
         #[snafu(display("Error registering {} {} in {}: {}", arch, name, region, source))]
@@ -310,6 +343,12 @@ mod error {
             source: ami::register::Error,
         },
 
+        #[snafu(display("Failed to serialize output to '{}': {}", path.display(), source))]
+        Serialize {
+            path: PathBuf,
+            source: serde_json::Error,
+        },
+
         #[snafu(display("AMI '{}' in {} did not become available: {}", id, region, source))]
         WaitAmi {
             id: String,

tools/pubsys/src/aws/client.rs

@@ -1,5 +1,5 @@
-use async_trait::async_trait;
 use crate::config::AwsConfig;
+use async_trait::async_trait;
 use rusoto_core::{request::DispatchSignedRequest, HttpClient, Region};
 use rusoto_credential::{
     AutoRefreshingProvider, AwsCredentials, CredentialsError, DefaultCredentialsProvider,
@@ -22,19 +22,19 @@ impl NewWith for EbsClient {
     where
         P: ProvideAwsCredentials + Send + Sync + 'static,
         D: DispatchSignedRequest + Send + Sync + 'static,
-        {
-            Self::new_with(request_dispatcher, credentials_provider, region)
-        }
+    {
+        Self::new_with(request_dispatcher, credentials_provider, region)
+    }
 }
 
 impl NewWith for Ec2Client {
     fn new_with<P, D>(request_dispatcher: D, credentials_provider: P, region: Region) -> Self
     where
         P: ProvideAwsCredentials + Send + Sync + 'static,
         D: DispatchSignedRequest + Send + Sync + 'static,
-        {
-            Self::new_with(request_dispatcher, credentials_provider, region)
-        }
+    {
+        Self::new_with(request_dispatcher, credentials_provider, region)
+    }
 }
 
 /// Create a rusoto client of the given type using the given region and configuration.

tools/pubsys/src/aws/mod.rs

@@ -1,4 +1,38 @@
+use crate::config::AwsConfig;
+use rusoto_core::Region;
+use snafu::ResultExt;
+
 #[macro_use]
 pub(crate) mod client;
 
 pub(crate) mod ami;
+pub(crate) mod publish_ami;
+
+/// Builds a Region from the given region name, and uses the custom endpoint from the AWS config,
+/// if specified in aws.region.REGION.endpoint.
+fn region_from_string(name: &str, aws: &AwsConfig) -> Result<Region> {
+    let maybe_endpoint = aws.region.get(name).and_then(|r| r.endpoint.clone());
+    Ok(match maybe_endpoint {
+        Some(endpoint) => Region::Custom {
+            name: name.to_string(),
+            endpoint,
+        },
+        None => name.parse().context(error::ParseRegion { name })?,
+    })
+}
+
+mod error {
+    use snafu::Snafu;
+
+    #[derive(Debug, Snafu)]
+    #[snafu(visibility = "pub(super)")]
+    pub(crate) enum Error {
+        #[snafu(display("Failed to parse region '{}': {}", name, source))]
+        ParseRegion {
+            name: String,
+            source: rusoto_signature::region::ParseRegionError,
+        },
+    }
+}
+pub(crate) use error::Error;
+type Result<T> = std::result::Result<T, error::Error>;