pipeline_spec.proto

syntax = "proto3";

package ml_pipelines;

import "google/protobuf/duration.proto";
import "google/protobuf/struct.proto";
import "google/rpc/status.proto";

option go_package = "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec";

// The spec of a pipeline job.
message PipelineJob {
  string name = 1;  // Name of the job.

  // User friendly display name
  string display_name = 2;

  reserved 3, 4, 5, 6;

  // Definition of the pipeline that is being executed.
  google.protobuf.Struct pipeline_spec = 7;

  reserved 8, 9, 10;

  // The labels with user-defined metadata to organize PipelineJob.
  map<string, string> labels = 11;

  // The runtime config of a PipelineJob.
  message RuntimeConfig {
    // Deprecated. Use [RuntimeConfig.parameter_values][] instead.
    map<string, Value> parameters = 1 [deprecated = true];

    // A path in a Cloud Storage bucket which will be treated as the root
    // output directory of the pipeline. It is used by the system to
    // generate the paths of output artifacts.
    // This is a GCP-specific optimization.
    string gcs_output_directory = 2;

    // The runtime parameters of the PipelineJob. The parameters will be
    // passed into [PipelineJob.pipeline_spec][] to replace the placeholders
    // at runtime.
    map<string, google.protobuf.Value> parameter_values = 3;
  }

  // Runtime config of the pipeline.
  RuntimeConfig runtime_config = 12;
}

// The spec of a pipeline.
message PipelineSpec {
  // The metadata of the pipeline.
  PipelineInfo pipeline_info = 1;

  // The deployment config of the pipeline.
  // The deployment config can be extended to provide platform specific configs.
  google.protobuf.Struct deployment_spec = 7;

  // The version of the sdk, which compiles the spec.
  string sdk_version = 4;

  // The version of the schema.
  string schema_version = 5;

  // The definition of the runtime parameter.
  message RuntimeParameter {
    // Required field. The type of the runtime parameter.
    PrimitiveType.PrimitiveTypeEnum type = 1;
    // Optional field. Default value of the runtime parameter. If not set and
    // the runtime parameter value is not provided during runtime, an error will
    // be raised.
    Value default_value = 2;
  }

  // The map of name to definition of all components used in this pipeline.
  map<string, ComponentSpec> components = 8;

  // The definition of the main pipeline.  Execution of the pipeline is
  // completed upon the completion of this component.
  ComponentSpec root = 9;

  // Optional field. The default root output directory of the pipeline.
  string default_pipeline_root = 10;
}

// Definition of a component.
message ComponentSpec {
  // Definition of the input parameters and artifacts of the component.
  ComponentInputsSpec input_definitions = 1;
  // Definition of the output parameters and artifacts of the component.
  ComponentOutputsSpec output_definitions = 2;
  // Either a DAG or a single execution.
  oneof implementation {
    DagSpec dag = 3;
    string executor_label = 4;
  }
  // Supports platform-specific component features.
  repeated SinglePlatformSpec single_platform_specs = 5;
}

// A DAG contains multiple tasks.
message DagSpec {
  // The tasks inside the dag.
  map<string, PipelineTaskSpec> tasks = 1;

  // Defines how the outputs of the dag are linked to the sub tasks.
  DagOutputsSpec outputs = 2;
}

// Definition of the output artifacts and parameters of the DAG component.
message DagOutputsSpec {
  // Selects a defined output artifact from a sub task of the DAG.
  message ArtifactSelectorSpec {
    // The name of the sub task which produces the output that matches with
    // the `output_artifact_key`.
    string producer_subtask = 1;

    // The key of [ComponentOutputsSpec.artifacts][] map of the producer task.
    string output_artifact_key = 2;
  }

  // Selects a list of output artifacts that will be aggregated to the single
  // output artifact channel of the DAG.
  message DagOutputArtifactSpec {
    // The selected artifacts will be aggregated as output as a single
    // output channel of the DAG.
    repeated ArtifactSelectorSpec artifact_selectors = 1;
  }

  // Name to the output artifact channel of the DAG.
  map<string, DagOutputArtifactSpec> artifacts = 1;

  // Selects a defined output parameter from a sub task of the DAG.
  message ParameterSelectorSpec {
    // The name of the sub task which produces the output that matches with
    // the `output_parameter_key`.
    string producer_subtask = 1;

    // The key of [ComponentOutputsSpec.parameters][] map of the producer task.
    string output_parameter_key = 2;
  }

  // Aggregate output parameters from sub tasks into a list object.
  message ParameterSelectorsSpec {
    repeated ParameterSelectorSpec parameter_selectors = 1;
  }

  // Aggregates output parameters from sub tasks into a map object.
  message MapParameterSelectorsSpec {
    map<string, ParameterSelectorSpec> mapped_parameters = 2;
  }

  // We support four ways to fan-in output parameters from sub tasks to the DAG
  // parent task.
  // 1. Directly expose a single output parameter from a sub task,
  // 2. (Conditional flow) Expose a list of output from multiple tasks
  // (some might be skipped) but allows only one of the output being generated.
  // 3. Expose a list of outputs from multiple tasks (e.g. iterator flow).
  // 4. Expose the aggregation of output parameters as a name-value map.
  message DagOutputParameterSpec {
    oneof kind {
      // Returns the sub-task parameter as a DAG parameter.  The selected
      // parameter must have the same type as the DAG parameter type.
      ParameterSelectorSpec value_from_parameter = 1;
      // Returns one of the sub-task parameters as a DAG parameter. If there are
      // multiple values are available to select, the DAG will fail. All the
      // selected parameters must have the same type as the DAG parameter type.
      ParameterSelectorsSpec value_from_oneof = 2;
    }
    reserved 3;
  }

  // The name to the output parameter.
  map<string, DagOutputParameterSpec> parameters = 2;
}

// Definition specification of the component input parameters and artifacts.
message ComponentInputsSpec {
  // Definition of an artifact input.
  message ArtifactSpec {
    ArtifactTypeSchema artifact_type = 1;

    // Indicates whether input is a single artifact or list of artifacts
    bool is_artifact_list = 2;

    // Whether this input artifact is optional or not.
    // - If required, the artifact must be able to resolve to an artifact
    // at runtime.
    // - If it's optional, it can be missing from the
    // PipelineTaskInputsSpec.InputArtifactSpec (if it's instantiated into a
    // task), or can be missing from the runtimeArtifact (if it's the root
    // component).
    bool is_optional = 3;

    // The description for this input artifact of the component.
    // Should not exceed 1024 characters.
    string description = 4;
  }
  // Definition of a parameter input.
  message ParameterSpec {
    // Specifies an input parameter's type.
    // Deprecated. Use [ParameterSpec.parameter_type][] instead.
    PrimitiveType.PrimitiveTypeEnum type = 1 [deprecated = true];

    // Specifies an input parameter's type.
    ParameterType.ParameterTypeEnum parameter_type = 2;

    // Optional field. Default value of the input parameter.
    google.protobuf.Value default_value = 3;

    // Whether this input parameter is optional or not.
    // - If required, the parameter should either have a default value, or have
    // to be able to resolve to a concrete value at runtime.
    // - If it's optional, it can be missing from the
    // PipelineTaskInputsSpec.InputParameterSpec (if it's instantiated into a
    // task), or can be missing from the runtimeParameter (if it's the root
    // component). If the value is missing, the default_value will be used. Or
    // if default_value is not provided, the default value of the parameter's
    // type will be used.
    bool is_optional = 4;

    // The description for this input parameter of the component.
    // Should not exceed 1024 characters.
    string description = 5;
  }
  // Name to artifact input.
  map<string, ArtifactSpec> artifacts = 1;
  // Name to parameter input.
  map<string, ParameterSpec> parameters = 2;
}

// Definition specification of the component output parameters and artifacts.
message ComponentOutputsSpec {
  // Definition of an artifact output.
  message ArtifactSpec {
    ArtifactTypeSchema artifact_type = 1;
    // Deprecated. Use [ArtifactSpec.metadata][] instead.
    map<string, ValueOrRuntimeParameter> properties = 2 [deprecated = true];
    // Deprecated. Use [ArtifactSpec.metadata][] instead.
    map<string, ValueOrRuntimeParameter> custom_properties = 3
        [deprecated = true];
    // Properties of the Artifact.
    google.protobuf.Struct metadata = 4;

    // Indicates whether output is a single artifact or list of artifacts
    bool is_artifact_list = 5;

    // The description for this output artifact of the component.
    // Should not exceed 1024 characters.
    string description = 6;
  }
  // Definition of a parameter output.
  message ParameterSpec {
    // Specifies an input parameter's type.
    // Deprecated. Use [ParameterSpec.parameter_type][] instead.
    PrimitiveType.PrimitiveTypeEnum type = 1 [deprecated = true];

    // Specifies an output parameter's type.
    ParameterType.ParameterTypeEnum parameter_type = 2;

    // The description for this output parameter of the component.
    // Should not exceed 1024 characters.
    string description = 3;
  }
  // Name to artifact output.
  map<string, ArtifactSpec> artifacts = 1;
  // Name to parameter output.
  map<string, ParameterSpec> parameters = 2;
}

// The spec of task inputs.
message TaskInputsSpec {
  // The specification of a task input artifact.
  message InputArtifactSpec {
    message TaskOutputArtifactSpec {
      // The name of the upstream task which produces the output that matches
      // with the `output_artifact_key`.
      string producer_task = 1;

      // The key of [TaskOutputsSpec.artifacts][] map of the producer task.
      string output_artifact_key = 2;
    }

    oneof kind {
      // Pass the input artifact from another task within the same parent
      // component.
      TaskOutputArtifactSpec task_output_artifact = 3;
      // Pass the input artifact from parent component input artifact.
      string component_input_artifact = 4;
    }
    reserved 5;
  }

  // Represents an input parameter. The value can be taken from an upstream
  // task's output parameter (if specifying `producer_task` and
  // `output_parameter_key`), or it can be a runtime value, which can either be
  // determined at compile-time, or from a pipeline parameter.
  message InputParameterSpec {
    // Represents an upstream task's output parameter.
    message TaskOutputParameterSpec {
      // The name of the upstream task which produces the output parameter that
      // matches with the `output_parameter_key`.
      string producer_task = 1;

      // The key of [TaskOutputsSpec.parameters][] map of the producer task.
      string output_parameter_key = 2;
    }

    // Represents an upstream task's final status. The field can only be set if
    // the schema version is `2.0.0`. The resolved input parameter will be a
    // JSON payload in string type.
    message TaskFinalStatus {
      // The name of the upstream task where the final status is coming from.
      string producer_task = 1;
    }

    oneof kind {
      // Output parameter from an upstream task.
      TaskOutputParameterSpec task_output_parameter = 1;
      // A constant value or runtime parameter.
      ValueOrRuntimeParameter runtime_value = 2;
      // Pass the input parameter from parent component input parameter.
      string component_input_parameter = 3;
      // The final status of an uptream task.
      TaskFinalStatus task_final_status = 5;
    }

    // Selector expression of Common Expression Language (CEL)
    // that applies to the parameter found from above kind.
    //
    // The expression is applied to the Value type
    // [Value][].  For example,
    // 'size(string_value)' will return the size of the Value.string_value.
    //
    // After applying the selection, the parameter will be returned as a
    // [Value][].  The type of the Value is either deferred from the input
    // definition in the corresponding
    // [ComponentSpec.input_definitions.parameters][], or if not found,
    // automatically deferred as either string value or double value.
    //
    // In addition to the builtin functions in CEL, The value.string_value can
    // be treated as a json string and parsed to the [google.protobuf.Value][]
    // proto message. Then, the CEL expression provided in this field will be
    // used to get the requested field. For examples,
    //  - if Value.string_value is a json array of "[1.1, 2.2, 3.3]",
    //  'parseJson(string_value)[i]' will pass the ith parameter from the list
    //  to the current task, or
    //  - if the Value.string_value is a json map of "{"a": 1.1, "b": 2.2,
    //  "c": 3.3}, 'parseJson(string_value)[key]' will pass the map value from
    //  the struct map to the current task.
    //
    // If unset, the value will be passed directly to the current task.
    string parameter_expression_selector = 4;
  }

  // A map of input parameters which are small values, stored by the system and
  // can be queriable.
  map<string, InputParameterSpec> parameters = 1;
  // A map of input artifacts.
  map<string, InputArtifactSpec> artifacts = 2;
}

// The spec of task outputs.
message TaskOutputsSpec {
  // The specification of a task output artifact.
  message OutputArtifactSpec {
    // The type of the artifact.
    ArtifactTypeSchema artifact_type = 1;

    // The properties of the artifact, which are determined either at
    // compile-time, or at pipeline submission time through runtime parameters
    map<string, ValueOrRuntimeParameter> properties = 2;

    // The custom properties of the artifact, which are determined either at
    // compile-time, or at pipeline submission time through runtime parameters
    map<string, ValueOrRuntimeParameter> custom_properties = 3;
  }

  // Specification for output parameters produced by the task.
  message OutputParameterSpec {
    // Required field. The type of the output parameter.
    PrimitiveType.PrimitiveTypeEnum type = 1;
  }

  // A map of output parameters which are small values, stored by the system and
  // can be queriable. The output key is used
  // by [TaskInputsSpec.InputParameterSpec][] of the downstream task to specify
  // the data dependency. The same key will also be used by
  // [ExecutorInput.Inputs][] to reference the output parameter.
  map<string, OutputParameterSpec> parameters = 1;
  // A map of output artifacts. Keyed by output key. The output key is used
  // by [TaskInputsSpec.InputArtifactSpec][] of the downstream task to specify
  // the data dependency. The same key will also be used by
  // [ExecutorInput.Inputs][] to reference the output artifact.
  map<string, OutputArtifactSpec> artifacts = 2;
}

// Represent primitive types. The wrapper is needed to give a namespace of
// enum value so we don't need add `PRIMITIVE_TYPE_` prefix of each enum value.
message PrimitiveType {
  option deprecated = true;

  // The primitive types.
  // Deprecated. Use [ParameterType.ParameterTypeEnum][] instead.
  enum PrimitiveTypeEnum {
    option deprecated = true;

    PRIMITIVE_TYPE_UNSPECIFIED = 0;
    INT = 1;
    DOUBLE = 2;
    STRING = 3;
  }
}

// Represent parameter types. The wrapper is needed to give a namespace of
// enum value so we don't need add `PARAMETER_TYPE_` prefix of each enum value.
message ParameterType {
  // The parameter types.
  enum ParameterTypeEnum {
    // Indicates that the parameter type was not specified.
    PARAMETER_TYPE_ENUM_UNSPECIFIED = 0;
    // Indicates that a parameter is a number that is stored in a field of type
    // `double`.
    NUMBER_DOUBLE = 1;
    // Indicates that a parameter is an integer stored in the `number_field`,
    // which is of type `double`. NUMBER_INTEGER values must be within the range
    // of JavaScript safe integers (-(2^53 - 1) to (2^53 - 1)). If you need to
    // support integers outside the range of JavaScript safe integers, use the
    // `STRING` parameter type to describe your parameter.
    NUMBER_INTEGER = 2;
    // Indicates that a parameter is a string.
    STRING = 3;
    // Indicates that a parameters is a boolean value.
    BOOLEAN = 4;
    // Indicates that a parameter is a list of values. LIST parameters are
    // serialized to JSON when passed as an input or output of a pipeline step.
    LIST = 5;
    // Indicates that a parameter is a struct value; structs represent a data
    // structure like a Python dictionary or a JSON object. STRUCT parameters
    // are serialized to JSON when passed as an input or output of a pipeline
    // step.
    STRUCT = 6;
    // Indicates that a parameter is a TaskFinalStatus type; these types can only accept inputs
    // specified by InputParameterSpec.task_final_status
    TASK_FINAL_STATUS = 7;
  }
}

// The spec of a pipeline task.
message PipelineTaskSpec {
  // Basic info of a pipeline task.
  PipelineTaskInfo task_info = 1;

  // Specification for task inputs which contains parameters and artifacts.
  TaskInputsSpec inputs = 2;

  // A list of names of upstream tasks that do not provide input
  // artifacts for this task, but nonetheless whose completion this task depends
  // on.
  repeated string dependent_tasks = 5;

  message CachingOptions {
    // Whether or not to enable cache for this task. Defaults to false.
    bool enable_cache = 1;
    // Customized cache key for this task. If set, the cache_key will be used
    // as the key for the task's cache.
    string cache_key = 2;
  }
  CachingOptions caching_options = 6;

  // Reference to a component.  Use this field to define either a DAG or an
  // executor.
  ComponentRef component_ref = 7;

  // Trigger policy defines how the task gets triggered. If a task is not
  // triggered, it will run into SKIPPED state.
  message TriggerPolicy {
    // An expression which will be evaluated into a boolean value. True to
    // trigger the task to run. The expression follows the language of
    // [CEL Spec][/~https://github.com/google/cel-spec]. It can access the data
    // from [ExecutorInput][] message of the task.
    // For example:
    // - `inputs.artifacts['model'][0].properties['accuracy']*100 > 90`
    // - `inputs.parameters['type'] == 'foo' && inputs.parameters['num'] == 1`
    string condition = 1;

    // An enum defines the trigger strategy of when the task will be ready to be
    // triggered.
    // ALL_UPSTREAM_TASKS_SUCCEEDED - all upstream tasks in succeeded state.
    // ALL_UPSTREAM_TASKS_COMPLETED - all upstream tasks in any final state.
    // (Note that CANCELLED is also a final state but job will not trigger new
    // tasks when job is in CANCELLING state, so that the task with the trigger
    // policy at ALL_UPSTREAM_TASKS_COMPLETED will not start when job
    // cancellation is in progress.)
    enum TriggerStrategy {
      // Unspecified.  Behave the same as ALL_UPSTREAM_TASKS_SUCCEEDED.
      TRIGGER_STRATEGY_UNSPECIFIED = 0;
      // Specifies that all upstream tasks are in succeeded state.
      ALL_UPSTREAM_TASKS_SUCCEEDED = 1;
      // Specifies that all upstream tasks are in any final state.
      ALL_UPSTREAM_TASKS_COMPLETED = 2;
    }
    // The trigger strategy of this task.  The `strategy` and `condition` are
    // in logic "AND", as a task will only be tested for the `condition` when
    // the `strategy` is meet.
    // Unset or set to default value of TRIGGER_STRATEGY_UNDEFINED behaves the
    // same as ALL_UPSTREAM_TASKS_SUCCEEDED.
    TriggerStrategy strategy = 2;
  }
  // Trigger policy of the task.
  TriggerPolicy trigger_policy = 8;

  // Iterator supports fanning out the task into multiple sub-tasks based on the
  // values of input artifact or parameter. The current task will become the
  // parent of all the fan-out tasks. The output of the current task follows
  // the following conventions:
  // * Output artifacts with the same name of each iteration will be merged
  //   into one output artifact channel of the parent iterator task.
  // * Output parameters with the same name of each iteration will be merged
  //   into a string output parameter with the same name with content being a
  //   JSON array.
  //
  // For example, if an iterator starts two sub-tasks (t1 and t2) with the
  // following outputs.
  //
  // t1.outputs.parameters = { 'p': 'v1' }
  // t1.outputs.artifacts = { 'a': [a1] }
  // t2.outputs.parameters = { 'p': 'v2' }
  // t2.outputs.artifacts = { 'a': [a2] }
  // parent_task.outputs.parameters = { 'p': '["v1", "v2"]' }
  // parent_task.outputs.aritfacts = { 'a': [a1, a2] }
  oneof iterator {
    // Iterator to iterate over an artifact input.
    ArtifactIteratorSpec artifact_iterator = 9;
    // Iterator to iterate over a parameter input.
    ParameterIteratorSpec parameter_iterator = 10;
  }

  // User-configured task-level retry.
  message RetryPolicy {
    // Number of retries before considering a task as failed. Set to 0 or
    // unspecified to disallow retry."
    int32 max_retry_count  = 1;

    // The time interval between retries. Defaults to zero (an immediate retry).
    google.protobuf.Duration backoff_duration = 2;

    // The exponential backoff factor applied to backoff_duration. If
    // unspecified, will default to 2.
    double backoff_factor = 3;

    // The maximum duration during which the task will be retried according to
    // the backoff strategy. Max allowed is 1 hour - higher value will be capped
    // to this limit. If unspecified, will set to 1 hour.
    google.protobuf.Duration backoff_max_duration = 4;
  }

  // User-configured task-level retry.
  // Applicable only to component tasks.
  RetryPolicy retry_policy = 11;

  // Iterator related settings.
  message IteratorPolicy {
    // The limit for the number of concurrent sub-tasks spawned by an iterator
    // task. The value should be a non-negative integer. A value of 0 represents
    // unconstrained parallelism.
    int32 parallelism_limit = 1;
  }

  // Iterator related settings.
  IteratorPolicy iterator_policy = 12;
}

// The spec of an artifact iterator. It supports fan-out a workflow from a list
// of artifacts.
message ArtifactIteratorSpec {
  // Specifies the name of the artifact channel which contains the collection of
  // items to iterate. The iterator will create a sub-task for each item of
  // the collection and pass the item as a new input artifact channel as
  // specified by [item_input][].
  message ItemsSpec {
    // The name of the input artifact.
    string input_artifact = 1;
  }
  // The items to iterate.
  ItemsSpec items = 1;
  // The name of the input artifact channel which has the artifact item from the
  // [items][] collection.
  string item_input = 2;
}

// The spec of a parameter iterator. It supports fan-out a workflow from a
// string parameter which contains a JSON array.
message ParameterIteratorSpec {
  // Specifies the spec to decribe the parameter items to iterate.
  message ItemsSpec {
    // Specifies where to get the collection of items to iterate. The iterator
    // will create a sub-task for each item of the collection and pass the item
    // as a new input parameter as specified by [item_input][].
    oneof kind {
      // The raw JSON array.
      string raw = 1;
      // The name of the input parameter whose value has the items collection.
      // The parameter must be in STRING type and its content can be parsed
      // as a JSON array.
      string input_parameter = 2;
    }
  }
  // The items to iterate.
  ItemsSpec items = 1;
  // The name of the input parameter which has the item value from the
  // [items][] collection.
  string item_input = 2;
}

message ComponentRef {
  // The name of a component. Refer to the key of the
  // [PipelineSpec.components][] map.
  string name = 1;
}

// Basic info of a pipeline.
message PipelineInfo {
  // Required field. The name of the pipeline.
  // The name will be used to create or find pipeline context in MLMD.
  string name = 1;

  // Optional fields. The readable display name for the pipeline template.
  // Should not exceed 1024 characters.
  string display_name = 2;

  // Optional fields. The readable description for the pipeline template.
  // Should not exceed 1024 characters.
  string description = 3;
}

// The definition of a artifact type in MLMD.
message ArtifactTypeSchema {
  oneof kind {
    // The name of the type. The format of the title must be:
    // `<namespace>.<title>`.
    // Examples:
    //  - `aiplatform.Model`
    //  - `acme.CustomModel`
    // When this field is set, the type must be pre-registered in the MLMD
    // store.
    string schema_title = 1;

    // Points to a YAML file stored on Google Cloud Storage describing the
    // format.
    // Deprecated. Use [PipelineArtifactTypeSchema.schema_title][] or
    // [PipelineArtifactTypeSchema.instance_schema][] instead.
    string schema_uri = 2 [deprecated = true];

    // Contains a raw YAML string, describing the format of
    // the properties of the type.
    string instance_schema = 3;
  }

  // The schema version of the artifact. If the value is not set, it defaults
  // to the the latest version in the system.
  string schema_version = 4;
}

// The basic info of a task.
message PipelineTaskInfo {
  // The display name of the task.
  string name = 1;
}

// Definition for a value or reference to a runtime parameter. A
// ValueOrRuntimeParameter instance can be either a field value that is
// determined during compilation time, or a runtime parameter which will be
// determined during runtime.
message ValueOrRuntimeParameter {
  oneof value {
    // Constant value which is determined in compile time.
    // Deprecated. Use [ValueOrRuntimeParameter.constant][] instead.
    Value constant_value = 1 [deprecated = true];
    // The runtime parameter refers to the parent component input parameter.
    string runtime_parameter = 2;
    // Constant value which is determined in compile time.
    google.protobuf.Value constant = 3;
  }
}

// The definition of the deployment config of the pipeline. It contains the
// the platform specific executor configs for KFP OSS.
message PipelineDeploymentConfig {
  // The specification on a container invocation.
  // The string fields of the message support string based placeholder contract
  // defined in [ExecutorInput](). The output of the container follows the
  // contract of [ExecutorOutput]().
  message PipelineContainerSpec {
    // The image uri of the container.
    string image = 1;
    // The main entrypoint commands of the container to run. If not provided,
    // fallback to use the entry point command defined in the container image.
    repeated string command = 2;
    // The arguments to pass into the main entrypoint of the container.
    repeated string args = 3;

    // The lifecycle hooks of the container.
    // Each hook follows the same I/O contract as the main container entrypoint.
    // See [ExecutorInput]() and [ExecutorOutput]() for details.
    // (-- TODO(b/165323565): add more documentation on caching and lifecycle
    // hooks. --)
    message Lifecycle {
      // The command and args to execute a program.
      message Exec {
        // The command of the exec program.
        repeated string command = 2;
        // The args of the exec program.
        repeated string args = 3;
      }
      // This hook is invoked before caching check. It can change the properties
      // of the execution and output artifacts before they are used to compute
      // the cache key. The updated metadata will be passed into the main
      // container entrypoint.
      Exec pre_cache_check = 1;
    }
    // The lifecycle hooks of the container executor.
    Lifecycle lifecycle = 4;

    // The specification on the resource requirements of a container execution.
    // This can include specification of vCPU, memory requirements, as well as
    // accelerator types and counts.
    message ResourceSpec {
      // The limit of the number of vCPU cores. This container execution needs
      // at most cpu_limit vCPU to run.
      // Deprecated. Use [ResourceSpec.resource_cpu_limit] instead.
      double cpu_limit = 1 [deprecated = true];

      // The memory limit in GB. This container execution needs at most
      // memory_limit RAM to run.
      // Deprecated. Use [ResourceSpec.resource_memory_limit] instead.
      double memory_limit = 2 [deprecated = true];

      // The request of the number of vCPU cores. This container execution
      // needs at least cpu_request vCPU to run.
      // Deprecated. Use [ResourceSpec.resource_cpu_request] instead.
      double cpu_request = 5 [deprecated = true];

      // The memory request in GB. This container execution needs at least
      // memory_request RAM to run.
      // Deprecated. Use [ResourceSpec.resource_memory_request] instead.
      double memory_request = 6 [deprecated = true];

      // The limit of the number of vCPU cores. This container execution needs
      // at most resource_cpu_limit vCPU to run. Handles static values and
      // placeholders.
      string resource_cpu_limit = 7;

      // The memory limit in GB. This container execution needs
      // at most resource_memory_limit RAM to run. Handles static values and
      // placeholders.
      string resource_memory_limit = 8;

      // The request of the number of vCPU cores. This container
      // execution needs at least resource_cpu_request vCPU to run. Handles
      // static values and placeholders.
      string resource_cpu_request = 9;

      // The memory request in GB. This container execution
      // needs at least resource_memory_request RAM to run. Handles static
      // values and placeholders.
      string resource_memory_request = 10;

      // The specification on the accelerators being attached to this container.
      message AcceleratorConfig {
        // The type of accelerators.
        // Deprecated. Use [ResourceSpec.AcceleratorConfig.resource_type]
        // instead.
        string type = 1 [deprecated = true];

        // The number of accelerators.
        // Deprecated. Use [ResourceSpec.AcceleratorConfig.resource_count]
        // instead.
        int64 count = 2 [deprecated = true];

        // The type of accelerators. Handles static values and
        // placeholders.
        string resource_type = 3;

        // The number of accelerators. Handles static values and
        // placeholders.
        string resource_count = 4;
      }
      AcceleratorConfig accelerator = 3;

      reserved 4;
    }
    ResourceSpec resources = 5;

    // Environment variables to be passed to the container.
    // Represents an environment variable present in a container.
    message EnvVar {
      // Name of the environment variable. Must be a valid C identifier. It can
      // be composed of characters such as uppercase, lowercase characters,
      // underscore, digits, but the leading character should be either a
      // letter or an underscore.
      string name = 1;

      // Variables that reference a $(VAR_NAME) are expanded using the previous
      // defined environment variables in the container and any environment
      // variables defined by the platform runtime that executes this pipeline.
      // If a variable cannot be resolved, the reference in the input string
      // will be unchanged. The $(VAR_NAME) syntax can be escaped with a double
      // $$, ie: $$(VAR_NAME). Escaped references will never be expanded,
      // regardless of whether the variable exists or not.
      string value = 2;
    }
    // Environment variables to be passed to the container.
    repeated EnvVar env = 6;
  }

  // The specification to import or reimport a new artifact to the pipeline.
  message ImporterSpec {
    // The URI of the artifact.
    ValueOrRuntimeParameter artifact_uri = 1;

    // The type of the artifact.
    ArtifactTypeSchema type_schema = 2;

    // The properties of the artifact.
    // Deprecated. Use [ImporterSpec.metadata][] instead.
    map<string, ValueOrRuntimeParameter> properties = 3 [deprecated = true];

    // The custom properties of the artifact.
    // Deprecated. Use [ImporterSpec.metadata][] instead.
    map<string, ValueOrRuntimeParameter> custom_properties = 4
        [deprecated = true];

    // Properties of the Artifact.
    google.protobuf.Struct metadata = 6;

    // Whether or not import an artifact regardless it has been imported before.
    bool reimport = 5;
  }

  // ResolverSpec resolves artifacts from historical metadata and returns them
  // to the pipeline as output artifacts of the resolver task. The downstream
  // tasks can consume them as their input artifacts.
  message ResolverSpec {
    // The query to fetch artifacts.
    message ArtifactQuerySpec {
      // The filter of the artifact query. The supported syntax are:
      // - `in_context("<context name>")`
      // - `artifact_type="<artifact type name>"`
      // - `uri="<uri>"`
      // - `state=<state>`
      // - `name="value"`
      // - `AND` to combine two conditions and returns when both are true.
      // If no `in_context` filter is set, the query will be scoped to the
      // the current pipeline context.
      string filter = 1;
      // The maximum number of the artifacts to be returned from the
      // query. If not defined, the default limit is `1`.
      int32 limit = 2;
    }

    // A list of resolver output definitions. The
    // key of the map must be exactly the same as
    // the keys in the [PipelineTaskOutputsSpec.artifacts][] map.
    // At least one output must be defined.
    map<string, ArtifactQuerySpec> output_artifact_queries = 1;
  }

  message AIPlatformCustomJobSpec {
    option deprecated = true;

    // API Specification for invoking a Google Cloud AI Platform CustomJob.
    // The fields must match the field names and structures of CustomJob
    // defined in
    // https://cloud.google.com/ai-platform-unified/docs/reference/rest/v1beta1/projects.locations.customJobs.
    // The field types must be either the same, or be a string containing the
    // string based placeholder contract defined in [ExecutorInput](). The
    // placeholders will be replaced with the actual value during the runtime
    // before the job is launched.
    google.protobuf.Struct custom_job = 1;
  }

  // The specification of the executor.
  message ExecutorSpec {
    oneof spec {
      // Starts a container.
      PipelineContainerSpec container = 1;
      // Import an artifact.
      ImporterSpec importer = 2;
      // Resolves an existing artifact.
      ResolverSpec resolver = 3;
      // Starts a Google Cloud AI Platform CustomJob.
      AIPlatformCustomJobSpec custom_job = 4 [deprecated = true];
    }
  }
  // Map from executor label to executor spec.
  map<string, ExecutorSpec> executors = 1;
}

// Value is the value of the field.
message Value {
  oneof value {
    // An integer value
    int64 int_value = 1;
    // A double value
    double double_value = 2;
    // A string value
    string string_value = 3;
  }
}

// The definition of a runtime artifact.
message RuntimeArtifact {
  // The name of an artifact.
  string name = 1;

  // The type of the artifact.
  ArtifactTypeSchema type = 2;

  // The URI of the artifact.
  string uri = 3;

  // The properties of the artifact.
  // Deprecated. Use [RuntimeArtifact.metadata][] instead.
  map<string, Value> properties = 4 [deprecated = true];

  // The custom properties of the artifact.
  // Deprecated. Use [RuntimeArtifact.metadata][] instead.
  map<string, Value> custom_properties = 5 [deprecated = true];

  // Properties of the Artifact.
  google.protobuf.Struct metadata = 6;
}

// Message that represents a list of artifacts.
message ArtifactList {
  // A list of artifacts.
  repeated RuntimeArtifact artifacts = 1;
}

// The input of an executor, which includes all the data that
// can be passed into the executor spec by a string based placeholder.
//
// The string based placeholder uses a JSON path to reference to the data
// in the [ExecutionInput]().
//
// `{{$}}`: prints the full [ExecutorInput]() as a JSON string.
// `{{$.inputs.artifacts['<name>'].uri}}`: prints the URI of an input
// artifact.
// `{{$.inputs.artifacts['<name>'].properties['<property name>']}}`: prints
// the
//   property of an input artifact.
// `{{$.inputs.parameters['<name>']}}`: prints the value of an input
// parameter.
// `{{$.outputs.artifacts['<name>'].uri}}: prints the URI of an output artifact.
// `{{$.outputs.artifacts['<name>'].properties['<property name>']}}`: prints the
//   property of an output artifact.
// `{{$.outputs.parameters['<name>'].output_file}}`: prints a file path which
// points to a file and container can write to it to return the value of the
// parameter..
// `{{$.outputs.output_file}}`: prints a file path of the output metadata file
// which is used to send output metadata from executor to orchestrator. The
// contract of the output metadata is [ExecutorOutput](). When both parameter
// output file and executor output metadata files are set by the container, the
// output metadata file will have higher precedence to set output parameters.
message ExecutorInput {
  // The runtime inputs data of the execution.
  message Inputs {
    // Input parameters of the execution.
    // Deprecated. Use [ExecutorInput.Inputs.parameter_values][] instead.
    map<string, Value> parameters = 1 [deprecated = true];

    // Input artifacts of the execution.
    map<string, ArtifactList> artifacts = 2;

    // Input parameters of the execution.
    map<string, google.protobuf.Value> parameter_values = 3;
  }

  // The runtime input artifacts of the task invocation.
  Inputs inputs = 1;

  // The runtime output parameter.
  message OutputParameter {
    // The file path which is used by the executor to pass the parameter value
    // to the system.
    string output_file = 1;
  }

  // The runtime outputs data of the execution.
  message Outputs {
    // The runtime output parameters.
    map<string, OutputParameter> parameters = 1;

    // The runtime output artifacts.
    map<string, ArtifactList> artifacts = 2;

    // The file path of the full output metadata JSON. The schema of the output
    // file is [ExecutorOutput][].
    //
    // When the full output metadata file is set by the container, the output
    // parameter files will be ignored.
    string output_file = 3;
  }

  // The runtime output artifacts of the task invocation.
  Outputs outputs = 2;
}

// The schema of the output metadata of an execution. It will be used to parse
// the output metadata file.
message ExecutorOutput {
  // The values for output parameters.
  // Deprecated. Use [ExecutorOutput.parameter_values][] instead.
  map<string, Value> parameters = 1 [deprecated = true];

  // The updated metadata for output artifact.
  map<string, ArtifactList> artifacts = 2;

  // The values for output parameters.
  map<string, google.protobuf.Value> parameter_values = 3;
}

// The final status of a task. The structure will be passed to input parameter
// of kind `task_final_status`.
message PipelineTaskFinalStatus {
  // The final state of the task.
  // The value is the string version of [PipelineStateEnum.PipelineTaskState][]
  string state = 1;

  // The error of the task.
  google.rpc.Status error = 2;

  // The pipeline job unique id.
  int64 pipeline_job_uuid = 3 [deprecated = true];

  // The pipeline job name from the [PipelineJob.name][].
  string pipeline_job_name = 4 [deprecated = true];

  // The pipeline job resource name, in the format of
  // `projects/{project}/locations/{location}/pipelineJobs/{pipeline_job}`.
  string pipeline_job_resource_name = 5;

  // The pipeline task that produces this status.
  string pipeline_task_name = 6;
}

message PipelineStateEnum {
  enum PipelineTaskState {
    TASK_STATE_UNSPECIFIED = 0;
    PENDING = 1;
    RUNNING_DRIVER = 2;
    DRIVER_SUCCEEDED = 3;
    RUNNING_EXECUTOR = 4;
    SUCCEEDED = 5;
    CANCEL_PENDING = 6;
    CANCELLING = 7;
    CANCELLED = 8;
    FAILED = 9;
    // Indicates that the task is skipped to run due to a cache hit.
    SKIPPED = 10;
    // Indicates that the task was just populated to the DB but not ready to
    // be scheduled.  Once job handler determined the task being ready to
    // be scheduled, the task state will change to PENDING.  The state
    // transition is depicted below:
    //  * QUEUED(not ready to run) --> PENDING(ready to run) --> RUNNING
    QUEUED = 11;
    // Indicates that the task is not triggered based on the
    // [PipelineTaskSpec.TriggerPolicy.condition][] config.
    NOT_TRIGGERED = 12;
    // Indicates that the tasks will no longer be schedulable.  Usually a task
    // was set to this state because its all upstream tasks are in final state
    // but the [PipelineTaskSpec.TriggerPolicy.strategy][] disallows the task to
    // be triggered.
    // The difference between `NOT_TRIGGERED` is that `UNSCHEDULABLE` must met
    // [PipelineTaskSpec.TriggerPolicy.strategy][], but must not met the
    // [PipelineTaskSpec.TriggerPolicy.condition][].
    UNSCHEDULABLE = 13;
  }
}

// Spec for all platforms; second document in IR
message PlatformSpec {
  // Platform key to full platform config
  map<string, SinglePlatformSpec> platforms = 1;
}

message SinglePlatformSpec {
  // Mirrors PipelineSpec.deployment_spec structure
  PlatformDeploymentConfig deployment_spec = 1;

  // Name of the platform. For example, "google_cloud"
  string platform = 2;

  // Arbitrary configuration, which will be defined by the platform
  // protos/libraries.
  google.protobuf.Struct config = 3;

  PipelineConfig pipelineConfig = 4;
}

message PlatformDeploymentConfig {
  // Map of executor label to executor-level config
  // Mirrors PipelineSpec.deployment_spec.executors structure
  map<string, google.protobuf.Struct> executors = 1;
}

// Spec for pipeline-level config options. See PipelineConfig DSL class.
message PipelineConfig {
  // Name of the semaphore key to control pipeline concurrency
  string semaphore_key = 1;

  // Name of the mutex to ensure mutual exclusion
  string mutex_name = 2;
}