Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support running scripts in topological order with melos exec #440

Merged
merged 4 commits into from
Jan 4, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions docs/configuration/scripts.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,14 @@ Whether `exec` should fail fast and not execute the script in further packages i
in an individual package.
Defaults to `false`.

## `scripts/*/exec/orderDependents`

Whether `exec` should order the execution of the script in multiple packages based on the
dependency graph of the packages. The script will be executed in leaf packages first and then
in packages that depend on them and so on. This is useful for example, for a script that generates
code in multiple packages, which depend on each other.
Defaults to `false`.

## `scripts/*/env`

A map of environment variables that will be passed to the executed command.
Expand Down
12 changes: 12 additions & 0 deletions packages/melos/lib/src/command_runner/exec.dart
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,16 @@ class ExecCommand extends MelosCommand {
'Whether exec should fail fast and not execute the script in further '
'packages if the script fails in a individual package.',
);
argParser.addFlag(
'order-dependents',
abbr: 'o',
help: 'Whether exec should order the execution of the script in multiple '
'packages based on the dependency graph of the packages. The script '
'will be executed in leaf packages first and then in packages that '
'depend on them and so on. This is useful for example, for a script '
'that generates code in multiple packages, which depend on each '
'other.',
);
}

@override
Expand Down Expand Up @@ -59,11 +69,13 @@ class ExecCommand extends MelosCommand {
final packageFilter = parsePackageFilter(config.path);
final concurrency = int.parse(argResults!['concurrency'] as String);
final failFast = argResults!['fail-fast'] as bool;
final orderDependents = argResults!['order-dependents'] as bool;

return melos.exec(
execArgs,
concurrency: concurrency,
failFast: failFast,
orderDependents: orderDependents,
global: global,
filter: packageFilter,
);
Expand Down
43 changes: 37 additions & 6 deletions packages/melos/lib/src/commands/exec.dart
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ mixin _ExecMixin on _Melos {
PackageFilter? filter,
int concurrency = 5,
bool failFast = false,
bool orderDependents = false,
}) async {
final workspace = await createWorkspace(global: global, filter: filter);
final packages = workspace.filteredPackages.values;
Expand All @@ -17,6 +18,7 @@ mixin _ExecMixin on _Melos {
execArgs,
failFast: failFast,
concurrency: concurrency,
orderDependents: orderDependents,
);
}

Expand All @@ -40,8 +42,7 @@ mixin _ExecMixin on _Melos {
'PATH': workspace.childProcessPath!,
};

// TODO what if it's not called 'example'?
if (package.path.endsWith('example')) {
if (package.isExample) {
final exampleParentPackagePath = p.normalize('${package.path}/..');
final exampleParentPubspecPath =
p.normalize('$exampleParentPackagePath/pubspec.yaml');
Expand Down Expand Up @@ -91,8 +92,9 @@ mixin _ExecMixin on _Melos {
List<String> execArgs, {
required int concurrency,
required bool failFast,
required bool orderDependents,
}) async {
final failures = <String, int>{};
final failures = <String, int?>{};
final pool = Pool(concurrency);
final execArgsString = execArgs.join(' ');
final prefixLogs = concurrency != 1 && packages.length != 1;
Expand All @@ -106,9 +108,35 @@ mixin _ExecMixin on _Melos {
logger.horizontalLine();
}

await pool.forEach<Package, void>(packages, (package) async {
final sortedPackages = packages.toList(growable: false);

if (orderDependents) {
sortPackagesTopologically(sortedPackages);
}

final packageResults = Map.fromEntries(
packages.map((package) => MapEntry(package.name, Completer<int?>())),
);

await pool.forEach<Package, void>(sortedPackages, (package) async {
if (failFast && failures.isNotEmpty) {
return Future.value();
return;
}

if (orderDependents) {
final dependenciesResults = await Future.wait(
package.allDependenciesInWorkspace.values
.map((package) => packageResults[package.name]?.future)
.whereNotNull(),
);

final dependencyFailed = dependenciesResults
.any((exitCode) => exitCode == null || exitCode > 0);
if (dependencyFailed) {
packageResults[package.name]?.complete();
failures[package.name] = null;
return;
}
}

if (!prefixLogs) {
Expand All @@ -124,6 +152,8 @@ mixin _ExecMixin on _Melos {
prefixLogs: prefixLogs,
);

packageResults[package.name]?.complete(packageExitCode);

if (packageExitCode > 0) {
failures[package.name] = packageExitCode;
} else if (!prefixLogs) {
Expand All @@ -147,7 +177,8 @@ mixin _ExecMixin on _Melos {
for (final packageName in failures.keys) {
failuresLogger.child(
'${errorPackageNameStyle(packageName)} '
'(with exit code ${failures[packageName]})',
'${failures[packageName] == null ? '(dependency failed)' : '('
'with exit code ${failures[packageName]})'}',
);
}
exitCode = 1;
Expand Down
1 change: 1 addition & 0 deletions packages/melos/lib/src/commands/publish.dart
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,7 @@ mixin _PublishMixin on _ExecMixin {
execArgs,
concurrency: 1,
failFast: true,
orderDependents: false,
);

if (exitCode != 1) {
Expand Down
24 changes: 21 additions & 3 deletions packages/melos/lib/src/scripts.dart
Original file line number Diff line number Diff line change
Expand Up @@ -95,32 +95,40 @@ class ExecOptions {
ExecOptions({
this.concurrency,
this.failFast,
this.orderDependents,
});

final int? concurrency;
final bool? failFast;
final bool? orderDependents;

Map<String, Object?> toJson() => {
if (concurrency != null) 'concurrency': concurrency,
if (failFast != null) 'failFast': failFast,
if (orderDependents != null) 'orderDependents': orderDependents,
};

@override
bool operator ==(Object other) =>
other is ExecOptions &&
runtimeType == other.runtimeType &&
concurrency == other.concurrency &&
failFast == other.failFast;
failFast == other.failFast &&
orderDependents == other.orderDependents;

@override
int get hashCode =>
runtimeType.hashCode ^ concurrency.hashCode ^ failFast.hashCode;
runtimeType.hashCode ^
concurrency.hashCode ^
failFast.hashCode ^
orderDependents.hashCode;

@override
String toString() => '''
ExecOptions(
concurrency: $concurrency,
failFast: $failFast,
orderDependents: $orderDependents,
)''';
}

Expand Down Expand Up @@ -248,9 +256,16 @@ class Script {
path: execPath,
);

final orderDependents = assertKeyIsA<bool?>(
key: 'orderDependents',
map: yaml,
path: execPath,
);

return ExecOptions(
concurrency: concurrency,
failFast: failFast,
orderDependents: orderDependents,
);
}

Expand Down Expand Up @@ -288,11 +303,14 @@ class Script {
parts.addAll(['--concurrency', '${exec.concurrency}']);
}

// --fail-fast is a flag and as such does not accept any value
if (exec.failFast ?? false) {
parts.add('--fail-fast');
}

if (exec.orderDependents ?? false) {
parts.add('--order-dependents');
}

parts.addAll(['--', quoteScript(run)]);

return parts.join(' ');
Expand Down
Loading