From 75ccf13c04e27210093b9b40a90d1c4c208119c5 Mon Sep 17 00:00:00 2001 From: Marc Paine Date: Thu, 5 Dec 2024 15:55:48 -0800 Subject: [PATCH 01/10] Force timeouts --- .../SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs b/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs index 99f111f3c177..7e173d0d8ea7 100644 --- a/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs +++ b/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs @@ -152,7 +152,7 @@ private async Task> PrepareWorkItem(ITaskItem xunitProject) var testFilter = string.IsNullOrEmpty(assemblyPartitionInfo.ClassListArgumentString) ? "" : $"--filter \"{assemblyPartitionInfo.ClassListArgumentString}\""; command = $"{driver} test {assemblyName} -e HELIX_WORK_ITEM_TIMEOUT={timeout} {testExecutionDirectory} {msbuildAdditionalSdkResolverFolder} " + - $"{(XUnitArguments != null ? " " + XUnitArguments : "")} --results-directory .{Path.DirectorySeparatorChar} --logger trx --blame-hang --blame-hang-timeout 30m {testFilter} -- {arguments}"; + $"{(XUnitArguments != null ? " " + XUnitArguments : "")} --results-directory .{Path.DirectorySeparatorChar} --logger trx --blame-hang --blame-hang-timeout 5m {testFilter} -- {arguments}"; Log.LogMessage($"Creating work item with properties Identity: {assemblyName}, PayloadDirectory: {publishDirectory}, Command: {command}"); From ecb6e1092a87a535a6d43afb54767f79f403a5d0 Mon Sep 17 00:00:00 2001 From: Marc Paine Date: Thu, 5 Dec 2024 15:56:00 -0800 Subject: [PATCH 02/10] Find where the blame dump is saved to --- test/UnitTests.proj | 1 + 1 file changed, 1 insertion(+) diff --git a/test/UnitTests.proj b/test/UnitTests.proj index 4483173428dc..1d18d14496f4 100644 --- a/test/UnitTests.proj +++ b/test/UnitTests.proj @@ -121,6 +121,7 @@ call %HELIX_CORRELATION_PAYLOAD%\t\RunTestsOnHelix.cmd $(TestFullMSBuild);$(HelixPreCommands) . $HELIX_CORRELATION_PAYLOAD/t/RunTestsOnHelix.sh;$(HelixPreCommands) PowerShell -ExecutionPolicy ByPass "dotnet nuget locals all -l | ForEach-Object { $_.Split(' ')[1]} | Where-Object{$_ -like '*cache'} | Get-ChildItem -Recurse -File -Filter '*.dat' | Measure";$(HelixPostCommands) + PowerShell -ExecutionPolicy ByPass "Get-ChildItem -Recurse -File -Filter '*.dmp'";$(HelixPostCommands) $(RepoRoot)artifacts\bin\redist\$(Configuration)\dotnet $(Version) $(RepoRoot)artifacts\bin\Microsoft.DotNet.MSBuildSdkResolver From 83fdde52c9bc3238f7150e7b0671dbf5c635bb56 Mon Sep 17 00:00:00 2001 From: Marc Paine Date: Fri, 6 Dec 2024 11:42:43 -0800 Subject: [PATCH 03/10] lower the threshold to try to force a timeout --- .../SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs b/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs index 7e173d0d8ea7..5e494c79bcf9 100644 --- a/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs +++ b/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs @@ -152,7 +152,7 @@ private async Task> PrepareWorkItem(ITaskItem xunitProject) var testFilter = string.IsNullOrEmpty(assemblyPartitionInfo.ClassListArgumentString) ? "" : $"--filter \"{assemblyPartitionInfo.ClassListArgumentString}\""; command = $"{driver} test {assemblyName} -e HELIX_WORK_ITEM_TIMEOUT={timeout} {testExecutionDirectory} {msbuildAdditionalSdkResolverFolder} " + - $"{(XUnitArguments != null ? " " + XUnitArguments : "")} --results-directory .{Path.DirectorySeparatorChar} --logger trx --blame-hang --blame-hang-timeout 5m {testFilter} -- {arguments}"; + $"{(XUnitArguments != null ? " " + XUnitArguments : "")} --results-directory .{Path.DirectorySeparatorChar} --logger trx --blame-hang --blame-hang-timeout 1m {testFilter} -- {arguments}"; Log.LogMessage($"Creating work item with properties Identity: {assemblyName}, PayloadDirectory: {publishDirectory}, Command: {command}"); From 3514d70fc2b7baaa90cead8b70cc01d27c02f18a Mon Sep 17 00:00:00 2001 From: Marc Paine Date: Fri, 6 Dec 2024 15:13:49 -0800 Subject: [PATCH 04/10] Trying copying the dump files to the helix work item upload folder --- test/UnitTests.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/UnitTests.proj b/test/UnitTests.proj index 1d18d14496f4..45dc8b10d412 100644 --- a/test/UnitTests.proj +++ b/test/UnitTests.proj @@ -121,7 +121,7 @@ call %HELIX_CORRELATION_PAYLOAD%\t\RunTestsOnHelix.cmd $(TestFullMSBuild);$(HelixPreCommands) . $HELIX_CORRELATION_PAYLOAD/t/RunTestsOnHelix.sh;$(HelixPreCommands) PowerShell -ExecutionPolicy ByPass "dotnet nuget locals all -l | ForEach-Object { $_.Split(' ')[1]} | Where-Object{$_ -like '*cache'} | Get-ChildItem -Recurse -File -Filter '*.dat' | Measure";$(HelixPostCommands) - PowerShell -ExecutionPolicy ByPass "Get-ChildItem -Recurse -File -Filter '*.dmp'";$(HelixPostCommands) + PowerShell -ExecutionPolicy ByPass "Get-ChildItem -Recurse -File -Filter '*hangdump.dmp' | Copy-Item -Destination $env:HELIX_WORKITEM_UPLOAD_ROOT";$(HelixPostCommands) $(RepoRoot)artifacts\bin\redist\$(Configuration)\dotnet $(Version) $(RepoRoot)artifacts\bin\Microsoft.DotNet.MSBuildSdkResolver From 877e63a19b7e01664941e181c2b49b944e306296 Mon Sep 17 00:00:00 2001 From: Marc Paine Date: Mon, 9 Dec 2024 09:12:10 -0800 Subject: [PATCH 05/10] Set the per test timeout to 15 minutes so we get a dump if there is a hang but don't fail for expected longer tests --- .../SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs b/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs index 5e494c79bcf9..3378a56b4d30 100644 --- a/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs +++ b/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs @@ -152,7 +152,7 @@ private async Task> PrepareWorkItem(ITaskItem xunitProject) var testFilter = string.IsNullOrEmpty(assemblyPartitionInfo.ClassListArgumentString) ? "" : $"--filter \"{assemblyPartitionInfo.ClassListArgumentString}\""; command = $"{driver} test {assemblyName} -e HELIX_WORK_ITEM_TIMEOUT={timeout} {testExecutionDirectory} {msbuildAdditionalSdkResolverFolder} " + - $"{(XUnitArguments != null ? " " + XUnitArguments : "")} --results-directory .{Path.DirectorySeparatorChar} --logger trx --blame-hang --blame-hang-timeout 1m {testFilter} -- {arguments}"; + $"{(XUnitArguments != null ? " " + XUnitArguments : "")} --results-directory .{Path.DirectorySeparatorChar} --logger trx --blame-hang --blame-hang-timeout 15m {testFilter} -- {arguments}"; Log.LogMessage($"Creating work item with properties Identity: {assemblyName}, PayloadDirectory: {publishDirectory}, Command: {command}"); From f3ad3e79d44446e32f591edbfef80099f1150be7 Mon Sep 17 00:00:00 2001 From: Marc Paine Date: Thu, 12 Dec 2024 11:55:47 -0800 Subject: [PATCH 06/10] Add a dotnet test diag log file --- .../SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs b/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs index 3378a56b4d30..9daf4da15357 100644 --- a/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs +++ b/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs @@ -149,10 +149,11 @@ private async Task> PrepareWorkItem(ITaskItem xunitProject) foreach (var assemblyPartitionInfo in assemblyPartitionInfos) { string command; + string logFileLocation = Path.Join(Environment.GetEnvironmentVariable("HELIX_WORKITEM_UPLOAD_ROOT"),"dotnetTestLog.log"); var testFilter = string.IsNullOrEmpty(assemblyPartitionInfo.ClassListArgumentString) ? "" : $"--filter \"{assemblyPartitionInfo.ClassListArgumentString}\""; command = $"{driver} test {assemblyName} -e HELIX_WORK_ITEM_TIMEOUT={timeout} {testExecutionDirectory} {msbuildAdditionalSdkResolverFolder} " + - $"{(XUnitArguments != null ? " " + XUnitArguments : "")} --results-directory .{Path.DirectorySeparatorChar} --logger trx --blame-hang --blame-hang-timeout 15m {testFilter} -- {arguments}"; + $"{(XUnitArguments != null ? " " + XUnitArguments : "")} --results-directory .{Path.DirectorySeparatorChar} --logger trx --blame-hang --blame-hang-timeout 15m {testFilter} -d {logFileLocation} -- {arguments}"; Log.LogMessage($"Creating work item with properties Identity: {assemblyName}, PayloadDirectory: {publishDirectory}, Command: {command}"); From 0cc58a2814ff9e69d4a04f311853d739efec99e3 Mon Sep 17 00:00:00 2001 From: Marc Paine Date: Fri, 13 Dec 2024 10:10:05 -0800 Subject: [PATCH 07/10] Switch to Path.Combine since join is not supported on framework --- .../SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs b/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs index 9daf4da15357..053ccd9bfb39 100644 --- a/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs +++ b/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs @@ -149,7 +149,7 @@ private async Task> PrepareWorkItem(ITaskItem xunitProject) foreach (var assemblyPartitionInfo in assemblyPartitionInfos) { string command; - string logFileLocation = Path.Join(Environment.GetEnvironmentVariable("HELIX_WORKITEM_UPLOAD_ROOT"),"dotnetTestLog.log"); + string logFileLocation = Path.Combine(Environment.GetEnvironmentVariable("HELIX_WORKITEM_UPLOAD_ROOT"),"dotnetTestLog.log"); var testFilter = string.IsNullOrEmpty(assemblyPartitionInfo.ClassListArgumentString) ? "" : $"--filter \"{assemblyPartitionInfo.ClassListArgumentString}\""; command = $"{driver} test {assemblyName} -e HELIX_WORK_ITEM_TIMEOUT={timeout} {testExecutionDirectory} {msbuildAdditionalSdkResolverFolder} " + From ff0709db80ac013f81e6e1ac8f77c7d11ff8321f Mon Sep 17 00:00:00 2001 From: Marc Paine Date: Mon, 16 Dec 2024 11:37:07 -0800 Subject: [PATCH 08/10] Switch to using the environment variable directly in the CLI call --- .../SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs b/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs index 053ccd9bfb39..adf2fdc7be8d 100644 --- a/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs +++ b/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs @@ -149,11 +149,11 @@ private async Task> PrepareWorkItem(ITaskItem xunitProject) foreach (var assemblyPartitionInfo in assemblyPartitionInfos) { string command; - string logFileLocation = Path.Combine(Environment.GetEnvironmentVariable("HELIX_WORKITEM_UPLOAD_ROOT"),"dotnetTestLog.log"); + string enableDiagLogging = IsPosixShell ? "-d $HELIX_WORKITEM_UPLOAD_ROOT//dotnetTestLog.log" : "-d %HELIX_WORKITEM_UPLOAD_ROOT%\\dotnetTestLog.log"; var testFilter = string.IsNullOrEmpty(assemblyPartitionInfo.ClassListArgumentString) ? "" : $"--filter \"{assemblyPartitionInfo.ClassListArgumentString}\""; command = $"{driver} test {assemblyName} -e HELIX_WORK_ITEM_TIMEOUT={timeout} {testExecutionDirectory} {msbuildAdditionalSdkResolverFolder} " + - $"{(XUnitArguments != null ? " " + XUnitArguments : "")} --results-directory .{Path.DirectorySeparatorChar} --logger trx --blame-hang --blame-hang-timeout 15m {testFilter} -d {logFileLocation} -- {arguments}"; + $"{(XUnitArguments != null ? " " + XUnitArguments : "")} --results-directory .{Path.DirectorySeparatorChar} --logger trx --blame-hang --blame-hang-timeout 15m {testFilter} {enableDiagLogging} -- {arguments}"; Log.LogMessage($"Creating work item with properties Identity: {assemblyName}, PayloadDirectory: {publishDirectory}, Command: {command}"); From 505a50074735de13da2dcc7c453d6ec842eedf41 Mon Sep 17 00:00:00 2001 From: Marc Paine Date: Wed, 18 Dec 2024 11:36:53 -0800 Subject: [PATCH 09/10] clean up the code. Remove the -- if there are no arguments. --- .../SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs b/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs index fbcbce1e236d..41e391b3689b 100644 --- a/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs +++ b/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs @@ -148,12 +148,13 @@ private async Task> PrepareWorkItem(ITaskItem xunitProject) var partitionedWorkItem = new List(); foreach (var assemblyPartitionInfo in assemblyPartitionInfos) { - string command; string enableDiagLogging = IsPosixShell ? "-d $HELIX_WORKITEM_UPLOAD_ROOT//dotnetTestLog.log" : "-d %HELIX_WORKITEM_UPLOAD_ROOT%\\dotnetTestLog.log"; + arguments = string.IsNullOrEmpty(arguments) ? "" : "-- " + arguments; var testFilter = string.IsNullOrEmpty(assemblyPartitionInfo.ClassListArgumentString) ? "" : $"--filter \"{assemblyPartitionInfo.ClassListArgumentString}\""; - command = $"{driver} test {assemblyName} -e HELIX_WORK_ITEM_TIMEOUT={timeout} {testExecutionDirectory} {msbuildAdditionalSdkResolverFolder} " + - $"{(XUnitArguments != null ? " " + XUnitArguments : "")} --results-directory .{Path.DirectorySeparatorChar} --logger trx --logger \"console;verbosity=detailed\" --blame-hang --blame-hang-timeout 15m {testFilter} {enableDiagLogging} -- {arguments}"; + + string command = $"{driver} test {assemblyName} -e HELIX_WORK_ITEM_TIMEOUT={timeout} {testExecutionDirectory} {msbuildAdditionalSdkResolverFolder} " + + $"{(XUnitArguments != null ? " " + XUnitArguments : "")} --results-directory .{Path.DirectorySeparatorChar} --logger trx --logger \"console;verbosity=detailed\" --blame-hang --blame-hang-timeout 15m {testFilter} {enableDiagLogging} {arguments}"; Log.LogMessage($"Creating work item with properties Identity: {assemblyName}, PayloadDirectory: {publishDirectory}, Command: {command}"); From fd425d748d68a44a6feba9f07c79f4b5021f0dbb Mon Sep 17 00:00:00 2001 From: Forgind <12969783+Forgind@users.noreply.github.com> Date: Wed, 8 Jan 2025 11:03:08 -0800 Subject: [PATCH 10/10] Add posix dmp copy command (#45704) --- test/UnitTests.proj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/UnitTests.proj b/test/UnitTests.proj index 45dc8b10d412..77c75cc1203b 100644 --- a/test/UnitTests.proj +++ b/test/UnitTests.proj @@ -121,7 +121,8 @@ call %HELIX_CORRELATION_PAYLOAD%\t\RunTestsOnHelix.cmd $(TestFullMSBuild);$(HelixPreCommands) . $HELIX_CORRELATION_PAYLOAD/t/RunTestsOnHelix.sh;$(HelixPreCommands) PowerShell -ExecutionPolicy ByPass "dotnet nuget locals all -l | ForEach-Object { $_.Split(' ')[1]} | Where-Object{$_ -like '*cache'} | Get-ChildItem -Recurse -File -Filter '*.dat' | Measure";$(HelixPostCommands) - PowerShell -ExecutionPolicy ByPass "Get-ChildItem -Recurse -File -Filter '*hangdump.dmp' | Copy-Item -Destination $env:HELIX_WORKITEM_UPLOAD_ROOT";$(HelixPostCommands) + PowerShell -ExecutionPolicy ByPass "Get-ChildItem -Recurse -File -Filter '*hangdump.dmp' | Copy-Item -Destination $env:HELIX_WORKITEM_UPLOAD_ROOT";$(HelixPostCommands) + find . -name '*hangdump.dmp' -exec cp {} "$HELIX_WORKITEM_UPLOAD_ROOT" \%3B;$(HelixPostCommands) $(RepoRoot)artifacts\bin\redist\$(Configuration)\dotnet $(Version) $(RepoRoot)artifacts\bin\Microsoft.DotNet.MSBuildSdkResolver