Skip to content

JobMonitor: preserve source metadata from job properties when resubmitting Helix work items #16908

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Copilot wants to merge 5 commits into
base: main
Choose a base branch
from
+130 −5
Draft

JobMonitor: preserve source metadata from job properties when resubmitting Helix work items #16908

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4a93d27
Initial plan
Copilot May 28, 2026
fe95053
Fix JobMonitor resubmit source metadata fields
Copilot May 28, 2026
0ac825f
Use job properties for resubmission source metadata
Copilot Jun 2, 2026
e04b7cb
Add explicit source-metadata fallback coverage
Copilot Jun 2, 2026
caa8529
Remove source parsing fallback for JobMonitor resubmission
Copilot Jun 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 29 additions & 2 deletions src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -272,15 +272,25 @@ await RetryHelper.RetryAsync(

string newJobListUri = AppendSasIfPresent(jobListBlobClient.Uri, container.ReadToken);

// 5. Build the new job creation request, copying over Source / Properties / Creator
// 5. Build the new job creation request, copying over source metadata / properties / creator
// so the resubmitted job remains discoverable (BuildId, System.StageName, TestRunName, etc.).
var creationRequest = new JobCreationRequest(details.Type, newJobListUri, details.QueueId)
{
Source = details.Source,
Creator = details.Creator,
Properties = ConvertPropertiesToImmutableDictionary(details.Properties)
.SetItem(HelixJobInfo.PreviousHelixJobNamePropertyName, originalJobName),
};
if (TryGetSourceMetadataFromProperties(details.Properties, out string sourcePrefix, out string teamProject, out string repository, out string branch))
{
creationRequest.SourcePrefix = sourcePrefix;
Copy link
Copy Markdown
Member

@mmitche mmitche Jun 2, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@copilot Why would the source not also be set here?

Copy link
Copy Markdown
Contributor Author

Copilot AI Jun 2, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because Helix rejects requests that include both Source and any of SourcePrefix/TeamProject/Repository/Branch (400). We mirror the SDK behavior here: use the split source fields when available; otherwise set Source.

creationRequest.TeamProject = teamProject;
creationRequest.Repository = repository;
creationRequest.Branch = branch;
}
else
{
creationRequest.Source = details.Source;
}

string idempotencyKey = Guid.NewGuid().ToString("N");
JobCreationResult newJob = await RetryHelper.RetryAsync(
Expand Down Expand Up @@ -335,6 +345,23 @@ private static ImmutableDictionary<string, string> ConvertPropertiesToImmutableD
return builder.ToImmutable();
}

private static bool TryGetSourceMetadataFromProperties(
JToken properties,
out string sourcePrefix,
out string teamProject,
out string repository,
out string branch)
{
sourcePrefix = GetStringPropertyFromProperties(properties, "SourcePrefix");
teamProject = GetStringPropertyFromProperties(properties, "TeamProject");
repository = GetStringPropertyFromProperties(properties, "Repository");
branch = GetStringPropertyFromProperties(properties, "Branch");
return !string.IsNullOrWhiteSpace(sourcePrefix)
&& !string.IsNullOrWhiteSpace(teamProject)
&& !string.IsNullOrWhiteSpace(repository)
&& !string.IsNullOrWhiteSpace(branch);
}

private static string GetStringPropertyFromProperties(JToken properties, string name)
{
if (properties is JObject obj && obj.TryGetValue(name, out JToken token))
Expand Down
104 changes: 101 additions & 3 deletions src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,11 @@ public async Task ResubmitWorkItemsAsync_UploadsFilteredJobListAndCreatesJobWith
Assert.Equal("helix-type", capturedRequest.Type);
Assert.Equal("queue-id", capturedRequest.QueueId);
Assert.Equal("https://account.blob.core.windows.net/container/job-list.json?read", capturedRequest.ListUri);
Assert.Equal("source", capturedRequest.Source);
Assert.Null(capturedRequest.Source);
Assert.Equal("ci", capturedRequest.SourcePrefix);
Assert.Equal("public", capturedRequest.TeamProject);
Assert.Equal("dotnet/arcade", capturedRequest.Repository);
Assert.Equal("refs/heads/main", capturedRequest.Branch);
Assert.Equal("creator", capturedRequest.Creator);
Assert.Equal("123", capturedRequest.Properties["BuildId"]);
Assert.Equal("custom run", capturedRequest.Properties["TestRunName"]);
Expand All @@ -257,6 +261,96 @@ public async Task ResubmitWorkItemsAsync_UploadsFilteredJobListAndCreatesJobWith
Times.Once);
}

[Fact]
public async Task ResubmitWorkItemsAsync_FallsBackToSourceWhenSourceCannotBeParsed()
{
var api = CreateApi();
api.Job
.Setup(j => j.DetailsAsync("original-job", It.IsAny<CancellationToken>()))
.ReturnsAsync(JobDetails(source: "source", includeSourceMetadataInProperties: false));
api.Storage
.Setup(s => s.NewAsync(It.IsAny<ContainerCreationRequest>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(new ContainerInformation(
DateTimeOffset.Parse("2026-04-30T00:00:00Z"),
DateTimeOffset.Parse("2026-05-30T00:00:00Z"),
"creator",
"container",
"account",
Guid.Empty,
"region")
{
ReadToken = "?read",
WriteToken = "?write",
});

JobCreationRequest capturedRequest = null;
api.Job
.Setup(j => j.NewAsync(It.IsAny<JobCreationRequest>(), It.IsAny<string>(), null, It.IsAny<CancellationToken>()))
.Callback<JobCreationRequest, string, bool?, CancellationToken>((request, _, _, _) => capturedRequest = request)
.ReturnsAsync(new JobCreationResult("new-job", "summary", "results", null));

var blobClientFactory = new FakeBlobClientFactory
{
DownloadedText = """[{ "WorkItemId": "work-a", "Command": "run a", "PayloadUri": "payload-a" }]""",
UploadedBlobUri = new Uri("https://account.blob.core.windows.net/container/job-list.json"),
};

await CreateService(api.Api.Object, blobClientFactory)
.ResubmitWorkItemsAsync(new HelixJobInfo("original-job", "finished"), [WorkItem("work-a")], CancellationToken.None);

Assert.NotNull(capturedRequest);
Assert.Equal("source", capturedRequest.Source);
Assert.Null(capturedRequest.SourcePrefix);
Assert.Null(capturedRequest.TeamProject);
Assert.Null(capturedRequest.Repository);
Assert.Null(capturedRequest.Branch);
}

[Fact]
public async Task ResubmitWorkItemsAsync_FallsBackToSourceWhenSourceMetadataPropertiesAreMissing()
{
var api = CreateApi();
api.Job
.Setup(j => j.DetailsAsync("original-job", It.IsAny<CancellationToken>()))
.ReturnsAsync(JobDetails(source: "ci/public/dotnet/arcade/refs/heads/main", includeSourceMetadataInProperties: false));
api.Storage
.Setup(s => s.NewAsync(It.IsAny<ContainerCreationRequest>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(new ContainerInformation(
DateTimeOffset.Parse("2026-04-30T00:00:00Z"),
DateTimeOffset.Parse("2026-05-30T00:00:00Z"),
"creator",
"container",
"account",
Guid.Empty,
"region")
{
ReadToken = "?read",
WriteToken = "?write",
});

JobCreationRequest capturedRequest = null;
api.Job
.Setup(j => j.NewAsync(It.IsAny<JobCreationRequest>(), It.IsAny<string>(), null, It.IsAny<CancellationToken>()))
.Callback<JobCreationRequest, string, bool?, CancellationToken>((request, _, _, _) => capturedRequest = request)
.ReturnsAsync(new JobCreationResult("new-job", "summary", "results", null));

var blobClientFactory = new FakeBlobClientFactory
{
DownloadedText = """[{ "WorkItemId": "work-a", "Command": "run a", "PayloadUri": "payload-a" }]""",
UploadedBlobUri = new Uri("https://account.blob.core.windows.net/container/job-list.json"),
};

await CreateService(api.Api.Object, blobClientFactory)
.ResubmitWorkItemsAsync(new HelixJobInfo("original-job", "finished"), [WorkItem("work-a")], CancellationToken.None);

Assert.NotNull(capturedRequest);
Assert.Equal("ci/public/dotnet/arcade/refs/heads/main", capturedRequest.Source);
Assert.Null(capturedRequest.SourcePrefix);
Assert.Null(capturedRequest.TeamProject);
Assert.Null(capturedRequest.Repository);
Assert.Null(capturedRequest.Branch);
}

private static HelixService CreateService(IHelixApi api, IBlobClientFactory blobClientFactory = null, IFileSystem fileSystem = null)
=> new(
api,
Expand All @@ -283,8 +377,8 @@ private static JobSummary Job(string name, string finished, JToken properties)
Properties = properties,
};

private static JobDetails JobDetails()
=> new("https://storage/job-list.json", null, "original-job", "wait", "source", "helix-type", "build")
private static JobDetails JobDetails(string source = "source", bool includeSourceMetadataInProperties = true)
=> new("https://storage/job-list.json", null, "original-job", "wait", source, "helix-type", "build")
{
Creator = "creator",
QueueId = "queue-id",
Expand All @@ -293,6 +387,10 @@ private static JobDetails JobDetails()
["BuildId"] = "123",
["TestRunName"] = "custom run",
["System.StageName"] = "test stage",
["SourcePrefix"] = includeSourceMetadataInProperties ? "ci" : null,
["TeamProject"] = includeSourceMetadataInProperties ? "public" : null,
["Repository"] = includeSourceMetadataInProperties ? "dotnet/arcade" : null,
["Branch"] = includeSourceMetadataInProperties ? "refs/heads/main" : null,
["ObjectProperty"] = new JObject
{
["nested"] = true,
Expand Down
Loading