Skip to content

Implement better property expansion #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gcampbell-msft merged 1 commit into from
Jul 21, 2026
+152 −17
Merged

Implement better property expansion #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 28 additions & 9 deletions InProcessExtractor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ public class InProcessExtractor : ICompileCommandsExtractor
private string[] _externalIncludePaths = [];
private string[] _includePaths = [];
private string? _fixupPropsPath;
private IReadOnlyDictionary<string, string>? _evaluatedProperties;

// Resolve unevaluated MSBuild expressions like $([MSBuild]::NormalizePath('base', 'rel'))
// that appear in IncludePath when GetPropertyValue doesn't fully evaluate intrinsics.
Expand Down Expand Up @@ -496,13 +497,25 @@ private List<ClCommandLineEntry> GetClCommandLines()
// and adds them to ClCompile.AdditionalIncludeDirectories before GetClCommandLines runs.
// - ReplaceExistingProjectInstance prevents stale cached ProjectInstances
// across --all-configurations iterations.
// - ProvideProjectStateAfterBuild returns the ProjectInstance with the FINAL
// property values, including properties defined dynamically while targets run
// (e.g. GeneratedFilesDir from the C++/WinRT targets). The input projectInstance
// only holds statically-evaluated properties, so residual $(...) references to
// target-defined properties would otherwise stay unexpanded.
var buildRequestData = new BuildRequestData(
projectInstance,
new[] { "ComputeReferenceCLInput", "GetProjectDirectories", "GetClCommandLines" },
hostServices: null,
BuildRequestDataFlags.ReplaceExistingProjectInstance);
BuildRequestDataFlags.ReplaceExistingProjectInstance
| BuildRequestDataFlags.ProvideProjectStateAfterBuild);
var buildResult = BuildManager.DefaultBuildManager.Build(buildParams, buildRequestData);

// Snapshot the evaluated property values so residual $(...) references that survive
// into the captured command lines can be expanded later in ToCompileCommands.
// Prefer the post-build project state (has target-defined properties); fall back to
// the input instance when the build didn't provide it.
_evaluatedProperties = MsBuildPropertyExpander.SnapshotProperties(buildResult.ProjectStateAfterBuild ?? projectInstance);

// Harvest GetProjectDirectories even on failure. It's a light target
// and often succeeds even when GetClCommandLines doesn't.
if (buildResult.ResultsByTarget.TryGetValue("GetProjectDirectories", out var projDirsResult)
Expand Down Expand Up @@ -791,19 +804,22 @@ string GetElementValue(string name) =>

// Build compile commands for each source file
var commands = new List<CompileCommand>();
foreach (var file in sourceFiles)
foreach (var rawFile in sourceFiles)
{
// Expand residual $(...) BEFORE rooting.
var file = MsBuildPropertyExpander.ExpandMsBuildProperties(rawFile, _evaluatedProperties);
var fullPath = Path.IsPathRooted(file)
? file
: Path.GetFullPath(Path.Combine(projectDir, file));

var fileArgs = new List<string>(baseArgs) { fullPath };
var commandLine = SanitizeFallbackPaths(fileArgs.ToArray(), _vcToolsInstallDir);
var expandedArgs = MsBuildPropertyExpander.ExpandMsBuildProperties(fileArgs.ToArray(), _evaluatedProperties);
var commandLine = SanitizeFallbackPaths(expandedArgs, _vcToolsInstallDir);

commands.Add(new CompileCommand(
File: fullPath,
Arguments: commandLine,
Directory: projectDir,
Directory: MsBuildPropertyExpander.ExpandMsBuildProperties(projectDir, _evaluatedProperties),
ProjectPath: _projectPath,
ProjectName: _projectName,
Configuration: _configuration,
Expand Down Expand Up @@ -984,12 +1000,14 @@ private List<CompileCommand> ToCompileCommands(List<ClCommandLineEntry> entries)
// Emit with a synthetic filename so consumers can inspect
// the project-wide baseline.
var defaultsFile = Path.Combine(entry.WorkingDirectory, "__project_defaults.cpp");
defaultsFile = MsBuildPropertyExpander.ExpandMsBuildProperties(defaultsFile, _evaluatedProperties);
var defaultsArgs = new List<string>(baseArgs) { defaultsFile };
var sanitized = SanitizeFallbackPaths(defaultsArgs.ToArray(), _vcToolsInstallDir);
var expandedDefaults = MsBuildPropertyExpander.ExpandMsBuildProperties(defaultsArgs.ToArray(), _evaluatedProperties);
var sanitized = SanitizeFallbackPaths(expandedDefaults, _vcToolsInstallDir);
commands.Add(new CompileCommand(
File: defaultsFile,
Arguments: sanitized,
Directory: entry.WorkingDirectory,
Directory: MsBuildPropertyExpander.ExpandMsBuildProperties(entry.WorkingDirectory, _evaluatedProperties),
ProjectPath: _projectPath,
ProjectName: _projectName,
Configuration: _configuration,
Expand Down Expand Up @@ -1019,12 +1037,13 @@ private List<CompileCommand> ToCompileCommands(List<ClCommandLineEntry> entries)
MergeDefaultFlags(fileArgs, defaultArgs);

// Resolve any remaining unresolved fallback paths in the arguments
var commandLine = SanitizeFallbackPaths(fileArgs.ToArray(), _vcToolsInstallDir);
var expandedFileArgs = MsBuildPropertyExpander.ExpandMsBuildProperties(fileArgs.ToArray(), _evaluatedProperties);
var commandLine = SanitizeFallbackPaths(expandedFileArgs, _vcToolsInstallDir);

commands.Add(new CompileCommand(
File: file,
File: MsBuildPropertyExpander.ExpandMsBuildProperties(file, _evaluatedProperties),
Arguments: commandLine,
Directory: entry.WorkingDirectory,
Directory: MsBuildPropertyExpander.ExpandMsBuildProperties(entry.WorkingDirectory, _evaluatedProperties),
ProjectPath: _projectPath,
ProjectName: _projectName,
Configuration: _configuration,
Expand Down
99 changes: 99 additions & 0 deletions MsBuildPropertyExpander.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
using Microsoft.Build.Execution;
using System.Text.RegularExpressions;

namespace MSBuild.CompileCommands.Extractor
{
/// <summary>
/// Expands residual MSBuild property references (e.g. <c>$(OpenConsoleDir)</c>,
/// <c>$(GeneratedFilesDir)</c>) that survive into extracted compile commands, using a project's
/// evaluated property values. MSBuild sometimes captures a command line, file path, or directory
/// before every <c>$(...)</c> in metadata like AdditionalIncludeDirectories has been expanded, so
/// this pass substitutes the authoritative evaluated value the project already computed.
/// Shared by both the in-process and out-of-process extractors.
/// </summary>
public static class MsBuildPropertyExpander
{
// Matches a single MSBuild property reference, e.g. $(OpenConsoleDir).
private static readonly Regex MsBuildPropertyRef = new(@"\$\(([^)]+)\)", RegexOptions.Compiled);

private const int MaxPropertyExpansionDepth = 10;

/// <summary>
/// Expands residual MSBuild property references in each element of an argument array.
/// Expansion is case-insensitive and re-scans its own output (bounded by
/// <see cref="MaxPropertyExpansionDepth"/>) so nested references resolve too.
/// Returns the input unchanged when no properties are available.
/// </summary>
public static string[] ExpandMsBuildProperties(string[] args, IReadOnlyDictionary<string, string>? evaluatedProperties)
{
if (evaluatedProperties == null || evaluatedProperties.Count == 0)
return args;

var result = new string[args.Length];
for (int i = 0; i < args.Length; i++)
result[i] = ExpandOne(args[i], evaluatedProperties);
return result;
}

/// <summary>
/// Expands residual MSBuild property references in a single string (e.g. a source file
/// path or working directory). Returns the input unchanged when no properties are available.
/// </summary>
public static string ExpandMsBuildProperties(string value, IReadOnlyDictionary<string, string>? evaluatedProperties)
{
if (string.IsNullOrEmpty(value) || evaluatedProperties == null || evaluatedProperties.Count == 0)
return value;
return ExpandOne(value, evaluatedProperties);
}

private static string ExpandOne(string value, IReadOnlyDictionary<string, string> evaluatedProperties)
{
for (int depth = 0; depth < MaxPropertyExpansionDepth && value.Contains("$("); depth++)
{
bool replacedAny = false;
value = MsBuildPropertyRef.Replace(value, match =>
{
var name = match.Groups[1].Value;
// Substitute whenever the property is DEFINED, even if it evaluates to an empty
// string. A defined-but-empty property (common at design time for values that
// depend on build-time state, e.g. $(GeneratedFilesDir) deriving from $(IntDir))
// should still have its $(...) removed rather than leaking into the output.
if (evaluatedProperties.TryGetValue(name, out var resolved))
{
replacedAny = true;
return resolved ?? string.Empty;
}

// Unresolved residual: the property is entirely absent from the evaluated
// project properties, so we intentionally leave the literal $(...) untouched
// here rather than guessing or dropping the argument. If you are chasing an
// unexpanded variable in compile_commands.json (e.g. $(OpenConsoleDir) or
// $(GeneratedFilesDir)), this is the spot: the property name did not appear in
// the evaluated project properties passed in (post-build ProjectInstance state
// in-process / binlog Property nodes out-of-process).
return match.Value;
});

if (!replacedAny)
break;
}

return value;
}

/// <summary>
/// Builds a case-insensitive map of evaluated MSBuild property name to value from a
/// <see cref="ProjectInstance"/>, used to expand residual $(...) references (in-process).
/// </summary>
public static Dictionary<string, string> SnapshotProperties(ProjectInstance projectInstance)
{
var map = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
foreach (var prop in projectInstance.Properties)
{
if (!string.IsNullOrEmpty(prop.Name))
map[prop.Name] = prop.EvaluatedValue ?? string.Empty;
}
return map;
}
}
}
33 changes: 25 additions & 8 deletions OutOfProcessExtractor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ public OutOfProcessExtractor(

private string[] _externalIncludePaths = [];
private string[] _includePaths = [];
private IReadOnlyDictionary<string, string>? _evaluatedProperties;

private static readonly char[] CmdMetaChars =
{ ' ', '\t', ';', '&', '|', '^', '<', '>', '(', ')' };
Expand Down Expand Up @@ -508,6 +509,16 @@ private List<ClCommandLineEntry> ParseBinlog(string binlogPath)
var build = BinaryLog.ReadBuild(binlogPath);
var target = build.FindFirstDescendant<Target>(t => t.Name == "GetClCommandLines");

// Snapshot evaluated property values (last-wins) so residual $(...) references that
// survive into the captured command lines can be expanded in ToCompileCommands.
var propertyMap = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
foreach (var prop in build.FindChildrenRecursive<Property>())
{
if (!string.IsNullOrEmpty(prop.Name) && !string.IsNullOrEmpty(prop.Value))
propertyMap[prop.Name] = prop.Value;
}
_evaluatedProperties = propertyMap;

// Prefer GetProjectDirectories target output over raw property reads,
// it accounts for UseEnv=true and Makefile configuration resolution that
// raw property values miss.
Expand Down Expand Up @@ -769,12 +780,14 @@ private List<CompileCommand> ToCompileCommands(List<ClCommandLineEntry> entries)
if (_emitDefaults)
{
var defaultsFile = Path.Combine(entry.WorkingDirectory, "__project_defaults.cpp");
defaultsFile = MsBuildPropertyExpander.ExpandMsBuildProperties(defaultsFile, _evaluatedProperties);
var defaultsArgs = new List<string>(baseArgs) { defaultsFile };
var sanitized = InProcessExtractor.SanitizeFallbackPaths(defaultsArgs.ToArray(), _vcToolsInstallDir);
var expandedDefaults = MsBuildPropertyExpander.ExpandMsBuildProperties(defaultsArgs.ToArray(), _evaluatedProperties);
var sanitized = InProcessExtractor.SanitizeFallbackPaths(expandedDefaults, _vcToolsInstallDir);
commands.Add(new CompileCommand(
File: defaultsFile,
Arguments: sanitized,
Directory: entry.WorkingDirectory,
Directory: MsBuildPropertyExpander.ExpandMsBuildProperties(entry.WorkingDirectory, _evaluatedProperties),
ProjectPath: _projectPath,
ProjectName: projectName,
Configuration: _configuration,
Expand All @@ -791,12 +804,13 @@ private List<CompileCommand> ToCompileCommands(List<ClCommandLineEntry> entries)
if (_mergeDefaults && defaultArgs != null)
InProcessExtractor.MergeDefaultFlags(fileArgs, defaultArgs);

var commandLine = InProcessExtractor.SanitizeFallbackPaths(fileArgs.ToArray(), _vcToolsInstallDir);
var expandedFileArgs = MsBuildPropertyExpander.ExpandMsBuildProperties(fileArgs.ToArray(), _evaluatedProperties);
var commandLine = InProcessExtractor.SanitizeFallbackPaths(expandedFileArgs, _vcToolsInstallDir);

commands.Add(new CompileCommand(
File: file,
File: MsBuildPropertyExpander.ExpandMsBuildProperties(file, _evaluatedProperties),
Arguments: commandLine,
Directory: entry.WorkingDirectory,
Directory: MsBuildPropertyExpander.ExpandMsBuildProperties(entry.WorkingDirectory, _evaluatedProperties),
ProjectPath: _projectPath,
ProjectName: projectName,
Configuration: _configuration,
Expand Down Expand Up @@ -1093,19 +1107,22 @@ string GetElementValue(string name) =>
a.StartsWith("/errorReport", StringComparison.OrdinalIgnoreCase));

var commands = new List<CompileCommand>();
foreach (var file in sourceFiles)
foreach (var rawFile in sourceFiles)
{
// Expand residual $(...) BEFORE rooting.
var file = MsBuildPropertyExpander.ExpandMsBuildProperties(rawFile, _evaluatedProperties);
var fullPath = Path.IsPathRooted(file)
? file
: Path.GetFullPath(Path.Combine(projectDir, file));

var fileArgs = new List<string>(baseArgs) { fullPath };
var commandLine = InProcessExtractor.SanitizeFallbackPaths(fileArgs.ToArray(), _vcToolsInstallDir);
var expandedArgs = MsBuildPropertyExpander.ExpandMsBuildProperties(fileArgs.ToArray(), _evaluatedProperties);
var commandLine = InProcessExtractor.SanitizeFallbackPaths(expandedArgs, _vcToolsInstallDir);

commands.Add(new CompileCommand(
File: fullPath,
Arguments: commandLine,
Directory: projectDir,
Directory: MsBuildPropertyExpander.ExpandMsBuildProperties(projectDir, _evaluatedProperties),
ProjectPath: _projectPath,
ProjectName: projectName,
Configuration: _configuration,
Expand Down
Loading