diff --git a/DotPrompt.Tests/DotPrompt.Tests.csproj b/DotPrompt.Tests/DotPrompt.Tests.csproj index 02ba079..bd81673 100644 --- a/DotPrompt.Tests/DotPrompt.Tests.csproj +++ b/DotPrompt.Tests/DotPrompt.Tests.csproj @@ -10,9 +10,9 @@ - + - + runtime; build; native; contentfiles; analyzers; buildtransitive all @@ -35,6 +35,9 @@ Always + + Always + diff --git a/DotPrompt.Tests/Extensions/OpenAi/OpenAiExtensionsTests.cs b/DotPrompt.Tests/Extensions/OpenAi/OpenAiExtensionsTests.cs index a894be0..c1c513c 100644 --- a/DotPrompt.Tests/Extensions/OpenAi/OpenAiExtensionsTests.cs +++ b/DotPrompt.Tests/Extensions/OpenAi/OpenAiExtensionsTests.cs @@ -1,3 +1,4 @@ +using System.Reflection; using DotPrompt.Extensions.OpenAi; using OpenAI.Chat; @@ -105,13 +106,10 @@ public void ToOpenAiChatCompletionOptions_WithJsonSchemaFormat_ReturnsAValidOpti const string expectedSchema = """{"type":"object","required":["field1"],"properties":{"field1":{"type":"string","description":"An example description for the field"},"field2":{"type":"array","items":{"type":"string"}}},"additionalProperties":false}"""; - var jsonSchemaProperty = options.ResponseFormat.GetType().GetProperty("JsonSchema"); - var jsonSchemaValue = jsonSchemaProperty!.GetValue(options.ResponseFormat); + var jsonSchemaValue = GetInternalProperty(options.ResponseFormat, "JsonSchema"); + var schemaValue = GetInternalProperty(jsonSchemaValue, "Schema"); - var schemaProperty = jsonSchemaValue!.GetType().GetProperty("Schema"); - var schemaValue = schemaProperty!.GetValue(jsonSchemaValue) as BinaryData; - - var optionsSchema = schemaValue!.ToString(); + var optionsSchema = schemaValue.ToString(); Assert.Equal(expectedSchema, optionsSchema); @@ -159,4 +157,24 @@ public void ToOpenAiChatCompletionOptions_WithInvalidFormat_ThrowsAnException() var exception = Assert.Throws(() => promptFileMock.ToOpenAiChatCompletionOptions()); Assert.Contains("The requested output format is not available", exception.Message); } + + private static T GetInternalProperty(object obj, string propertyName) + { + var property = obj.GetType().GetProperty(propertyName, + BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Instance); + + if (property == null) + { + throw new InvalidOperationException($"Property '{propertyName}' not found on type '{obj.GetType().Name}'"); + } + + var value = property.GetValue(obj); + + if (value is T result) + { + return result; + } + + throw new InvalidOperationException($"Property '{propertyName}' is not of expected type '{typeof(T).Name}'"); + } } \ No newline at end of file diff --git a/DotPrompt.Tests/PromptFileTests.cs b/DotPrompt.Tests/PromptFileTests.cs index a4ea5c4..80d6e65 100644 --- a/DotPrompt.Tests/PromptFileTests.cs +++ b/DotPrompt.Tests/PromptFileTests.cs @@ -21,6 +21,7 @@ public void FromFile_BasicPrompt_ProducesValidPromptFile() }; Assert.Equal("basic", promptFile.Name); + Assert.Equal(1, promptFile.Version); Assert.NotNull(promptFile.Model); Assert.Equal("claude-3-5-sonnet-latest", promptFile.Model); @@ -213,6 +214,30 @@ public void FromStream_WithMissingModelName_IsPersistedAsNullValue() Assert.Null(promptFile.Model); } + [Theory] + [InlineData(int.MinValue, true)] + [InlineData(-1, true)] + [InlineData(0, false)] + [InlineData(1, false)] + public void FromStream_WithInvalidVersion_ThrowsAnException(int version, bool throwsException) + { + var content = $"name: test\nversion: {version}\nprompts:\n system: System prompt\n user: User prompt"; + using var ms = new MemoryStream(Encoding.UTF8.GetBytes(content)); + ms.Seek(0, SeekOrigin.Begin); + + var act = () => PromptFile.FromStream("test", ms); + + if (throwsException) + { + var exception = Assert.Throws(act); + Assert.Contains("The version of the prompt file cannot be negative", exception.Message); + } + else + { + act(); + } + } + [Fact] public void GenerateUserPrompt_UsingDefaults_CorrectlyGeneratesPromptFromTemplate() { @@ -368,7 +393,7 @@ public void ToStream_WithValidInput_ProducesExpectedContent() } }; - var expected = "name: test\nconfig:\n input:\n parameters:\n test?: string\n outputFormat: Json\n maxTokens: 500\nprompts:\n system: system prompt\n user: user prompt\n"; + const string expected = "name: test\nversion: 1\nconfig:\n input:\n parameters:\n test?: string\n outputFormat: Json\n maxTokens: 500\nprompts:\n system: system prompt\n user: user prompt\n"; var ms = new MemoryStream(); promptFile.ToStream(ms); diff --git a/DotPrompt.Tests/PromptManagerTests.cs b/DotPrompt.Tests/PromptManagerTests.cs index bda5e12..9cd5aca 100644 --- a/DotPrompt.Tests/PromptManagerTests.cs +++ b/DotPrompt.Tests/PromptManagerTests.cs @@ -41,6 +41,21 @@ public void PromptManager_WithPathSpecified_LoadsPromptsFromSpecifiedLocation() Assert.Contains("basic", actualPrompts); Assert.Contains("example-with-name", actualPrompts); } + + [Fact] + public void PromptManager_ListPromptFileNamesWithVersions_ReturnsListOfPromptFileNamesAndVersions() + { + var manager = new PromptManager(); + + var expectedPrompts = new List { "basic:1", "example-with-name:1" }; + var actualPrompts = manager.ListPromptFileNamesWithVersions().ToList(); + + Assert.Equal(expectedPrompts.Count, actualPrompts.Count); + foreach (var expectedPrompt in expectedPrompts) + { + Assert.Contains(expectedPrompt, actualPrompts); + } + } [Fact] public void PromptManager_WithDuplicateNames_ThrowsException() @@ -53,6 +68,54 @@ public void PromptManager_WithDuplicateNames_ThrowsException() Assert.Contains("a duplicate exists", exception.Message); } + [Fact] + public void PromptManager_WithDifferentVersions_LoadsSuccessfully() + { + var manager = new PromptManager("multiple-version-prompts"); + + var expectedPrompts = new List { "basic:1", "basic:2" }; + var actualPrompts = manager.ListPromptFileNamesWithVersions().ToList(); + + Assert.Equal(expectedPrompts.Count, actualPrompts.Count); + foreach (var expectedPrompt in expectedPrompts) + { + Assert.Contains(expectedPrompt, actualPrompts); + } + } + + [Fact] + public void PromptManager_WhenRequestedByName_ReturnsLatestVersion() + { + var manager = new PromptManager("multiple-version-prompts"); + + var expectedPrompt = PromptFile.FromFile("multiple-version-prompts/basic-new.prompt"); + var actualPrompt = manager.GetPromptFile("basic"); + + Assert.Equivalent(expectedPrompt, actualPrompt, strict: true); + } + + [Fact] + public void PromptManager_WhenRequestedByNameAndVersion_ReturnsCorrectVersion() + { + var manager = new PromptManager("multiple-version-prompts"); + + var expectedPrompt = PromptFile.FromFile("multiple-version-prompts/basic.prompt"); + var actualPrompt = manager.GetPromptFile("basic", 1); + + Assert.Equivalent(expectedPrompt, actualPrompt, strict: true); + } + + [Fact] + public void PromptManager_WhenRequestedByNameAndInvalidVersion_ThrowsException() + { + var manager = new PromptManager("multiple-version-prompts"); + + var act = () => manager.GetPromptFile("basic", 3); + + var exception = Assert.Throws(act); + Assert.Equal("No prompt file with that name and version has been loaded", exception.Message); + } + [Fact] public void GetPromptFile_WhenRequestedWithValidName_LoadsExpectedPromptFile() { diff --git a/DotPrompt.Tests/multiple-version-prompts/basic-new.prompt b/DotPrompt.Tests/multiple-version-prompts/basic-new.prompt new file mode 100644 index 0000000..e772d46 --- /dev/null +++ b/DotPrompt.Tests/multiple-version-prompts/basic-new.prompt @@ -0,0 +1,20 @@ +name: basic +version: 2 +config: + outputFormat: text + temperature: 0.9 + maxTokens: 500 + input: + parameters: + country: string + style?: string + default: + country: Malta +prompts: + system: | + You are a helpful AI assistant that enjoys making capybara related puns. You should work as many into your response as possible + user: | + I am looking at going on holiday to {{ country }} and would like to know more about it, what can you tell me? + {% if style -%} + Can you answer in the style of a {{ style }} + {% endif -%} diff --git a/DotPrompt.Tests/multiple-version-prompts/basic.prompt b/DotPrompt.Tests/multiple-version-prompts/basic.prompt new file mode 100644 index 0000000..d191ab7 --- /dev/null +++ b/DotPrompt.Tests/multiple-version-prompts/basic.prompt @@ -0,0 +1,20 @@ +name: basic +version: 1 +config: + outputFormat: text + temperature: 0.9 + maxTokens: 500 + input: + parameters: + country: string + style?: string + default: + country: Malta +prompts: + system: | + You are a helpful AI assistant that enjoys making penguin related puns. You should work as many into your response as possible + user: | + I am looking at going on holiday to {{ country }} and would like to know more about it, what can you tell me? + {% if style -%} + Can you answer in the style of a {{ style }} + {% endif -%} diff --git a/DotPrompt/DotPrompt.csproj b/DotPrompt/DotPrompt.csproj index d091b13..7525f8a 100644 --- a/DotPrompt/DotPrompt.csproj +++ b/DotPrompt/DotPrompt.csproj @@ -21,9 +21,9 @@ - - - + + + diff --git a/DotPrompt/PromptFile.cs b/DotPrompt/PromptFile.cs index 288b7a0..eb9b548 100644 --- a/DotPrompt/PromptFile.cs +++ b/DotPrompt/PromptFile.cs @@ -23,6 +23,11 @@ public partial class PromptFile /// public required string Name { get; set; } + /// + /// Gets, sets the version of the prompt file. + /// + public int Version { get; set; } = 1; + /// /// Gets, sets the name of the model (or deployment) the prompt should be executed using /// @@ -113,6 +118,12 @@ public static PromptFile FromStream(string name, Stream inputStream) { promptFile.Name = name; } + + // If the prompt version is negative, then throw an exception + if (promptFile.Version < 0) + { + throw new DotPromptException("The version of the prompt file cannot be negative"); + } // If the prompt output configuration is null then create a new one and set the output format. This is to handle // instances where the output format is slightly older and is set at the top level diff --git a/DotPrompt/PromptManager.cs b/DotPrompt/PromptManager.cs index 16860d5..5687f49 100644 --- a/DotPrompt/PromptManager.cs +++ b/DotPrompt/PromptManager.cs @@ -2,12 +2,19 @@ namespace DotPrompt; +/// +/// Identifies a .prompt file uniquely using its name and version +/// +/// The name of the prompt file +/// The prompt file version +public record PromptFileIdentifier(string Name, int Version); + /// /// Manages loading and accessing of .prompt files from a specified directory. /// public class PromptManager : IPromptManager { - private readonly ConcurrentDictionary _promptFiles = new(); + private readonly ConcurrentDictionary _promptFiles = new(); /// /// Creates a new instance of the using a default instance of the @@ -30,35 +37,67 @@ public PromptManager(IPromptStore promptStore) { foreach (var promptFile in promptStore.Load()) { - if (!_promptFiles.TryAdd(promptFile.Name, promptFile)) + if (!_promptFiles.TryAdd(new PromptFileIdentifier(promptFile.Name, promptFile.Version), promptFile)) { - throw new DotPromptException($"Unable to add prompt file with name '{promptFile.Name}' as a duplicate exists"); + throw new DotPromptException($"Unable to add prompt file with name '{promptFile.Name}' and version {promptFile.Version} as a duplicate exists"); } } } /// - /// Retrieves a by its name + /// Retrieves a by its name. If multiple versions of the prompt file are found, the + /// latest version is returned. /// /// The name of the prompt file to retrieve /// The with the specified name /// Thrown when no prompt file with the specified name is found public PromptFile GetPromptFile(string name) { - if (_promptFiles.TryGetValue(name, out var promptFile)) - { - return promptFile; - } + var promptFilesWithName = _promptFiles + .Where(kvp => kvp.Key.Name == name) + .OrderByDescending(kvp => kvp.Key.Version) + .ToList(); - throw new DotPromptException("No prompt file with that name has been loaded"); + return promptFilesWithName.Count == 0 + ? throw new DotPromptException("No prompt file with that name has been loaded") + : promptFilesWithName[0].Value; } + /// + /// Retrieves a by its name and version. + /// + /// The name of the prompt file to retrieve + /// The version of the prompt file to retrieve + /// The with the specified name and version + /// Thrown when no prompt file with the specified name and version is found + public PromptFile GetPromptFile(string name, int version) + { + return _promptFiles.TryGetValue(new PromptFileIdentifier(name, version), out var promptFile) + ? promptFile + : throw new DotPromptException("No prompt file with that name and version has been loaded"); + } + /// /// Lists the names of all loaded prompt files. /// /// An enumerable collection of prompt file names. public IEnumerable ListPromptFileNames() { - return _promptFiles.Keys.Select(k => k); + return _promptFiles + .DistinctBy(kvp => kvp.Key.Name) + .OrderBy(kvp => kvp.Key.Name) + .Select(kvp => $"{kvp.Key.Name}"); + } + + /// + /// Lists the names of all loaded prompts with their versions. + /// + /// An enumerable collection of prompt file names and versions. + public IEnumerable ListPromptFileNamesWithVersions() + { + return _promptFiles + .OrderBy(kvp => kvp.Key.Name) + .ThenByDescending(kvp => kvp.Key.Version) + .Select(kvp => $"{kvp.Key.Name}:{kvp.Key.Version}"); } } \ No newline at end of file diff --git a/README.md b/README.md index a6d89b1..d7306d5 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ A complete prompt file would look like this. ```yaml name: Example +version: 1 model: gpt-4o config: outputFormat: text @@ -71,6 +72,10 @@ The `name` is optional in the configuration, if it's not provided then the name If you use this property then when the file is loaded the name is converted to lowercase and spaces are replaced with hyphens. So a name of `My cool Prompt` would become `my-cool-prompt`. This is done to make sure the name is easily accessible from the code. +### Version + +The `version` is optional in the configuration, if it's not provided then the version is set to `1`. This is used to allow you to update the prompt file without breaking existing code. Different versions can be loaded by specifying the version number when loading the prompt file through the prompt manager. If no version is specified then the latest version is loaded. + ### Model This is another optional item in the configuration, but it provides information to the user of the prompt file which model (or deployment for Azure Open AI) it should use. As this can be null if not specified this the consumer should make sure to check before usage. For example: @@ -211,6 +216,9 @@ var promptManager = new PromptManager("another-location"); var promptFile = promptManager.GetPromptFile("example"); +// Loading a specific version +// var promptFile = promptManager.GetPromptFile("example", 2); + // List all of the prompts loaded var promptNames = promptManager.ListPromptFileNames(); ``` diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000..2a4a839 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,10 @@ +coverage: + status: + project: + default: + target: auto + threshold: 3% + patch: + default: + target: auto + threshold: 3%