diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index adea645..d85195f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: - name: Environment Setup uses: actions/setup-dotnet@v4 with: - dotnet-version: 8.0.x + dotnet-version: 9.0.x - name: Build run: dotnet build ./src/FlakeId.sln - name: Test diff --git a/README.md b/README.md index 365171f..9d98262 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Snowflake IDs were originally introduced by Twitter in 2010 as unique, decentralized IDs for Tweets. Their 8-byte size, ordered nature and guaranteed uniqueness make them ideal to use as resource identifiers. Since then, many applications at various scale have adopted Snowflake-esque identifiers. -This repository contains an implementation of decentralized, K-ordered Snowflake IDs based on [the Discord Snowflake specification](https://discord.com/developers/docs/reference). The implementation heavily focuses on high-throughput, supporting upwards of 10.000 unique generations per second on commodity hardware. +This repository contains an implementation of decentralized, K-ordered Snowflake IDs based on [the Discord Snowflake specification](https://discord.com/developers/docs/reference). The implementation heavily focuses on high-throughput, supporting upwards of 10.000 unique generations per second on commodity hardware (up to the theoretical limit of around 4 million per second). You can grab the latest stable version from NuGet: @@ -40,13 +40,6 @@ Where the original Discord reference mentions worker ID and process ID, we subst thread and process ID respectively, as the combination of these two provide sufficient uniqueness, and they are the closest we can get to the original specification within the .NET ecosystem. -The Increment component is a monotonically incrementing number, which is incremented every time a snowflake is generated. -This is in contrast with some other flake-ish implementations, which only increment the counter any time a snowflake is -generated twice at the exact same instant in time. - -We have opted to increment every time an ID is generated, rather than when two or more IDs are generated at the exact same millisecond. -The reasoning behind this is that it's vastly simpler - we can avoid locking altogether - and thus, more performant. It is also closer to Discord's implementation, which was referenced when designing this library. - # Epoch The timestamp component is a delta from a predefined instant in time, this instant is known as the **epoch**. @@ -79,9 +72,8 @@ When exposing your IDs to web clients, it is recommended to use the `ToBase64Str ## Performance -We've benchmarked FlakeId on .NET 8 against [MassTransit's NewId](https://github.com/phatboyg/NewId) library, and [IdGen](https://github.com/RobThree/IdGen) both libraries are widely used. It is worth noting that NewId generates 128-bit integers. +We've benchmarked FlakeId on .NET 8 against [IdGen](https://github.com/RobThree/IdGen), which is another implementation of Snowflake IDs in .NET. FlakeId performs significantly better. -We've also included `Guid.NewGuid` as a baseline benchmark, as it is very well optimized, and arguably the most widely used identifier generator in .NET. ``` BenchmarkDotNet v0.13.12, Windows 11 (10.0.22631.3155/23H2/2023Update/SunValley3) @@ -93,9 +85,7 @@ AMD Ryzen 5 5600X, 1 CPU, 12 logical and 6 physical cores | Method | Mean | Error | StdDev | Code Size | |--------------- |------------:|----------:|-----------:|----------:| -| Single_FlakeId | 26.48 ns | 0.020 ns | 0.019 ns | 358 B | -| Single_Guid | 41.85 ns | 0.481 ns | 0.450 ns | 245 B | -| Single_NewId | 31.83 ns | 0.013 ns | 0.012 ns | 303 B | +| Single_FlakeId | 349.2 ns | 6.58 ns | 6.16 ns | 358 B | | Single_IdGen | 3,473.96 ns | 69.295 ns | 168.673 ns | 671 B | ``` @@ -104,22 +94,21 @@ In this benchmark, IdGen was configured to `SpinWait` in the event multiple IDs Below are the benchmark results for FlakeId running on multiple runtimes. ``` -BenchmarkDotNet v0.13.12, Windows 11 (10.0.22631.3155/23H2/2023Update/SunValley3) -AMD Ryzen 5 5600X, 1 CPU, 12 logical and 6 physical cores -.NET SDK 8.0.201 - [Host] : .NET 8.0.2 (8.0.224.6711), X64 RyuJIT AVX2 - .NET 5.0 : .NET 5.0.17 (5.0.1722.21314), X64 RyuJIT AVX2 - .NET 6.0 : .NET 6.0.27 (6.0.2724.6912), X64 RyuJIT AVX2 - .NET 7.0 : .NET 7.0.16 (7.0.1624.6629), X64 RyuJIT AVX2 - .NET 8.0 : .NET 8.0.2 (8.0.224.6711), X64 RyuJIT AVX2 - - -| Method | Job | Runtime | Mean | Error | StdDev | Code Size | -|--------------- |--------- |--------- |---------:|---------:|---------:|----------:| -| Single_FlakeId | .NET 5.0 | .NET 5.0 | 27.85 ns | 0.111 ns | 0.103 ns | 254 B | -| Single_FlakeId | .NET 6.0 | .NET 6.0 | 26.37 ns | 0.056 ns | 0.053 ns | 215 B | -| Single_FlakeId | .NET 7.0 | .NET 7.0 | 26.72 ns | 0.211 ns | 0.176 ns | 209 B | -| Single_FlakeId | .NET 8.0 | .NET 8.0 | 26.56 ns | 0.085 ns | 0.071 ns | 358 B | +BenchmarkDotNet v0.13.12, macOS 15.6.1 (24G90) [Darwin 24.6.0] +Apple M1 Pro, 1 CPU, 10 logical and 10 physical cores +.NET SDK 9.0.100 + [Host] : .NET 8.0.2 (8.0.224.6711), Arm64 RyuJIT AdvSIMD + .NET 7.0 : .NET 7.0.11 (7.0.1123.42427), Arm64 RyuJIT AdvSIMD + .NET 8.0 : .NET 8.0.2 (8.0.224.6711), Arm64 RyuJIT AdvSIMD + .NET 9.0 : .NET 9.0.0 (9.0.24.52809), Arm64 RyuJIT AdvSIMD + + +| Method | Job | Runtime | Mean | Error | StdDev | +|--------------- |--------- |--------- |---------:|--------:|---------:| +| Single_FlakeId | .NET 7.0 | .NET 7.0 | 354.0 ns | 6.87 ns | 13.89 ns | +| Single_FlakeId | .NET 8.0 | .NET 8.0 | 356.6 ns | 7.17 ns | 9.81 ns | +| Single_FlakeId | .NET 9.0 | .NET 9.0 | 349.2 ns | 6.58 ns | 6.16 ns | + ``` ## Issues diff --git a/src/FlakeId.Benchmarks/FlakeId.Benchmarks.csproj b/src/FlakeId.Benchmarks/FlakeId.Benchmarks.csproj index eb886d6..4e7902b 100644 --- a/src/FlakeId.Benchmarks/FlakeId.Benchmarks.csproj +++ b/src/FlakeId.Benchmarks/FlakeId.Benchmarks.csproj @@ -2,7 +2,7 @@ Exe - net8.0;net7.0;net6.0 + net6.0;net7.0;net8.0;net9.0 diff --git a/src/FlakeId.Benchmarks/FlakeIdMultipleRuntimesBenchmarks.cs b/src/FlakeId.Benchmarks/FlakeIdMultipleRuntimesBenchmarks.cs index 08412c7..41ff958 100644 --- a/src/FlakeId.Benchmarks/FlakeIdMultipleRuntimesBenchmarks.cs +++ b/src/FlakeId.Benchmarks/FlakeIdMultipleRuntimesBenchmarks.cs @@ -1,5 +1,6 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Diagnostics.Windows.Configs; +using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; namespace FlakeId.Benchmarks @@ -8,8 +9,11 @@ namespace FlakeId.Benchmarks [SimpleJob(RuntimeMoniker.Net60)] [SimpleJob(RuntimeMoniker.Net70)] [SimpleJob(RuntimeMoniker.Net80)] + [SimpleJob(RuntimeMoniker.Net90)] +#if _WINDOWS [DisassemblyDiagnoser] [InliningDiagnoser(true, null)] +#endif public class FlakeIdMultipleRuntimesBenchmarks { [Benchmark] diff --git a/src/FlakeId.Benchmarks/IdCreationBenchmarks.cs b/src/FlakeId.Benchmarks/IdCreationBenchmarks.cs index a423d69..a04ec65 100644 --- a/src/FlakeId.Benchmarks/IdCreationBenchmarks.cs +++ b/src/FlakeId.Benchmarks/IdCreationBenchmarks.cs @@ -6,8 +6,10 @@ namespace FlakeId.Benchmarks { +#if _WINDOWS [DisassemblyDiagnoser] [InliningDiagnoser(true, null)] +#endif public class IdCreationBenchmarks { private static readonly IdGenerator s_idGenerator = new IdGenerator(10, @@ -30,7 +32,7 @@ public void Single_NewId() { NewId.Next(); } - + [Benchmark] public void Single_IdGen() { diff --git a/src/FlakeId.Tests/ConcurrencyTests.cs b/src/FlakeId.Tests/ConcurrencyTests.cs new file mode 100644 index 0000000..32027c0 --- /dev/null +++ b/src/FlakeId.Tests/ConcurrencyTests.cs @@ -0,0 +1,125 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FlakeId.Tests; + +[TestClass] +public class ConcurrencyTests +{ + private const int ThreadIdBits = 5; + private const int ProcessIdBits = 5; + private const int IncrementBits = 12; + private const int TimestampShift = ThreadIdBits + ProcessIdBits + IncrementBits; + private const int IncrementMask = (1 << IncrementBits) - 1; + + private static readonly FieldInfo s_prevIdField = + typeof(Id).GetField("s_prevId", BindingFlags.NonPublic | BindingFlags.Static); + + [TestCleanup] + public void TestCleanup() + { + Assert.IsNotNull(s_prevIdField, "Could not find the private static field 's_prevId' for testing."); + s_prevIdField.SetValue(null, 0L); + } + + [TestMethod] + public void Create_ShouldGenerateNonZeroId() + { + long id = Id.Create(); + + Assert.AreNotEqual(0L, id); + } + + [TestMethod] + public void Create_ShouldGenerateMonotonicallyIncreasingIds() + { + const int idCount = 500_000; + long[] ids = new long[idCount]; + + for (int i = 0; i < idCount; i++) + { + ids[i] = Id.Create(); + } + + for (int i = 1; i < idCount; i++) + { + Assert.IsTrue(ids[i] > ids[i - 1], $"ID at index {i} was not greater than the previous one."); + } + } + + [TestMethod] + public void Create_ShouldGenerateUniqueIds_WhenCalledConcurrently() + { + const int idsPerTask = 100_000; + int taskCount = Environment.ProcessorCount; + int totalIds = idsPerTask * taskCount; + var generatedIds = new ConcurrentBag(); + + var tasks = new List(); + + for (int i = 0; i < taskCount; i++) + { + tasks.Add(Task.Run(() => + { + for (int j = 0; j < idsPerTask; j++) + { + generatedIds.Add(Id.Create()); + } + })); + } + + Task.WaitAll(tasks.ToArray()); + + Assert.AreEqual(totalIds, generatedIds.Count, "The number of generated IDs does not match the expected total."); + + var distinctIds = new HashSet(generatedIds); + Assert.AreEqual(totalIds, distinctIds.Count, "Duplicate IDs were found when generating concurrently."); + } + + [TestMethod] + public void Create_ShouldThrow_WhenClockMovesBackwards() + { + Assert.IsNotNull(s_prevIdField, "Could not find the private static field 's_prevId' for testing."); + + long baseId = Id.Create(); + long baseTimestamp = baseId >> TimestampShift; + + long futureTimestamp = baseTimestamp + 5000; + long futureId = futureTimestamp << TimestampShift; + s_prevIdField.SetValue(null, futureId); + + var ex = Assert.ThrowsException(() => Id.Create()); + StringAssert.Contains(ex.Message, "Clock shifted backwards"); + } + + [TestMethod] + public void Increment_ShouldResetToZero_OnNewMillisecond() + {Assert.IsNotNull(s_prevIdField, "Could not find the private static field 's_prevId' for testing."); + + long baseId = Id.Create(); + long baseTimestamp = baseId >> TimestampShift; + + // create a fake "previous ID" that is at the same timestamp but has a high increment. + // we preserve the other parts of the ID (thread, process) to make the state realistic. + const int highIncrement = 4000; + long otherParts = baseId & ~((1L << 42) - 1); // masks out the timestamp + long prevId = otherParts | (baseTimestamp << TimestampShift) | highIncrement; + + s_prevIdField.SetValue(null, prevId); + + Thread.Sleep(5); + + long newIdValue = Id.Create(); + + long newTimestamp = newIdValue >> TimestampShift; + int newIncrement = (int)(newIdValue & IncrementMask); + + Assert.IsTrue(newTimestamp > baseTimestamp, "Timestamp should have advanced."); + Assert.AreEqual(0, newIncrement, "Increment should reset to 0 on a new millisecond."); + } +} diff --git a/src/FlakeId.Tests/FlakeId.Tests.csproj b/src/FlakeId.Tests/FlakeId.Tests.csproj index 19a4233..bc8e202 100644 --- a/src/FlakeId.Tests/FlakeId.Tests.csproj +++ b/src/FlakeId.Tests/FlakeId.Tests.csproj @@ -1,9 +1,10 @@ - net8.0 false + + net9.0 diff --git a/src/FlakeId.Tests/IdTests.cs b/src/FlakeId.Tests/IdTests.cs index 1559f73..6bb8c96 100644 --- a/src/FlakeId.Tests/IdTests.cs +++ b/src/FlakeId.Tests/IdTests.cs @@ -90,14 +90,5 @@ public void Id_ToString() Assert.AreEqual(id.ToString(), id.ToString()); } - - [TestMethod] - public void Id_ContainsTimeZoneComponent() - { - DateTimeOffset timeStamp = new DateTimeOffset(2020, 1, 1, 0, 0, 0, TimeSpan.FromHours(7)); - Id id = Id.Create(timeStamp.ToUnixTimeMilliseconds()); - - Assert.AreEqual(timeStamp.ToUnixTimeMilliseconds(), id.ToUnixTimeMilliseconds()); - } } } diff --git a/src/FlakeId/FlakeId.csproj b/src/FlakeId/FlakeId.csproj index ae6a97c..263c9ba 100644 --- a/src/FlakeId/FlakeId.csproj +++ b/src/FlakeId/FlakeId.csproj @@ -1,7 +1,7 @@ - net8.0;netstandard2.0 + netstandard2.0;net9.0 true Discord inspired, Twitter like implementation of Snowflake IDs, focused on performance. Snowflakes are 64 bit, highly optimized, decentralized, K-ordered, unique identifiers. https://github.com/aevitas/flakeid @@ -15,6 +15,12 @@ Apache-2.0 + + + <_Parameter1>FlakeId.Tests + + + True diff --git a/src/FlakeId/Id.cs b/src/FlakeId/Id.cs index 2f32f7d..924fef0 100644 --- a/src/FlakeId/Id.cs +++ b/src/FlakeId/Id.cs @@ -9,7 +9,7 @@ namespace FlakeId /// Represents a unique, K-ordered, sortable identifier. /// [DebuggerDisplay("{_value}")] - public struct Id : IComparable + public struct Id : IComparable, IEquatable { // This implementation of Snowflake ID is based on the specification as published by Discord: // https://discord.com/developers/docs/reference @@ -21,7 +21,7 @@ public struct Id : IComparable // 111111111111111111111111111111111111111111 11111 11111 111111111111 // 64 22 17 12 0 // - // The Timestamp component is represented as the milliseconds since the first second of 2015. + // The Timestamp component is represented as the milliseconds since the first second of 2015. // Since we're using all 64 bits available, this epoch can be any point in time, as long as it's in the past. // If the epoch is set to a point in time in the future, it may result in negative snowflakes being generated. // @@ -30,16 +30,16 @@ public struct Id : IComparable // the closest we can get to the original specification within the .NET ecosystem. // // The Increment component is a monotonically incrementing number, which is incremented every time a snowflake is generated. - // This is in contrast with some other flake-ish implementations, which only increment the counter any time a snowflake is + // This is in contrast with some other flake-ish implementations, which only increment the counter any time a snowflake is // generated twice at the exact same instant in time. We believe Discord's implementation is more correct here, // as even two snowflakes that are generated at the exact same point in time will not be identical, because of their increments. // - // This implementation is optimised for high-throughput applications, while providing IDs that are roughly sortable, and + // This implementation is optimised for high-throughput applications, while providing IDs that are roughly sortable, and // with a very high degree of uniqueness. private long _value; - private static int s_increment; + private static long s_prevId; // Calling Process.GetCurrentProcess() is a very slow operation, as it has to query the operating system. // Because it's highly unlikely the process ID will change (if at all possible) during our run time, we'll cache it. @@ -71,7 +71,7 @@ public static Id Create() } /// - /// Creates a new ID based on the provided timestamp in milliseconds. + /// Creates a new ID based on the provided timestamp in milliseconds. /// When using this overload, make sure you take the timezone of the provided timestamp into consideration. /// /// @@ -87,9 +87,10 @@ public static Id Create(long timeStampMs) Id id = new Id(); long relativeTimeStamp = timeStampMs - MonotonicTimer.Epoch.ToUnixTimeMilliseconds(); - if (relativeTimeStamp < 0) + if (relativeTimeStamp < 0) { - throw new ArgumentException("Specified timestamp would result in a negative ID (it's before instance epoch)"); + throw new ArgumentException( + "Specified timestamp would result in a negative ID (it's before instance epoch)"); } id.CreateInternal(relativeTimeStamp); @@ -139,22 +140,58 @@ public static Id Parse(long value) private void CreateInternal(long timeStampMs = 0) { - long milliseconds = timeStampMs == 0 ? MonotonicTimer.ElapsedMilliseconds : timeStampMs; - long timestamp = milliseconds & TimestampMask; - int threadId = Thread.CurrentThread.ManagedThreadId & ThreadIdMask; - // int processId = s_processId ??= Process.GetCurrentProcess().Id & ProcessIdMask; if (s_processId is null) + { s_processId = Process.GetCurrentProcess().Id & ProcessIdMask; - int processId = s_processId.Value; + } - int increment = Interlocked.Increment(ref s_increment) & IncrementMask; + int processId = s_processId.Value; + SpinWait spinner = default; - unchecked + while (true) { - _value = (timestamp << (ThreadIdBits + ProcessIdBits + IncrementBits)) - + (threadId << (ProcessIdBits + IncrementBits)) - + (processId << IncrementBits) - + increment; + long prev = Interlocked.Read(ref s_prevId); + + long lastTimestamp = (prev >> (ThreadIdBits + ProcessIdBits + IncrementBits)); + long currentTimestamp = timeStampMs == 0 ? MonotonicTimer.ElapsedMilliseconds : timeStampMs; + + if (currentTimestamp < lastTimestamp) + { + throw new InvalidOperationException( + "Clock shifted backwards; can't reliably generate ID"); + } + + int increment; + if (currentTimestamp == lastTimestamp) + { + increment = (int)(prev & IncrementMask) + 1; + + // Increment overflows, wait for the next ms to avoid it being 0 for ages + if (increment > IncrementMask) + { + spinner.SpinOnce(); + continue; + } + } + else + { + increment = 0; + } + + int threadId = Environment.CurrentManagedThreadId & ThreadIdMask; + + long newValue = (currentTimestamp << (ThreadIdBits + ProcessIdBits + IncrementBits)) + | (long)(threadId << (ProcessIdBits + IncrementBits)) + | (long)(processId << IncrementBits) + | (long)increment; + + // Atomically update the last value. If the original value was changed by another + // thread, this will fail and we'll loop again. + if (Interlocked.CompareExchange(ref s_prevId, newValue, prev) == prev) + { + _value = newValue; + break; + } } }