diff --git a/Sources/VimKit/Database+Importer.swift b/Sources/VimKit/Database+Importer.swift index 5eaee2b..514ee3a 100644 --- a/Sources/VimKit/Database+Importer.swift +++ b/Sources/VimKit/Database+Importer.swift @@ -53,7 +53,7 @@ extension Database { if shouldImport { ImportTaskTracker.shared.tasks[sha256Hash] = true let importer = Database.ImportActor(self) - await importer.import(limit) + await importer.import() } } @@ -95,10 +95,8 @@ extension Database { }.store(in: &subscribers) } - /// Starts the import process. - /// - Parameter limit: the max limit of models per entity to import - func `import`(_ limit: Int = .max) { + func `import`() { let group = DispatchGroup() let start = Date.now @@ -122,7 +120,7 @@ extension Database { debugPrint("􁃎 [\(modelName)] - skipping cache warming") return } - warmCache(modelType, limit) + warmCache(modelType) } } @@ -148,7 +146,7 @@ extension Database { debugPrint("􁃎 [\(modelName)] - skipping import") return } - importModel(modelType, limit) + importModel(modelType) } } @@ -177,36 +175,35 @@ extension Database { /// Warms the cache for the specified model type. /// - Parameters: /// - modelType: the type of model - /// - limit: the cache size limit - private func warmCache(_ modelType: any IndexedPersistentModel.Type, _ limit: Int) { + private func warmCache(_ modelType: any IndexedPersistentModel.Type) { guard let table = database.tables[modelType.modelName] else { return } - let count = min(table.rows.count, limit) - _ = modelType.warm(size: count, cache: cache) + modelType.warm(table: table, cache: cache) } /// Imports models of the specified type into the model container. /// - Parameters: /// - modelType: the model type - /// - limit: the max limit of models to import - private func importModel(_ modelType: any IndexedPersistentModel.Type, _ limit: Int) { + private func importModel(_ modelType: any IndexedPersistentModel.Type) { let modelName = modelType.modelName guard let table = database.tables[modelName] else { return } + guard let modelCache = cache.caches[modelName] else { return } + let keys = modelCache.keys let start = Date.now - let rowCount = table.rows.count + let rowCount = modelCache.keys.count//table.rows.count var state: ModelMetadata.State = .unknown defer { - let timeInterval = abs(start.timeIntervalSinceNow) - debugPrint("􂂼 [\(modelName)] - [\(state)] [\(rowCount)] in [\(timeInterval.stringFromTimeInterval())]") - updateMeta(modelName, state: state) - } - - debugPrint("􀈄 [\(modelType.modelName)] - importing [\(rowCount)] models") - for i in 0..= limit || Task.isCancelled { break } - let index = Int64(i) - let row = table.rows[i] + let timeInterval = abs(start.timeIntervalSinceNow) + debugPrint("􂂼 [\(modelName)] - [\(state)] [\(rowCount)] in [\(timeInterval.stringFromTimeInterval())]") + updateMeta(modelName, state: state) + } + + debugPrint("􀈄 [\(modelName)] - importing [\(rowCount)] models") + + for index in keys { + if Task.isCancelled { break } + let row = table.rows[Int(index)] update(index: index, modelType, data: row) count += 1 } @@ -232,21 +229,21 @@ extension Database { try? modelContext.transaction { for cacheKey in cacheKeys { - guard let cache = cache.caches[cacheKey] else { continue } + guard let modelCache = cache.caches[cacheKey] else { continue } let start = Date.now - let keys = cache.keys + let keys = modelCache.keys defer { let timeInterval = abs(start.timeIntervalSinceNow) debugPrint("􂂼 [Batch] - inserted [\(cacheKey)] [\(keys.count)] in [\(timeInterval.stringFromTimeInterval())]") - cache.empty() + modelCache.empty() } for key in keys { - guard let model = cache[key] else { continue } + guard let model = modelCache[key] else { continue } modelContext.insert(model) batchCount += 1 - cache.removeValue(for: key) + modelCache.removeValue(for: key) } } } @@ -326,13 +323,13 @@ extension Database { init() {} /// Warms the cache to a specific size - /// - Parameter size: the size of the cache + /// - Parameter table: the database table data /// - Returns: a list of models that have been cached. @discardableResult - func warm(_ size: Int) -> [T] where T: IndexedPersistentModel { + func warm(_ table: Database.Table) -> [T] where T: IndexedPersistentModel { let cacheKey: CacheKey = T.modelName let cache = findOrCreateCache(cacheKey) - return cache.warm(size) + return cache.warm(table) } /// Finds or creates a model with the specified index and type. @@ -368,7 +365,7 @@ extension Database { fileprivate final class ModelCache: @unchecked Sendable { /// The backing storage cache. - private lazy var cache: Cache = { + fileprivate lazy var cache: Cache = { let cache = Cache() cache.totalCostLimit = cacheTotalCostLimit cache.evictsObjectsWithDiscardedContent = true @@ -383,31 +380,35 @@ extension Database { /// Initializer. init() { } - /// Warms the cache up to the specified index size. Any entities that - /// have cache index misses are stubbed out skeletons that can later be filled in with `.update(data:cache:)`. + /// Warms the cache for the specified table. The entities are stubbed out skeletons that can later be filled in with `.update(data:cache:)`. /// Please note that he models that are inserted into the cache are not inserted into the model context. As an import optimization, /// all of the models are are inserted via the `.batchInsert()` method. - /// - Parameter size: the upper bounds of the model index size + /// - Parameter table: the database table to cache from /// - Returns: empty results for now, simply used to infer type from the generic - could be reworked @discardableResult - func warm(_ size: Int) -> [T] where T: IndexedPersistentModel { + func warm(_ table: Database.Table) -> [T] where T: IndexedPersistentModel { let cacheKey: CacheKey = T.modelName + let size = table.rows.count if size <= .zero { debugPrint("􂂼 [\(cacheKey)] - skipping warm - [\(models.count)] [\(size)]") return [] } debugPrint("􁰹 [\(cacheKey)] - warming cache [\(size)]") + var count = 0 let start = Date.now + defer { + let timeInterval = abs(start.timeIntervalSinceNow) + debugPrint("􂂼 [\(cacheKey)] - cache created [\(count)] in [\(timeInterval.stringFromTimeInterval())]") + } - let range: Range = 0.. [Self] { - cache.warm(size) + static func warm(table: Database.Table, cache: Database.ImportCache) -> [Self] { + cache.warm(table) } /// Performs a fetch request for all models in the specified context. @@ -539,7 +539,6 @@ extension Database { public var room: Room? public var group: Group? public var workset: Workset? - public var parameters: [Parameter] /// Returns the elements instance type public var instanceType: Element? { @@ -552,25 +551,10 @@ extension Database { return results[0] } - /// Returns a hash of instance parameters grouped by name - public var instanceParameters: [String: [Parameter]] { - var groups = [String: [Parameter]]() - for parameter in parameters { - guard let descriptor = parameter.descriptor else { continue } - if groups[descriptor.group] != nil { - groups[descriptor.group]?.append(parameter) - } else { - groups[descriptor.group] = [parameter] - } - } - return groups - } - /// Initializer. public required init() { index = .empty elementId = .empty - parameters = [] } public func update(from data: [String: AnyHashable], cache: ImportCache) { @@ -890,12 +874,13 @@ extension Database { } @Transient - public static let importPriority: ModelImportPriority = .normal + public static let importPriority: ModelImportPriority = .veryHigh @Attribute(.unique) public var index: Int64 public var value: String public var descriptor: ParameterDescriptor? + public var element: Int64 /// Provides a convenience formatted value if the value is pipe delimited. @Transient @@ -903,22 +888,21 @@ extension Database { value.contains("|") ? String(value.split(separator: "|").last!) : value } - /// Initializer. public required init() { index = .empty value = .empty + element = .empty } public func update(from data: [String: AnyHashable], cache: ImportCache) { if let idx = data["ParameterDescriptor"] as? Int64, idx != .empty { descriptor = cache.findOrCreate(idx) } + value = data["Value"] as? String ?? .empty if let idx = data["Element"] as? Int64, idx != .empty { - let element: Element = cache.findOrCreate(idx) - element.parameters.append(self) + element = idx } - value = data["Value"] as? String ?? .empty } } diff --git a/Sources/VimKit/Extensions/NSCache+Extensions.swift b/Sources/VimKit/Extensions/NSCache+Extensions.swift index 8594b11..0257b6c 100644 --- a/Sources/VimKit/Extensions/NSCache+Extensions.swift +++ b/Sources/VimKit/Extensions/NSCache+Extensions.swift @@ -71,6 +71,19 @@ public final class Cache: @unchecked Sendable { return storage.object(forKey: WrappedKey(key))?.value } + /// Returns a list of values for the given set of keys + /// - Parameter keys: the set of keys + /// - Returns: a list of values for the given keys + public func values(in keys: Set) -> [Value] { + lock.lock() + defer { lock.unlock() } + var values: [Value?] = [] + for key in keys { + values.append(storage.object(forKey: WrappedKey(key))?.value) + } + return values.compactMap{ $0 } + } + /// Removes the value of the specified key in the cache. /// - Parameter key: the key to remove public func removeValue(for key: Key) { diff --git a/Sources/VimKitShaders/Resources/Indirect.metal b/Sources/VimKitShaders/Resources/Indirect.metal index 733bc8c..36b75d0 100644 --- a/Sources/VimKitShaders/Resources/Indirect.metal +++ b/Sources/VimKitShaders/Resources/Indirect.metal @@ -13,29 +13,16 @@ using namespace metal; // - Parameters: // - camera: The per frame camera data. // - instance: The instance to check if inside the view frustum. +// - corners: The instance bounding box corners. // - Returns: true if the instance is inside the view frustum, otherwise false __attribute__((always_inline)) static bool isInsideViewFrustumAndClipPlanes(const Camera camera, - const Instance instance) { + const Instance instance, + const float4 corners[8]) { if (instance.state == InstanceStateHidden) { return false; } - const float3 minBounds = instance.minBounds; - const float3 maxBounds = instance.maxBounds; - - // Extract the box corners - const float4 corners[8] = { - float4(minBounds, 1.0), - float4(minBounds.x, minBounds.y, maxBounds.z, 1.0), - float4(minBounds.x, maxBounds.y, minBounds.z, 1.0), - float4(minBounds.x, maxBounds.y, maxBounds.z, 1.0), - float4(maxBounds.x, minBounds.y, minBounds.z, 1.0), - float4(maxBounds.x, minBounds.y, maxBounds.z, 1.0), - float4(maxBounds.x, maxBounds.y, minBounds.z, 1.0), - float4(maxBounds, 1.0) - }; - // Loop through the frustum + clip planes and check the box corners for (int i = 0; i < 6; i++) { @@ -90,6 +77,7 @@ static float2 textureCoordinates(const Frame frame, // - Parameters: // - camera: The per frame data. // - instance: The instance to check. +// - corners: The instance bounding box corners. // - textureSize: The texture size. // - textureSampler: The texture sampler. // - depthTexture: The depth texture. @@ -97,6 +85,7 @@ static float2 textureCoordinates(const Frame frame, __attribute__((always_inline)) static bool isInstanceVisible(const Frame frame, const Instance instance, + const float4 corners[8], const uint2 textureSize, const sampler textureSampler, depth2d depthTexture) { @@ -132,18 +121,6 @@ static bool isInstanceVisible(const Frame frame, // Depth z culling (eliminate instances that are behind other instances) if (enableDepthTesting) { - // Extract the box corners - const float4 corners[8] = { - float4(instance.minBounds, 1.0), - float4(instance.minBounds.x, instance.minBounds.y, instance.maxBounds.z, 1.0), - float4(instance.minBounds.x, instance.maxBounds.y, instance.minBounds.z, 1.0), - float4(instance.minBounds.x, instance.maxBounds.y, instance.maxBounds.z, 1.0), - float4(instance.maxBounds.x, instance.minBounds.y, instance.minBounds.z, 1.0), - float4(instance.maxBounds.x, instance.minBounds.y, instance.maxBounds.z, 1.0), - float4(instance.maxBounds.x, instance.maxBounds.y, instance.minBounds.z, 1.0), - float4(instance.maxBounds, 1.0) - }; - for (int i = 0; i < 8; i++) { const float4 corner = projectionViewMatrix * corners[i]; const float2 sampleCoords = textureCoordinates(frame, corner); @@ -192,11 +169,24 @@ static bool isInstancedMeshVisible(const Frame frame, for (int i = lowerBound; i < upperBound; i++) { const Instance instance = instances[i]; - const bool insideFrustum = isInsideViewFrustumAndClipPlanes(camera, instance); + + // Extract the box corners + const float4 corners[8] = { + float4(instance.minBounds, 1.0), + float4(instance.minBounds.x, instance.minBounds.y, instance.maxBounds.z, 1.0), + float4(instance.minBounds.x, instance.maxBounds.y, instance.minBounds.z, 1.0), + float4(instance.minBounds.x, instance.maxBounds.y, instance.maxBounds.z, 1.0), + float4(instance.maxBounds.x, instance.minBounds.y, instance.minBounds.z, 1.0), + float4(instance.maxBounds.x, instance.minBounds.y, instance.maxBounds.z, 1.0), + float4(instance.maxBounds.x, instance.maxBounds.y, instance.minBounds.z, 1.0), + float4(instance.maxBounds, 1.0) + }; + + const bool insideFrustum = isInsideViewFrustumAndClipPlanes(camera, instance, corners); if (insideFrustum) { // Check if the instance passes the depth & contribution test - const bool isVisible = isInstanceVisible(frame, instance, textureSize, textureSampler, depthTexture); + const bool isVisible = isInstanceVisible(frame, instance, corners, textureSize, textureSampler, depthTexture); if (isVisible) { return true; } } }