From 709f8fbb2e5d526d4062111e27b8356a1897008a Mon Sep 17 00:00:00 2001 From: Peter Friese Date: Mon, 8 Jun 2026 23:03:20 -0700 Subject: [PATCH 1/3] feat(apple-ai): add support for Apple Foundation Models integration --- .../project.pbxproj | 53 ++- .../FirebaseAIExample/ContentView.swift | 6 + .../AppleFoundationModels/Models/Models.swift | 95 +++++ .../Screens/AppleAIScreen.swift | 346 ++++++++++++++++++ .../Tools/FindLocalPlacesTool.swift | 91 +++++ .../ViewModels/AppleAIViewModel.swift | 211 +++++++++++ .../Shared/Models/Sample.swift | 6 + .../Shared/Util/MLErrorHelper.swift | 72 ++++ .../Shared/Views/ErrorDetailsView.swift | 13 +- 9 files changed, 874 insertions(+), 19 deletions(-) create mode 100644 firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift create mode 100644 firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift create mode 100644 firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift create mode 100644 firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift create mode 100644 firebaseai/FirebaseAIExample/Shared/Util/MLErrorHelper.swift diff --git a/firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj b/firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj index c0c267ae4..7124e3c93 100644 --- a/firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj +++ b/firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj @@ -9,7 +9,10 @@ /* Begin PBXBuildFile section */ 88151ADC2EC9345700775CFB /* MarkdownUI in Frameworks */ = {isa = PBXBuildFile; productRef = 88151ADB2EC9345700775CFB /* MarkdownUI */; }; 88779D902EC8AA920080D023 /* ConversationKit in Frameworks */ = {isa = PBXBuildFile; productRef = 88779D8F2EC8AA920080D023 /* ConversationKit */; }; - 88779D932EC8AC460080D023 /* FirebaseAILogic in Frameworks */ = {isa = PBXBuildFile; productRef = 88779D922EC8AC460080D023 /* FirebaseAILogic */; }; + 88862D5D2FD7C19F003702C7 /* FirebaseAI in Frameworks */ = {isa = PBXBuildFile; productRef = 88862D5C2FD7C19F003702C7 /* FirebaseAI */; }; + 88862D5F2FD7C19F003702C7 /* FirebaseAILogic in Frameworks */ = {isa = PBXBuildFile; productRef = 88862D5E2FD7C19F003702C7 /* FirebaseAILogic */; }; + 88862D612FD7C19F003702C7 /* FirebaseAppCheck in Frameworks */ = {isa = PBXBuildFile; productRef = 88862D602FD7C19F003702C7 /* FirebaseAppCheck */; }; + 88862D632FD7C19F003702C7 /* FirebaseCore in Frameworks */ = {isa = PBXBuildFile; productRef = 88862D622FD7C19F003702C7 /* FirebaseCore */; }; /* End PBXBuildFile section */ /* Begin PBXFileReference section */ @@ -29,9 +32,12 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( + 88862D5F2FD7C19F003702C7 /* FirebaseAILogic in Frameworks */, 88151ADC2EC9345700775CFB /* MarkdownUI in Frameworks */, - 88779D932EC8AC460080D023 /* FirebaseAILogic in Frameworks */, 88779D902EC8AA920080D023 /* ConversationKit in Frameworks */, + 88862D612FD7C19F003702C7 /* FirebaseAppCheck in Frameworks */, + 88862D632FD7C19F003702C7 /* FirebaseCore in Frameworks */, + 88862D5D2FD7C19F003702C7 /* FirebaseAI in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -75,8 +81,11 @@ name = FirebaseAIExample; packageProductDependencies = ( 88779D8F2EC8AA920080D023 /* ConversationKit */, - 88779D922EC8AC460080D023 /* FirebaseAILogic */, 88151ADB2EC9345700775CFB /* MarkdownUI */, + 88862D5C2FD7C19F003702C7 /* FirebaseAI */, + 88862D5E2FD7C19F003702C7 /* FirebaseAILogic */, + 88862D602FD7C19F003702C7 /* FirebaseAppCheck */, + 88862D622FD7C19F003702C7 /* FirebaseCore */, ); productName = FirebaseAIExample; productReference = 88779D352EC8A9CF0080D023 /* FirebaseAIExample.app */; @@ -108,8 +117,8 @@ minimizedProjectReferenceProxies = 1; packageReferences = ( 88779D8E2EC8AA920080D023 /* XCRemoteSwiftPackageReference "ConversationKit" */, - 88779D912EC8AC460080D023 /* XCRemoteSwiftPackageReference "firebase-ios-sdk" */, 88151ADA2EC9345700775CFB /* XCRemoteSwiftPackageReference "swift-markdown-ui" */, + 88862D5B2FD7C19F003702C7 /* XCLocalSwiftPackageReference "../../firebase-ios-sdk" */, ); preferredProjectObjectVersion = 77; productRefGroup = 88779D362EC8A9CF0080D023 /* Products */; @@ -268,7 +277,7 @@ ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = ""; + DEVELOPMENT_TEAM = YGAZHQXHH4; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = YES; INFOPLIST_KEY_NSMicrophoneUsageDescription = "The app needs access to your microphone to enable live voice conversations with Gemini."; @@ -302,7 +311,7 @@ ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = ""; + DEVELOPMENT_TEAM = YGAZHQXHH4; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = YES; INFOPLIST_KEY_NSMicrophoneUsageDescription = "The app needs access to your microphone to enable live voice conversations with Gemini."; @@ -352,6 +361,13 @@ }; /* End XCConfigurationList section */ +/* Begin XCLocalSwiftPackageReference section */ + 88862D5B2FD7C19F003702C7 /* XCLocalSwiftPackageReference "../../firebase-ios-sdk" */ = { + isa = XCLocalSwiftPackageReference; + relativePath = "../../firebase-ios-sdk"; + }; +/* End XCLocalSwiftPackageReference section */ + /* Begin XCRemoteSwiftPackageReference section */ 88151ADA2EC9345700775CFB /* XCRemoteSwiftPackageReference "swift-markdown-ui" */ = { isa = XCRemoteSwiftPackageReference; @@ -364,17 +380,9 @@ 88779D8E2EC8AA920080D023 /* XCRemoteSwiftPackageReference "ConversationKit" */ = { isa = XCRemoteSwiftPackageReference; repositoryURL = "https://github.com/peterfriese/ConversationKit"; - requirement = { - kind = exactVersion; - version = 0.0.4; - }; - }; - 88779D912EC8AC460080D023 /* XCRemoteSwiftPackageReference "firebase-ios-sdk" */ = { - isa = XCRemoteSwiftPackageReference; - repositoryURL = "https://github.com/firebase/firebase-ios-sdk"; requirement = { kind = upToNextMajorVersion; - minimumVersion = 12.13.0; + minimumVersion = 0.0.4; }; }; /* End XCRemoteSwiftPackageReference section */ @@ -390,11 +398,22 @@ package = 88779D8E2EC8AA920080D023 /* XCRemoteSwiftPackageReference "ConversationKit" */; productName = ConversationKit; }; - 88779D922EC8AC460080D023 /* FirebaseAILogic */ = { + 88862D5C2FD7C19F003702C7 /* FirebaseAI */ = { + isa = XCSwiftPackageProductDependency; + productName = FirebaseAI; + }; + 88862D5E2FD7C19F003702C7 /* FirebaseAILogic */ = { isa = XCSwiftPackageProductDependency; - package = 88779D912EC8AC460080D023 /* XCRemoteSwiftPackageReference "firebase-ios-sdk" */; productName = FirebaseAILogic; }; + 88862D602FD7C19F003702C7 /* FirebaseAppCheck */ = { + isa = XCSwiftPackageProductDependency; + productName = FirebaseAppCheck; + }; + 88862D622FD7C19F003702C7 /* FirebaseCore */ = { + isa = XCSwiftPackageProductDependency; + productName = FirebaseCore; + }; /* End XCSwiftPackageProductDependency section */ }; rootObject = 88779D2D2EC8A9CF0080D023 /* Project object */; diff --git a/firebaseai/FirebaseAIExample/ContentView.swift b/firebaseai/FirebaseAIExample/ContentView.swift index 62808c579..103b4d69c 100644 --- a/firebaseai/FirebaseAIExample/ContentView.swift +++ b/firebaseai/FirebaseAIExample/ContentView.swift @@ -112,6 +112,12 @@ struct ContentView: View { GroundingScreen(backendType: selectedBackend, sample: sample) case "LiveScreen": LiveScreen(backendType: selectedBackend, sample: sample) + case "AppleAIScreen": + if #available(iOS 27.0, *) { + AppleAIScreen() + } else { + Text("Apple Intelligence is only available on iOS 27 or newer.") + } default: EmptyView() } diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift new file mode 100644 index 000000000..543a05ccf --- /dev/null +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift @@ -0,0 +1,95 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import FoundationModels + +@available(iOS 26.0, *) +@Generable +public struct Itinerary: Equatable, Codable { + @Guide(description: "An exciting name for the trip.") + public let title: String + + @Guide(description: "The name of the destination.") + public let destinationName: String + + @Guide(description: "A brief, catchy description of the trip.") + public let description: String + + @Guide(description: "An explanation of why this plan fits the user's request.") + public let rationale: String + + @Guide(description: "A list of day plans.") + public let days: [DayPlan] +} + +@available(iOS 26.0, *) +@Generable +public struct DayPlan: Equatable, Codable { + @Guide(description: "A unique title for this day.") + public let title: String + + public let subtitle: String + + @Guide(description: "A list of activities planned for this day.") + public let activities: [Activity] +} + +@available(iOS 26.0, *) +@Generable +public struct Activity: Equatable, Codable { + public let type: ActivityKind + public let title: String + public let description: String + public let latitude: Double? + public let longitude: Double? +} + +@available(iOS 26.0, *) +@Generable +public enum ActivityKind: String, Codable { + case sightseeing + case foodAndDining + case shopping + case hotelAndLodging + + public var displayName: String { + switch self { + case .sightseeing: return "Sightseeing" + case .foodAndDining: return "Food & Dining" + case .shopping: return "Shopping" + case .hotelAndLodging: return "Hotel & Lodging" + } + } +} + +@available(iOS 26.0, *) +@Generable +public struct IdentifiedObject: Equatable, Codable { + @Guide(description: "The name of the primary object or landmark detected.") + public let name: String + + @Guide(description: "The category of the object (e.g. Landmark, Plant, Food, Animal, Device, Clothing).") + public let category: String + + @Guide(description: "A short, 2-sentence description of the object and interesting facts.") + public let description: String +} + +@available(iOS 26.0, *) +@Generable +public struct TextSummary: Equatable, Codable { + @Guide(description: "A list of exactly 2 key summary points.") + public let summaryPoints: [String] +} diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift new file mode 100644 index 000000000..4673e5123 --- /dev/null +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift @@ -0,0 +1,346 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import SwiftUI +import PhotosUI +import FoundationModels +import FirebaseAILogic + +@available(iOS 27.0, *) +struct AppleAIScreen: View { + @StateObject private var viewModel = AppleAIViewModel() + @State private var selectedTab = 0 + @State private var photosPickerItem: PhotosPickerItem? = nil + @State private var presentErrorDetails = false + + var body: some View { + ZStack { + VStack(spacing: 0) { + // Segmented Control + Picker("Feature", selection: $selectedTab) { + Text("Hybrid AI").tag(0) + Text("Planner").tag(1) + Text("Vision ID").tag(2) + } + .pickerStyle(.segmented) + .padding() + .background(Color(.systemBackground)) + + Divider() + + // Content Views + ScrollView { + VStack(spacing: 20) { + if selectedTab == 0 { + hybridAIView + } else if selectedTab == 1 { + plannerView + } else { + visionIDView + } + } + .padding() + } + } + + if viewModel.inProgress { + ProgressOverlay() + } + } + .background(Color(.systemGroupedBackground)) + .navigationTitle("Apple Intelligence") + .navigationBarTitleDisplayMode(.inline) + .toolbar { + if viewModel.inProgress { + ToolbarItem(placement: .navigationBarTrailing) { + Button("Stop") { + viewModel.stopActiveTask() + } + } + } + } + .sheet(isPresented: $presentErrorDetails) { + if let error = viewModel.error { + ErrorDetailsView(error: error) + } + } + .onChange(of: viewModel.error != nil) { oldValue, newValue in + if newValue { + presentErrorDetails = true + } + } + } + + // MARK: - Subviews + + // Feature 1: Hybrid AI + private var hybridAIView: some View { + VStack(alignment: .leading, spacing: 16) { + VStack(alignment: .leading, spacing: 8) { + Text("Input Text") + .font(.headline) + TextEditor(text: $viewModel.inputText) + .frame(height: 120) + .padding(4) + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(8) + } + + Button(action: { + Task { + await viewModel.runSummarization() + } + }) { + HStack { + Spacer() + Image(systemName: "sparkles") + Text("Summarize with Apple SDK") + Spacer() + } + .padding() + .foregroundColor(.white) + .background(Color.blue) + .cornerRadius(10) + } + .disabled(viewModel.inProgress) + + if let summary = viewModel.outputSummary { + VStack(alignment: .leading, spacing: 12) { + HStack { + Text("Summary Points") + .font(.headline) + Spacer() + + // Badge indicating where it was executed + HStack(spacing: 4) { + Image(systemName: viewModel.isUsingLocalModel ? "iphone" : "cloud.fill") + Text(viewModel.isUsingLocalModel ? "Local (Apple)" : "Cloud (Gemini)") + } + .font(.caption) + .padding(.horizontal, 8) + .padding(.vertical, 4) + .background(viewModel.isUsingLocalModel ? Color.green.opacity(0.2) : Color.purple.opacity(0.2)) + .foregroundColor(viewModel.isUsingLocalModel ? .green : .purple) + .cornerRadius(8) + } + + VStack(alignment: .leading, spacing: 8) { + ForEach(summary.summaryPoints, id: \.self) { point in + HStack(alignment: .top, spacing: 6) { + Text("•") + .bold() + Text(point) + .font(.subheadline) + } + } + } + .padding() + .frame(maxWidth: .infinity, alignment: .leading) + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(10) + } + .transition(.opacity.combined(with: .slide)) + } + } + } + + // Feature 2: Planner View + private var plannerView: some View { + VStack(alignment: .leading, spacing: 16) { + VStack(spacing: 12) { + HStack { + Text("Destination") + .frame(width: 100, alignment: .leading) + TextField("e.g. Paris, Tokyo", text: $viewModel.destination) + .textFieldStyle(.roundedBorder) + } + HStack { + Text("Interests") + .frame(width: 100, alignment: .leading) + TextField("e.g. art, cafes, parks", text: $viewModel.interests) + .textFieldStyle(.roundedBorder) + } + } + .padding() + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(10) + + Button(action: { + Task { + await viewModel.generateItinerary() + } + }) { + HStack { + Spacer() + Image(systemName: "map.fill") + Text("Create Itinerary Plan") + Spacer() + } + .padding() + .foregroundColor(.white) + .background(Color.blue) + .cornerRadius(10) + } + .disabled(viewModel.inProgress) + + if let itinerary = viewModel.itinerary { + VStack(alignment: .leading, spacing: 12) { + Text(itinerary.title ?? "Generating plan...") + .font(.title3) + .bold() + + if let desc = itinerary.description { + Text(desc) + .font(.subheadline) + .foregroundColor(.secondary) + } + + if let rationale = itinerary.rationale { + Text("Rationale: \(rationale)") + .font(.caption) + .padding() + .background(Color.blue.opacity(0.1)) + .cornerRadius(8) + } + + // Day Plans + if let days = itinerary.days { + ForEach(days, id: \.title) { day in + VStack(alignment: .leading, spacing: 8) { + Text(day.title ?? "Day Plan") + .font(.headline) + .padding(.top, 4) + + if let activities = day.activities { + ForEach(activities, id: \.title) { activity in + HStack(alignment: .top, spacing: 12) { + Image(systemName: getSymbol(for: activity.type)) + .foregroundColor(.blue) + .frame(width: 24, height: 24) + .background(Color.blue.opacity(0.1)) + .cornerRadius(6) + + VStack(alignment: .leading, spacing: 2) { + Text(activity.title ?? "Activity") + .font(.subheadline) + .bold() + if let desc = activity.description { + Text(desc) + .font(.caption) + .foregroundColor(.secondary) + } + if let lat = activity.latitude, let lon = activity.longitude { + Text(String(format: "Location: %.4f, %.4f", lat, lon)) + .font(.system(.caption, design: .monospaced)) + .foregroundColor(.gray) + } + } + } + .padding(.vertical, 4) + } + } + } + .padding() + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(10) + } + } + + + } + } + } + } + + // Feature 3: Vision ID View + private var visionIDView: some View { + VStack(alignment: .leading, spacing: 16) { + Text("Select or Snap a Photo to Identify") + .font(.headline) + + PhotosPicker(selection: $photosPickerItem, matching: .images) { + VStack(spacing: 12) { + if let image = viewModel.selectedImage { + Image(uiImage: image) + .resizable() + .aspectRatio(contentMode: .fit) + .frame(maxHeight: 200) + .cornerRadius(12) + } else { + VStack(spacing: 8) { + Image(systemName: "photo.badge.plus") + .font(.system(size: 40)) + Text("Select an Image") + } + .frame(maxWidth: .infinity) + .frame(height: 180) + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(12) + } + } + } + .onChange(of: photosPickerItem) { oldItem, newItem in + Task { + if let data = try? await newItem?.loadTransferable(type: Data.self), + let image = UIImage(data: data) { + viewModel.selectedImage = image + await viewModel.identifySelectedImage() + } + } + } + + if let identified = viewModel.identifiedObject { + VStack(alignment: .leading, spacing: 12) { + Text("Identification Result") + .font(.headline) + + VStack(alignment: .leading, spacing: 8) { + HStack { + Text("Name:") + .bold() + Text(identified.name) + } + HStack { + Text("Category:") + .bold() + Text(identified.category) + } + VStack(alignment: .leading, spacing: 4) { + Text("Description:") + .bold() + Text(identified.description) + .foregroundColor(.secondary) + } + } + .padding() + .frame(maxWidth: .infinity, alignment: .leading) + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(10) + } + .transition(.opacity) + } + } + } + + // Helper to pick icons for activity kinds + private func getSymbol(for kind: ActivityKind?) -> String { + guard let kind = kind else { return "info.circle" } + switch kind { + case .sightseeing: return "binoculars.fill" + case .foodAndDining: return "fork.knife" + case .shopping: return "bag.fill" + case .hotelAndLodging: return "bed.double.fill" + } + } +} diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift new file mode 100644 index 000000000..716aec456 --- /dev/null +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift @@ -0,0 +1,91 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import FoundationModels +import FirebaseAILogic + +@available(iOS 26.0, *) +@Generable +public struct PlacesResponse: Codable { + public let places: [String] +} + +@available(iOS 26.0, *) +@Generable +public struct ToolResult: Codable { + public let summary: String +} + +@available(iOS 26.0, *) +public final class FindLocalPlacesTool { + public let name = "findLocalPlaces" + public let description = "Finds points of interest and local businesses matching a query for a specified destination." + + public init() {} + + @available(iOS 26.0, *) + @Generable + public struct Arguments { + @Guide(description: "The destination city/place to look up for (e.g. 'Paris', 'New York').") + public let destination: String + + @Guide(description: "The category or query to search (e.g. 'hotels', 'Italian restaurants', 'museums').") + public let category: String + } + + public func call(arguments: Arguments) async throws -> ToolResult { + print("FindLocalPlacesTool: call called with destination: \(arguments.destination), category: \(arguments.category)") + + // Setup Firebase AI with Google Search grounding + let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) + let session = ai.generativeModelSession( + model: "gemini-3.1-flash-lite", + tools: [Tool.googleSearch()] + ) + + let prompt = "Find 3 real popular \(arguments.category) in \(arguments.destination). Use Google Search to make sure the places are real and currently open." + + do { + let response = try await session.respond( + to: prompt, + generating: PlacesResponse.self + ) + + let results = response.content.places + print("FindLocalPlacesTool: found places: \(results.joined(separator: ", "))") + + return ToolResult(summary: "Here are some popular \(arguments.category) in \(arguments.destination): \(results.joined(separator: ", "))") + } catch { + print("FindLocalPlacesTool error: \(error.localizedDescription)") + throw error + } + } +} + +// Struct wrapper to conform to FoundationModels.Tool AND ToolRepresentable +@available(iOS 26.0, *) +public struct FindLocalPlacesToolWrapper: FoundationModels.Tool, ToolRepresentable, @unchecked Sendable { + let tool: FindLocalPlacesTool + + public var name: String { tool.name } + public var description: String { tool.description } + + public typealias Arguments = FindLocalPlacesTool.Arguments + public typealias Output = ToolResult + + public func call(arguments: Arguments) async throws -> ToolResult { + try await tool.call(arguments: arguments) + } +} diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift new file mode 100644 index 000000000..31d6b38f6 --- /dev/null +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift @@ -0,0 +1,211 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import SwiftUI +import Combine +import FoundationModels +import FirebaseAILogic + +@available(iOS 27.0, *) +@MainActor +public final class AppleAIViewModel: ObservableObject { + // Shared state + @Published public var inProgress = false + @Published public var error: Error? + + // Feature 1: Hybrid Summary/Translation + @Published public var inputText: String = "It is the quintessential autumn harvest fruit, famously baked into warm cinnamon pastries, dipped in sticky caramel on Halloween, and traditionally rumored to keep medical professionals at bay if eaten once a day." + @Published public var outputSummary: TextSummary? + @Published public var isUsingLocalModel: Bool = false + + // Feature 2: Smart Planner + @Published public var destination: String = "San Francisco" + @Published public var interests: String = "tech history, good coffee, bay views" + @Published public var itinerary: Itinerary.PartiallyGenerated? + + // Feature 3: Visual Identifier + @Published public var selectedImage: UIImage? + @Published public var identifiedObject: IdentifiedObject? + + private var activeTask: Task? + + public init() {} + + public func stopActiveTask() { + activeTask?.cancel() + activeTask = nil + inProgress = false + } + + // MARK: - Feature 1: Hybrid Summary/Translation + public func runSummarization() async { + stopActiveTask() + inProgress = true + error = nil + outputSummary = nil + + activeTask = Task { + let availability = SystemLanguageModel.default.availability + let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) + + // Try local model first if it reports available + if availability == .available { + isUsingLocalModel = true + do { + let session = LanguageModelSession( + model: SystemLanguageModel.default, + instructions: Instructions { + "Your job is to summarize the provided text in exactly 2 bullet points." + } + ) + let response = try await session.respond(to: inputText, generating: TextSummary.self) + if !Task.isCancelled { + self.outputSummary = response.content + } + inProgress = false + return + } catch { + // Fall back to cloud model if local model fails (e.g. assets not downloaded on simulator) + if error.isMLAssetUnavailable { + print("Local ML assets unavailable. Falling back to cloud model...") + } else { + print("Local model failed: \(error.localizedDescription). Falling back to cloud model...") + } + } + } + + // Fallback to cloud model + isUsingLocalModel = false + do { + let model = ai.geminiLanguageModel(name: "gemini-3.1-flash-lite") + let session = LanguageModelSession( + model: model, + instructions: Instructions { + "Your job is to summarize the provided text in exactly 2 bullet points." + } + ) + let response = try await session.respond(to: inputText, generating: TextSummary.self) + if !Task.isCancelled { + self.outputSummary = response.content + } + } catch { + if !Task.isCancelled { + // If even the cloud model via LanguageModelSession fails (likely due to missing guardrail assets), + // we show a descriptive error message. + if let mlMessage = error.mlAssetErrorMessage { + print("ML Asset Error detected in cloud fallback: \(mlMessage)") + // We can either set the error here or try one last direct fallback. + // Let's set a custom error that wraps the message. + self.error = NSError(domain: "FirebaseAIExample", + code: 1, + userInfo: [NSLocalizedDescriptionKey: mlMessage]) + } else { + self.error = error + } + } + } + inProgress = false + } + } + + // MARK: - Feature 2: Smart Planner + public func generateItinerary() async { + stopActiveTask() + inProgress = true + error = nil + itinerary = nil + + activeTask = Task { + let localPlacesTool = FindLocalPlacesToolWrapper(tool: FindLocalPlacesTool()) + let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) + + let model = ai.geminiLanguageModel( + name: "gemini-3.5-flash" + ) + + let session = LanguageModelSession( + model: model, + tools: [localPlacesTool], + instructions: Instructions { + "Your job is to create a structured 1-day itinerary for the user." + "Ensure you use the findLocalPlaces tool to search for real places and recommendations in the destination city." + } + ) + + let promptText = "Generate a structured 1-day itinerary for \(destination) focused on \(interests). Include exactly 3 activities (morning, afternoon, evening) with real recommendations." + + let stream = session.streamResponse( + to: promptText, + generating: Itinerary.self + ) + + do { + for try await response in stream { + if Task.isCancelled { break } + self.itinerary = response.content + } + } catch { + if !Task.isCancelled { + self.error = error + } + } + inProgress = false + } + } + + // MARK: - Feature 3: Visual Identifier + public func identifySelectedImage() async { + guard let image = selectedImage else { return } + stopActiveTask() + inProgress = true + error = nil + identifiedObject = nil + + activeTask = Task { + let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) + let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") + + guard let cgImage = image.cgImage else { + inProgress = false + return + } + + let session = LanguageModelSession( + model: model, + instructions: Instructions { + "You are a visual object identifier." + } + ) + + do { + let response = try await session.respond( + generating: IdentifiedObject.self + ) { + "Identify the primary object in this image. Be as specific as possible, categorize it, and provide a short 2-sentence description." + Attachment(cgImage) + } + + if !Task.isCancelled { + self.identifiedObject = response.content + } + } catch { + if !Task.isCancelled { + self.error = error + } + } + inProgress = false + } + } +} diff --git a/firebaseai/FirebaseAIExample/Shared/Models/Sample.swift b/firebaseai/FirebaseAIExample/Shared/Models/Sample.swift index 35518d55a..eee6200cd 100644 --- a/firebaseai/FirebaseAIExample/Shared/Models/Sample.swift +++ b/firebaseai/FirebaseAIExample/Shared/Models/Sample.swift @@ -309,6 +309,12 @@ extension Sample { ), tip: InlineTip(text: "Try asking the model to change the background color"), ), + Sample( + title: "Apple Foundation Models", + description: "Demonstrates how to integrate Firebase AI Logic (Gemini) with Apple's Foundation Models framework (Apple Intelligence).", + useCases: [.text, .image, .functionCalling], + navRoute: "AppleAIScreen" + ), ] public static var sample = samples[0] diff --git a/firebaseai/FirebaseAIExample/Shared/Util/MLErrorHelper.swift b/firebaseai/FirebaseAIExample/Shared/Util/MLErrorHelper.swift new file mode 100644 index 000000000..bfaf543f7 --- /dev/null +++ b/firebaseai/FirebaseAIExample/Shared/Util/MLErrorHelper.swift @@ -0,0 +1,72 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import FoundationModels + +extension Error { + /// Returns true if this error indicates that required ML assets (like safety guardrails) + /// are missing from the device or simulator. + public var isMLAssetUnavailable: Bool { + // 1. Check for LanguageModelSession.GenerationError.assetsUnavailable + if #available(iOS 26.0, *) { + if let genError = self as? LanguageModelSession.GenerationError { + if case .assetsUnavailable = genError { + return true + } + } + } + + let nsError = self as NSError + + // 2. Check for the specific SensitiveContentAnalysisML error (Code 15) + if nsError.domain == "com.apple.SensitiveContentAnalysisML" && nsError.code == 15 { + return true + } + + // 3. Check for underlying GenerativeFunctionsFoundation error (Code 1020000) + // This is often found in the UserInfo of higher-level errors. + if nsError.domain == "com.apple.GenerativeFunctionsFoundation.GenerativeError" && nsError.code == 1020000 { + return true + } + + // 4. Recursive check for underlying errors + if let underlying = nsError.userInfo[NSUnderlyingErrorKey] as? Error { + return underlying.isMLAssetUnavailable + } + + return false + } + + /// Returns a user-friendly message for ML asset errors, particularly for simulators. + public var mlAssetErrorMessage: String? { + guard isMLAssetUnavailable else { return nil } + + #if targetEnvironment(simulator) + return """ + Apple Intelligence assets are missing in this simulator. + + To fix this: + 1. Open Settings in the simulator. + 2. Go to Apple Intelligence & Siri. + 3. Toggle Apple Intelligence ON. + 4. Wait for assets to download (check the status in Settings). + + Alternatively, use a physical device with Apple Intelligence support. + """ + #else + return "Apple Intelligence assets are not yet ready on this device. Please ensure Apple Intelligence is enabled in Settings and that model assets have finished downloading." + #endif + } +} diff --git a/firebaseai/FirebaseAIExample/Shared/Views/ErrorDetailsView.swift b/firebaseai/FirebaseAIExample/Shared/Views/ErrorDetailsView.swift index cd1e3f60e..7ba2b1788 100644 --- a/firebaseai/FirebaseAIExample/Shared/Views/ErrorDetailsView.swift +++ b/firebaseai/FirebaseAIExample/Shared/Views/ErrorDetailsView.swift @@ -148,11 +148,20 @@ struct ErrorDetailsView: View { default: Section("Error Type") { - Text("Some other error") + if error.isMLAssetUnavailable { + Text("Apple Intelligence Assets Missing") + .foregroundColor(.orange) + } else { + Text("Some other error") + } } Section("Details") { - SubtitleFormRow(title: "Error description", value: error.localizedDescription) + if let mlMessage = error.mlAssetErrorMessage { + SubtitleFormRow(title: "How to fix", value: mlMessage) + } else { + SubtitleFormRow(title: "Error description", value: error.localizedDescription) + } } } } From 4767029da28711b0de4ae6910e09bc12d18e85af Mon Sep 17 00:00:00 2001 From: Peter Friese Date: Tue, 9 Jun 2026 06:41:04 -0700 Subject: [PATCH 2/3] refactor(apple-ai): clean up tasks and session handling in AppleAIViewModel --- .../ViewModels/AppleAIViewModel.swift | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift index 31d6b38f6..8bc81bd83 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift @@ -57,8 +57,13 @@ public final class AppleAIViewModel: ObservableObject { outputSummary = nil activeTask = Task { + defer { self.inProgress = false } + + let instructions = Instructions { + "Your job is to summarize the provided text in exactly 2 bullet points." + } + let availability = SystemLanguageModel.default.availability - let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) // Try local model first if it reports available if availability == .available { @@ -66,15 +71,12 @@ public final class AppleAIViewModel: ObservableObject { do { let session = LanguageModelSession( model: SystemLanguageModel.default, - instructions: Instructions { - "Your job is to summarize the provided text in exactly 2 bullet points." - } + instructions: instructions ) let response = try await session.respond(to: inputText, generating: TextSummary.self) if !Task.isCancelled { self.outputSummary = response.content } - inProgress = false return } catch { // Fall back to cloud model if local model fails (e.g. assets not downloaded on simulator) @@ -89,12 +91,11 @@ public final class AppleAIViewModel: ObservableObject { // Fallback to cloud model isUsingLocalModel = false do { + let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) let model = ai.geminiLanguageModel(name: "gemini-3.1-flash-lite") let session = LanguageModelSession( model: model, - instructions: Instructions { - "Your job is to summarize the provided text in exactly 2 bullet points." - } + instructions: instructions ) let response = try await session.respond(to: inputText, generating: TextSummary.self) if !Task.isCancelled { @@ -116,7 +117,6 @@ public final class AppleAIViewModel: ObservableObject { } } } - inProgress = false } } @@ -128,6 +128,8 @@ public final class AppleAIViewModel: ObservableObject { itinerary = nil activeTask = Task { + defer { self.inProgress = false } + let localPlacesTool = FindLocalPlacesToolWrapper(tool: FindLocalPlacesTool()) let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) @@ -161,7 +163,6 @@ public final class AppleAIViewModel: ObservableObject { self.error = error } } - inProgress = false } } @@ -174,14 +175,13 @@ public final class AppleAIViewModel: ObservableObject { identifiedObject = nil activeTask = Task { + defer { self.inProgress = false } + + guard let cgImage = image.cgImage else { return } + let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") - guard let cgImage = image.cgImage else { - inProgress = false - return - } - let session = LanguageModelSession( model: model, instructions: Instructions { @@ -205,7 +205,6 @@ public final class AppleAIViewModel: ObservableObject { self.error = error } } - inProgress = false } } } From e71635549fe4c19a9fc1e77824645d0cffd6bfa5 Mon Sep 17 00:00:00 2001 From: Peter Friese Date: Tue, 9 Jun 2026 06:45:52 -0700 Subject: [PATCH 3/3] feat(apple-ai): add support for Google Maps grounding attributions in planner --- .../AppleFoundationModels/Models/Models.swift | 10 +++++++ .../Screens/AppleAIScreen.swift | 26 ++++++++++++++++++- .../Tools/FindLocalPlacesTool.swift | 21 ++++++++++++--- .../ViewModels/AppleAIViewModel.swift | 2 +- 4 files changed, 53 insertions(+), 6 deletions(-) diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift index 543a05ccf..821cef536 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift @@ -32,6 +32,16 @@ public struct Itinerary: Equatable, Codable { @Guide(description: "A list of day plans.") public let days: [DayPlan] + + @Guide(description: "Any source attributions or links for the recommended places.") + public let attributions: [Attribution]? +} + +@available(iOS 26.0, *) +@Generable +public struct Attribution: Equatable, Codable { + public let title: String + public let url: String } @available(iOS 26.0, *) diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift index 4673e5123..e942051f4 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift @@ -257,7 +257,31 @@ struct AppleAIScreen: View { } } - + if let attributions = itinerary.attributions, !attributions.isEmpty { + VStack(alignment: .leading, spacing: 8) { + Text("Sources & Maps Links") + .font(.headline) + + ForEach(attributions, id: \.url) { attribution in + if let urlString = attribution.url, let url = URL(string: urlString), let title = attribution.title { + Link(destination: url) { + HStack { + Image(systemName: "mappin.and.ellipse") + Text(title) + .underline() + Spacer() + } + .font(.caption) + .foregroundColor(.blue) + } + .padding(.vertical, 2) + } + } + } + .padding() + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(10) + } } } } diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift index 716aec456..e2efbc92b 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift @@ -26,6 +26,7 @@ public struct PlacesResponse: Codable { @Generable public struct ToolResult: Codable { public let summary: String + public let attributions: [Attribution] } @available(iOS 26.0, *) @@ -48,14 +49,14 @@ public final class FindLocalPlacesTool { public func call(arguments: Arguments) async throws -> ToolResult { print("FindLocalPlacesTool: call called with destination: \(arguments.destination), category: \(arguments.category)") - // Setup Firebase AI with Google Search grounding + // Setup Firebase AI with Google Maps grounding let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) let session = ai.generativeModelSession( model: "gemini-3.1-flash-lite", - tools: [Tool.googleSearch()] + tools: [Tool.googleMaps()] ) - let prompt = "Find 3 real popular \(arguments.category) in \(arguments.destination). Use Google Search to make sure the places are real and currently open." + let prompt = "Find 3 real popular \(arguments.category) in \(arguments.destination). Use Google Maps to make sure the places are real and currently open." do { let response = try await session.respond( @@ -63,10 +64,22 @@ public final class FindLocalPlacesTool { generating: PlacesResponse.self ) + var attributions: [Attribution] = [] + if let metadata = response.rawResponse.candidates.first?.groundingMetadata { + for chunk in metadata.groundingChunks { + if let maps = chunk.maps, let title = maps.title, let url = maps.url?.absoluteString { + attributions.append(Attribution(title: title, url: url)) + } + } + } + let results = response.content.places print("FindLocalPlacesTool: found places: \(results.joined(separator: ", "))") - return ToolResult(summary: "Here are some popular \(arguments.category) in \(arguments.destination): \(results.joined(separator: ", "))") + return ToolResult( + summary: "Here are some popular \(arguments.category) in \(arguments.destination): \(results.joined(separator: ", "))", + attributions: attributions + ) } catch { print("FindLocalPlacesTool error: \(error.localizedDescription)") throw error diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift index 8bc81bd83..00b71d41c 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift @@ -146,7 +146,7 @@ public final class AppleAIViewModel: ObservableObject { } ) - let promptText = "Generate a structured 1-day itinerary for \(destination) focused on \(interests). Include exactly 3 activities (morning, afternoon, evening) with real recommendations." + let promptText = "Generate a structured 1-day itinerary for \(destination) focused on \(interests). Include exactly 3 activities (morning, afternoon, evening) with real recommendations. For any place returned by the findLocalPlaces tool, populate the 'attributions' array in the response with the exact name (title) and Google Maps URL (url) returned by the tool." let stream = session.streamResponse( to: promptText,