diff --git a/firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj b/firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj index c0c267ae4..7124e3c93 100644 --- a/firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj +++ b/firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj @@ -9,7 +9,10 @@ /* Begin PBXBuildFile section */ 88151ADC2EC9345700775CFB /* MarkdownUI in Frameworks */ = {isa = PBXBuildFile; productRef = 88151ADB2EC9345700775CFB /* MarkdownUI */; }; 88779D902EC8AA920080D023 /* ConversationKit in Frameworks */ = {isa = PBXBuildFile; productRef = 88779D8F2EC8AA920080D023 /* ConversationKit */; }; - 88779D932EC8AC460080D023 /* FirebaseAILogic in Frameworks */ = {isa = PBXBuildFile; productRef = 88779D922EC8AC460080D023 /* FirebaseAILogic */; }; + 88862D5D2FD7C19F003702C7 /* FirebaseAI in Frameworks */ = {isa = PBXBuildFile; productRef = 88862D5C2FD7C19F003702C7 /* FirebaseAI */; }; + 88862D5F2FD7C19F003702C7 /* FirebaseAILogic in Frameworks */ = {isa = PBXBuildFile; productRef = 88862D5E2FD7C19F003702C7 /* FirebaseAILogic */; }; + 88862D612FD7C19F003702C7 /* FirebaseAppCheck in Frameworks */ = {isa = PBXBuildFile; productRef = 88862D602FD7C19F003702C7 /* FirebaseAppCheck */; }; + 88862D632FD7C19F003702C7 /* FirebaseCore in Frameworks */ = {isa = PBXBuildFile; productRef = 88862D622FD7C19F003702C7 /* FirebaseCore */; }; /* End PBXBuildFile section */ /* Begin PBXFileReference section */ @@ -29,9 +32,12 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( + 88862D5F2FD7C19F003702C7 /* FirebaseAILogic in Frameworks */, 88151ADC2EC9345700775CFB /* MarkdownUI in Frameworks */, - 88779D932EC8AC460080D023 /* FirebaseAILogic in Frameworks */, 88779D902EC8AA920080D023 /* ConversationKit in Frameworks */, + 88862D612FD7C19F003702C7 /* FirebaseAppCheck in Frameworks */, + 88862D632FD7C19F003702C7 /* FirebaseCore in Frameworks */, + 88862D5D2FD7C19F003702C7 /* FirebaseAI in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -75,8 +81,11 @@ name = FirebaseAIExample; packageProductDependencies = ( 88779D8F2EC8AA920080D023 /* ConversationKit */, - 88779D922EC8AC460080D023 /* FirebaseAILogic */, 88151ADB2EC9345700775CFB /* MarkdownUI */, + 88862D5C2FD7C19F003702C7 /* FirebaseAI */, + 88862D5E2FD7C19F003702C7 /* FirebaseAILogic */, + 88862D602FD7C19F003702C7 /* FirebaseAppCheck */, + 88862D622FD7C19F003702C7 /* FirebaseCore */, ); productName = FirebaseAIExample; productReference = 88779D352EC8A9CF0080D023 /* FirebaseAIExample.app */; @@ -108,8 +117,8 @@ minimizedProjectReferenceProxies = 1; packageReferences = ( 88779D8E2EC8AA920080D023 /* XCRemoteSwiftPackageReference "ConversationKit" */, - 88779D912EC8AC460080D023 /* XCRemoteSwiftPackageReference "firebase-ios-sdk" */, 88151ADA2EC9345700775CFB /* XCRemoteSwiftPackageReference "swift-markdown-ui" */, + 88862D5B2FD7C19F003702C7 /* XCLocalSwiftPackageReference "../../firebase-ios-sdk" */, ); preferredProjectObjectVersion = 77; productRefGroup = 88779D362EC8A9CF0080D023 /* Products */; @@ -268,7 +277,7 @@ ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = ""; + DEVELOPMENT_TEAM = YGAZHQXHH4; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = YES; INFOPLIST_KEY_NSMicrophoneUsageDescription = "The app needs access to your microphone to enable live voice conversations with Gemini."; @@ -302,7 +311,7 @@ ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = ""; + DEVELOPMENT_TEAM = YGAZHQXHH4; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = YES; INFOPLIST_KEY_NSMicrophoneUsageDescription = "The app needs access to your microphone to enable live voice conversations with Gemini."; @@ -352,6 +361,13 @@ }; /* End XCConfigurationList section */ +/* Begin XCLocalSwiftPackageReference section */ + 88862D5B2FD7C19F003702C7 /* XCLocalSwiftPackageReference "../../firebase-ios-sdk" */ = { + isa = XCLocalSwiftPackageReference; + relativePath = "../../firebase-ios-sdk"; + }; +/* End XCLocalSwiftPackageReference section */ + /* Begin XCRemoteSwiftPackageReference section */ 88151ADA2EC9345700775CFB /* XCRemoteSwiftPackageReference "swift-markdown-ui" */ = { isa = XCRemoteSwiftPackageReference; @@ -364,17 +380,9 @@ 88779D8E2EC8AA920080D023 /* XCRemoteSwiftPackageReference "ConversationKit" */ = { isa = XCRemoteSwiftPackageReference; repositoryURL = "https://github.com/peterfriese/ConversationKit"; - requirement = { - kind = exactVersion; - version = 0.0.4; - }; - }; - 88779D912EC8AC460080D023 /* XCRemoteSwiftPackageReference "firebase-ios-sdk" */ = { - isa = XCRemoteSwiftPackageReference; - repositoryURL = "https://github.com/firebase/firebase-ios-sdk"; requirement = { kind = upToNextMajorVersion; - minimumVersion = 12.13.0; + minimumVersion = 0.0.4; }; }; /* End XCRemoteSwiftPackageReference section */ @@ -390,11 +398,22 @@ package = 88779D8E2EC8AA920080D023 /* XCRemoteSwiftPackageReference "ConversationKit" */; productName = ConversationKit; }; - 88779D922EC8AC460080D023 /* FirebaseAILogic */ = { + 88862D5C2FD7C19F003702C7 /* FirebaseAI */ = { + isa = XCSwiftPackageProductDependency; + productName = FirebaseAI; + }; + 88862D5E2FD7C19F003702C7 /* FirebaseAILogic */ = { isa = XCSwiftPackageProductDependency; - package = 88779D912EC8AC460080D023 /* XCRemoteSwiftPackageReference "firebase-ios-sdk" */; productName = FirebaseAILogic; }; + 88862D602FD7C19F003702C7 /* FirebaseAppCheck */ = { + isa = XCSwiftPackageProductDependency; + productName = FirebaseAppCheck; + }; + 88862D622FD7C19F003702C7 /* FirebaseCore */ = { + isa = XCSwiftPackageProductDependency; + productName = FirebaseCore; + }; /* End XCSwiftPackageProductDependency section */ }; rootObject = 88779D2D2EC8A9CF0080D023 /* Project object */; diff --git a/firebaseai/FirebaseAIExample/ContentView.swift b/firebaseai/FirebaseAIExample/ContentView.swift index 62808c579..103b4d69c 100644 --- a/firebaseai/FirebaseAIExample/ContentView.swift +++ b/firebaseai/FirebaseAIExample/ContentView.swift @@ -112,6 +112,12 @@ struct ContentView: View { GroundingScreen(backendType: selectedBackend, sample: sample) case "LiveScreen": LiveScreen(backendType: selectedBackend, sample: sample) + case "AppleAIScreen": + if #available(iOS 27.0, *) { + AppleAIScreen() + } else { + Text("Apple Intelligence is only available on iOS 27 or newer.") + } default: EmptyView() } diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift new file mode 100644 index 000000000..821cef536 --- /dev/null +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift @@ -0,0 +1,105 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import FoundationModels + +@available(iOS 26.0, *) +@Generable +public struct Itinerary: Equatable, Codable { + @Guide(description: "An exciting name for the trip.") + public let title: String + + @Guide(description: "The name of the destination.") + public let destinationName: String + + @Guide(description: "A brief, catchy description of the trip.") + public let description: String + + @Guide(description: "An explanation of why this plan fits the user's request.") + public let rationale: String + + @Guide(description: "A list of day plans.") + public let days: [DayPlan] + + @Guide(description: "Any source attributions or links for the recommended places.") + public let attributions: [Attribution]? +} + +@available(iOS 26.0, *) +@Generable +public struct Attribution: Equatable, Codable { + public let title: String + public let url: String +} + +@available(iOS 26.0, *) +@Generable +public struct DayPlan: Equatable, Codable { + @Guide(description: "A unique title for this day.") + public let title: String + + public let subtitle: String + + @Guide(description: "A list of activities planned for this day.") + public let activities: [Activity] +} + +@available(iOS 26.0, *) +@Generable +public struct Activity: Equatable, Codable { + public let type: ActivityKind + public let title: String + public let description: String + public let latitude: Double? + public let longitude: Double? +} + +@available(iOS 26.0, *) +@Generable +public enum ActivityKind: String, Codable { + case sightseeing + case foodAndDining + case shopping + case hotelAndLodging + + public var displayName: String { + switch self { + case .sightseeing: return "Sightseeing" + case .foodAndDining: return "Food & Dining" + case .shopping: return "Shopping" + case .hotelAndLodging: return "Hotel & Lodging" + } + } +} + +@available(iOS 26.0, *) +@Generable +public struct IdentifiedObject: Equatable, Codable { + @Guide(description: "The name of the primary object or landmark detected.") + public let name: String + + @Guide(description: "The category of the object (e.g. Landmark, Plant, Food, Animal, Device, Clothing).") + public let category: String + + @Guide(description: "A short, 2-sentence description of the object and interesting facts.") + public let description: String +} + +@available(iOS 26.0, *) +@Generable +public struct TextSummary: Equatable, Codable { + @Guide(description: "A list of exactly 2 key summary points.") + public let summaryPoints: [String] +} diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift new file mode 100644 index 000000000..e942051f4 --- /dev/null +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift @@ -0,0 +1,370 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import SwiftUI +import PhotosUI +import FoundationModels +import FirebaseAILogic + +@available(iOS 27.0, *) +struct AppleAIScreen: View { + @StateObject private var viewModel = AppleAIViewModel() + @State private var selectedTab = 0 + @State private var photosPickerItem: PhotosPickerItem? = nil + @State private var presentErrorDetails = false + + var body: some View { + ZStack { + VStack(spacing: 0) { + // Segmented Control + Picker("Feature", selection: $selectedTab) { + Text("Hybrid AI").tag(0) + Text("Planner").tag(1) + Text("Vision ID").tag(2) + } + .pickerStyle(.segmented) + .padding() + .background(Color(.systemBackground)) + + Divider() + + // Content Views + ScrollView { + VStack(spacing: 20) { + if selectedTab == 0 { + hybridAIView + } else if selectedTab == 1 { + plannerView + } else { + visionIDView + } + } + .padding() + } + } + + if viewModel.inProgress { + ProgressOverlay() + } + } + .background(Color(.systemGroupedBackground)) + .navigationTitle("Apple Intelligence") + .navigationBarTitleDisplayMode(.inline) + .toolbar { + if viewModel.inProgress { + ToolbarItem(placement: .navigationBarTrailing) { + Button("Stop") { + viewModel.stopActiveTask() + } + } + } + } + .sheet(isPresented: $presentErrorDetails) { + if let error = viewModel.error { + ErrorDetailsView(error: error) + } + } + .onChange(of: viewModel.error != nil) { oldValue, newValue in + if newValue { + presentErrorDetails = true + } + } + } + + // MARK: - Subviews + + // Feature 1: Hybrid AI + private var hybridAIView: some View { + VStack(alignment: .leading, spacing: 16) { + VStack(alignment: .leading, spacing: 8) { + Text("Input Text") + .font(.headline) + TextEditor(text: $viewModel.inputText) + .frame(height: 120) + .padding(4) + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(8) + } + + Button(action: { + Task { + await viewModel.runSummarization() + } + }) { + HStack { + Spacer() + Image(systemName: "sparkles") + Text("Summarize with Apple SDK") + Spacer() + } + .padding() + .foregroundColor(.white) + .background(Color.blue) + .cornerRadius(10) + } + .disabled(viewModel.inProgress) + + if let summary = viewModel.outputSummary { + VStack(alignment: .leading, spacing: 12) { + HStack { + Text("Summary Points") + .font(.headline) + Spacer() + + // Badge indicating where it was executed + HStack(spacing: 4) { + Image(systemName: viewModel.isUsingLocalModel ? "iphone" : "cloud.fill") + Text(viewModel.isUsingLocalModel ? "Local (Apple)" : "Cloud (Gemini)") + } + .font(.caption) + .padding(.horizontal, 8) + .padding(.vertical, 4) + .background(viewModel.isUsingLocalModel ? Color.green.opacity(0.2) : Color.purple.opacity(0.2)) + .foregroundColor(viewModel.isUsingLocalModel ? .green : .purple) + .cornerRadius(8) + } + + VStack(alignment: .leading, spacing: 8) { + ForEach(summary.summaryPoints, id: \.self) { point in + HStack(alignment: .top, spacing: 6) { + Text("•") + .bold() + Text(point) + .font(.subheadline) + } + } + } + .padding() + .frame(maxWidth: .infinity, alignment: .leading) + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(10) + } + .transition(.opacity.combined(with: .slide)) + } + } + } + + // Feature 2: Planner View + private var plannerView: some View { + VStack(alignment: .leading, spacing: 16) { + VStack(spacing: 12) { + HStack { + Text("Destination") + .frame(width: 100, alignment: .leading) + TextField("e.g. Paris, Tokyo", text: $viewModel.destination) + .textFieldStyle(.roundedBorder) + } + HStack { + Text("Interests") + .frame(width: 100, alignment: .leading) + TextField("e.g. art, cafes, parks", text: $viewModel.interests) + .textFieldStyle(.roundedBorder) + } + } + .padding() + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(10) + + Button(action: { + Task { + await viewModel.generateItinerary() + } + }) { + HStack { + Spacer() + Image(systemName: "map.fill") + Text("Create Itinerary Plan") + Spacer() + } + .padding() + .foregroundColor(.white) + .background(Color.blue) + .cornerRadius(10) + } + .disabled(viewModel.inProgress) + + if let itinerary = viewModel.itinerary { + VStack(alignment: .leading, spacing: 12) { + Text(itinerary.title ?? "Generating plan...") + .font(.title3) + .bold() + + if let desc = itinerary.description { + Text(desc) + .font(.subheadline) + .foregroundColor(.secondary) + } + + if let rationale = itinerary.rationale { + Text("Rationale: \(rationale)") + .font(.caption) + .padding() + .background(Color.blue.opacity(0.1)) + .cornerRadius(8) + } + + // Day Plans + if let days = itinerary.days { + ForEach(days, id: \.title) { day in + VStack(alignment: .leading, spacing: 8) { + Text(day.title ?? "Day Plan") + .font(.headline) + .padding(.top, 4) + + if let activities = day.activities { + ForEach(activities, id: \.title) { activity in + HStack(alignment: .top, spacing: 12) { + Image(systemName: getSymbol(for: activity.type)) + .foregroundColor(.blue) + .frame(width: 24, height: 24) + .background(Color.blue.opacity(0.1)) + .cornerRadius(6) + + VStack(alignment: .leading, spacing: 2) { + Text(activity.title ?? "Activity") + .font(.subheadline) + .bold() + if let desc = activity.description { + Text(desc) + .font(.caption) + .foregroundColor(.secondary) + } + if let lat = activity.latitude, let lon = activity.longitude { + Text(String(format: "Location: %.4f, %.4f", lat, lon)) + .font(.system(.caption, design: .monospaced)) + .foregroundColor(.gray) + } + } + } + .padding(.vertical, 4) + } + } + } + .padding() + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(10) + } + } + + if let attributions = itinerary.attributions, !attributions.isEmpty { + VStack(alignment: .leading, spacing: 8) { + Text("Sources & Maps Links") + .font(.headline) + + ForEach(attributions, id: \.url) { attribution in + if let urlString = attribution.url, let url = URL(string: urlString), let title = attribution.title { + Link(destination: url) { + HStack { + Image(systemName: "mappin.and.ellipse") + Text(title) + .underline() + Spacer() + } + .font(.caption) + .foregroundColor(.blue) + } + .padding(.vertical, 2) + } + } + } + .padding() + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(10) + } + } + } + } + } + + // Feature 3: Vision ID View + private var visionIDView: some View { + VStack(alignment: .leading, spacing: 16) { + Text("Select or Snap a Photo to Identify") + .font(.headline) + + PhotosPicker(selection: $photosPickerItem, matching: .images) { + VStack(spacing: 12) { + if let image = viewModel.selectedImage { + Image(uiImage: image) + .resizable() + .aspectRatio(contentMode: .fit) + .frame(maxHeight: 200) + .cornerRadius(12) + } else { + VStack(spacing: 8) { + Image(systemName: "photo.badge.plus") + .font(.system(size: 40)) + Text("Select an Image") + } + .frame(maxWidth: .infinity) + .frame(height: 180) + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(12) + } + } + } + .onChange(of: photosPickerItem) { oldItem, newItem in + Task { + if let data = try? await newItem?.loadTransferable(type: Data.self), + let image = UIImage(data: data) { + viewModel.selectedImage = image + await viewModel.identifySelectedImage() + } + } + } + + if let identified = viewModel.identifiedObject { + VStack(alignment: .leading, spacing: 12) { + Text("Identification Result") + .font(.headline) + + VStack(alignment: .leading, spacing: 8) { + HStack { + Text("Name:") + .bold() + Text(identified.name) + } + HStack { + Text("Category:") + .bold() + Text(identified.category) + } + VStack(alignment: .leading, spacing: 4) { + Text("Description:") + .bold() + Text(identified.description) + .foregroundColor(.secondary) + } + } + .padding() + .frame(maxWidth: .infinity, alignment: .leading) + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(10) + } + .transition(.opacity) + } + } + } + + // Helper to pick icons for activity kinds + private func getSymbol(for kind: ActivityKind?) -> String { + guard let kind = kind else { return "info.circle" } + switch kind { + case .sightseeing: return "binoculars.fill" + case .foodAndDining: return "fork.knife" + case .shopping: return "bag.fill" + case .hotelAndLodging: return "bed.double.fill" + } + } +} diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift new file mode 100644 index 000000000..e2efbc92b --- /dev/null +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift @@ -0,0 +1,104 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import FoundationModels +import FirebaseAILogic + +@available(iOS 26.0, *) +@Generable +public struct PlacesResponse: Codable { + public let places: [String] +} + +@available(iOS 26.0, *) +@Generable +public struct ToolResult: Codable { + public let summary: String + public let attributions: [Attribution] +} + +@available(iOS 26.0, *) +public final class FindLocalPlacesTool { + public let name = "findLocalPlaces" + public let description = "Finds points of interest and local businesses matching a query for a specified destination." + + public init() {} + + @available(iOS 26.0, *) + @Generable + public struct Arguments { + @Guide(description: "The destination city/place to look up for (e.g. 'Paris', 'New York').") + public let destination: String + + @Guide(description: "The category or query to search (e.g. 'hotels', 'Italian restaurants', 'museums').") + public let category: String + } + + public func call(arguments: Arguments) async throws -> ToolResult { + print("FindLocalPlacesTool: call called with destination: \(arguments.destination), category: \(arguments.category)") + + // Setup Firebase AI with Google Maps grounding + let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) + let session = ai.generativeModelSession( + model: "gemini-3.1-flash-lite", + tools: [Tool.googleMaps()] + ) + + let prompt = "Find 3 real popular \(arguments.category) in \(arguments.destination). Use Google Maps to make sure the places are real and currently open." + + do { + let response = try await session.respond( + to: prompt, + generating: PlacesResponse.self + ) + + var attributions: [Attribution] = [] + if let metadata = response.rawResponse.candidates.first?.groundingMetadata { + for chunk in metadata.groundingChunks { + if let maps = chunk.maps, let title = maps.title, let url = maps.url?.absoluteString { + attributions.append(Attribution(title: title, url: url)) + } + } + } + + let results = response.content.places + print("FindLocalPlacesTool: found places: \(results.joined(separator: ", "))") + + return ToolResult( + summary: "Here are some popular \(arguments.category) in \(arguments.destination): \(results.joined(separator: ", "))", + attributions: attributions + ) + } catch { + print("FindLocalPlacesTool error: \(error.localizedDescription)") + throw error + } + } +} + +// Struct wrapper to conform to FoundationModels.Tool AND ToolRepresentable +@available(iOS 26.0, *) +public struct FindLocalPlacesToolWrapper: FoundationModels.Tool, ToolRepresentable, @unchecked Sendable { + let tool: FindLocalPlacesTool + + public var name: String { tool.name } + public var description: String { tool.description } + + public typealias Arguments = FindLocalPlacesTool.Arguments + public typealias Output = ToolResult + + public func call(arguments: Arguments) async throws -> ToolResult { + try await tool.call(arguments: arguments) + } +} diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift new file mode 100644 index 000000000..00b71d41c --- /dev/null +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift @@ -0,0 +1,210 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import SwiftUI +import Combine +import FoundationModels +import FirebaseAILogic + +@available(iOS 27.0, *) +@MainActor +public final class AppleAIViewModel: ObservableObject { + // Shared state + @Published public var inProgress = false + @Published public var error: Error? + + // Feature 1: Hybrid Summary/Translation + @Published public var inputText: String = "It is the quintessential autumn harvest fruit, famously baked into warm cinnamon pastries, dipped in sticky caramel on Halloween, and traditionally rumored to keep medical professionals at bay if eaten once a day." + @Published public var outputSummary: TextSummary? + @Published public var isUsingLocalModel: Bool = false + + // Feature 2: Smart Planner + @Published public var destination: String = "San Francisco" + @Published public var interests: String = "tech history, good coffee, bay views" + @Published public var itinerary: Itinerary.PartiallyGenerated? + + // Feature 3: Visual Identifier + @Published public var selectedImage: UIImage? + @Published public var identifiedObject: IdentifiedObject? + + private var activeTask: Task? + + public init() {} + + public func stopActiveTask() { + activeTask?.cancel() + activeTask = nil + inProgress = false + } + + // MARK: - Feature 1: Hybrid Summary/Translation + public func runSummarization() async { + stopActiveTask() + inProgress = true + error = nil + outputSummary = nil + + activeTask = Task { + defer { self.inProgress = false } + + let instructions = Instructions { + "Your job is to summarize the provided text in exactly 2 bullet points." + } + + let availability = SystemLanguageModel.default.availability + + // Try local model first if it reports available + if availability == .available { + isUsingLocalModel = true + do { + let session = LanguageModelSession( + model: SystemLanguageModel.default, + instructions: instructions + ) + let response = try await session.respond(to: inputText, generating: TextSummary.self) + if !Task.isCancelled { + self.outputSummary = response.content + } + return + } catch { + // Fall back to cloud model if local model fails (e.g. assets not downloaded on simulator) + if error.isMLAssetUnavailable { + print("Local ML assets unavailable. Falling back to cloud model...") + } else { + print("Local model failed: \(error.localizedDescription). Falling back to cloud model...") + } + } + } + + // Fallback to cloud model + isUsingLocalModel = false + do { + let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) + let model = ai.geminiLanguageModel(name: "gemini-3.1-flash-lite") + let session = LanguageModelSession( + model: model, + instructions: instructions + ) + let response = try await session.respond(to: inputText, generating: TextSummary.self) + if !Task.isCancelled { + self.outputSummary = response.content + } + } catch { + if !Task.isCancelled { + // If even the cloud model via LanguageModelSession fails (likely due to missing guardrail assets), + // we show a descriptive error message. + if let mlMessage = error.mlAssetErrorMessage { + print("ML Asset Error detected in cloud fallback: \(mlMessage)") + // We can either set the error here or try one last direct fallback. + // Let's set a custom error that wraps the message. + self.error = NSError(domain: "FirebaseAIExample", + code: 1, + userInfo: [NSLocalizedDescriptionKey: mlMessage]) + } else { + self.error = error + } + } + } + } + } + + // MARK: - Feature 2: Smart Planner + public func generateItinerary() async { + stopActiveTask() + inProgress = true + error = nil + itinerary = nil + + activeTask = Task { + defer { self.inProgress = false } + + let localPlacesTool = FindLocalPlacesToolWrapper(tool: FindLocalPlacesTool()) + let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) + + let model = ai.geminiLanguageModel( + name: "gemini-3.5-flash" + ) + + let session = LanguageModelSession( + model: model, + tools: [localPlacesTool], + instructions: Instructions { + "Your job is to create a structured 1-day itinerary for the user." + "Ensure you use the findLocalPlaces tool to search for real places and recommendations in the destination city." + } + ) + + let promptText = "Generate a structured 1-day itinerary for \(destination) focused on \(interests). Include exactly 3 activities (morning, afternoon, evening) with real recommendations. For any place returned by the findLocalPlaces tool, populate the 'attributions' array in the response with the exact name (title) and Google Maps URL (url) returned by the tool." + + let stream = session.streamResponse( + to: promptText, + generating: Itinerary.self + ) + + do { + for try await response in stream { + if Task.isCancelled { break } + self.itinerary = response.content + } + } catch { + if !Task.isCancelled { + self.error = error + } + } + } + } + + // MARK: - Feature 3: Visual Identifier + public func identifySelectedImage() async { + guard let image = selectedImage else { return } + stopActiveTask() + inProgress = true + error = nil + identifiedObject = nil + + activeTask = Task { + defer { self.inProgress = false } + + guard let cgImage = image.cgImage else { return } + + let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) + let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") + + let session = LanguageModelSession( + model: model, + instructions: Instructions { + "You are a visual object identifier." + } + ) + + do { + let response = try await session.respond( + generating: IdentifiedObject.self + ) { + "Identify the primary object in this image. Be as specific as possible, categorize it, and provide a short 2-sentence description." + Attachment(cgImage) + } + + if !Task.isCancelled { + self.identifiedObject = response.content + } + } catch { + if !Task.isCancelled { + self.error = error + } + } + } + } +} diff --git a/firebaseai/FirebaseAIExample/Shared/Models/Sample.swift b/firebaseai/FirebaseAIExample/Shared/Models/Sample.swift index 35518d55a..eee6200cd 100644 --- a/firebaseai/FirebaseAIExample/Shared/Models/Sample.swift +++ b/firebaseai/FirebaseAIExample/Shared/Models/Sample.swift @@ -309,6 +309,12 @@ extension Sample { ), tip: InlineTip(text: "Try asking the model to change the background color"), ), + Sample( + title: "Apple Foundation Models", + description: "Demonstrates how to integrate Firebase AI Logic (Gemini) with Apple's Foundation Models framework (Apple Intelligence).", + useCases: [.text, .image, .functionCalling], + navRoute: "AppleAIScreen" + ), ] public static var sample = samples[0] diff --git a/firebaseai/FirebaseAIExample/Shared/Util/MLErrorHelper.swift b/firebaseai/FirebaseAIExample/Shared/Util/MLErrorHelper.swift new file mode 100644 index 000000000..bfaf543f7 --- /dev/null +++ b/firebaseai/FirebaseAIExample/Shared/Util/MLErrorHelper.swift @@ -0,0 +1,72 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import FoundationModels + +extension Error { + /// Returns true if this error indicates that required ML assets (like safety guardrails) + /// are missing from the device or simulator. + public var isMLAssetUnavailable: Bool { + // 1. Check for LanguageModelSession.GenerationError.assetsUnavailable + if #available(iOS 26.0, *) { + if let genError = self as? LanguageModelSession.GenerationError { + if case .assetsUnavailable = genError { + return true + } + } + } + + let nsError = self as NSError + + // 2. Check for the specific SensitiveContentAnalysisML error (Code 15) + if nsError.domain == "com.apple.SensitiveContentAnalysisML" && nsError.code == 15 { + return true + } + + // 3. Check for underlying GenerativeFunctionsFoundation error (Code 1020000) + // This is often found in the UserInfo of higher-level errors. + if nsError.domain == "com.apple.GenerativeFunctionsFoundation.GenerativeError" && nsError.code == 1020000 { + return true + } + + // 4. Recursive check for underlying errors + if let underlying = nsError.userInfo[NSUnderlyingErrorKey] as? Error { + return underlying.isMLAssetUnavailable + } + + return false + } + + /// Returns a user-friendly message for ML asset errors, particularly for simulators. + public var mlAssetErrorMessage: String? { + guard isMLAssetUnavailable else { return nil } + + #if targetEnvironment(simulator) + return """ + Apple Intelligence assets are missing in this simulator. + + To fix this: + 1. Open Settings in the simulator. + 2. Go to Apple Intelligence & Siri. + 3. Toggle Apple Intelligence ON. + 4. Wait for assets to download (check the status in Settings). + + Alternatively, use a physical device with Apple Intelligence support. + """ + #else + return "Apple Intelligence assets are not yet ready on this device. Please ensure Apple Intelligence is enabled in Settings and that model assets have finished downloading." + #endif + } +} diff --git a/firebaseai/FirebaseAIExample/Shared/Views/ErrorDetailsView.swift b/firebaseai/FirebaseAIExample/Shared/Views/ErrorDetailsView.swift index cd1e3f60e..7ba2b1788 100644 --- a/firebaseai/FirebaseAIExample/Shared/Views/ErrorDetailsView.swift +++ b/firebaseai/FirebaseAIExample/Shared/Views/ErrorDetailsView.swift @@ -148,11 +148,20 @@ struct ErrorDetailsView: View { default: Section("Error Type") { - Text("Some other error") + if error.isMLAssetUnavailable { + Text("Apple Intelligence Assets Missing") + .foregroundColor(.orange) + } else { + Text("Some other error") + } } Section("Details") { - SubtitleFormRow(title: "Error description", value: error.localizedDescription) + if let mlMessage = error.mlAssetErrorMessage { + SubtitleFormRow(title: "How to fix", value: mlMessage) + } else { + SubtitleFormRow(title: "Error description", value: error.localizedDescription) + } } } }