diff --git a/.github/agents/apify-integration-expert.md b/.github/agents/apify-integration-expert.md index 0eeeca9..458f6c9 100644 --- a/.github/agents/apify-integration-expert.md +++ b/.github/agents/apify-integration-expert.md @@ -53,9 +53,6 @@ Your job is to help integrate Actors into codebases based on what the user needs ## Recommended Workflow -0. **Prepare the repo** - - Ensure the base branch is available locally before making changes. Run `git fetch origin main:main --depth=1 || git fetch origin main` so `git diff refs/heads/main` succeeds in Copilot runs. - 1. **Understand Context** - Look at the project's README and how they currently handle data ingestion. - Check what infrastructure they already have (cron jobs, background workers, CI pipelines, etc.). @@ -69,13 +66,10 @@ Your job is to help integrate Actors into codebases based on what the user needs - Decide how to trigger the Actor (manually, on a schedule, or when something happens). - Plan where the results should be stored (database, file, etc.). - Think about what happens if the same data comes back twice or if something fails. - - Audit any external assets or links the Actor may return (images, files, media). Decide whether the target stack needs host allowlists, proxying, or graceful fallbacks if assets are blocked. 4. **Implement It** - Use `call-actor` to test running the Actor. - Provide working code examples (see language-specific guides below) they can copy and modify. - - Normalize the Actor output so consumers handle missing or malformed fields safely. Prefer explicit defaults over assuming the data is complete. - - Build data-access layers that can downgrade functionality (e.g., fall back to placeholders) when a platform constraint such as CSP, SSR limitations, or `next/image` host checks blocks remote assets. 5. **Test & Document** - Run a few test cases to make sure the integration works. @@ -99,17 +93,6 @@ Always tell the user what tools you're using and what you found. - **Be careful with data:** Don't scrape or process data that's protected or regulated without the user's knowledge. - **Respect limits:** Watch out for API rate limits and costs. Start with small test runs before going big. - **Don't break things:** Avoid operations that permanently delete or modify data (like dropping tables) unless explicitly told to do so. -- **Validate external resources:** Check framework-level restrictions (image/CDN allowlists, CORS, CSP, mixed-content rules) before surfacing URLs from Actor results. Provide clear fallbacks if resources cannot be fetched safely. - -## Integration Checklist - -Use this lightweight checklist to catch common edge cases before handing work back to the user: - -- ✅ Confirm environment variables and secrets (`APIFY_TOKEN`, API keys, etc.) are documented and validated at runtime. -- ✅ Note any framework or hosting constraints (e.g., asset allowlists, cold-start limits, execution timeouts) and describe how the integration adapts. -- ✅ Ensure outputs are typed, sanitized, and have default values for missing fields. -- ✅ Outline manual or automated tests that prove the Actor workflow succeeds and failure states are covered. -- ✅ Highlight post-integration maintenance tasks such as monitoring, quota usage, and update cadence for the Actor or downstream APIs. # Running an Actor on Apify (JavaScript/TypeScript) diff --git a/ecomm-demo/.env.example b/ecomm-demo/.env.example deleted file mode 100644 index 496d1c2..0000000 --- a/ecomm-demo/.env.example +++ /dev/null @@ -1,3 +0,0 @@ -# Apify API Token -# Get your token from: https://console.apify.com/account#/integrations -APIFY_TOKEN=your_apify_token_here diff --git a/ecomm-demo/.gitignore b/ecomm-demo/.gitignore index de22420..2017053 100644 --- a/ecomm-demo/.gitignore +++ b/ecomm-demo/.gitignore @@ -31,9 +31,7 @@ yarn-error.log* .pnpm-debug.log* # env files (can opt-in for committing if needed) -.env -.env.local -.env.*.local +.env* # vercel .vercel @@ -44,4 +42,5 @@ next-env.d.ts # internal task-manager.md -PRD.md \ No newline at end of file +PRD.md +README.md \ No newline at end of file diff --git a/ecomm-demo/README.md b/ecomm-demo/README.md deleted file mode 100644 index 69ef61a..0000000 --- a/ecomm-demo/README.md +++ /dev/null @@ -1,129 +0,0 @@ -# E-Commerce Demo with Apify Integration - -A Next.js application that demonstrates integration with the Apify E-Commerce Scraper to search and display real product data from e-commerce websites. - -## Features - -- 🔍 Real-time product search using Apify E-Commerce Scraper -- 📊 Dynamic statistics including product count and average price -- 📋 Product data displayed in table and card formats -- ⚡ Loading states and error handling -- 🎨 Built with Next.js, TypeScript, Tailwind CSS, and shadcn/ui - -## Prerequisites - -- Node.js 20.x or higher -- An Apify account with an API token - -## Setup - -### 1. Install Dependencies - -```bash -npm install -``` - -### 2. Configure Apify Token - -1. Create a free Apify account at [https://console.apify.com](https://console.apify.com) -2. Get your API token from [https://console.apify.com/account#/integrations](https://console.apify.com/account#/integrations) -3. Create a `.env.local` file in the root directory: - -```bash -cp .env.example .env.local -``` - -4. Add your Apify token to `.env.local`: - -``` -NEXT_PUBLIC_APIFY_TOKEN=your_apify_token_here -``` - -⚠️ **Important**: The token must be prefixed with `NEXT_PUBLIC_` to be accessible in the browser. - -### 3. Run the Development Server - -```bash -npm run dev -``` - -Open [http://localhost:3000](http://localhost:3000) in your browser. - -## Usage - -1. Enter a product keyword in the search bar (e.g., "laptop", "headphones", "camera") -2. Click "Submit" to search for products -3. The app will use Apify E-Commerce Scraper to fetch real product data from Amazon -4. Results will be displayed in both table and card formats -5. Statistics cards show the total number of products and average price - -## How It Works - -The application uses the Apify E-Commerce Scraping Tool (`apify/e-commerce-scraping-tool`) to: - -1. Search for products by keyword on Amazon (default marketplace) -2. Extract product information including: - - Product name - - Price - - Image URL - - Description - - Product URL -3. Display the results in a user-friendly interface - -## Project Structure - -``` -ecomm-demo/ -├── app/ -│ ├── page.tsx # Main page component with search logic -│ ├── layout.tsx # Root layout -│ └── globals.css # Global styles -├── components/ -│ ├── SearchBar.tsx # Search input component -│ ├── StatsCards.tsx # Statistics display -│ ├── ProductTable.tsx # Product data table -│ ├── ProductCards.tsx # Product card grid -│ └── ui/ # shadcn/ui components -├── lib/ -│ ├── apify-service.ts # Apify integration service -│ └── types.ts # TypeScript type definitions -├── data/ -│ └── products.ts # Mock data (used as fallback) -└── .env.example # Environment variable template -``` - -## Available Scripts - -- `npm run dev` - Start development server -- `npm run build` - Build for production -- `npm run start` - Start production server -- `npm run lint` - Run ESLint - -## Pricing - -The E-Commerce Scraping Tool uses a pay-per-event model. The $5 credit included in the Apify free plan lets you scrape approximately 800 product URLs. For more details, see the [Actor's pricing page](https://apify.com/apify/e-commerce-scraping-tool/pricing). - -## Troubleshooting - -### "Failed to search products" Error - -- **Check your API token**: Make sure it's correctly set in `.env.local` -- **Verify token prefix**: The environment variable must start with `NEXT_PUBLIC_` -- **Restart the dev server**: After changing `.env.local`, restart `npm run dev` - -### No Results Found - -- Try different search keywords -- Check that your Apify account has sufficient credits -- Review the browser console for detailed error messages - -## Learn More - -- [Apify Documentation](https://docs.apify.com) -- [E-Commerce Scraping Tool](https://apify.com/apify/e-commerce-scraping-tool) -- [Next.js Documentation](https://nextjs.org/docs) -- [Apify JavaScript Client](https://docs.apify.com/api/client/js) - -## License - -This project is provided as a demonstration of Apify integration. diff --git a/ecomm-demo/app/page.tsx b/ecomm-demo/app/page.tsx index ac2d3b3..5561482 100644 --- a/ecomm-demo/app/page.tsx +++ b/ecomm-demo/app/page.tsx @@ -1,50 +1,12 @@ -"use client"; - -import { useState } from "react"; -import { products as mockProducts } from "@/data/products"; +import { products } from "@/data/products"; import { StatsCards } from "@/components/StatsCards"; import { ProductTable } from "@/components/ProductTable"; import { ProductCards } from "@/components/ProductCards"; import { SearchBar } from "@/components/SearchBar"; import { Separator } from "@/components/ui/separator"; -import { searchProducts } from "@/lib/apify-service"; -import { Product } from "@/lib/types"; import Image from "next/image"; export default function Home() { - const [products, setProducts] = useState(mockProducts); - const [isLoading, setIsLoading] = useState(false); - const [error, setError] = useState(null); - const [dataSource, setDataSource] = useState<"Mock" | "Apify">("Mock"); - - const handleSearch = async (query: string) => { - if (!query.trim()) { - setError("Please enter a search query"); - return; - } - - setIsLoading(true); - setError(null); - - try { - const results = await searchProducts({ keyword: query }); - - if (results.length === 0) { - setError(`No products found for "${query}". Try a different search term.`); - setProducts([]); - } else { - setProducts(results); - setDataSource("Apify"); - } - } catch (err) { - const errorMessage = err instanceof Error ? err.message : "An unknown error occurred"; - setError(`Failed to search products: ${errorMessage}`); - console.error("Search error:", err); - } finally { - setIsLoading(false); - } - }; - return (
{/* Header */} @@ -75,39 +37,19 @@ export default function Home() { Product Catalog Demo

- Search for products using Apify E-Commerce Scraper. - Enter a keyword to scrape real product data from Amazon. + Showcase of e-commerce product data ready for Apify integration. + This demo displays scraped product information in multiple formats.

{/* Search Bar */}
- +
- - {/* Loading State */} - {isLoading && ( -
-

- 🔄 Searching for products... This may take a moment. -

-
- )} - - {/* Error State */} - {error && ( -
-

❌ {error}

-
- )}
{/* Stats Section */}
- +
diff --git a/ecomm-demo/components/SearchBar.tsx b/ecomm-demo/components/SearchBar.tsx index 15651d8..02f1e80 100644 --- a/ecomm-demo/components/SearchBar.tsx +++ b/ecomm-demo/components/SearchBar.tsx @@ -7,17 +7,18 @@ import { Search } from "lucide-react"; interface SearchBarProps { onSearch?: (query: string) => void; - disabled?: boolean; } -export function SearchBar({ onSearch, disabled = false }: SearchBarProps) { +export function SearchBar({ onSearch }: SearchBarProps) { const [query, setQuery] = useState(""); const handleSubmit = (e: React.FormEvent) => { e.preventDefault(); - if (onSearch && !disabled) { + if (onSearch) { onSearch(query); } + // Placeholder for actual search functionality + console.log("Search query:", query); }; return ( @@ -31,11 +32,10 @@ export function SearchBar({ onSearch, disabled = false }: SearchBarProps) { value={query} onChange={(e) => setQuery(e.target.value)} className="pl-10" - disabled={disabled} /> - diff --git a/ecomm-demo/components/StatsCards.tsx b/ecomm-demo/components/StatsCards.tsx index 06148c2..b37ae92 100644 --- a/ecomm-demo/components/StatsCards.tsx +++ b/ecomm-demo/components/StatsCards.tsx @@ -1,25 +1,11 @@ -import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { Package } from "lucide-react"; -import { Product } from "@/lib/types"; interface StatsCardsProps { productCount: number; - products?: Product[]; - dataSource?: "Mock" | "Apify"; } -export function StatsCards({ productCount, products = [], dataSource = "Mock" }: StatsCardsProps) { - // Calculate average price from products - const calculateAveragePrice = () => { - if (products.length === 0) return 0; - const validPrices = products.filter(p => p.price > 0); - if (validPrices.length === 0) return 0; - const sum = validPrices.reduce((acc, p) => acc + p.price, 0); - return sum / validPrices.length; - }; - - const averagePrice = calculateAveragePrice(); - +export function StatsCards({ productCount }: StatsCardsProps) { return (
@@ -41,11 +27,9 @@ export function StatsCards({ productCount, products = [], dataSource = "Mock" }: $ -
- {averagePrice > 0 ? `$${averagePrice.toFixed(2)}` : "-"} -
+
-

- {averagePrice > 0 ? "Calculated from scraped data" : "No price data available"} + Will be calculated from data

@@ -56,9 +40,9 @@ export function StatsCards({ productCount, products = [], dataSource = "Mock" }: 🔗 -
{dataSource}
+
Mock

- {dataSource === "Apify" ? "Live scraped data" : "Ready for Apify integration"} + Ready for Apify integration

diff --git a/ecomm-demo/lib/apify-service.ts b/ecomm-demo/lib/apify-service.ts deleted file mode 100644 index 8ed25f3..0000000 --- a/ecomm-demo/lib/apify-service.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { ApifyClient } from "apify-client"; -import { Product } from "./types"; - -/** - * Service for interacting with Apify E-Commerce Scraper - */ - -// Initialize the Apify client -const client = new ApifyClient({ - token: process.env.NEXT_PUBLIC_APIFY_TOKEN, -}); - -/** - * Apify Actor output item structure - */ -interface ApifyProductItem { - url?: string; - name?: string; - image?: string; - description?: string; - offers?: { - price?: number; - priceCurrency?: string; - }; - brand?: { - slogan?: string; - }; -} - -/** - * Maps Apify Actor output to our Product type - */ -function mapApifyProductToProduct(item: ApifyProductItem): Product { - return { - url: item.url || "", - title: item.name || "Untitled Product", - image: item.image || "https://picsum.photos/seed/default/400/400", - description: item.description || "", - price: item.offers?.price || 0, - }; -} - -export interface SearchProductsOptions { - keyword: string; - marketplace?: string; - maxProducts?: number; -} - -/** - * Search for products using Apify E-Commerce Scraper - */ -export async function searchProducts( - options: SearchProductsOptions -): Promise { - const { keyword, marketplace = "www.amazon.com", maxProducts = 20 } = options; - - try { - // Call the Apify E-Commerce Scraping Tool - const run = await client.actor("apify/e-commerce-scraping-tool").call({ - keyword, - marketplaces: [marketplace], - }); - - // Wait for the run to finish - await client.run(run.id).waitForFinish(); - - // Get the dataset with results - if (!run.defaultDatasetId) { - throw new Error("No dataset ID returned from Actor run"); - } - - const dataset = client.dataset(run.defaultDatasetId); - const { items } = await dataset.listItems({ limit: maxProducts }); - - // Map the Apify output to our Product type - return items.map(mapApifyProductToProduct); - } catch (error) { - console.error("Error searching products with Apify:", error); - throw new Error( - `Failed to search products: ${ - error instanceof Error ? error.message : "Unknown error" - }` - ); - } -} diff --git a/ecomm-demo/package.json b/ecomm-demo/package.json index 3cef9a4..44871bf 100644 --- a/ecomm-demo/package.json +++ b/ecomm-demo/package.json @@ -12,7 +12,6 @@ "@radix-ui/react-scroll-area": "^1.2.10", "@radix-ui/react-separator": "^1.1.8", "@radix-ui/react-slot": "^1.2.4", - "apify-client": "^2.19.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "lucide-react": "^0.553.0",