diff --git a/README.md b/README.md index 5478537..22eea03 100644 --- a/README.md +++ b/README.md @@ -36,17 +36,34 @@ Each example is self-contained. No shared dependencies to install. | Use case | Description | Languages | Frameworks | | --- | --- | --- | --- | | [Auto-Accept Cookie Consent Banners](./examples/auto-accept-cookie-banners) | Detect and dismiss cookie consent banners before scraping or capturing a page | | | +| [Batch DOM Queries](./examples/batch-dom-queries) | Run multiple DOM queries in a single BQL request — all fields execute in the same browser session with no repeated page loads | | | | [Basic Playwright Connection](./examples/basic-playwright-connection) | Connect an existing Playwright script to Browserless by swapping `chromium.launch()` for `connectOverCDP()` | | | | [Browse Cloudflare Access-Protected Pages](./examples/browse-cloudflare-access-pages) | Access pages protected by Cloudflare Access zero-trust policies using Service Token headers or a saved authenticated profile | | | +| [Close a Session](./examples/close-session) | Terminate a BQL or Session API browser session when your automation is complete | | | | [Run Concurrent Browser Sessions](./examples/concurrent-browser-sessions) | Launch multiple independent browser sessions in parallel to speed up large-scale scraping or automation | | | +| [E2E Testing](./examples/e2e-testing) | Run end-to-end tests against a live remote browser using Playwright, Puppeteer, or BQL | | | +| [Export a Slide Deck](./examples/export-slide-deck) | Export a Google Slides presentation as a PDF by navigating to its export URL | | | | [Export Pages and Assets](./examples/export-pages-and-assets) | Export a page and all its linked resources (CSS, JS, images) as a ZIP file using the Browserless /export endpoint | | | | [Fill and Submit a Form](./examples/fill-and-submit-a-form) | Automate form interactions — typing into fields, selecting options, solving CAPTCHAs, and clicking submit | | | | [Generate a PDF](./examples/generate-a-pdf) | Export any webpage or HTML content as a PDF using the Browserless PDF API | | | | [Log In and Reuse Sessions](./examples/login-and-reuse-sessions) | Log in once and reuse that authenticated state across many browser sessions without re-entering credentials | | | | [Log In via Email OTP](./examples/login-via-email-otp) | Automate login flows that send a one-time passcode to an email address — trigger the OTP, read the code, and enter it | | | | [Log In with BQL and Browser Automation](./examples/login-with-bql-and-browser-automation) | Automate a login flow — filling credentials, solving CAPTCHAs, and handling post-login redirects | | | +| [Navigate After Click](./examples/navigate-after-click) | Click a link and wait for the resulting page navigation to complete before continuing | | | +| [Persist a Session](./examples/persist-session) | Save and restore browser state across requests for multi-step automation workflows | | | | [Record a Browser Session](./examples/record-browser-session) | Capture a browser session as a `.webm` video file using the Browserless recording API | | | +| [Reconnect to a Browser](./examples/reconnect-to-a-browser) | Reattach to an existing running browser session after disconnecting | | | +| [Retry with Exponential Backoff](./examples/retry-backoff) | Implement retry logic with exponential backoff around Browserless requests to handle transient failures | | | +| [Route Traffic Through a Proxy](./examples/proxy) | Route browser sessions through a residential or custom proxy | | | | [Save Logins to Authenticated Profiles](./examples/save-logins-to-authenticated-profiles) | Capture a logged-in browser state once and reuse it across parallel sessions without re-entering credentials | | | +| [Scrape Booking.com](./examples/scrape-booking) | Extract hotel listings from Booking.com using stealth mode and a residential proxy | | | +| [Scrape Etsy](./examples/scrape-etsy) | Search Etsy and extract product titles, prices, and links | | | +| [Scrape Glassdoor](./examples/scrape-glassdoor) | Extract job listings from Glassdoor using stealth mode and a residential proxy | | | +| [Scrape Google Shopping](./examples/scrape-google-shopping) | Pull product results from Google Shopping search pages using stealth mode | | | +| [Scrape Reddit](./examples/scrape-reddit) | Extract posts from a subreddit using stealth mode | | | +| [Scrape Walmart](./examples/scrape-walmart) | Extract product listings from Walmart search results using stealth mode and a residential proxy | | | +| [Scrape YouTube](./examples/scrape-youtube) | Extract video results from YouTube search pages using stealth mode | | | +| [Scrape Zillow](./examples/scrape-zillow) | Extract property listings from Zillow using stealth mode and a residential proxy | | | | [Solving Cloudflare Challenges](./examples/solve-cloudflare-challenges) | Bypass Cloudflare Turnstile and JS challenges using the `/unblock` endpoint or BQL's `solve` mutation | | | | [Solving reCAPTCHAs](./examples/solve-recaptchas) | Automatically detect and solve reCAPTCHA v2, v3, invisible, and other CAPTCHA challenges during browser automation | | | | [Take a Screenshot](./examples/take-a-screenshot) | Capture any webpage as a PNG or JPEG using the Browserless screenshot API | | | diff --git a/examples/batch-dom-queries/bql/batch-dom-queries.graphql b/examples/batch-dom-queries/bql/batch-dom-queries.graphql new file mode 100644 index 0000000..d7144c7 --- /dev/null +++ b/examples/batch-dom-queries/bql/batch-dom-queries.graphql @@ -0,0 +1,26 @@ +# Runs multiple DOM queries in a single BQL request. +# All fields execute in the same browser session — no repeated page loads. +mutation BatchDOMQueries { + goto(url: "https://example.com", waitUntil: networkIdle) { + status + } + + title { + title + } + + heading: text(selector: "h1") { + text + } + + description: attribute(selector: "meta[name='description']", name: "content") { + value + } + + links: mapSelector(selector: "a") { + text: innerText + href: attribute(name: "href") { + value + } + } +} diff --git a/examples/batch-dom-queries/frameworks/playwright/nodejs/batch-dom-queries.mjs b/examples/batch-dom-queries/frameworks/playwright/nodejs/batch-dom-queries.mjs new file mode 100644 index 0000000..3d2fc94 --- /dev/null +++ b/examples/batch-dom-queries/frameworks/playwright/nodejs/batch-dom-queries.mjs @@ -0,0 +1,36 @@ +// Runs multiple DOM queries in a single browser session using Playwright. +// +// Install: npm install playwright-core +// Run: node batch-dom-queries.mjs + +import { chromium } from 'playwright-core'; + +const TOKEN = 'YOUR_API_TOKEN_HERE'; + +const browser = await chromium.connectOverCDP( + `wss://production-sfo.browserless.io?token=${TOKEN}` +); + +try { + const context = browser.contexts()[0] ?? await browser.newContext(); + const page = await context.newPage(); + await page.goto('https://example.com', { waitUntil: 'networkidle' }); + + // Run all queries in a single evaluate call to minimise round-trips. + const results = await page.evaluate(() => ({ + title: document.title, + heading: document.querySelector('h1')?.innerText ?? '', + description: document.querySelector('meta[name="description"]')?.content ?? '', + links: Array.from(document.querySelectorAll('a')).map((a) => ({ + text: a.innerText.trim(), + href: a.href, + })), + })); + + console.log('Title:', results.title); + console.log('H1:', results.heading); + console.log('Description:', results.description); + console.log('Links:', results.links); +} finally { + await browser.close(); +} diff --git a/examples/batch-dom-queries/frameworks/puppeteer/batch-dom-queries.mjs b/examples/batch-dom-queries/frameworks/puppeteer/batch-dom-queries.mjs new file mode 100644 index 0000000..d5feb1b --- /dev/null +++ b/examples/batch-dom-queries/frameworks/puppeteer/batch-dom-queries.mjs @@ -0,0 +1,35 @@ +// Runs multiple DOM queries in a single browser session using Puppeteer. +// +// Install: npm install puppeteer-core +// Run: node batch-dom-queries.mjs + +import puppeteer from 'puppeteer-core'; + +const TOKEN = 'YOUR_API_TOKEN_HERE'; + +const browser = await puppeteer.connect({ + browserWSEndpoint: `wss://production-sfo.browserless.io?token=${TOKEN}`, +}); + +try { + const page = await browser.newPage(); + await page.goto('https://example.com', { waitUntil: 'networkidle2' }); + + // Run all queries in a single evaluate call to minimise round-trips. + const results = await page.evaluate(() => ({ + title: document.title, + heading: document.querySelector('h1')?.innerText ?? '', + description: document.querySelector('meta[name="description"]')?.content ?? '', + links: Array.from(document.querySelectorAll('a')).map((a) => ({ + text: a.innerText.trim(), + href: a.href, + })), + })); + + console.log('Title:', results.title); + console.log('H1:', results.heading); + console.log('Description:', results.description); + console.log('Links:', results.links); +} finally { + await browser.close(); +} diff --git a/examples/batch-dom-queries/rest/csharp/BatchDOMQueries.cs b/examples/batch-dom-queries/rest/csharp/BatchDOMQueries.cs new file mode 100644 index 0000000..3efed94 --- /dev/null +++ b/examples/batch-dom-queries/rest/csharp/BatchDOMQueries.cs @@ -0,0 +1,35 @@ +// Runs multiple DOM queries in a single BQL request. +// +// Run: dotnet run + +using System.Net.Http; +using System.Text; +using System.Text.Json; + +string token = "YOUR_API_TOKEN_HERE"; +string endpoint = $"https://production-sfo.browserless.io/bql?token={token}"; + +var payload = new +{ + query = @"mutation BatchDOMQueries { + goto(url: ""https://example.com"", waitUntil: networkIdle) { status } + title { title } + heading: text(selector: ""h1"") { text } + description: attribute(selector: ""meta[name='description']"", name: ""content"") { value } + links: mapSelector(selector: ""a"") { + text: innerText + href: attribute(name: ""href"") { value } + } + }", + variables = new { }, + operationName = "BatchDOMQueries", +}; + +using (HttpClient httpClient = new HttpClient()) +{ + var content = new StringContent( + JsonSerializer.Serialize(payload), Encoding.UTF8, "application/json"); + var response = await httpClient.PostAsync(endpoint, content); + string body = await response.Content.ReadAsStringAsync(); + Console.WriteLine(body); +} diff --git a/examples/batch-dom-queries/rest/curl/batch-dom-queries.sh b/examples/batch-dom-queries/rest/curl/batch-dom-queries.sh new file mode 100644 index 0000000..af35615 --- /dev/null +++ b/examples/batch-dom-queries/rest/curl/batch-dom-queries.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Runs multiple DOM queries in a single BQL request — title, h1, meta description, and links. +# Run: bash batch-dom-queries.sh + +curl -X POST \ + "https://production-sfo.browserless.io/bql?token=YOUR_API_TOKEN_HERE" \ + -H "Content-Type: application/json" \ + -d '{ + "query": "mutation BatchDOMQueries { goto(url: \"https://example.com\", waitUntil: networkIdle) { status } title { title } heading: text(selector: \"h1\") { text } description: attribute(selector: \"meta[name='\''description'\'']\", name: \"content\") { value } links: mapSelector(selector: \"a\") { text: innerText href: attribute(name: \"href\") { value } } }", + "variables": {}, + "operationName": "BatchDOMQueries" + }' diff --git a/examples/batch-dom-queries/rest/java/BatchDOMQueries.java b/examples/batch-dom-queries/rest/java/BatchDOMQueries.java new file mode 100644 index 0000000..5b7f414 --- /dev/null +++ b/examples/batch-dom-queries/rest/java/BatchDOMQueries.java @@ -0,0 +1,33 @@ +// Runs multiple DOM queries in a single BQL request. +// +// Run: javac BatchDOMQueries.java && java BatchDOMQueries + +import java.net.URI; +import java.net.http.*; + +public class BatchDOMQueries { + public static void main(String[] args) throws Exception { + String token = "YOUR_API_TOKEN_HERE"; + String endpoint = "https://production-sfo.browserless.io/bql?token=" + token; + + String query = "mutation BatchDOMQueries {" + + " goto(url: \\\"https://example.com\\\", waitUntil: networkIdle) { status }" + + " title { title }" + + " heading: text(selector: \\\"h1\\\") { text }" + + " description: attribute(selector: \\\"meta[name='description']\\\", name: \\\"content\\\") { value }" + + " links: mapSelector(selector: \\\"a\\\") { text: innerText href: attribute(name: \\\"href\\\") { value } }" + + " }"; + + String payload = "{\"query\": \"" + query + "\", \"variables\": {}, \"operationName\": \"BatchDOMQueries\"}"; + + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(endpoint)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(payload)) + .build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + System.out.println(response.body()); + } +} diff --git a/examples/batch-dom-queries/rest/nodejs/batch-dom-queries.mjs b/examples/batch-dom-queries/rest/nodejs/batch-dom-queries.mjs new file mode 100644 index 0000000..0e66fa2 --- /dev/null +++ b/examples/batch-dom-queries/rest/nodejs/batch-dom-queries.mjs @@ -0,0 +1,40 @@ +// Runs multiple DOM queries in a single BQL request — title, h1, meta description, and links. +// BQL executes all fields in one browser session, avoiding multiple round-trips. +// +// Run: node batch-dom-queries.mjs + +const query = `mutation BatchDOMQueries { + goto(url: "https://example.com", waitUntil: networkIdle) { + status + } + title { + title + } + heading: text(selector: "h1") { + text + } + description: attribute(selector: "meta[name='description']", name: "content") { + value + } + links: mapSelector(selector: "a") { + text: innerText + href: attribute(name: "href") { + value + } + } +}`; + +const response = await fetch( + 'https://production-sfo.browserless.io/bql?token=YOUR_API_TOKEN_HERE', + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ query, variables: {}, operationName: 'BatchDOMQueries' }), + } +); + +const { data } = await response.json(); +console.log('Title:', data.title.title); +console.log('H1:', data.heading.text); +console.log('Description:', data.description.value); +console.log('Links:', data.links.map((l) => ({ text: l.text, href: l.href?.value }))); diff --git a/examples/batch-dom-queries/rest/python/batch_dom_queries.py b/examples/batch-dom-queries/rest/python/batch_dom_queries.py new file mode 100644 index 0000000..57308a6 --- /dev/null +++ b/examples/batch-dom-queries/rest/python/batch_dom_queries.py @@ -0,0 +1,43 @@ +# Runs multiple DOM queries in a single BQL request — title, h1, meta description, and links. +# BQL executes all fields in one browser session, avoiding multiple round-trips. +# +# Install: pip install requests +# Run: python batch_dom_queries.py + +import requests + +query = """ +mutation BatchDOMQueries { + goto(url: "https://example.com", waitUntil: networkIdle) { + status + } + title { + title + } + heading: text(selector: "h1") { + text + } + description: attribute(selector: "meta[name='description']", name: "content") { + value + } + links: mapSelector(selector: "a") { + text: innerText + href: attribute(name: "href") { + value + } + } +} +""" + +response = requests.post( + 'https://production-sfo.browserless.io/bql', + params={'token': 'YOUR_API_TOKEN_HERE'}, + json={'query': query, 'variables': {}, 'operationName': 'BatchDOMQueries'}, +) + +data = response.json()['data'] +print('Title:', data['title']['title']) +print('H1:', data['heading']['text']) +print('Description:', data['description']['value']) +for link in data['links']: + print(f" {link['text']} -> {link['href']['value'] if link['href'] else ''}") diff --git a/examples/close-session/bql/close-session.graphql b/examples/close-session/bql/close-session.graphql new file mode 100644 index 0000000..c58f5f4 --- /dev/null +++ b/examples/close-session/bql/close-session.graphql @@ -0,0 +1,10 @@ +# BQL sessions are automatically closed when the mutation completes. +# For Session API sessions using BQL, terminate them with a DELETE request to the stop URL. +mutation CloseExample { + goto(url: "https://example.com", waitUntil: networkIdle) { + status + } + text(selector: "h1") { + text + } +} diff --git a/examples/close-session/frameworks/playwright/nodejs/close-session.mjs b/examples/close-session/frameworks/playwright/nodejs/close-session.mjs new file mode 100644 index 0000000..61b59ff --- /dev/null +++ b/examples/close-session/frameworks/playwright/nodejs/close-session.mjs @@ -0,0 +1,23 @@ +// Connects to Browserless via Playwright, does work, then closes the session. +// +// Install: npm install playwright-core +// Run: node close-session.mjs + +import { chromium } from "playwright-core"; + +const TOKEN = "YOUR_API_TOKEN_HERE"; +const browser = await chromium.connectOverCDP( + `wss://production-sfo.browserless.io?token=${TOKEN}&stealth` +); + +try { + const context = browser.contexts()[0] ?? await browser.newContext(); + const page = await context.newPage(); + await page.goto("https://example.com", { waitUntil: "networkidle" }); + + const title = await page.title(); + console.log("Page title:", title); +} finally { + await browser.close(); + console.log("Browser closed — session resources released."); +} diff --git a/examples/close-session/frameworks/puppeteer/close-session.mjs b/examples/close-session/frameworks/puppeteer/close-session.mjs new file mode 100644 index 0000000..ff2fa0c --- /dev/null +++ b/examples/close-session/frameworks/puppeteer/close-session.mjs @@ -0,0 +1,22 @@ +// Connects to Browserless via Puppeteer, does work, then closes the session. +// +// Install: npm install puppeteer-core +// Run: node close-session.mjs + +import puppeteer from "puppeteer-core"; + +const TOKEN = "YOUR_API_TOKEN_HERE"; +const browser = await puppeteer.connect({ + browserWSEndpoint: `wss://production-sfo.browserless.io?token=${TOKEN}`, +}); + +try { + const page = await browser.newPage(); + await page.goto("https://example.com", { waitUntil: "networkidle2" }); + + const title = await page.title(); + console.log("Page title:", title); +} finally { + await browser.close(); + console.log("Browser closed — session resources released."); +} diff --git a/examples/close-session/rest/csharp/CloseSession.cs b/examples/close-session/rest/csharp/CloseSession.cs new file mode 100644 index 0000000..f28343c --- /dev/null +++ b/examples/close-session/rest/csharp/CloseSession.cs @@ -0,0 +1,29 @@ +// Creates a Browserless session and closes it via the Session API. +// +// Run: dotnet run + +using System.Net.Http; +using System.Text; +using System.Text.Json; + +var token = "YOUR_API_TOKEN_HERE"; +var client = new HttpClient(); + +var content = new StringContent( + JsonSerializer.Serialize(new { ttl = 60000, stealth = true }), + Encoding.UTF8, + "application/json" +); + +var response = await client.PostAsync( + $"https://production-sfo.browserless.io/session?token={token}", + content +); +var body = await response.Content.ReadAsStringAsync(); +var session = JsonSerializer.Deserialize(body); + +var stopUrl = session.GetProperty("stop").GetString(); +Console.WriteLine($"Session created: {session.GetProperty("id")}"); + +var closeResponse = await client.DeleteAsync($"{stopUrl}&force=true"); +Console.WriteLine($"Session closed: {closeResponse.StatusCode}"); diff --git a/examples/close-session/rest/curl/close-session.sh b/examples/close-session/rest/curl/close-session.sh new file mode 100644 index 0000000..937d804 --- /dev/null +++ b/examples/close-session/rest/curl/close-session.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Creates a Browserless session and immediately closes it. +# Run: bash close-session.sh + +TOKEN="YOUR_API_TOKEN_HERE" + +# Step 1: Create a session +SESSION=$(curl -s -X POST \ + "https://production-sfo.browserless.io/session?token=${TOKEN}" \ + -H "Content-Type: application/json" \ + -d '{"ttl": 60000, "stealth": true}') + +echo "Session created: $SESSION" + +# Step 2: Extract the stop URL and close the session +STOP_URL=$(echo "$SESSION" | python3 -c "import sys,json; print(json.load(sys.stdin)['stop'])") +curl -X DELETE "${STOP_URL}&force=true" +echo "Session closed." diff --git a/examples/close-session/rest/java/CloseSession.java b/examples/close-session/rest/java/CloseSession.java new file mode 100644 index 0000000..8ef41c4 --- /dev/null +++ b/examples/close-session/rest/java/CloseSession.java @@ -0,0 +1,41 @@ +// Creates a Browserless session and closes it via the Session API. +// +// Run: javac CloseSession.java && java CloseSession + +import java.net.URI; +import java.net.http.*; + +public class CloseSession { + public static void main(String[] args) throws Exception { + String token = "YOUR_API_TOKEN_HERE"; + String endpoint = "https://production-sfo.browserless.io/session?token=" + token; + + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(endpoint)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString( + "{\"ttl\": 60000, \"stealth\": true}" + )) + .build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + String body = response.body(); + System.out.println("Session created: " + body); + + // Parse stop URL from response + int stopStart = body.indexOf("\"stop\":\"") + 8; + int stopEnd = body.indexOf("\"", stopStart); + String stopUrl = body.substring(stopStart, stopEnd); + + HttpRequest deleteRequest = HttpRequest.newBuilder() + .uri(URI.create(stopUrl + "&force=true")) + .DELETE() + .build(); + + HttpResponse deleteResponse = client.send( + deleteRequest, HttpResponse.BodyHandlers.ofString() + ); + System.out.println("Session closed: " + deleteResponse.statusCode()); + } +} diff --git a/examples/close-session/rest/nodejs/close-session.mjs b/examples/close-session/rest/nodejs/close-session.mjs new file mode 100644 index 0000000..38caaba --- /dev/null +++ b/examples/close-session/rest/nodejs/close-session.mjs @@ -0,0 +1,21 @@ +// Creates a Browserless session and closes it via the Session API. +// +// Run: node close-session.mjs + +const TOKEN = "YOUR_API_TOKEN_HERE"; + +const sessionResponse = await fetch( + `https://production-sfo.browserless.io/session?token=${TOKEN}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ ttl: 60000, stealth: true }), + } +); +const session = await sessionResponse.json(); +console.log("Session created:", session.id); + +const closeResponse = await fetch(`${session.stop}&force=true`, { + method: "DELETE", +}); +console.log("Session closed:", closeResponse.status === 200 ? "success" : "failed"); diff --git a/examples/close-session/rest/python/close_session.py b/examples/close-session/rest/python/close_session.py new file mode 100644 index 0000000..e9eab3f --- /dev/null +++ b/examples/close-session/rest/python/close_session.py @@ -0,0 +1,18 @@ +# Creates a Browserless session and closes it via the Session API. +# +# Install: pip install requests +# Run: python close_session.py + +import requests + +TOKEN = "YOUR_API_TOKEN_HERE" + +session_response = requests.post( + f"https://production-sfo.browserless.io/session?token={TOKEN}", + json={"ttl": 60000, "stealth": True}, +) +session = session_response.json() +print("Session created:", session["id"]) + +close_response = requests.delete(f"{session['stop']}&force=true") +print("Session closed:", "success" if close_response.ok else "failed") diff --git a/examples/e2e-testing/bql/e2e-testing.graphql b/examples/e2e-testing/bql/e2e-testing.graphql new file mode 100644 index 0000000..c5cd971 --- /dev/null +++ b/examples/e2e-testing/bql/e2e-testing.graphql @@ -0,0 +1,14 @@ +# End-to-end test using BQL to verify page behavior. +mutation E2ETest { + goto(url: "https://automationexercise.com", waitUntil: networkIdle) { + status + } + + title { + title + } + + verify: text(selector: "h2") { + text + } +} diff --git a/examples/e2e-testing/frameworks/playwright/csharp/ExampleTests.cs b/examples/e2e-testing/frameworks/playwright/csharp/ExampleTests.cs new file mode 100644 index 0000000..54e04d3 --- /dev/null +++ b/examples/e2e-testing/frameworks/playwright/csharp/ExampleTests.cs @@ -0,0 +1,58 @@ +// Playwright E2E tests running against a remote Browserless browser. +// Install: dotnet add package Microsoft.Playwright.NUnit +// Run: dotnet test + +using Microsoft.Playwright; +using NUnit.Framework; + +[TestFixture] +public class ExampleTests +{ + private IPlaywright _playwright; + private IBrowser _browser; + private IBrowserContext _context; + private IPage _page; + + [OneTimeSetUp] + public async Task Setup() + { + string token = "YOUR_API_TOKEN_HERE"; + _playwright = await Playwright.CreateAsync(); + _browser = await _playwright.Chromium.ConnectAsync( + $"wss://production-sfo.browserless.io/chromium/playwright?token={token}" + ); + } + + [SetUp] + public async Task CreatePage() + { + _context = await _browser.NewContextAsync(); + _page = await _context.NewPageAsync(); + } + + [TearDown] + public async Task ClosePage() => await _context.CloseAsync(); + + [OneTimeTearDown] + public async Task Teardown() + { + await _browser.CloseAsync(); + _playwright.Dispose(); + } + + [Test] + public async Task HomepageLoads() + { + await _page.GotoAsync("https://automationexercise.com"); + StringAssert.Contains("Automation", await _page.TitleAsync()); + } + + [Test] + public async Task ProductsPageShowsItems() + { + await _page.GotoAsync("https://automationexercise.com/products"); + await _page.WaitForSelectorAsync(".features_items"); + var products = await _page.QuerySelectorAllAsync(".product-image-wrapper"); + Assert.That(products.Count, Is.GreaterThan(0)); + } +} diff --git a/examples/e2e-testing/frameworks/playwright/java/ExampleTest.java b/examples/e2e-testing/frameworks/playwright/java/ExampleTest.java new file mode 100644 index 0000000..193b415 --- /dev/null +++ b/examples/e2e-testing/frameworks/playwright/java/ExampleTest.java @@ -0,0 +1,54 @@ +// Playwright E2E tests running against a remote Browserless browser. +// Requires: com.microsoft.playwright:playwright in pom.xml or build.gradle +// Run: mvn test (or gradle test) + +import com.microsoft.playwright.*; +import org.junit.jupiter.api.*; +import static org.junit.jupiter.api.Assertions.*; + +public class ExampleTest { + static Playwright playwright; + static Browser browser; + BrowserContext context; + Page page; + + @BeforeAll + static void launchBrowser() { + String token = "YOUR_API_TOKEN_HERE"; + playwright = Playwright.create(); + browser = playwright.chromium().connect( + "wss://production-sfo.browserless.io/chromium/playwright?token=" + token + ); + } + + @BeforeEach + void createContextAndPage() { + context = browser.newContext(); + page = context.newPage(); + } + + @AfterEach + void closeContext() { + context.close(); + } + + @AfterAll + static void closeBrowser() { + browser.close(); + playwright.close(); + } + + @Test + void homepageLoads() { + page.navigate("https://automationexercise.com"); + assertTrue(page.title().contains("Automation"), "Title should contain 'Automation'"); + } + + @Test + void productsPageShowsItems() { + page.navigate("https://automationexercise.com/products"); + page.waitForSelector(".features_items"); + int count = page.querySelectorAll(".product-image-wrapper").size(); + assertTrue(count > 0, "Should find at least one product"); + } +} diff --git a/examples/e2e-testing/frameworks/playwright/nodejs/playwright.config.js b/examples/e2e-testing/frameworks/playwright/nodejs/playwright.config.js new file mode 100644 index 0000000..f873f9e --- /dev/null +++ b/examples/e2e-testing/frameworks/playwright/nodejs/playwright.config.js @@ -0,0 +1,16 @@ +// Playwright configuration for running tests against a remote Browserless browser. +// Run: npx playwright test + +import { defineConfig } from '@playwright/test'; + +const TOKEN = process.env.BROWSERLESS_TOKEN ?? 'YOUR_API_TOKEN_HERE'; + +export default defineConfig({ + testDir: './tests', + timeout: 60000, + use: { + connectOptions: { + wsEndpoint: `wss://production-sfo.browserless.io/chromium/playwright?token=${TOKEN}`, + }, + }, +}); diff --git a/examples/e2e-testing/frameworks/playwright/nodejs/tests/example.spec.js b/examples/e2e-testing/frameworks/playwright/nodejs/tests/example.spec.js new file mode 100644 index 0000000..5734b3d --- /dev/null +++ b/examples/e2e-testing/frameworks/playwright/nodejs/tests/example.spec.js @@ -0,0 +1,27 @@ +// Playwright test spec running against a remote Browserless browser. +// Run: npx playwright test + +import { test, expect } from '@playwright/test'; + +test('homepage loads and shows products', async ({ page }) => { + await page.goto('https://automationexercise.com', { waitUntil: 'networkidle' }); + + await expect(page).toHaveTitle(/Automation/); + + await page.click('a[href="/products"]'); + await page.waitForSelector('.features_items'); + + const products = page.locator('.product-image-wrapper'); + await expect(products).toHaveCount(await products.count()); + expect(await products.count()).toBeGreaterThan(0); +}); + +test('can add item to cart', async ({ page }) => { + await page.goto('https://automationexercise.com/products', { waitUntil: 'networkidle' }); + + await page.hover('.product-image-wrapper:first-child'); + await page.click('.product-image-wrapper:first-child .add-to-cart'); + await page.waitForSelector('#cartModal'); + + await expect(page.locator('#cartModal')).toBeVisible(); +}); diff --git a/examples/e2e-testing/frameworks/playwright/python/conftest.py b/examples/e2e-testing/frameworks/playwright/python/conftest.py new file mode 100644 index 0000000..d534652 --- /dev/null +++ b/examples/e2e-testing/frameworks/playwright/python/conftest.py @@ -0,0 +1,25 @@ +# Pytest fixture for connecting Playwright to a remote Browserless browser. +# Install: pip install pytest-playwright +# Run: pytest + +import pytest +from playwright.sync_api import sync_playwright + +TOKEN = "YOUR_API_TOKEN_HERE" +WS_ENDPOINT = f"wss://production-sfo.browserless.io/chromium/playwright?token={TOKEN}" + + +@pytest.fixture(scope="session") +def browser(): + with sync_playwright() as p: + browser = p.chromium.connect(WS_ENDPOINT) + yield browser + browser.close() + + +@pytest.fixture +def page(browser): + context = browser.new_context() + page = context.new_page() + yield page + context.close() diff --git a/examples/e2e-testing/frameworks/playwright/python/tests/test_example.py b/examples/e2e-testing/frameworks/playwright/python/tests/test_example.py new file mode 100644 index 0000000..8e21a5a --- /dev/null +++ b/examples/e2e-testing/frameworks/playwright/python/tests/test_example.py @@ -0,0 +1,22 @@ +# Playwright E2E tests running against a remote Browserless browser. +# Install: pip install pytest-playwright +# Run: pytest + +def test_homepage_loads(page): + page.goto("https://automationexercise.com", wait_until="networkidle") + assert "Automation" in page.title() + + +def test_products_page(page): + page.goto("https://automationexercise.com/products", wait_until="networkidle") + page.wait_for_selector(".features_items") + products = page.query_selector_all(".product-image-wrapper") + assert len(products) > 0, "No products found" + + +def test_add_to_cart(page): + page.goto("https://automationexercise.com/products", wait_until="networkidle") + page.hover(".product-image-wrapper:first-child") + page.click(".product-image-wrapper:first-child .add-to-cart") + page.wait_for_selector("#cartModal") + assert page.is_visible("#cartModal"), "Cart modal not shown" diff --git a/examples/e2e-testing/frameworks/puppeteer/e2e-testing.mjs b/examples/e2e-testing/frameworks/puppeteer/e2e-testing.mjs new file mode 100644 index 0000000..89ee74d --- /dev/null +++ b/examples/e2e-testing/frameworks/puppeteer/e2e-testing.mjs @@ -0,0 +1,40 @@ +// End-to-end test using Puppeteer connected to a remote Browserless browser. +// +// Install: npm install puppeteer-core +// Run: node e2e-testing.mjs + +import puppeteer from 'puppeteer-core'; + +const TOKEN = 'YOUR_API_TOKEN_HERE'; + +const browser = await puppeteer.connect({ + browserWSEndpoint: `wss://production-sfo.browserless.io?token=${TOKEN}`, +}); + +try { + const page = await browser.newPage(); + await page.goto('https://automationexercise.com', { waitUntil: 'networkidle2' }); + + const title = await page.title(); + console.assert(title.includes('Automation'), `Title check failed: ${title}`); + + // Test: navigate to products page + await page.click('a[href="/products"]'); + await page.waitForSelector('.features_items'); + + const productCount = await page.$$eval('.product-image-wrapper', (els) => els.length); + console.assert(productCount > 0, 'No products found'); + + // Test: add first product to cart + await page.hover('.product-image-wrapper:first-child'); + await page.click('.product-image-wrapper:first-child .add-to-cart'); + await page.waitForSelector('#cartModal'); + + const modalVisible = await page.$('#cartModal'); + console.assert(modalVisible, 'Cart modal not shown'); + + console.log('All E2E tests passed.'); + console.log(`Products found: ${productCount}`); +} finally { + await browser.close(); +} diff --git a/examples/e2e-testing/rest/csharp/E2ETesting.cs b/examples/e2e-testing/rest/csharp/E2ETesting.cs new file mode 100644 index 0000000..a1016ce --- /dev/null +++ b/examples/e2e-testing/rest/csharp/E2ETesting.cs @@ -0,0 +1,39 @@ +// Runs an end-to-end test against a remote Browserless browser using BQL. +// +// Run: dotnet run + +using System.Net.Http; +using System.Text; +using System.Text.Json; + +string token = "YOUR_API_TOKEN_HERE"; +string endpoint = $"https://production-sfo.browserless.io/bql?token={token}"; + +var payload = new +{ + query = @"mutation E2ETest { + goto(url: ""https://automationexercise.com"", waitUntil: networkIdle) { status } + title { title } + verify: text(selector: ""h2"") { text } + }", + variables = new { }, + operationName = "E2ETest", +}; + +using (HttpClient httpClient = new HttpClient()) +{ + var content = new StringContent( + JsonSerializer.Serialize(payload), Encoding.UTF8, "application/json"); + var response = await httpClient.PostAsync(endpoint, content); + string body = await response.Content.ReadAsStringAsync(); + + var doc = JsonDocument.Parse(body); + int status = doc.RootElement.GetProperty("data").GetProperty("goto").GetProperty("status").GetInt32(); + if (status != 200) throw new Exception($"Expected status 200, got {status}"); + + string title = doc.RootElement.GetProperty("data").GetProperty("title").GetProperty("title").GetString() ?? ""; + if (!title.Contains("Automation")) throw new Exception($"Title check failed: {title}"); + + Console.WriteLine("E2E test passed."); + Console.WriteLine($"Page title: {title}"); +} diff --git a/examples/e2e-testing/rest/curl/e2e-testing.sh b/examples/e2e-testing/rest/curl/e2e-testing.sh new file mode 100644 index 0000000..d2db6b2 --- /dev/null +++ b/examples/e2e-testing/rest/curl/e2e-testing.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Runs an end-to-end test against a remote Browserless browser using BQL. +# Run: bash e2e-testing.sh + +curl -X POST \ + "https://production-sfo.browserless.io/bql?token=YOUR_API_TOKEN_HERE" \ + -H "Content-Type: application/json" \ + -d '{ + "query": "mutation E2ETest { goto(url: \"https://automationexercise.com\", waitUntil: networkIdle) { status } title { title } verify: text(selector: \"h2\") { text } }", + "variables": {}, + "operationName": "E2ETest" + }' diff --git a/examples/e2e-testing/rest/java/E2ETesting.java b/examples/e2e-testing/rest/java/E2ETesting.java new file mode 100644 index 0000000..7c60921 --- /dev/null +++ b/examples/e2e-testing/rest/java/E2ETesting.java @@ -0,0 +1,37 @@ +// Runs an end-to-end test against a remote Browserless browser using BQL. +// +// Run: javac E2ETesting.java && java E2ETesting + +import java.net.URI; +import java.net.http.*; + +public class E2ETesting { + public static void main(String[] args) throws Exception { + String token = "YOUR_API_TOKEN_HERE"; + String endpoint = "https://production-sfo.browserless.io/bql?token=" + token; + + String query = "mutation E2ETest {" + + " goto(url: \\\"https://automationexercise.com\\\", waitUntil: networkIdle) { status }" + + " title { title }" + + " verify: text(selector: \\\"h2\\\") { text }" + + " }"; + + String payload = "{\"query\": \"" + query + "\", \"variables\": {}, \"operationName\": \"E2ETest\"}"; + + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(endpoint)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(payload)) + .build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + String body = response.body(); + System.out.println(body); + + if (!body.contains("\"status\":200")) { + throw new AssertionError("Expected status 200"); + } + System.out.println("E2E test passed."); + } +} diff --git a/examples/e2e-testing/rest/nodejs/e2e-testing.mjs b/examples/e2e-testing/rest/nodejs/e2e-testing.mjs new file mode 100644 index 0000000..45e721e --- /dev/null +++ b/examples/e2e-testing/rest/nodejs/e2e-testing.mjs @@ -0,0 +1,36 @@ +// Runs an end-to-end test against a remote Browserless browser using BQL. +// +// Run: node e2e-testing.mjs + +const query = `mutation E2ETest { + goto(url: "https://automationexercise.com", waitUntil: networkIdle) { + status + } + title { + title + } + verify: text(selector: "h2") { + text + } +}`; + +const response = await fetch( + 'https://production-sfo.browserless.io/bql?token=YOUR_API_TOKEN_HERE', + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ query, variables: {}, operationName: 'E2ETest' }), + } +); + +const { data } = await response.json(); + +console.assert(data.goto.status === 200, `Expected status 200, got ${data.goto.status}`); +console.assert( + data.title.title.includes('Automation'), + `Expected title to include "Automation", got: ${data.title.title}` +); + +console.log('All assertions passed.'); +console.log('Page title:', data.title.title); +console.log('H2 text:', data.verify.text); diff --git a/examples/e2e-testing/rest/python/e2e_testing.py b/examples/e2e-testing/rest/python/e2e_testing.py new file mode 100644 index 0000000..488f1d5 --- /dev/null +++ b/examples/e2e-testing/rest/python/e2e_testing.py @@ -0,0 +1,35 @@ +# Runs an end-to-end test against a remote Browserless browser using BQL. +# +# Install: pip install requests +# Run: python e2e_testing.py + +import requests + +query = """ +mutation E2ETest { + goto(url: "https://automationexercise.com", waitUntil: networkIdle) { + status + } + title { + title + } + verify: text(selector: "h2") { + text + } +} +""" + +response = requests.post( + 'https://production-sfo.browserless.io/bql', + params={'token': 'YOUR_API_TOKEN_HERE'}, + json={'query': query, 'variables': {}, 'operationName': 'E2ETest'}, +) + +data = response.json()['data'] + +assert data['goto']['status'] == 200, f"Expected status 200, got {data['goto']['status']}" +assert 'Automation' in data['title']['title'], f"Title check failed: {data['title']['title']}" + +print('All assertions passed.') +print('Page title:', data['title']['title']) +print('H2 text:', data['verify']['text']) diff --git a/examples/export-slide-deck/bql/export-slide-deck.graphql b/examples/export-slide-deck/bql/export-slide-deck.graphql new file mode 100644 index 0000000..5df0d44 --- /dev/null +++ b/examples/export-slide-deck/bql/export-slide-deck.graphql @@ -0,0 +1,14 @@ +# Exports a Google Slides presentation as a PDF. +# Replace YOUR_PRESENTATION_ID with your actual presentation ID. +mutation ExportSlideDeck { + goto( + url: "https://docs.google.com/presentation/d/YOUR_PRESENTATION_ID/export/pdf" + waitUntil: networkIdle + ) { + status + } + + pdf(options: { printBackground: true, format: "A4", landscape: true }) { + base64 + } +} diff --git a/examples/export-slide-deck/frameworks/playwright/nodejs/export-slide-deck.mjs b/examples/export-slide-deck/frameworks/playwright/nodejs/export-slide-deck.mjs new file mode 100644 index 0000000..b13ffe4 --- /dev/null +++ b/examples/export-slide-deck/frameworks/playwright/nodejs/export-slide-deck.mjs @@ -0,0 +1,34 @@ +// Exports a Google Slides presentation as a PDF using Playwright. +// +// Install: npm install playwright-core +// Run: node export-slide-deck.mjs + +import { chromium } from 'playwright-core'; +import { writeFileSync } from 'fs'; + +const TOKEN = 'YOUR_API_TOKEN_HERE'; +const PRESENTATION_ID = 'YOUR_PRESENTATION_ID'; + +const browser = await chromium.connectOverCDP( + `wss://production-sfo.browserless.io?token=${TOKEN}` +); + +try { + const context = browser.contexts()[0] ?? await browser.newContext(); + const page = await context.newPage(); + await page.goto( + `https://docs.google.com/presentation/d/${PRESENTATION_ID}/pub?start=false&loop=false&delayms=3000`, + { waitUntil: 'networkidle' } + ); + + const pdf = await page.pdf({ + format: 'A4', + landscape: true, + printBackground: true, + }); + + writeFileSync('slide-deck.pdf', pdf); + console.log('Exported to slide-deck.pdf'); +} finally { + await browser.close(); +} diff --git a/examples/export-slide-deck/frameworks/puppeteer/export-slide-deck.mjs b/examples/export-slide-deck/frameworks/puppeteer/export-slide-deck.mjs new file mode 100644 index 0000000..989bf04 --- /dev/null +++ b/examples/export-slide-deck/frameworks/puppeteer/export-slide-deck.mjs @@ -0,0 +1,33 @@ +// Exports a Google Slides presentation as a PDF using Puppeteer. +// +// Install: npm install puppeteer-core +// Run: node export-slide-deck.mjs + +import puppeteer from 'puppeteer-core'; + +const TOKEN = 'YOUR_API_TOKEN_HERE'; +const PRESENTATION_ID = 'YOUR_PRESENTATION_ID'; + +const browser = await puppeteer.connect({ + browserWSEndpoint: `wss://production-sfo.browserless.io?token=${TOKEN}`, +}); + +try { + const page = await browser.newPage(); + await page.goto( + `https://docs.google.com/presentation/d/${PRESENTATION_ID}/pub?start=false&loop=false&delayms=3000`, + { waitUntil: 'networkidle2' } + ); + + const pdf = await page.pdf({ + format: 'A4', + landscape: true, + printBackground: true, + }); + + import { writeFileSync } from 'fs'; + writeFileSync('slide-deck.pdf', pdf); + console.log('Exported to slide-deck.pdf'); +} finally { + await browser.close(); +} diff --git a/examples/export-slide-deck/rest/csharp/ExportSlideDeck.cs b/examples/export-slide-deck/rest/csharp/ExportSlideDeck.cs new file mode 100644 index 0000000..e6aac2d --- /dev/null +++ b/examples/export-slide-deck/rest/csharp/ExportSlideDeck.cs @@ -0,0 +1,27 @@ +// Exports a Google Slides presentation as a PDF using Browserless. +// +// Run: dotnet run + +using System.Net.Http; +using System.Text; +using System.Text.Json; + +string token = "YOUR_API_TOKEN_HERE"; +string presentationId = "YOUR_PRESENTATION_ID"; + +var payload = new +{ + url = $"https://docs.google.com/presentation/d/{presentationId}/export/pdf", + options = new { printBackground = true, format = "A4", landscape = true }, +}; + +using (HttpClient httpClient = new HttpClient()) +{ + var content = new StringContent( + JsonSerializer.Serialize(payload), Encoding.UTF8, "application/json"); + var response = await httpClient.PostAsync( + $"https://production-sfo.browserless.io/pdf?token={token}", content); + var bytes = await response.Content.ReadAsByteArrayAsync(); + await File.WriteAllBytesAsync("slide-deck.pdf", bytes); + Console.WriteLine("Exported to slide-deck.pdf"); +} diff --git a/examples/export-slide-deck/rest/curl/export-slide-deck.sh b/examples/export-slide-deck/rest/curl/export-slide-deck.sh new file mode 100644 index 0000000..2e95657 --- /dev/null +++ b/examples/export-slide-deck/rest/curl/export-slide-deck.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Exports a Google Slides presentation as a PDF using Browserless. +# Run: bash export-slide-deck.sh + +curl -X POST \ + "https://production-sfo.browserless.io/pdf?token=YOUR_API_TOKEN_HERE" \ + -H "Content-Type: application/json" \ + -d '{ + "url": "https://docs.google.com/presentation/d/YOUR_PRESENTATION_ID/export/pdf", + "options": { + "printBackground": true, + "format": "A4", + "landscape": true + } + }' \ + --output slide-deck.pdf + +echo "Exported to slide-deck.pdf" diff --git a/examples/export-slide-deck/rest/java/ExportSlideDeck.java b/examples/export-slide-deck/rest/java/ExportSlideDeck.java new file mode 100644 index 0000000..eb3e4ab --- /dev/null +++ b/examples/export-slide-deck/rest/java/ExportSlideDeck.java @@ -0,0 +1,31 @@ +// Exports a Google Slides presentation as a PDF using Browserless. +// +// Run: javac ExportSlideDeck.java && java ExportSlideDeck + +import java.net.URI; +import java.net.http.*; +import java.nio.file.*; + +public class ExportSlideDeck { + public static void main(String[] args) throws Exception { + String token = "YOUR_API_TOKEN_HERE"; + String presentationId = "YOUR_PRESENTATION_ID"; + String endpoint = "https://production-sfo.browserless.io/pdf?token=" + token; + + String payload = "{" + + "\"url\": \"https://docs.google.com/presentation/d/" + presentationId + "/export/pdf\"," + + "\"options\": {\"printBackground\": true, \"format\": \"A4\", \"landscape\": true}" + + "}"; + + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(endpoint)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(payload)) + .build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofByteArray()); + Files.write(Path.of("slide-deck.pdf"), response.body()); + System.out.println("Exported to slide-deck.pdf"); + } +} diff --git a/examples/export-slide-deck/rest/nodejs/export-slide-deck.mjs b/examples/export-slide-deck/rest/nodejs/export-slide-deck.mjs new file mode 100644 index 0000000..0168b6b --- /dev/null +++ b/examples/export-slide-deck/rest/nodejs/export-slide-deck.mjs @@ -0,0 +1,28 @@ +// Exports a Google Slides presentation as a PDF using Browserless. +// +// Run: node export-slide-deck.mjs + +import { writeFileSync } from 'fs'; + +const PRESENTATION_ID = 'YOUR_PRESENTATION_ID'; +const TOKEN = 'YOUR_API_TOKEN_HERE'; + +const response = await fetch( + `https://production-sfo.browserless.io/pdf?token=${TOKEN}`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + url: `https://docs.google.com/presentation/d/${PRESENTATION_ID}/export/pdf`, + options: { + printBackground: true, + format: 'A4', + landscape: true, + }, + }), + } +); + +const buffer = await response.arrayBuffer(); +writeFileSync('slide-deck.pdf', Buffer.from(buffer)); +console.log('Exported to slide-deck.pdf'); diff --git a/examples/export-slide-deck/rest/python/export_slide_deck.py b/examples/export-slide-deck/rest/python/export_slide_deck.py new file mode 100644 index 0000000..02ce205 --- /dev/null +++ b/examples/export-slide-deck/rest/python/export_slide_deck.py @@ -0,0 +1,26 @@ +# Exports a Google Slides presentation as a PDF using Browserless. +# +# Install: pip install requests +# Run: python export_slide_deck.py + +import requests + +PRESENTATION_ID = 'YOUR_PRESENTATION_ID' +TOKEN = 'YOUR_API_TOKEN_HERE' + +response = requests.post( + f'https://production-sfo.browserless.io/pdf?token={TOKEN}', + json={ + 'url': f'https://docs.google.com/presentation/d/{PRESENTATION_ID}/export/pdf', + 'options': { + 'printBackground': True, + 'format': 'A4', + 'landscape': True, + }, + }, +) + +with open('slide-deck.pdf', 'wb') as f: + f.write(response.content) + +print('Exported to slide-deck.pdf') diff --git a/examples/navigate-after-click/bql/navigate-after-click.graphql b/examples/navigate-after-click/bql/navigate-after-click.graphql new file mode 100644 index 0000000..57fb878 --- /dev/null +++ b/examples/navigate-after-click/bql/navigate-after-click.graphql @@ -0,0 +1,19 @@ +# Clicks a link and waits for navigation to complete. +# The waitForNavigation flag on click ensures the mutation waits for the new page to load. +mutation NavigateAfterClick { + goto(url: "https://example.com", waitUntil: networkIdle) { + status + } + + click(selector: "a", waitForNavigation: true) { + time + } + + title { + title + } + + currentURL { + url + } +} diff --git a/examples/navigate-after-click/frameworks/playwright/nodejs/navigate-after-click.mjs b/examples/navigate-after-click/frameworks/playwright/nodejs/navigate-after-click.mjs new file mode 100644 index 0000000..1a6a647 --- /dev/null +++ b/examples/navigate-after-click/frameworks/playwright/nodejs/navigate-after-click.mjs @@ -0,0 +1,29 @@ +// Clicks a link and waits for navigation to complete using Playwright. +// Playwright automatically waits for navigation after click — no special handling needed. +// +// Install: npm install playwright-core +// Run: node navigate-after-click.mjs + +import { chromium } from 'playwright-core'; + +const TOKEN = 'YOUR_API_TOKEN_HERE'; + +const browser = await chromium.connectOverCDP( + `wss://production-sfo.browserless.io?token=${TOKEN}` +); + +try { + const context = browser.contexts()[0] ?? await browser.newContext(); + const page = await context.newPage(); + + await page.goto('https://example.com', { waitUntil: 'networkidle' }); + + // Playwright waits for navigation automatically after click. + await page.click('a'); + await page.waitForLoadState('networkidle'); + + console.log('Navigated to:', page.url()); + console.log('Page title:', await page.title()); +} finally { + await browser.close(); +} diff --git a/examples/navigate-after-click/frameworks/puppeteer/navigate-after-click.mjs b/examples/navigate-after-click/frameworks/puppeteer/navigate-after-click.mjs new file mode 100644 index 0000000..ecd28cd --- /dev/null +++ b/examples/navigate-after-click/frameworks/puppeteer/navigate-after-click.mjs @@ -0,0 +1,28 @@ +// Clicks a link and waits for navigation to complete using Puppeteer. +// +// Install: npm install puppeteer-core +// Run: node navigate-after-click.mjs + +import puppeteer from 'puppeteer-core'; + +const TOKEN = 'YOUR_API_TOKEN_HERE'; + +const browser = await puppeteer.connect({ + browserWSEndpoint: `wss://production-sfo.browserless.io?token=${TOKEN}`, +}); + +try { + const page = await browser.newPage(); + await page.goto('https://example.com', { waitUntil: 'networkidle2' }); + + // Click the link and wait for navigation in parallel. + await Promise.all([ + page.waitForNavigation({ waitUntil: 'networkidle2' }), + page.click('a'), + ]); + + console.log('Navigated to:', page.url()); + console.log('Page title:', await page.title()); +} finally { + await browser.close(); +} diff --git a/examples/navigate-after-click/rest/csharp/NavigateAfterClick.cs b/examples/navigate-after-click/rest/csharp/NavigateAfterClick.cs new file mode 100644 index 0000000..54ab8ef --- /dev/null +++ b/examples/navigate-after-click/rest/csharp/NavigateAfterClick.cs @@ -0,0 +1,31 @@ +// Clicks a link and waits for navigation to complete using BQL. +// +// Run: dotnet run + +using System.Net.Http; +using System.Text; +using System.Text.Json; + +string token = "YOUR_API_TOKEN_HERE"; +string endpoint = $"https://production-sfo.browserless.io/bql?token={token}"; + +var payload = new +{ + query = @"mutation NavigateAfterClick { + goto(url: ""https://example.com"", waitUntil: networkIdle) { status } + click(selector: ""a"", waitForNavigation: true) { time } + title { title } + currentURL { url } + }", + variables = new { }, + operationName = "NavigateAfterClick", +}; + +using (HttpClient httpClient = new HttpClient()) +{ + var content = new StringContent( + JsonSerializer.Serialize(payload), Encoding.UTF8, "application/json"); + var response = await httpClient.PostAsync(endpoint, content); + string body = await response.Content.ReadAsStringAsync(); + Console.WriteLine(body); +} diff --git a/examples/navigate-after-click/rest/curl/navigate-after-click.sh b/examples/navigate-after-click/rest/curl/navigate-after-click.sh new file mode 100644 index 0000000..8401ebd --- /dev/null +++ b/examples/navigate-after-click/rest/curl/navigate-after-click.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Clicks a link and waits for navigation to complete using BQL. +# Run: bash navigate-after-click.sh + +curl -X POST \ + "https://production-sfo.browserless.io/bql?token=YOUR_API_TOKEN_HERE" \ + -H "Content-Type: application/json" \ + -d '{ + "query": "mutation NavigateAfterClick { goto(url: \"https://example.com\", waitUntil: networkIdle) { status } click(selector: \"a\", waitForNavigation: true) { time } title { title } currentURL { url } }", + "variables": {}, + "operationName": "NavigateAfterClick" + }' diff --git a/examples/navigate-after-click/rest/java/NavigateAfterClick.java b/examples/navigate-after-click/rest/java/NavigateAfterClick.java new file mode 100644 index 0000000..53edf7a --- /dev/null +++ b/examples/navigate-after-click/rest/java/NavigateAfterClick.java @@ -0,0 +1,32 @@ +// Clicks a link and waits for navigation to complete using BQL. +// +// Run: javac NavigateAfterClick.java && java NavigateAfterClick + +import java.net.URI; +import java.net.http.*; + +public class NavigateAfterClick { + public static void main(String[] args) throws Exception { + String token = "YOUR_API_TOKEN_HERE"; + String endpoint = "https://production-sfo.browserless.io/bql?token=" + token; + + String query = "mutation NavigateAfterClick {" + + " goto(url: \\\"https://example.com\\\", waitUntil: networkIdle) { status }" + + " click(selector: \\\"a\\\", waitForNavigation: true) { time }" + + " title { title }" + + " currentURL { url }" + + " }"; + + String payload = "{\"query\": \"" + query + "\", \"variables\": {}, \"operationName\": \"NavigateAfterClick\"}"; + + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(endpoint)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(payload)) + .build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + System.out.println(response.body()); + } +} diff --git a/examples/navigate-after-click/rest/nodejs/navigate-after-click.mjs b/examples/navigate-after-click/rest/nodejs/navigate-after-click.mjs new file mode 100644 index 0000000..296b64e --- /dev/null +++ b/examples/navigate-after-click/rest/nodejs/navigate-after-click.mjs @@ -0,0 +1,31 @@ +// Clicks a link and waits for navigation to complete using BQL. +// +// Run: node navigate-after-click.mjs + +const query = `mutation NavigateAfterClick { + goto(url: "https://example.com", waitUntil: networkIdle) { + status + } + click(selector: "a", waitForNavigation: true) { + time + } + title { + title + } + currentURL { + url + } +}`; + +const response = await fetch( + 'https://production-sfo.browserless.io/bql?token=YOUR_API_TOKEN_HERE', + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ query, variables: {}, operationName: 'NavigateAfterClick' }), + } +); + +const { data } = await response.json(); +console.log('Navigated to:', data.currentURL.url); +console.log('Page title:', data.title.title); diff --git a/examples/navigate-after-click/rest/python/navigate_after_click.py b/examples/navigate-after-click/rest/python/navigate_after_click.py new file mode 100644 index 0000000..2e2c42b --- /dev/null +++ b/examples/navigate-after-click/rest/python/navigate_after_click.py @@ -0,0 +1,33 @@ +# Clicks a link and waits for navigation to complete using BQL. +# +# Install: pip install requests +# Run: python navigate_after_click.py + +import requests + +query = """ +mutation NavigateAfterClick { + goto(url: "https://example.com", waitUntil: networkIdle) { + status + } + click(selector: "a", waitForNavigation: true) { + time + } + title { + title + } + currentURL { + url + } +} +""" + +response = requests.post( + 'https://production-sfo.browserless.io/bql', + params={'token': 'YOUR_API_TOKEN_HERE'}, + json={'query': query, 'variables': {}, 'operationName': 'NavigateAfterClick'}, +) + +data = response.json()['data'] +print('Navigated to:', data['currentURL']['url']) +print('Page title:', data['title']['title']) diff --git a/examples/persist-session/bql/persist-session.graphql b/examples/persist-session/bql/persist-session.graphql new file mode 100644 index 0000000..3e70eab --- /dev/null +++ b/examples/persist-session/bql/persist-session.graphql @@ -0,0 +1,17 @@ +# Step 1: Create a session via the REST API first, then use the returned browserQL endpoint. +# This mutation sets localStorage state — run it against the session's browserQL endpoint. +mutation SetState { + goto(url: "https://automationexercise.com", waitUntil: networkIdle) { + status + } + + evaluate( + expression: """ + localStorage.setItem('shoppingCart', JSON.stringify({items: [{id: 1}], totalItems: 1})); + localStorage.setItem('userPreferences', JSON.stringify({theme: 'dark'})); + 'state set' + """ + ) { + value + } +} diff --git a/examples/persist-session/frameworks/playwright/nodejs/persist-session.mjs b/examples/persist-session/frameworks/playwright/nodejs/persist-session.mjs new file mode 100644 index 0000000..ca61020 --- /dev/null +++ b/examples/persist-session/frameworks/playwright/nodejs/persist-session.mjs @@ -0,0 +1,59 @@ +// Creates a long-lived Browserless session, connects via Playwright to set state, +// disconnects, then reconnects and verifies state persisted. +// +// Install: npm install playwright-core +// Run: node persist-session.mjs + +import { chromium } from "playwright-core"; + +const TOKEN = "YOUR_API_TOKEN_HERE"; + +// Create a long-lived session +const sessionResponse = await fetch( + `https://production-sfo.browserless.io/session?token=${TOKEN}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ ttl: 300000, stealth: true }), + } +); +const session = await sessionResponse.json(); + +// Connect and set browser state +const browser = await chromium.connectOverCDP(session.connect); +try { + const context = browser.contexts()[0] ?? await browser.newContext(); + const page = await context.newPage(); + await page.goto("https://automationexercise.com", { waitUntil: "networkidle" }); + + await page.evaluate(() => { + localStorage.setItem( + "shoppingCart", + JSON.stringify({ items: [{ id: 1, name: "Blue Top" }], totalItems: 1, totalPrice: 500 }) + ); + localStorage.setItem("userPreferences", JSON.stringify({ theme: "dark", language: "en" })); + }); + + console.log("State set. Closing browser..."); +} finally { + await browser.close(); +} + +// Reconnect — a new browser process starts but loads persisted data from disk +const browser2 = await chromium.connectOverCDP(session.connect); +try { + const context2 = browser2.contexts()[0] ?? await browser2.newContext(); + const page2 = await context2.newPage(); + await page2.goto("https://automationexercise.com", { waitUntil: "networkidle" }); + + const cart = await page2.evaluate(() => localStorage.getItem("shoppingCart")); + const prefs = await page2.evaluate(() => localStorage.getItem("userPreferences")); + + console.log("Cart:", JSON.parse(cart)); + console.log("Preferences:", JSON.parse(prefs)); +} finally { + await browser2.close(); +} + +// Clean up +await fetch(`${session.stop}&force=true`, { method: "DELETE" }); diff --git a/examples/persist-session/frameworks/puppeteer/persist-session.mjs b/examples/persist-session/frameworks/puppeteer/persist-session.mjs new file mode 100644 index 0000000..0f4969f --- /dev/null +++ b/examples/persist-session/frameworks/puppeteer/persist-session.mjs @@ -0,0 +1,57 @@ +// Creates a long-lived Browserless session, connects via Puppeteer to set state, +// disconnects, then reconnects and verifies state persisted. +// +// Install: npm install puppeteer-core +// Run: node persist-session.mjs + +import puppeteer from "puppeteer-core"; + +const TOKEN = "YOUR_API_TOKEN_HERE"; + +// Create a long-lived session +const sessionResponse = await fetch( + `https://production-sfo.browserless.io/session?token=${TOKEN}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ ttl: 300000, stealth: true }), + } +); +const session = await sessionResponse.json(); + +// Connect and set browser state +const browser = await puppeteer.connect({ browserWSEndpoint: session.connect }); +try { + const page = await browser.newPage(); + await page.goto("https://automationexercise.com", { waitUntil: "networkidle2" }); + + await page.evaluate(() => { + localStorage.setItem( + "shoppingCart", + JSON.stringify({ items: [{ id: 1, name: "Blue Top" }], totalItems: 1, totalPrice: 500 }) + ); + localStorage.setItem("userPreferences", JSON.stringify({ theme: "dark", language: "en" })); + }); + + console.log("State set. Disconnecting..."); +} finally { + await browser.disconnect(); +} + +// Reconnect to the same session — state is preserved +const browser2 = await puppeteer.connect({ browserWSEndpoint: session.connect }); +try { + const page2 = await browser2.newPage(); + await page2.goto("https://automationexercise.com", { waitUntil: "networkidle2" }); + + const cart = await page2.evaluate(() => localStorage.getItem("shoppingCart")); + const prefs = await page2.evaluate(() => localStorage.getItem("userPreferences")); + + console.log("Cart:", JSON.parse(cart)); + console.log("Preferences:", JSON.parse(prefs)); +} finally { + await browser2.close(); +} + +// Clean up +await fetch(`${session.stop}&force=true`, { method: "DELETE" }); diff --git a/examples/persist-session/rest/csharp/PersistSession.cs b/examples/persist-session/rest/csharp/PersistSession.cs new file mode 100644 index 0000000..6083501 --- /dev/null +++ b/examples/persist-session/rest/csharp/PersistSession.cs @@ -0,0 +1,32 @@ +// Creates a long-lived Browserless session and demonstrates persisting state across connections. +// +// Run: dotnet run + +using System.Net.Http; +using System.Text; +using System.Text.Json; + +var token = "YOUR_API_TOKEN_HERE"; +var client = new HttpClient(); + +var content = new StringContent( + JsonSerializer.Serialize(new { ttl = 300000, stealth = true, headless = false }), + Encoding.UTF8, + "application/json" +); + +var response = await client.PostAsync( + $"https://production-sfo.browserless.io/session?token={token}", + content +); +var body = await response.Content.ReadAsStringAsync(); +var session = JsonSerializer.Deserialize(body); + +var connectUrl = session.GetProperty("connect").GetString(); +var stopUrl = session.GetProperty("stop").GetString(); +Console.WriteLine($"Session created: {session.GetProperty("id")}"); +Console.WriteLine($"Connect URL: {connectUrl}"); + +// Stop the session when done +await client.DeleteAsync($"{stopUrl}&force=true"); +Console.WriteLine("Session stopped."); diff --git a/examples/persist-session/rest/curl/persist-session.sh b/examples/persist-session/rest/curl/persist-session.sh new file mode 100644 index 0000000..6a47b84 --- /dev/null +++ b/examples/persist-session/rest/curl/persist-session.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Creates a long-lived Browserless session, connects to set state, disconnects, reconnects to verify. +# Run: bash persist-session.sh + +TOKEN="YOUR_API_TOKEN_HERE" + +# Step 1: Create a long-lived session +SESSION=$(curl -s -X POST \ + "https://production-sfo.browserless.io/session?token=${TOKEN}" \ + -H "Content-Type: application/json" \ + -d '{"ttl": 300000, "stealth": true, "headless": false}') + +echo "Session created: $SESSION" + +# Step 2: Extract stop URL for cleanup +STOP_URL=$(echo "$SESSION" | python3 -c "import sys,json; print(json.load(sys.stdin)['stop'])") + +# Step 3: Stop the session when done +curl -X DELETE "${STOP_URL}&force=true" +echo "Session stopped." diff --git a/examples/persist-session/rest/java/PersistSession.java b/examples/persist-session/rest/java/PersistSession.java new file mode 100644 index 0000000..69fd0f1 --- /dev/null +++ b/examples/persist-session/rest/java/PersistSession.java @@ -0,0 +1,46 @@ +// Creates a long-lived Browserless session and demonstrates persisting state across connections. +// +// Run: javac PersistSession.java && java PersistSession + +import java.net.URI; +import java.net.http.*; + +public class PersistSession { + public static void main(String[] args) throws Exception { + String token = "YOUR_API_TOKEN_HERE"; + String endpoint = "https://production-sfo.browserless.io/session?token=" + token; + + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(endpoint)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString( + "{\"ttl\": 300000, \"stealth\": true, \"headless\": false}" + )) + .build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + String body = response.body(); + System.out.println("Session created: " + body); + + // Parse connect and stop URLs from response + int connectStart = body.indexOf("\"connect\":\"") + 11; + int connectEnd = body.indexOf("\"", connectStart); + String connectUrl = body.substring(connectStart, connectEnd); + + int stopStart = body.indexOf("\"stop\":\"") + 8; + int stopEnd = body.indexOf("\"", stopStart); + String stopUrl = body.substring(stopStart, stopEnd); + + System.out.println("Connect URL: " + connectUrl); + + // Stop the session when done + HttpRequest deleteRequest = HttpRequest.newBuilder() + .uri(URI.create(stopUrl + "&force=true")) + .DELETE() + .build(); + + client.send(deleteRequest, HttpResponse.BodyHandlers.ofString()); + System.out.println("Session stopped."); + } +} diff --git a/examples/persist-session/rest/nodejs/persist-session.mjs b/examples/persist-session/rest/nodejs/persist-session.mjs new file mode 100644 index 0000000..4a69268 --- /dev/null +++ b/examples/persist-session/rest/nodejs/persist-session.mjs @@ -0,0 +1,52 @@ +// Creates a long-lived Browserless session, connects to set state, disconnects, +// then reconnects to verify the state persists. +// +// Install: npm install puppeteer-core +// Run: node persist-session.mjs + +import puppeteer from "puppeteer-core"; + +const TOKEN = "YOUR_API_TOKEN_HERE"; + +// Step 1: Create a long-lived session +const sessionResponse = await fetch( + `https://production-sfo.browserless.io/session?token=${TOKEN}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ ttl: 300000, stealth: true, headless: false }), + } +); +const session = await sessionResponse.json(); +console.log("Session created:", session.id); + +// Step 2: Connect via Puppeteer and store data +const browser = await puppeteer.connect({ browserWSEndpoint: session.connect }); +const page = await browser.newPage(); + +await page.goto("https://automationexercise.com", { waitUntil: "networkidle2" }); + +await page.evaluate(() => { + localStorage.setItem( + "shoppingCart", + JSON.stringify({ items: [{ id: 1 }], totalItems: 1, totalPrice: 500 }) + ); + localStorage.setItem("userPreferences", JSON.stringify({ theme: "dark" })); +}); + +await browser.disconnect(); +console.log("State set. Disconnected."); + +// Step 3: Reconnect and verify state persists +const browser2 = await puppeteer.connect({ browserWSEndpoint: session.connect }); +const page2 = await browser2.newPage(); +await page2.goto("https://automationexercise.com", { waitUntil: "networkidle2" }); + +const cart = await page2.evaluate(() => localStorage.getItem("shoppingCart")); +console.log("Cart persisted:", JSON.parse(cart)); + +await browser2.close(); + +// Step 4: Delete the session +await fetch(`${session.stop}&force=true`, { method: "DELETE" }); +console.log("Session stopped."); diff --git a/examples/persist-session/rest/python/persist_session.py b/examples/persist-session/rest/python/persist_session.py new file mode 100644 index 0000000..2e8c3ca --- /dev/null +++ b/examples/persist-session/rest/python/persist_session.py @@ -0,0 +1,47 @@ +# Creates a long-lived Browserless session, connects to set state, disconnects, +# then reconnects to verify the state persists. +# +# Install: pip install requests playwright +# Run: python persist_session.py + +import requests +from playwright.sync_api import sync_playwright + +TOKEN = "YOUR_API_TOKEN_HERE" + +# Step 1: Create a long-lived session +session_response = requests.post( + f"https://production-sfo.browserless.io/session?token={TOKEN}", + json={"ttl": 300000, "stealth": True, "headless": False}, +) +session = session_response.json() +print("Session created:", session["id"]) +print("Connect URL:", session["connect"]) + +# Step 2: Connect and set state via Playwright +with sync_playwright() as p: + browser = p.chromium.connect_over_cdp(session["connect"]) + context = browser.contexts[0] if browser.contexts else browser.new_context() + page = context.new_page() + page.goto("https://automationexercise.com") + + page.evaluate("""() => { + localStorage.setItem('shoppingCart', JSON.stringify({items: [{id: 1}]})); + }""") + + browser.close() + + # Step 3: Reconnect and verify + browser2 = p.chromium.connect_over_cdp(session["connect"]) + context2 = browser2.contexts[0] if browser2.contexts else browser2.new_context() + page2 = context2.new_page() + page2.goto("https://automationexercise.com") + + cart = page2.evaluate("() => localStorage.getItem('shoppingCart')") + print("Cart persisted:", cart) + + browser2.close() + +# Step 4: Stop the session +requests.delete(f"{session['stop']}&force=true") +print("Session stopped.") diff --git a/examples/proxy/bql/proxy.graphql b/examples/proxy/bql/proxy.graphql new file mode 100644 index 0000000..3a60d60 --- /dev/null +++ b/examples/proxy/bql/proxy.graphql @@ -0,0 +1,10 @@ +# Navigates to an IP-check endpoint via a residential proxy. +# Pass proxy parameters in the BQL endpoint URL query string. +mutation CheckIP { + goto(url: "https://api.ipify.org?format=json", waitUntil: domContentLoaded) { + status + } + ip: innerText(selector: "body") { + innerText + } +} diff --git a/examples/proxy/frameworks/playwright/nodejs/proxy.mjs b/examples/proxy/frameworks/playwright/nodejs/proxy.mjs new file mode 100644 index 0000000..e078ced --- /dev/null +++ b/examples/proxy/frameworks/playwright/nodejs/proxy.mjs @@ -0,0 +1,20 @@ +// Routes browser traffic through a residential proxy using Playwright. +// The proxy is configured at the connection level — all pages share the same proxy IP. +// +// Install: npm install playwright-core +// Run: node proxy.mjs + +import { chromium } from 'playwright-core'; + +const browser = await chromium.connect( + 'wss://production-sfo.browserless.io/chromium/playwright?token=YOUR_API_TOKEN_HERE&proxy=residential&proxyCountry=us' +); + +try { + const page = await browser.newPage(); + await page.goto('https://api.ipify.org?format=json', { waitUntil: 'networkidle' }); + const ip = await page.evaluate(() => JSON.parse(document.body.innerText).ip); + console.log('Proxy IP:', ip); +} finally { + await browser.close(); +} diff --git a/examples/proxy/frameworks/puppeteer/proxy.mjs b/examples/proxy/frameworks/puppeteer/proxy.mjs new file mode 100644 index 0000000..dd927d3 --- /dev/null +++ b/examples/proxy/frameworks/puppeteer/proxy.mjs @@ -0,0 +1,21 @@ +// Routes browser traffic through a residential proxy using Puppeteer. +// The proxy is configured at the connection level — all pages share the same proxy IP. +// +// Install: npm install puppeteer-core +// Run: node proxy.mjs + +import puppeteer from 'puppeteer-core'; + +const browser = await puppeteer.connect({ + browserWSEndpoint: + 'wss://production-sfo.browserless.io?token=YOUR_API_TOKEN_HERE&proxy=residential&proxyCountry=us', +}); + +try { + const page = await browser.newPage(); + await page.goto('https://api.ipify.org?format=json', { waitUntil: 'networkidle2' }); + const ip = await page.evaluate(() => JSON.parse(document.body.innerText).ip); + console.log('Proxy IP:', ip); +} finally { + await browser.close(); +} diff --git a/examples/proxy/rest/csharp/ProxyExample.cs b/examples/proxy/rest/csharp/ProxyExample.cs new file mode 100644 index 0000000..0fb0fa2 --- /dev/null +++ b/examples/proxy/rest/csharp/ProxyExample.cs @@ -0,0 +1,25 @@ +// Routes browser traffic through a residential proxy and prints the resulting IP. +// +// Run: dotnet run + +using System.Net.Http; +using System.Text; +using System.Text.Json; + +string token = "YOUR_API_TOKEN_HERE"; +string endpoint = $"https://production-sfo.browserless.io/scrape?token={token}&proxy=residential&proxyCountry=us"; + +var payload = new +{ + url = "https://api.ipify.org?format=json", + elements = new[] { new { selector = "body" } }, +}; + +using (HttpClient httpClient = new HttpClient()) +{ + var content = new StringContent( + JsonSerializer.Serialize(payload), Encoding.UTF8, "application/json"); + var response = await httpClient.PostAsync(endpoint, content); + string body = await response.Content.ReadAsStringAsync(); + Console.WriteLine(body); +} diff --git a/examples/proxy/rest/curl/proxy.sh b/examples/proxy/rest/curl/proxy.sh new file mode 100644 index 0000000..69dba11 --- /dev/null +++ b/examples/proxy/rest/curl/proxy.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Routes browser traffic through a residential proxy and checks the resulting IP. +# Run: bash proxy.sh + +curl -X POST \ + "https://production-sfo.browserless.io/scrape?token=YOUR_API_TOKEN_HERE&proxy=residential&proxyCountry=us" \ + -H "Content-Type: application/json" \ + -d '{ + "url": "https://api.ipify.org?format=json", + "elements": [{ "selector": "body" }] + }' diff --git a/examples/proxy/rest/java/ProxyExample.java b/examples/proxy/rest/java/ProxyExample.java new file mode 100644 index 0000000..e052c1f --- /dev/null +++ b/examples/proxy/rest/java/ProxyExample.java @@ -0,0 +1,26 @@ +// Routes browser traffic through a residential proxy and prints the resulting IP. +// +// Run: javac ProxyExample.java && java ProxyExample + +import java.net.URI; +import java.net.http.*; + +public class ProxyExample { + public static void main(String[] args) throws Exception { + String token = "YOUR_API_TOKEN_HERE"; + String endpoint = "https://production-sfo.browserless.io/scrape?token=" + token + + "&proxy=residential&proxyCountry=us"; + + String payload = "{\"url\": \"https://api.ipify.org?format=json\", \"elements\": [{\"selector\": \"body\"}]}"; + + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(endpoint)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(payload)) + .build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + System.out.println(response.body()); + } +} diff --git a/examples/proxy/rest/nodejs/proxy.mjs b/examples/proxy/rest/nodejs/proxy.mjs new file mode 100644 index 0000000..24cd225 --- /dev/null +++ b/examples/proxy/rest/nodejs/proxy.mjs @@ -0,0 +1,19 @@ +// Routes browser traffic through a residential proxy and logs the resulting IP. +// +// Run: node proxy.mjs + +const response = await fetch( + 'https://production-sfo.browserless.io/scrape?token=YOUR_API_TOKEN_HERE&proxy=residential&proxyCountry=us', + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + url: 'https://api.ipify.org?format=json', + elements: [{ selector: 'body' }], + }), + } +); + +const { data } = await response.json(); +const ip = JSON.parse(data[0].results[0].text).ip; +console.log('Proxy IP:', ip); diff --git a/examples/proxy/rest/python/proxy.py b/examples/proxy/rest/python/proxy.py new file mode 100644 index 0000000..b0be5b2 --- /dev/null +++ b/examples/proxy/rest/python/proxy.py @@ -0,0 +1,24 @@ +# Routes browser traffic through a residential proxy and logs the resulting IP. +# +# Install: pip install requests +# Run: python proxy.py + +import requests +import json + +response = requests.post( + 'https://production-sfo.browserless.io/scrape', + params={ + 'token': 'YOUR_API_TOKEN_HERE', + 'proxy': 'residential', + 'proxyCountry': 'us', + }, + json={ + 'url': 'https://api.ipify.org?format=json', + 'elements': [{'selector': 'body'}], + }, +) + +data = response.json()['data'] +ip = json.loads(data[0]['results'][0]['text'])['ip'] +print('Proxy IP:', ip) diff --git a/examples/reconnect-to-a-browser/bql/reconnect.graphql b/examples/reconnect-to-a-browser/bql/reconnect.graphql new file mode 100644 index 0000000..d202503 --- /dev/null +++ b/examples/reconnect-to-a-browser/bql/reconnect.graphql @@ -0,0 +1,12 @@ +# Step 1: Start a session and request a reconnection endpoint. +# The browser stays alive for 60 seconds after this request completes. +mutation StartSession { + goto(url: "https://example.com", waitUntil: domContentLoaded) { + status + } + + reconnect(timeout: 60000) { + browserQLEndpoint + browserWSEndpoint + } +} diff --git a/examples/reconnect-to-a-browser/frameworks/playwright/nodejs/reconnect.mjs b/examples/reconnect-to-a-browser/frameworks/playwright/nodejs/reconnect.mjs new file mode 100644 index 0000000..8ec4305 --- /dev/null +++ b/examples/reconnect-to-a-browser/frameworks/playwright/nodejs/reconnect.mjs @@ -0,0 +1,45 @@ +// Disconnects from a remote Playwright browser session and reconnects to the same session. +// Uses the Browserless.reconnect CDP command to keep the browser alive after disconnecting. +// +// Install: npm install playwright-core +// Run: node reconnect.mjs + +import { chromium } from 'playwright-core'; + +const TOKEN = 'YOUR_API_TOKEN_HERE'; + +const browser = await chromium.connect( + `wss://production-sfo.browserless.io/chromium/playwright?token=${TOKEN}` +); + +try { + const page = await browser.newPage(); + const cdp = await page.context().newCDPSession(page); + + await page.goto('https://example.com', { waitUntil: 'networkidle' }); + + // Keep the browser alive for 60 seconds after disconnecting. + const { browserWSEndpoint } = await cdp.send('Browserless.reconnect', { + timeout: 60000, + }); + + // Disconnect (does not kill the browser — it stays alive). + await browser.close(); + + console.log('Reconnect URL:', browserWSEndpoint); + + // Reconnect to the same browser session. + const reconnected = await chromium.connect( + `${browserWSEndpoint}?token=${TOKEN}` + ); + + const pages = reconnected.contexts()[0]?.pages() ?? []; + if (pages.length > 0) { + console.log('Still on:', await pages[0].title()); + } + + await reconnected.close(); +} catch (err) { + await browser.close(); + throw err; +} diff --git a/examples/reconnect-to-a-browser/frameworks/puppeteer/reconnect.mjs b/examples/reconnect-to-a-browser/frameworks/puppeteer/reconnect.mjs new file mode 100644 index 0000000..dd0caed --- /dev/null +++ b/examples/reconnect-to-a-browser/frameworks/puppeteer/reconnect.mjs @@ -0,0 +1,44 @@ +// Disconnects from a remote Puppeteer browser session and reconnects to the same session. +// Uses the Browserless.reconnect CDP command to keep the browser alive after disconnecting. +// +// Install: npm install puppeteer-core +// Run: node reconnect.mjs + +import puppeteer from 'puppeteer-core'; + +const TOKEN = 'YOUR_API_TOKEN_HERE'; + +const browser = await puppeteer.connect({ + browserWSEndpoint: `wss://production-sfo.browserless.io?token=${TOKEN}`, +}); + +try { + const page = await browser.newPage(); + const cdp = await page.createCDPSession(); + + await page.goto('https://example.com', { waitUntil: 'networkidle2' }); + + // Keep the browser alive for 60 seconds after disconnecting. + const { browserWSEndpoint } = await cdp.send('Browserless.reconnect', { + timeout: 60000, + }); + + // Disconnect (does not kill the browser — it stays alive). + await browser.close(); + + console.log('Reconnect URL:', browserWSEndpoint); + + // Reconnect to the same browser session. + const reconnected = await puppeteer.connect({ + browserWSEndpoint: `${browserWSEndpoint}?token=${TOKEN}`, + }); + + const pages = await reconnected.pages(); + const existingPage = pages[0]; + console.log('Still on:', await existingPage.title()); + + await reconnected.close(); +} catch (err) { + await browser.close(); + throw err; +} diff --git a/examples/reconnect-to-a-browser/rest/csharp/ReconnectBrowser.cs b/examples/reconnect-to-a-browser/rest/csharp/ReconnectBrowser.cs new file mode 100644 index 0000000..7dc1ba4 --- /dev/null +++ b/examples/reconnect-to-a-browser/rest/csharp/ReconnectBrowser.cs @@ -0,0 +1,49 @@ +// Disconnects from a Browserless browser session and reconnects to the same session. +// Uses BQL over HTTP — no browser library required. +// +// Run: dotnet run + +using System.Net.Http; +using System.Text; +using System.Text.Json; + +string token = "YOUR_API_TOKEN_HERE"; +string bqlUrl = $"https://production-sfo.browserless.io/stealth/bql?token={token}"; + +var startPayload = new +{ + query = @"mutation StartSession { + goto(url: ""https://example.com"", waitUntil: domContentLoaded) { status } + reconnect(timeout: 60000) { browserQLEndpoint } + }", + variables = new { }, + operationName = "StartSession", +}; + +using (HttpClient httpClient = new HttpClient()) +{ + var json = JsonSerializer.Serialize(startPayload); + var content = new StringContent(json, Encoding.UTF8, "application/json"); + var startResponse = await httpClient.PostAsync(bqlUrl, content); + string startBody = await startResponse.Content.ReadAsStringAsync(); + + var doc = JsonDocument.Parse(startBody); + string reconnectUrl = doc.RootElement + .GetProperty("data") + .GetProperty("reconnect") + .GetProperty("browserQLEndpoint") + .GetString() + $"?token={token}"; + + var continuePayload = new + { + query = "mutation ContinueSession { html { html } }", + variables = new { }, + operationName = "ContinueSession", + }; + + var continueContent = new StringContent( + JsonSerializer.Serialize(continuePayload), Encoding.UTF8, "application/json"); + var continueResponse = await httpClient.PostAsync(reconnectUrl, continueContent); + string continueBody = await continueResponse.Content.ReadAsStringAsync(); + Console.WriteLine(continueBody); +} diff --git a/examples/reconnect-to-a-browser/rest/curl/reconnect.sh b/examples/reconnect-to-a-browser/rest/curl/reconnect.sh new file mode 100644 index 0000000..d4c889a --- /dev/null +++ b/examples/reconnect-to-a-browser/rest/curl/reconnect.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Disconnect and reconnect to a Browserless browser session using BQL over HTTP. +# Run: bash reconnect.sh + +# Step 1: Start a session and get the reconnection endpoint +RESPONSE=$(curl -s -X POST \ + "https://production-sfo.browserless.io/stealth/bql?token=YOUR_API_TOKEN_HERE" \ + -H "Content-Type: application/json" \ + -d '{ + "query": "mutation StartSession { goto(url: \"https://example.com\", waitUntil: domContentLoaded) { status } reconnect(timeout: 60000) { browserQLEndpoint browserWSEndpoint } }", + "variables": {}, + "operationName": "StartSession" + }') + +echo "Session started:" +echo "$RESPONSE" + +# Step 2: Extract the browserQLEndpoint from the response +RECONNECT_URL=$(echo "$RESPONSE" | python3 -c "import sys,json; print(json.load(sys.stdin)['data']['reconnect']['browserQLEndpoint'])") + +# Step 3: Reconnect using the returned endpoint +curl -X POST \ + "${RECONNECT_URL}?token=YOUR_API_TOKEN_HERE" \ + -H "Content-Type: application/json" \ + -d '{ + "query": "mutation ContinueSession { html { html } }", + "variables": {}, + "operationName": "ContinueSession" + }' diff --git a/examples/reconnect-to-a-browser/rest/java/ReconnectBrowser.java b/examples/reconnect-to-a-browser/rest/java/ReconnectBrowser.java new file mode 100644 index 0000000..262a17f --- /dev/null +++ b/examples/reconnect-to-a-browser/rest/java/ReconnectBrowser.java @@ -0,0 +1,56 @@ +// Disconnects from a Browserless browser session and reconnects to the same session. +// Uses BQL over HTTP — no browser library required. +// +// Run: javac ReconnectBrowser.java && java ReconnectBrowser + +import java.net.URI; +import java.net.http.*; + +public class ReconnectBrowser { + public static void main(String[] args) throws Exception { + String token = "YOUR_API_TOKEN_HERE"; + String bqlUrl = "https://production-sfo.browserless.io/stealth/bql?token=" + token; + + String startPayload = """ + { + "query": "mutation StartSession { goto(url: \\"https://example.com\\", waitUntil: domContentLoaded) { status } reconnect(timeout: 60000) { browserQLEndpoint } }", + "variables": {}, + "operationName": "StartSession" + } + """; + + HttpClient client = HttpClient.newHttpClient(); + HttpRequest startRequest = HttpRequest.newBuilder() + .uri(URI.create(bqlUrl)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(startPayload)) + .build(); + + HttpResponse startResponse = client.send(startRequest, HttpResponse.BodyHandlers.ofString()); + String body = startResponse.body(); + + // Extract browserQLEndpoint from the JSON response + String marker = "\"browserQLEndpoint\":\""; + int start = body.indexOf(marker) + marker.length(); + int end = body.indexOf("\"", start); + String reconnectUrl = body.substring(start, end) + "?token=" + token; + + // Send follow-up mutation to the reconnected session + String continuePayload = """ + { + "query": "mutation ContinueSession { html { html } }", + "variables": {}, + "operationName": "ContinueSession" + } + """; + + HttpRequest continueRequest = HttpRequest.newBuilder() + .uri(URI.create(reconnectUrl)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(continuePayload)) + .build(); + + HttpResponse continueResponse = client.send(continueRequest, HttpResponse.BodyHandlers.ofString()); + System.out.println(continueResponse.body()); + } +} diff --git a/examples/reconnect-to-a-browser/rest/nodejs/reconnect.mjs b/examples/reconnect-to-a-browser/rest/nodejs/reconnect.mjs new file mode 100644 index 0000000..8327484 --- /dev/null +++ b/examples/reconnect-to-a-browser/rest/nodejs/reconnect.mjs @@ -0,0 +1,42 @@ +// Disconnects from a Browserless browser session and reconnects to the same session. +// Uses BQL over HTTP — no browser library required. +// +// Run: node reconnect.mjs + +const TOKEN = 'YOUR_API_TOKEN_HERE'; +const BQL_URL = `https://production-sfo.browserless.io/stealth/bql?token=${TOKEN}`; + +const startQuery = `mutation StartSession { + goto(url: "https://example.com", waitUntil: domContentLoaded) { + status + } + reconnect(timeout: 60000) { + browserQLEndpoint + } +}`; + +const startResponse = await fetch(BQL_URL, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + query: startQuery, + variables: {}, + operationName: 'StartSession', + }), +}); + +const { data } = await startResponse.json(); +const reconnectURL = `${data.reconnect.browserQLEndpoint}?token=${TOKEN}`; + +const continueResponse = await fetch(reconnectURL, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + query: 'mutation ContinueSession { html { html } }', + variables: {}, + operationName: 'ContinueSession', + }), +}); + +const result = await continueResponse.json(); +console.log(result.data.html.html.substring(0, 200)); diff --git a/examples/reconnect-to-a-browser/rest/python/reconnect.py b/examples/reconnect-to-a-browser/rest/python/reconnect.py new file mode 100644 index 0000000..63f46f0 --- /dev/null +++ b/examples/reconnect-to-a-browser/rest/python/reconnect.py @@ -0,0 +1,45 @@ +# Disconnects from a Browserless browser session and reconnects to the same session. +# Uses BQL over HTTP — no browser library required. +# +# Install: pip install requests +# Run: python reconnect.py + +import requests + +TOKEN = 'YOUR_API_TOKEN_HERE' +BQL_URL = f'https://production-sfo.browserless.io/stealth/bql?token={TOKEN}' + +start_query = """ +mutation StartSession { + goto(url: "https://example.com", waitUntil: domContentLoaded) { + status + } + reconnect(timeout: 60000) { + browserQLEndpoint + } +} +""" + +start_response = requests.post( + BQL_URL, + json={ + 'query': start_query, + 'variables': {}, + 'operationName': 'StartSession', + }, +) + +data = start_response.json()['data'] +reconnect_url = f"{data['reconnect']['browserQLEndpoint']}?token={TOKEN}" + +continue_response = requests.post( + reconnect_url, + json={ + 'query': 'mutation ContinueSession { html { html } }', + 'variables': {}, + 'operationName': 'ContinueSession', + }, +) + +result = continue_response.json() +print(result['data']['html']['html'][:200]) diff --git a/examples/retry-backoff/bql/retry-backoff.graphql b/examples/retry-backoff/bql/retry-backoff.graphql new file mode 100644 index 0000000..d733e10 --- /dev/null +++ b/examples/retry-backoff/bql/retry-backoff.graphql @@ -0,0 +1,11 @@ +# BQL does not have built-in retry logic — implement retries in the calling client. +# This is the query to retry; wrap it in your language's retry/backoff loop. +mutation RetryExample { + goto(url: "https://example.com", waitUntil: networkIdle) { + status + } + + title { + title + } +} diff --git a/examples/retry-backoff/frameworks/playwright/nodejs/retry-backoff.mjs b/examples/retry-backoff/frameworks/playwright/nodejs/retry-backoff.mjs new file mode 100644 index 0000000..5a12f73 --- /dev/null +++ b/examples/retry-backoff/frameworks/playwright/nodejs/retry-backoff.mjs @@ -0,0 +1,40 @@ +// Retries a Playwright session with exponential backoff. +// Only retries transient failures (connection/WebSocket errors) — logic errors are re-thrown immediately. +// +// Install: npm install playwright-core +// Run: node retry-backoff.mjs + +import { chromium } from 'playwright-core'; + +const TRANSIENT = ['ECONNREFUSED', 'WebSocket', 'Protocol error', 'Target closed', 'net::']; + +async function withRetry(fn, { retries = 3, baseDelay = 1000 } = {}) { + for (let attempt = 0; attempt <= retries; attempt++) { + try { + return await fn(); + } catch (err) { + const isTransient = TRANSIENT.some(s => err.message?.includes(s)); + if (attempt === retries || !isTransient) throw err; + const delay = baseDelay * 2 ** attempt; + console.warn(`Attempt ${attempt + 1} failed: ${err.message}. Retrying in ${delay}ms...`); + await new Promise(r => setTimeout(r, delay)); + } + } +} + +const result = await withRetry(async () => { + const browser = await chromium.connectOverCDP( + 'wss://production-sfo.browserless.io/chromium/playwright?token=YOUR_API_TOKEN_HERE' + ); + try { + const context = browser.contexts()[0] ?? await browser.newContext(); + const page = await context.newPage(); + await page.goto('https://example.com'); + return await page.title(); + } finally { + // Always close to release the session even on error. + await browser.close(); + } +}); + +console.log(result); diff --git a/examples/retry-backoff/frameworks/puppeteer/retry-backoff.mjs b/examples/retry-backoff/frameworks/puppeteer/retry-backoff.mjs new file mode 100644 index 0000000..31abd19 --- /dev/null +++ b/examples/retry-backoff/frameworks/puppeteer/retry-backoff.mjs @@ -0,0 +1,39 @@ +// Retries a Puppeteer session with exponential backoff. +// Only retries transient failures (connection/WebSocket errors) — logic errors are re-thrown immediately. +// +// Install: npm install puppeteer-core +// Run: node retry-backoff.mjs + +import puppeteer from 'puppeteer-core'; + +const TRANSIENT = ['ECONNREFUSED', 'WebSocket', 'Protocol error', 'Target closed', 'net::']; + +async function withRetry(fn, { retries = 3, baseDelay = 1000 } = {}) { + for (let attempt = 0; attempt <= retries; attempt++) { + try { + return await fn(); + } catch (err) { + const isTransient = TRANSIENT.some(s => err.message?.includes(s)); + if (attempt === retries || !isTransient) throw err; + const delay = baseDelay * 2 ** attempt; + console.warn(`Attempt ${attempt + 1} failed: ${err.message}. Retrying in ${delay}ms...`); + await new Promise(r => setTimeout(r, delay)); + } + } +} + +const result = await withRetry(async () => { + const browser = await puppeteer.connect({ + browserWSEndpoint: 'wss://production-sfo.browserless.io?token=YOUR_API_TOKEN_HERE', + }); + try { + const page = await browser.newPage(); + await page.goto('https://example.com'); + return await page.title(); + } finally { + // Always close to release the session even on error. + await browser.close(); + } +}); + +console.log(result); diff --git a/examples/retry-backoff/rest/csharp/RetryBackoff.cs b/examples/retry-backoff/rest/csharp/RetryBackoff.cs new file mode 100644 index 0000000..dbe63e2 --- /dev/null +++ b/examples/retry-backoff/rest/csharp/RetryBackoff.cs @@ -0,0 +1,46 @@ +// Retries a Browserless BQL request with exponential backoff on failure. +// +// Run: dotnet run + +using System.Net.Http; +using System.Text; +using System.Text.Json; + +string token = "YOUR_API_TOKEN_HERE"; +string endpoint = $"https://production-sfo.browserless.io/bql?token={token}"; + +const int maxRetries = 5; +int delayMs = 1000; + +var payload = JsonSerializer.Serialize(new +{ + query = @"mutation { + goto(url: ""https://example.com"", waitUntil: networkIdle) { status } + title { title } + }", + variables = new { }, +}); + +using HttpClient httpClient = new HttpClient(); + +for (int attempt = 1; attempt <= maxRetries; attempt++) +{ + try + { + var content = new StringContent(payload, Encoding.UTF8, "application/json"); + var response = await httpClient.PostAsync(endpoint, content); + response.EnsureSuccessStatusCode(); + + string body = await response.Content.ReadAsStringAsync(); + Console.WriteLine($"Success on attempt {attempt}:"); + Console.WriteLine(body); + return; + } + catch (Exception ex) + { + if (attempt == maxRetries) throw; + Console.WriteLine($"Attempt {attempt} failed: {ex.Message}. Retrying in {delayMs}ms..."); + await Task.Delay(delayMs); + delayMs *= 2; + } +} diff --git a/examples/retry-backoff/rest/curl/retry-backoff.sh b/examples/retry-backoff/rest/curl/retry-backoff.sh new file mode 100644 index 0000000..e668c15 --- /dev/null +++ b/examples/retry-backoff/rest/curl/retry-backoff.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Retries a Browserless BQL request with exponential backoff on failure. +# Run: bash retry-backoff.sh + +TOKEN="YOUR_API_TOKEN_HERE" +URL="https://production-sfo.browserless.io/bql?token=${TOKEN}" +MAX_RETRIES=5 +DELAY=1 + +for i in $(seq 1 $MAX_RETRIES); do + RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "$URL" \ + -H "Content-Type: application/json" \ + -d '{"query": "mutation { goto(url: \"https://example.com\", waitUntil: networkIdle) { status } title { title } }", "variables": {}}') + + HTTP_CODE=$(echo "$RESPONSE" | tail -1) + BODY=$(echo "$RESPONSE" | head -1) + + if [ "$HTTP_CODE" = "200" ]; then + echo "Success on attempt $i:" + echo "$BODY" + exit 0 + fi + + echo "Attempt $i failed (HTTP $HTTP_CODE). Retrying in ${DELAY}s..." + sleep $DELAY + DELAY=$((DELAY * 2)) +done + +echo "All $MAX_RETRIES attempts failed." +exit 1 diff --git a/examples/retry-backoff/rest/java/RetryBackoff.java b/examples/retry-backoff/rest/java/RetryBackoff.java new file mode 100644 index 0000000..13aa449 --- /dev/null +++ b/examples/retry-backoff/rest/java/RetryBackoff.java @@ -0,0 +1,46 @@ +// Retries a Browserless BQL request with exponential backoff on failure. +// +// Run: javac RetryBackoff.java && java RetryBackoff + +import java.net.URI; +import java.net.http.*; +import java.time.Duration; + +public class RetryBackoff { + static final int MAX_RETRIES = 5; + static final long BASE_DELAY_MS = 1000; + + public static void main(String[] args) throws Exception { + String token = "YOUR_API_TOKEN_HERE"; + String endpoint = "https://production-sfo.browserless.io/bql?token=" + token; + String payload = "{\"query\": \"mutation { goto(url: \\\"https://example.com\\\", waitUntil: networkIdle) { status } title { title } }\", \"variables\": {}}"; + + HttpClient client = HttpClient.newHttpClient(); + long delay = BASE_DELAY_MS; + + for (int attempt = 1; attempt <= MAX_RETRIES; attempt++) { + try { + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(endpoint)) + .header("Content-Type", "application/json") + .timeout(Duration.ofSeconds(30)) + .POST(HttpRequest.BodyPublishers.ofString(payload)) + .build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() == 200) { + System.out.println("Success on attempt " + attempt + ":"); + System.out.println(response.body()); + return; + } + throw new RuntimeException("HTTP " + response.statusCode()); + } catch (Exception e) { + if (attempt == MAX_RETRIES) throw e; + System.out.println("Attempt " + attempt + " failed: " + e.getMessage() + ". Retrying in " + delay + "ms..."); + Thread.sleep(delay); + delay *= 2; + } + } + } +} diff --git a/examples/retry-backoff/rest/nodejs/retry-backoff.mjs b/examples/retry-backoff/rest/nodejs/retry-backoff.mjs new file mode 100644 index 0000000..e24e6d3 --- /dev/null +++ b/examples/retry-backoff/rest/nodejs/retry-backoff.mjs @@ -0,0 +1,39 @@ +// Retries a Browserless BQL request with exponential backoff on failure. +// +// Run: node retry-backoff.mjs + +const TOKEN = 'YOUR_API_TOKEN_HERE'; +const BQL_URL = `https://production-sfo.browserless.io/bql?token=${TOKEN}`; + +const MAX_RETRIES = 5; +const BASE_DELAY_MS = 1000; + +async function fetchWithRetry(url, options, retries = MAX_RETRIES, delay = BASE_DELAY_MS) { + for (let attempt = 1; attempt <= retries; attempt++) { + try { + const response = await fetch(url, options); + if (response.ok) return response; + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } catch (err) { + if (attempt === retries) throw err; + console.warn(`Attempt ${attempt} failed: ${err.message}. Retrying in ${delay}ms...`); + await new Promise((r) => setTimeout(r, delay)); + delay *= 2; + } + } +} + +const response = await fetchWithRetry(BQL_URL, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + query: `mutation { + goto(url: "https://example.com", waitUntil: networkIdle) { status } + title { title } + }`, + variables: {}, + }), +}); + +const { data } = await response.json(); +console.log('Title:', data.title.title); diff --git a/examples/retry-backoff/rest/python/retry_backoff.py b/examples/retry-backoff/rest/python/retry_backoff.py new file mode 100644 index 0000000..c6d1b0b --- /dev/null +++ b/examples/retry-backoff/rest/python/retry_backoff.py @@ -0,0 +1,40 @@ +# Retries a Browserless BQL request with exponential backoff on failure. +# +# Install: pip install requests +# Run: python retry_backoff.py + +import requests +import time + +TOKEN = 'YOUR_API_TOKEN_HERE' +BQL_URL = f'https://production-sfo.browserless.io/bql?token={TOKEN}' + +MAX_RETRIES = 5 +BASE_DELAY = 1 # seconds + +query = """ +mutation { + goto(url: "https://example.com", waitUntil: networkIdle) { status } + title { title } +} +""" + +delay = BASE_DELAY +for attempt in range(1, MAX_RETRIES + 1): + try: + response = requests.post( + BQL_URL, + json={'query': query, 'variables': {}}, + timeout=30, + ) + response.raise_for_status() + data = response.json()['data'] + print(f'Success on attempt {attempt}.') + print('Title:', data['title']['title']) + break + except Exception as e: + if attempt == MAX_RETRIES: + raise + print(f'Attempt {attempt} failed: {e}. Retrying in {delay}s...') + time.sleep(delay) + delay *= 2 diff --git a/examples/scrape-booking/bql/scrape-booking.graphql b/examples/scrape-booking/bql/scrape-booking.graphql new file mode 100644 index 0000000..be7323a --- /dev/null +++ b/examples/scrape-booking/bql/scrape-booking.graphql @@ -0,0 +1,25 @@ +# Scrapes Booking.com hotel listings. Run against /stealth/bql with residential proxy. +mutation ScrapeBooking { + goto( + url: "https://www.booking.com/searchresults.html?ss=New+York&checkin=2025-09-01&checkout=2025-09-07&group_adults=2" + waitUntil: networkIdle + ) { + status + } + + waitForTimeout(time: 3000) { + time + } + + hotels: mapSelector(selector: "[data-testid='property-card']") { + name: mapSelector(selector: "[data-testid='title']") { + innerText + } + price: mapSelector(selector: "[data-testid='price-and-discounted-price']") { + innerText + } + rating: mapSelector(selector: "[data-testid='review-score']") { + innerText + } + } +} diff --git a/examples/scrape-booking/frameworks/playwright/nodejs/scrape-booking.mjs b/examples/scrape-booking/frameworks/playwright/nodejs/scrape-booking.mjs new file mode 100644 index 0000000..bac9ebc --- /dev/null +++ b/examples/scrape-booking/frameworks/playwright/nodejs/scrape-booking.mjs @@ -0,0 +1,33 @@ +// Scrapes Booking.com hotel listings using Playwright with stealth mode and residential proxy. +// +// Install: npm install playwright-core +// Run: node scrape-booking.mjs + +import { chromium } from 'playwright-core'; + +const browser = await chromium.connectOverCDP( + 'wss://production-sfo.browserless.io?token=YOUR_API_TOKEN_HERE&stealth&proxy=residential&proxyCountry=us' +); + +try { + const context = browser.contexts()[0]; + const page = await context.newPage(); + await page.goto( + 'https://www.booking.com/searchresults.html?ss=New+York&checkin=2025-09-01&checkout=2025-09-07&group_adults=2', + { waitUntil: 'networkidle' } + ); + + await page.waitForTimeout(3000); + + const hotels = await page.evaluate(() => + Array.from(document.querySelectorAll('[data-testid="property-card"]')).map((card) => ({ + name: card.querySelector('[data-testid="title"]')?.innerText?.trim() ?? '', + price: card.querySelector('[data-testid="price-and-discounted-price"]')?.innerText?.trim() ?? '', + rating: card.querySelector('[data-testid="review-score"]')?.innerText?.trim() ?? '', + })) + ); + + console.log(JSON.stringify(hotels, null, 2)); +} finally { + await browser.close(); +} diff --git a/examples/scrape-booking/frameworks/puppeteer/scrape-booking.mjs b/examples/scrape-booking/frameworks/puppeteer/scrape-booking.mjs new file mode 100644 index 0000000..3205691 --- /dev/null +++ b/examples/scrape-booking/frameworks/puppeteer/scrape-booking.mjs @@ -0,0 +1,33 @@ +// Scrapes Booking.com hotel listings using Puppeteer with stealth mode and residential proxy. +// +// Install: npm install puppeteer-core +// Run: node scrape-booking.mjs + +import puppeteer from 'puppeteer-core'; + +const browser = await puppeteer.connect({ + browserWSEndpoint: + 'wss://production-sfo.browserless.io/stealth?token=YOUR_API_TOKEN_HERE&proxy=residential&proxyCountry=us', +}); + +try { + const page = await browser.newPage(); + await page.goto( + 'https://www.booking.com/searchresults.html?ss=New+York&checkin=2025-09-01&checkout=2025-09-07&group_adults=2', + { waitUntil: 'networkidle2' } + ); + + await new Promise((r) => setTimeout(r, 3000)); + + const hotels = await page.evaluate(() => + Array.from(document.querySelectorAll('[data-testid="property-card"]')).map((card) => ({ + name: card.querySelector('[data-testid="title"]')?.innerText?.trim() ?? '', + price: card.querySelector('[data-testid="price-and-discounted-price"]')?.innerText?.trim() ?? '', + rating: card.querySelector('[data-testid="review-score"]')?.innerText?.trim() ?? '', + })) + ); + + console.log(JSON.stringify(hotels, null, 2)); +} finally { + await browser.close(); +} diff --git a/examples/scrape-booking/rest/csharp/ScrapeBooking.cs b/examples/scrape-booking/rest/csharp/ScrapeBooking.cs new file mode 100644 index 0000000..085af73 --- /dev/null +++ b/examples/scrape-booking/rest/csharp/ScrapeBooking.cs @@ -0,0 +1,34 @@ +// Scrapes Booking.com hotel listings using BQL with stealth mode and residential proxy. +// +// Run: dotnet run + +using System.Net.Http; +using System.Text; +using System.Text.Json; + +string token = "YOUR_API_TOKEN_HERE"; +string endpoint = $"https://production-sfo.browserless.io/stealth/bql?token={token}&proxy=residential&proxyCountry=us"; + +var payload = new +{ + query = @"mutation ScrapeBooking { + goto(url: ""https://www.booking.com/searchresults.html?ss=New+York&checkin=2025-09-01&checkout=2025-09-07&group_adults=2"", waitUntil: networkIdle) { status } + waitForTimeout(time: 3000) { time } + hotels: mapSelector(selector: ""[data-testid='property-card']"") { + name: mapSelector(selector: ""[data-testid='title']"") { innerText } + price: mapSelector(selector: ""[data-testid='price-and-discounted-price']"") { innerText } + rating: mapSelector(selector: ""[data-testid='review-score']"") { innerText } + } + }", + variables = new { }, + operationName = "ScrapeBooking", +}; + +using (HttpClient httpClient = new HttpClient()) +{ + var content = new StringContent( + JsonSerializer.Serialize(payload), Encoding.UTF8, "application/json"); + var response = await httpClient.PostAsync(endpoint, content); + string body = await response.Content.ReadAsStringAsync(); + Console.WriteLine(body); +} diff --git a/examples/scrape-booking/rest/curl/scrape-booking.sh b/examples/scrape-booking/rest/curl/scrape-booking.sh new file mode 100644 index 0000000..3006979 --- /dev/null +++ b/examples/scrape-booking/rest/curl/scrape-booking.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Scrapes Booking.com hotel listings using BQL with stealth mode and residential proxy. +# Run: bash scrape-booking.sh + +curl -X POST \ + "https://production-sfo.browserless.io/stealth/bql?token=YOUR_API_TOKEN_HERE&proxy=residential&proxyCountry=us" \ + -H "Content-Type: application/json" \ + -d '{ + "query": "mutation ScrapeBooking { goto(url: \"https://www.booking.com/searchresults.html?ss=New+York&checkin=2025-09-01&checkout=2025-09-07&group_adults=2\", waitUntil: networkIdle) { status } waitForTimeout(time: 3000) { time } hotels: mapSelector(selector: \"[data-testid='\''property-card'\'']\") { name: mapSelector(selector: \"[data-testid='\''title'\'']\") { innerText } price: mapSelector(selector: \"[data-testid='\''price-and-discounted-price'\'']\") { innerText } rating: mapSelector(selector: \"[data-testid='\''review-score'\'']\") { innerText } } }", + "variables": {}, + "operationName": "ScrapeBooking" + }' diff --git a/examples/scrape-booking/rest/java/ScrapeBooking.java b/examples/scrape-booking/rest/java/ScrapeBooking.java new file mode 100644 index 0000000..6da9c0f --- /dev/null +++ b/examples/scrape-booking/rest/java/ScrapeBooking.java @@ -0,0 +1,36 @@ +// Scrapes Booking.com hotel listings using BQL with stealth mode and residential proxy. +// +// Run: javac ScrapeBooking.java && java ScrapeBooking + +import java.net.URI; +import java.net.http.*; + +public class ScrapeBooking { + public static void main(String[] args) throws Exception { + String token = "YOUR_API_TOKEN_HERE"; + String endpoint = "https://production-sfo.browserless.io/stealth/bql?token=" + + token + "&proxy=residential&proxyCountry=us"; + + String query = "mutation ScrapeBooking {" + + " goto(url: \\\"https://www.booking.com/searchresults.html?ss=New+York&checkin=2025-09-01&checkout=2025-09-07&group_adults=2\\\", waitUntil: networkIdle) { status }" + + " waitForTimeout(time: 3000) { time }" + + " hotels: mapSelector(selector: \\\"[data-testid='property-card']\\\") {" + + " name: mapSelector(selector: \\\"[data-testid='title']\\\") { innerText }" + + " price: mapSelector(selector: \\\"[data-testid='price-and-discounted-price']\\\") { innerText }" + + " rating: mapSelector(selector: \\\"[data-testid='review-score']\\\") { innerText }" + + " }" + + " }"; + + String payload = "{\"query\": \"" + query + "\", \"variables\": {}, \"operationName\": \"ScrapeBooking\"}"; + + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(endpoint)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(payload)) + .build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + System.out.println(response.body()); + } +} diff --git a/examples/scrape-booking/rest/nodejs/scrape-booking.mjs b/examples/scrape-booking/rest/nodejs/scrape-booking.mjs new file mode 100644 index 0000000..5d524ca --- /dev/null +++ b/examples/scrape-booking/rest/nodejs/scrape-booking.mjs @@ -0,0 +1,43 @@ +// Scrapes Booking.com hotel listings using BQL with stealth mode and residential proxy. +// +// Run: node scrape-booking.mjs + +const query = `mutation ScrapeBooking { + goto( + url: "https://www.booking.com/searchresults.html?ss=New+York&checkin=2025-09-01&checkout=2025-09-07&group_adults=2" + waitUntil: networkIdle + ) { + status + } + waitForTimeout(time: 3000) { + time + } + hotels: mapSelector(selector: "[data-testid='property-card']") { + name: mapSelector(selector: "[data-testid='title']") { + innerText + } + price: mapSelector(selector: "[data-testid='price-and-discounted-price']") { + innerText + } + rating: mapSelector(selector: "[data-testid='review-score']") { + innerText + } + } +}`; + +const response = await fetch( + 'https://production-sfo.browserless.io/stealth/bql?token=YOUR_API_TOKEN_HERE&proxy=residential&proxyCountry=us', + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ query, variables: {}, operationName: 'ScrapeBooking' }), + } +); + +const { data } = await response.json(); +const hotels = data.hotels.map((h) => ({ + name: h.name?.[0]?.innerText ?? '', + price: h.price?.[0]?.innerText ?? '', + rating: h.rating?.[0]?.innerText ?? '', +})); +console.log(JSON.stringify(hotels, null, 2)); diff --git a/examples/scrape-booking/rest/python/scrape_booking.py b/examples/scrape-booking/rest/python/scrape_booking.py new file mode 100644 index 0000000..be5e05e --- /dev/null +++ b/examples/scrape-booking/rest/python/scrape_booking.py @@ -0,0 +1,48 @@ +# Scrapes Booking.com hotel listings using BQL with stealth mode and residential proxy. +# +# Install: pip install requests +# Run: python scrape_booking.py + +import requests + +query = """ +mutation ScrapeBooking { + goto( + url: "https://www.booking.com/searchresults.html?ss=New+York&checkin=2025-09-01&checkout=2025-09-07&group_adults=2" + waitUntil: networkIdle + ) { + status + } + waitForTimeout(time: 3000) { + time + } + hotels: mapSelector(selector: "[data-testid='property-card']") { + name: mapSelector(selector: "[data-testid='title']") { + innerText + } + price: mapSelector(selector: "[data-testid='price-and-discounted-price']") { + innerText + } + rating: mapSelector(selector: "[data-testid='review-score']") { + innerText + } + } +} +""" + +response = requests.post( + 'https://production-sfo.browserless.io/stealth/bql', + params={ + 'token': 'YOUR_API_TOKEN_HERE', + 'proxy': 'residential', + 'proxyCountry': 'us', + }, + json={'query': query, 'variables': {}, 'operationName': 'ScrapeBooking'}, +) + +data = response.json()['data'] +for hotel in data['hotels']: + name = hotel['name'][0]['innerText'] if hotel['name'] else '' + price = hotel['price'][0]['innerText'] if hotel['price'] else '' + rating = hotel['rating'][0]['innerText'] if hotel['rating'] else '' + print(f'{name} — {price} — {rating}') diff --git a/examples/scrape-etsy/bql/scrape-etsy.graphql b/examples/scrape-etsy/bql/scrape-etsy.graphql new file mode 100644 index 0000000..1dce274 --- /dev/null +++ b/examples/scrape-etsy/bql/scrape-etsy.graphql @@ -0,0 +1,35 @@ +# Searches Etsy for candles and extracts product titles, prices, and links. +mutation ScrapeEtsy { + goto(url: "https://www.etsy.com", waitUntil: domContentLoaded) { + status + } + + fillSearch: type( + selector: "input#global-enhancements-search-query" + text: "candles" + ) { + time + } + + submitSearch: click(selector: "button.wt-input-btn-group__btn") { + time + } + + waitForTimeout(time: 3000) { + time + } + + listings: mapSelector(selector: ".v2-listing-card") { + title: mapSelector(selector: "h3", wait: true) { + innerText + } + price: mapSelector(selector: ".currency-value", wait: true) { + innerText + } + link: mapSelector(selector: "a.listing-link") { + href: attribute(name: "href") { + value + } + } + } +} diff --git a/examples/scrape-etsy/frameworks/playwright/nodejs/scrape-etsy.mjs b/examples/scrape-etsy/frameworks/playwright/nodejs/scrape-etsy.mjs new file mode 100644 index 0000000..fbbcaf4 --- /dev/null +++ b/examples/scrape-etsy/frameworks/playwright/nodejs/scrape-etsy.mjs @@ -0,0 +1,30 @@ +// Scrapes Etsy product listings using Playwright with stealth mode and residential proxy. +// +// Install: npm install playwright-core +// Run: node scrape-etsy.mjs + +import { chromium } from 'playwright-core'; + +const browser = await chromium.connectOverCDP( + 'wss://production-sfo.browserless.io?token=YOUR_API_TOKEN_HERE&stealth&proxy=residential&proxyCountry=us' +); + +try { + const context = browser.contexts()[0]; + const page = await context.newPage(); + await page.goto('https://www.etsy.com/search?q=candles', { + waitUntil: 'networkidle', + }); + + const products = await page.evaluate(() => + Array.from(document.querySelectorAll('.v2-listing-card')).map((card) => ({ + title: card.querySelector('h3')?.innerText?.trim() ?? '', + price: card.querySelector('.currency-value')?.innerText?.trim() ?? '', + link: card.querySelector('a.listing-link')?.href ?? '', + })) + ); + + console.log(products); +} finally { + await browser.close(); +} diff --git a/examples/scrape-etsy/frameworks/puppeteer/scrape-etsy.mjs b/examples/scrape-etsy/frameworks/puppeteer/scrape-etsy.mjs new file mode 100644 index 0000000..029334b --- /dev/null +++ b/examples/scrape-etsy/frameworks/puppeteer/scrape-etsy.mjs @@ -0,0 +1,30 @@ +// Scrapes Etsy product listings using Puppeteer with stealth mode and residential proxy. +// +// Install: npm install puppeteer-core +// Run: node scrape-etsy.mjs + +import puppeteer from 'puppeteer-core'; + +const browser = await puppeteer.connect({ + browserWSEndpoint: + 'wss://production-sfo.browserless.io/stealth?token=YOUR_API_TOKEN_HERE&proxy=residential&proxyCountry=us', +}); + +try { + const page = await browser.newPage(); + await page.goto('https://www.etsy.com/search?q=candles', { + waitUntil: 'networkidle2', + }); + + const products = await page.evaluate(() => + Array.from(document.querySelectorAll('.v2-listing-card')).map((card) => ({ + title: card.querySelector('h3')?.innerText?.trim() ?? '', + price: card.querySelector('.currency-value')?.innerText?.trim() ?? '', + link: card.querySelector('a.listing-link')?.href ?? '', + })) + ); + + console.log(products); +} finally { + await browser.close(); +} diff --git a/examples/scrape-etsy/rest/csharp/ScrapeEtsy.cs b/examples/scrape-etsy/rest/csharp/ScrapeEtsy.cs new file mode 100644 index 0000000..18e3d44 --- /dev/null +++ b/examples/scrape-etsy/rest/csharp/ScrapeEtsy.cs @@ -0,0 +1,29 @@ +// Scrapes Etsy product titles and prices from search results. +// +// Run: dotnet run + +using System.Net.Http; +using System.Text; +using System.Text.Json; + +string token = "YOUR_API_TOKEN_HERE"; +string endpoint = $"https://production-sfo.browserless.io/scrape?token={token}"; + +var payload = new +{ + url = "https://www.etsy.com/search?q=candles", + elements = new[] + { + new { selector = ".v2-listing-card h3" }, + new { selector = ".v2-listing-card .currency-value" }, + }, +}; + +using (HttpClient httpClient = new HttpClient()) +{ + var jsonPayload = JsonSerializer.Serialize(payload); + var content = new StringContent(jsonPayload, Encoding.UTF8, "application/json"); + var response = await httpClient.PostAsync(endpoint, content); + string responseBody = await response.Content.ReadAsStringAsync(); + Console.WriteLine(responseBody); +} diff --git a/examples/scrape-etsy/rest/curl/scrape-etsy.sh b/examples/scrape-etsy/rest/curl/scrape-etsy.sh new file mode 100644 index 0000000..d074c2c --- /dev/null +++ b/examples/scrape-etsy/rest/curl/scrape-etsy.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Scrapes Etsy product titles and prices from search results. +# Run: bash scrape-etsy.sh + +curl -X POST \ + "https://production-sfo.browserless.io/scrape?token=YOUR_API_TOKEN_HERE" \ + -H "Content-Type: application/json" \ + -d '{ + "url": "https://www.etsy.com/search?q=candles", + "elements": [ + { "selector": ".v2-listing-card h3" }, + { "selector": ".v2-listing-card .currency-value" } + ] + }' diff --git a/examples/scrape-etsy/rest/java/ScrapeEtsy.java b/examples/scrape-etsy/rest/java/ScrapeEtsy.java new file mode 100644 index 0000000..b44d7e2 --- /dev/null +++ b/examples/scrape-etsy/rest/java/ScrapeEtsy.java @@ -0,0 +1,33 @@ +// Scrapes Etsy product titles and prices from search results. +// +// Run: javac ScrapeEtsy.java && java ScrapeEtsy + +import java.net.URI; +import java.net.http.*; + +public class ScrapeEtsy { + public static void main(String[] args) throws Exception { + String token = "YOUR_API_TOKEN_HERE"; + String endpoint = "https://production-sfo.browserless.io/scrape?token=" + token; + + String payload = """ + { + "url": "https://www.etsy.com/search?q=candles", + "elements": [ + { "selector": ".v2-listing-card h3" }, + { "selector": ".v2-listing-card .currency-value" } + ] + } + """; + + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(endpoint)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(payload)) + .build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + System.out.println(response.body()); + } +} diff --git a/examples/scrape-etsy/rest/nodejs/scrape-etsy.mjs b/examples/scrape-etsy/rest/nodejs/scrape-etsy.mjs new file mode 100644 index 0000000..69d7f43 --- /dev/null +++ b/examples/scrape-etsy/rest/nodejs/scrape-etsy.mjs @@ -0,0 +1,23 @@ +// Scrapes Etsy product titles and prices from search results. +// +// Run: node scrape-etsy.mjs + +const response = await fetch( + 'https://production-sfo.browserless.io/scrape?token=YOUR_API_TOKEN_HERE', + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + url: 'https://www.etsy.com/search?q=candles', + elements: [ + { selector: '.v2-listing-card h3' }, + { selector: '.v2-listing-card .currency-value' }, + ], + }), + } +); + +const { data } = await response.json(); +const titles = data[0].results.map((r) => r.text); +const prices = data[1].results.map((r) => r.text); +console.log(titles.map((title, i) => ({ title, price: prices[i] }))); diff --git a/examples/scrape-etsy/rest/python/scrape_etsy.py b/examples/scrape-etsy/rest/python/scrape_etsy.py new file mode 100644 index 0000000..8258db6 --- /dev/null +++ b/examples/scrape-etsy/rest/python/scrape_etsy.py @@ -0,0 +1,23 @@ +# Scrapes Etsy product titles and prices from search results. +# +# Install: pip install requests +# Run: python scrape_etsy.py + +import requests + +response = requests.post( + 'https://production-sfo.browserless.io/scrape?token=YOUR_API_TOKEN_HERE', + json={ + 'url': 'https://www.etsy.com/search?q=candles', + 'elements': [ + {'selector': '.v2-listing-card h3'}, + {'selector': '.v2-listing-card .currency-value'}, + ], + }, +) + +data = response.json()['data'] +titles = [item['text'] for item in data[0]['results']] +prices = [item['text'] for item in data[1]['results']] +for title, price in zip(titles, prices): + print(f'{title}: ${price}') diff --git a/examples/scrape-glassdoor/bql/scrape-glassdoor.graphql b/examples/scrape-glassdoor/bql/scrape-glassdoor.graphql new file mode 100644 index 0000000..74b916b --- /dev/null +++ b/examples/scrape-glassdoor/bql/scrape-glassdoor.graphql @@ -0,0 +1,24 @@ +# Scrapes Glassdoor job listings. Run against /stealth/bql with residential proxy. +mutation ScrapeGlassdoor { + goto( + url: "https://www.glassdoor.com/Job/new-york-software-engineer-jobs-SRCH_IL.0,8_IC1132348_KO9,26.htm" + waitUntil: networkIdle + ) { + status + } + + jobs: mapSelector(selector: "[data-test='jobListing']") { + title: mapSelector(selector: "a[data-test='job-title']") { + innerText + } + company: mapSelector(selector: "[data-test='employer-name']") { + innerText + } + location: mapSelector(selector: "[data-test='emp-location']") { + innerText + } + salary: mapSelector(selector: "[data-test='detailSalary']") { + innerText + } + } +} diff --git a/examples/scrape-glassdoor/frameworks/playwright/nodejs/scrape-glassdoor.mjs b/examples/scrape-glassdoor/frameworks/playwright/nodejs/scrape-glassdoor.mjs new file mode 100644 index 0000000..1b17835 --- /dev/null +++ b/examples/scrape-glassdoor/frameworks/playwright/nodejs/scrape-glassdoor.mjs @@ -0,0 +1,32 @@ +// Scrapes Glassdoor job listings using Playwright with stealth mode and residential proxy. +// +// Install: npm install playwright-core +// Run: node scrape-glassdoor.mjs + +import { chromium } from 'playwright-core'; + +const browser = await chromium.connectOverCDP( + 'wss://production-sfo.browserless.io?token=YOUR_API_TOKEN_HERE&stealth&proxy=residential&proxyCountry=us' +); + +try { + const context = browser.contexts()[0]; + const page = await context.newPage(); + await page.goto( + 'https://www.glassdoor.com/Job/new-york-software-engineer-jobs-SRCH_IL.0,8_IC1132348_KO9,26.htm', + { waitUntil: 'networkidle' } + ); + + const jobs = await page.evaluate(() => + Array.from(document.querySelectorAll('[data-test="jobListing"]')).map((card) => ({ + title: card.querySelector('a[data-test="job-title"]')?.innerText?.trim() ?? '', + company: card.querySelector('[data-test="employer-name"]')?.innerText?.trim() ?? '', + location: card.querySelector('[data-test="emp-location"]')?.innerText?.trim() ?? '', + salary: card.querySelector('[data-test="detailSalary"]')?.innerText?.trim() ?? '', + })) + ); + + console.log(JSON.stringify(jobs, null, 2)); +} finally { + await browser.close(); +} diff --git a/examples/scrape-glassdoor/frameworks/puppeteer/scrape-glassdoor.mjs b/examples/scrape-glassdoor/frameworks/puppeteer/scrape-glassdoor.mjs new file mode 100644 index 0000000..eda5d7c --- /dev/null +++ b/examples/scrape-glassdoor/frameworks/puppeteer/scrape-glassdoor.mjs @@ -0,0 +1,32 @@ +// Scrapes Glassdoor job listings using Puppeteer with stealth mode and residential proxy. +// +// Install: npm install puppeteer-core +// Run: node scrape-glassdoor.mjs + +import puppeteer from 'puppeteer-core'; + +const browser = await puppeteer.connect({ + browserWSEndpoint: + 'wss://production-sfo.browserless.io/stealth?token=YOUR_API_TOKEN_HERE&proxy=residential&proxyCountry=us', +}); + +try { + const page = await browser.newPage(); + await page.goto( + 'https://www.glassdoor.com/Job/new-york-software-engineer-jobs-SRCH_IL.0,8_IC1132348_KO9,26.htm', + { waitUntil: 'networkidle2' } + ); + + const jobs = await page.evaluate(() => + Array.from(document.querySelectorAll('[data-test="jobListing"]')).map((card) => ({ + title: card.querySelector('a[data-test="job-title"]')?.innerText?.trim() ?? '', + company: card.querySelector('[data-test="employer-name"]')?.innerText?.trim() ?? '', + location: card.querySelector('[data-test="emp-location"]')?.innerText?.trim() ?? '', + salary: card.querySelector('[data-test="detailSalary"]')?.innerText?.trim() ?? '', + })) + ); + + console.log(JSON.stringify(jobs, null, 2)); +} finally { + await browser.close(); +} diff --git a/examples/scrape-glassdoor/rest/csharp/ScrapeGlassdoor.cs b/examples/scrape-glassdoor/rest/csharp/ScrapeGlassdoor.cs new file mode 100644 index 0000000..f649647 --- /dev/null +++ b/examples/scrape-glassdoor/rest/csharp/ScrapeGlassdoor.cs @@ -0,0 +1,34 @@ +// Scrapes Glassdoor job listings using BQL with stealth mode and residential proxy. +// +// Run: dotnet run + +using System.Net.Http; +using System.Text; +using System.Text.Json; + +string token = "YOUR_API_TOKEN_HERE"; +string endpoint = $"https://production-sfo.browserless.io/stealth/bql?token={token}&proxy=residential&proxyCountry=us"; + +var payload = new +{ + query = @"mutation ScrapeGlassdoor { + goto(url: ""https://www.glassdoor.com/Job/new-york-software-engineer-jobs-SRCH_IL.0,8_IC1132348_KO9,26.htm"", waitUntil: networkIdle) { status } + jobs: mapSelector(selector: ""[data-test='jobListing']"") { + title: mapSelector(selector: ""a[data-test='job-title']"") { innerText } + company: mapSelector(selector: ""[data-test='employer-name']"") { innerText } + location: mapSelector(selector: ""[data-test='emp-location']"") { innerText } + salary: mapSelector(selector: ""[data-test='detailSalary']"") { innerText } + } + }", + variables = new { }, + operationName = "ScrapeGlassdoor", +}; + +using (HttpClient httpClient = new HttpClient()) +{ + var content = new StringContent( + JsonSerializer.Serialize(payload), Encoding.UTF8, "application/json"); + var response = await httpClient.PostAsync(endpoint, content); + string body = await response.Content.ReadAsStringAsync(); + Console.WriteLine(body); +} diff --git a/examples/scrape-glassdoor/rest/curl/scrape-glassdoor.sh b/examples/scrape-glassdoor/rest/curl/scrape-glassdoor.sh new file mode 100644 index 0000000..92cd154 --- /dev/null +++ b/examples/scrape-glassdoor/rest/curl/scrape-glassdoor.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Scrapes Glassdoor job listings using BQL with stealth mode and residential proxy. +# Run: bash scrape-glassdoor.sh + +curl -X POST \ + "https://production-sfo.browserless.io/stealth/bql?token=YOUR_API_TOKEN_HERE&proxy=residential&proxyCountry=us" \ + -H "Content-Type: application/json" \ + -d '{ + "query": "mutation ScrapeGlassdoor { goto(url: \"https://www.glassdoor.com/Job/new-york-software-engineer-jobs-SRCH_IL.0,8_IC1132348_KO9,26.htm\", waitUntil: networkIdle) { status } jobs: mapSelector(selector: \"[data-test='\''jobListing'\'']\") { title: mapSelector(selector: \"a[data-test='\''job-title'\'']\") { innerText } company: mapSelector(selector: \"[data-test='\''employer-name'\'']\") { innerText } location: mapSelector(selector: \"[data-test='\''emp-location'\'']\") { innerText } salary: mapSelector(selector: \"[data-test='\''detailSalary'\'']\") { innerText } } }", + "variables": {}, + "operationName": "ScrapeGlassdoor" + }' diff --git a/examples/scrape-glassdoor/rest/java/ScrapeGlassdoor.java b/examples/scrape-glassdoor/rest/java/ScrapeGlassdoor.java new file mode 100644 index 0000000..730b3d6 --- /dev/null +++ b/examples/scrape-glassdoor/rest/java/ScrapeGlassdoor.java @@ -0,0 +1,36 @@ +// Scrapes Glassdoor job listings using BQL with stealth mode and residential proxy. +// +// Run: javac ScrapeGlassdoor.java && java ScrapeGlassdoor + +import java.net.URI; +import java.net.http.*; + +public class ScrapeGlassdoor { + public static void main(String[] args) throws Exception { + String token = "YOUR_API_TOKEN_HERE"; + String endpoint = "https://production-sfo.browserless.io/stealth/bql?token=" + + token + "&proxy=residential&proxyCountry=us"; + + String query = "mutation ScrapeGlassdoor {" + + " goto(url: \\\"https://www.glassdoor.com/Job/new-york-software-engineer-jobs-SRCH_IL.0,8_IC1132348_KO9,26.htm\\\", waitUntil: networkIdle) { status }" + + " jobs: mapSelector(selector: \\\"[data-test='jobListing']\\\") {" + + " title: mapSelector(selector: \\\"a[data-test='job-title']\\\") { innerText }" + + " company: mapSelector(selector: \\\"[data-test='employer-name']\\\") { innerText }" + + " location: mapSelector(selector: \\\"[data-test='emp-location']\\\") { innerText }" + + " salary: mapSelector(selector: \\\"[data-test='detailSalary']\\\") { innerText }" + + " }" + + " }"; + + String payload = "{\"query\": \"" + query + "\", \"variables\": {}, \"operationName\": \"ScrapeGlassdoor\"}"; + + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(endpoint)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(payload)) + .build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + System.out.println(response.body()); + } +} diff --git a/examples/scrape-glassdoor/rest/nodejs/scrape-glassdoor.mjs b/examples/scrape-glassdoor/rest/nodejs/scrape-glassdoor.mjs new file mode 100644 index 0000000..a625f5e --- /dev/null +++ b/examples/scrape-glassdoor/rest/nodejs/scrape-glassdoor.mjs @@ -0,0 +1,44 @@ +// Scrapes Glassdoor job listings using BQL with stealth mode and residential proxy. +// +// Run: node scrape-glassdoor.mjs + +const query = `mutation ScrapeGlassdoor { + goto( + url: "https://www.glassdoor.com/Job/new-york-software-engineer-jobs-SRCH_IL.0,8_IC1132348_KO9,26.htm" + waitUntil: networkIdle + ) { + status + } + jobs: mapSelector(selector: "[data-test='jobListing']") { + title: mapSelector(selector: "a[data-test='job-title']") { + innerText + } + company: mapSelector(selector: "[data-test='employer-name']") { + innerText + } + location: mapSelector(selector: "[data-test='emp-location']") { + innerText + } + salary: mapSelector(selector: "[data-test='detailSalary']") { + innerText + } + } +}`; + +const response = await fetch( + 'https://production-sfo.browserless.io/stealth/bql?token=YOUR_API_TOKEN_HERE&proxy=residential&proxyCountry=us', + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ query, variables: {}, operationName: 'ScrapeGlassdoor' }), + } +); + +const { data } = await response.json(); +const jobs = data.jobs.map((job) => ({ + title: job.title?.[0]?.innerText ?? '', + company: job.company?.[0]?.innerText ?? '', + location: job.location?.[0]?.innerText ?? '', + salary: job.salary?.[0]?.innerText ?? '', +})); +console.log(JSON.stringify(jobs, null, 2)); diff --git a/examples/scrape-glassdoor/rest/python/scrape_glassdoor.py b/examples/scrape-glassdoor/rest/python/scrape_glassdoor.py new file mode 100644 index 0000000..1eb4965 --- /dev/null +++ b/examples/scrape-glassdoor/rest/python/scrape_glassdoor.py @@ -0,0 +1,49 @@ +# Scrapes Glassdoor job listings using BQL with stealth mode and residential proxy. +# +# Install: pip install requests +# Run: python scrape_glassdoor.py + +import requests + +query = """ +mutation ScrapeGlassdoor { + goto( + url: "https://www.glassdoor.com/Job/new-york-software-engineer-jobs-SRCH_IL.0,8_IC1132348_KO9,26.htm" + waitUntil: networkIdle + ) { + status + } + jobs: mapSelector(selector: "[data-test='jobListing']") { + title: mapSelector(selector: "a[data-test='job-title']") { + innerText + } + company: mapSelector(selector: "[data-test='employer-name']") { + innerText + } + location: mapSelector(selector: "[data-test='emp-location']") { + innerText + } + salary: mapSelector(selector: "[data-test='detailSalary']") { + innerText + } + } +} +""" + +response = requests.post( + 'https://production-sfo.browserless.io/stealth/bql', + params={ + 'token': 'YOUR_API_TOKEN_HERE', + 'proxy': 'residential', + 'proxyCountry': 'us', + }, + json={'query': query, 'variables': {}, 'operationName': 'ScrapeGlassdoor'}, +) + +data = response.json()['data'] +for job in data['jobs']: + title = job['title'][0]['innerText'] if job['title'] else '' + company = job['company'][0]['innerText'] if job['company'] else '' + location = job['location'][0]['innerText'] if job['location'] else '' + salary = job['salary'][0]['innerText'] if job['salary'] else '' + print(f'{title} at {company} — {location} — {salary}') diff --git a/examples/scrape-google-shopping/bql/scrape-google-shopping.graphql b/examples/scrape-google-shopping/bql/scrape-google-shopping.graphql new file mode 100644 index 0000000..fb6fbb5 --- /dev/null +++ b/examples/scrape-google-shopping/bql/scrape-google-shopping.graphql @@ -0,0 +1,24 @@ +# Scrapes Google Shopping results. Run against /stealth/bql. +mutation ScrapeGoogleShopping { + goto( + url: "https://www.google.com/search?q=wireless+headphones&tbm=shop" + waitUntil: networkIdle + ) { + status + } + + products: mapSelector(selector: ".sh-dgr__grid-result") { + title: mapSelector(selector: "h3.tAxDx") { + innerText + } + price: mapSelector(selector: ".a8Pemb") { + innerText + } + store: mapSelector(selector: ".aULzUe") { + innerText + } + rating: mapSelector(selector: ".Rsc7Yb") { + innerText + } + } +} diff --git a/examples/scrape-google-shopping/frameworks/playwright/nodejs/scrape-google-shopping.mjs b/examples/scrape-google-shopping/frameworks/playwright/nodejs/scrape-google-shopping.mjs new file mode 100644 index 0000000..f958a8d --- /dev/null +++ b/examples/scrape-google-shopping/frameworks/playwright/nodejs/scrape-google-shopping.mjs @@ -0,0 +1,32 @@ +// Scrapes Google Shopping results using Playwright with stealth mode. +// +// Install: npm install playwright-core +// Run: node scrape-google-shopping.mjs + +import { chromium } from 'playwright-core'; + +const browser = await chromium.connectOverCDP( + 'wss://production-sfo.browserless.io?token=YOUR_API_TOKEN_HERE&stealth' +); + +try { + const context = browser.contexts()[0]; + const page = await context.newPage(); + await page.goto( + 'https://www.google.com/search?q=wireless+headphones&tbm=shop', + { waitUntil: 'networkidle' } + ); + + const products = await page.evaluate(() => + Array.from(document.querySelectorAll('.sh-dgr__grid-result')).map((card) => ({ + title: card.querySelector('h3.tAxDx')?.innerText?.trim() ?? '', + price: card.querySelector('.a8Pemb')?.innerText?.trim() ?? '', + store: card.querySelector('.aULzUe')?.innerText?.trim() ?? '', + rating: card.querySelector('.Rsc7Yb')?.innerText?.trim() ?? '', + })) + ); + + console.log(JSON.stringify(products, null, 2)); +} finally { + await browser.close(); +} diff --git a/examples/scrape-google-shopping/frameworks/puppeteer/scrape-google-shopping.mjs b/examples/scrape-google-shopping/frameworks/puppeteer/scrape-google-shopping.mjs new file mode 100644 index 0000000..9be02a9 --- /dev/null +++ b/examples/scrape-google-shopping/frameworks/puppeteer/scrape-google-shopping.mjs @@ -0,0 +1,32 @@ +// Scrapes Google Shopping results using Puppeteer with stealth mode. +// +// Install: npm install puppeteer-core +// Run: node scrape-google-shopping.mjs + +import puppeteer from 'puppeteer-core'; + +const browser = await puppeteer.connect({ + browserWSEndpoint: + 'wss://production-sfo.browserless.io/stealth?token=YOUR_API_TOKEN_HERE', +}); + +try { + const page = await browser.newPage(); + await page.goto( + 'https://www.google.com/search?q=wireless+headphones&tbm=shop', + { waitUntil: 'networkidle2' } + ); + + const products = await page.evaluate(() => + Array.from(document.querySelectorAll('.sh-dgr__grid-result')).map((card) => ({ + title: card.querySelector('h3.tAxDx')?.innerText?.trim() ?? '', + price: card.querySelector('.a8Pemb')?.innerText?.trim() ?? '', + store: card.querySelector('.aULzUe')?.innerText?.trim() ?? '', + rating: card.querySelector('.Rsc7Yb')?.innerText?.trim() ?? '', + })) + ); + + console.log(JSON.stringify(products, null, 2)); +} finally { + await browser.close(); +} diff --git a/examples/scrape-google-shopping/rest/csharp/ScrapeGoogleShopping.cs b/examples/scrape-google-shopping/rest/csharp/ScrapeGoogleShopping.cs new file mode 100644 index 0000000..286ebe5 --- /dev/null +++ b/examples/scrape-google-shopping/rest/csharp/ScrapeGoogleShopping.cs @@ -0,0 +1,34 @@ +// Scrapes Google Shopping results using BQL with stealth mode. +// +// Run: dotnet run + +using System.Net.Http; +using System.Text; +using System.Text.Json; + +string token = "YOUR_API_TOKEN_HERE"; +string endpoint = $"https://production-sfo.browserless.io/stealth/bql?token={token}"; + +var payload = new +{ + query = @"mutation ScrapeGoogleShopping { + goto(url: ""https://www.google.com/search?q=wireless+headphones&tbm=shop"", waitUntil: networkIdle) { status } + products: mapSelector(selector: "".sh-dgr__grid-result"") { + title: mapSelector(selector: ""h3.tAxDx"") { innerText } + price: mapSelector(selector: "".a8Pemb"") { innerText } + store: mapSelector(selector: "".aULzUe"") { innerText } + rating: mapSelector(selector: "".Rsc7Yb"") { innerText } + } + }", + variables = new { }, + operationName = "ScrapeGoogleShopping", +}; + +using (HttpClient httpClient = new HttpClient()) +{ + var content = new StringContent( + JsonSerializer.Serialize(payload), Encoding.UTF8, "application/json"); + var response = await httpClient.PostAsync(endpoint, content); + string body = await response.Content.ReadAsStringAsync(); + Console.WriteLine(body); +} diff --git a/examples/scrape-google-shopping/rest/curl/scrape-google-shopping.sh b/examples/scrape-google-shopping/rest/curl/scrape-google-shopping.sh new file mode 100644 index 0000000..bb82b5a --- /dev/null +++ b/examples/scrape-google-shopping/rest/curl/scrape-google-shopping.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Scrapes Google Shopping results using BQL with stealth mode. +# Run: bash scrape-google-shopping.sh + +curl -X POST \ + "https://production-sfo.browserless.io/stealth/bql?token=YOUR_API_TOKEN_HERE" \ + -H "Content-Type: application/json" \ + -d '{ + "query": "mutation ScrapeGoogleShopping { goto(url: \"https://www.google.com/search?q=wireless+headphones&tbm=shop\", waitUntil: networkIdle) { status } products: mapSelector(selector: \".sh-dgr__grid-result\") { title: mapSelector(selector: \"h3.tAxDx\") { innerText } price: mapSelector(selector: \".a8Pemb\") { innerText } store: mapSelector(selector: \".aULzUe\") { innerText } rating: mapSelector(selector: \".Rsc7Yb\") { innerText } } }", + "variables": {}, + "operationName": "ScrapeGoogleShopping" + }' diff --git a/examples/scrape-google-shopping/rest/java/ScrapeGoogleShopping.java b/examples/scrape-google-shopping/rest/java/ScrapeGoogleShopping.java new file mode 100644 index 0000000..f776bfa --- /dev/null +++ b/examples/scrape-google-shopping/rest/java/ScrapeGoogleShopping.java @@ -0,0 +1,35 @@ +// Scrapes Google Shopping results using BQL with stealth mode. +// +// Run: javac ScrapeGoogleShopping.java && java ScrapeGoogleShopping + +import java.net.URI; +import java.net.http.*; + +public class ScrapeGoogleShopping { + public static void main(String[] args) throws Exception { + String token = "YOUR_API_TOKEN_HERE"; + String endpoint = "https://production-sfo.browserless.io/stealth/bql?token=" + token; + + String query = "mutation ScrapeGoogleShopping {" + + " goto(url: \\\"https://www.google.com/search?q=wireless+headphones&tbm=shop\\\", waitUntil: networkIdle) { status }" + + " products: mapSelector(selector: \\\".sh-dgr__grid-result\\\") {" + + " title: mapSelector(selector: \\\"h3.tAxDx\\\") { innerText }" + + " price: mapSelector(selector: \\\".a8Pemb\\\") { innerText }" + + " store: mapSelector(selector: \\\".aULzUe\\\") { innerText }" + + " rating: mapSelector(selector: \\\".Rsc7Yb\\\") { innerText }" + + " }" + + " }"; + + String payload = "{\"query\": \"" + query + "\", \"variables\": {}, \"operationName\": \"ScrapeGoogleShopping\"}"; + + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(endpoint)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(payload)) + .build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + System.out.println(response.body()); + } +} diff --git a/examples/scrape-google-shopping/rest/nodejs/scrape-google-shopping.mjs b/examples/scrape-google-shopping/rest/nodejs/scrape-google-shopping.mjs new file mode 100644 index 0000000..fd25ac7 --- /dev/null +++ b/examples/scrape-google-shopping/rest/nodejs/scrape-google-shopping.mjs @@ -0,0 +1,41 @@ +// Scrapes Google Shopping results using BQL with stealth mode. +// +// Run: node scrape-google-shopping.mjs + +const query = `mutation ScrapeGoogleShopping { + goto(url: "https://www.google.com/search?q=wireless+headphones&tbm=shop", waitUntil: networkIdle) { + status + } + products: mapSelector(selector: ".sh-dgr__grid-result") { + title: mapSelector(selector: "h3.tAxDx") { + innerText + } + price: mapSelector(selector: ".a8Pemb") { + innerText + } + store: mapSelector(selector: ".aULzUe") { + innerText + } + rating: mapSelector(selector: ".Rsc7Yb") { + innerText + } + } +}`; + +const response = await fetch( + 'https://production-sfo.browserless.io/stealth/bql?token=YOUR_API_TOKEN_HERE', + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ query, variables: {}, operationName: 'ScrapeGoogleShopping' }), + } +); + +const { data } = await response.json(); +const products = data.products.map((p) => ({ + title: p.title?.[0]?.innerText ?? '', + price: p.price?.[0]?.innerText ?? '', + store: p.store?.[0]?.innerText ?? '', + rating: p.rating?.[0]?.innerText ?? '', +})); +console.log(JSON.stringify(products, null, 2)); diff --git a/examples/scrape-google-shopping/rest/python/scrape_google_shopping.py b/examples/scrape-google-shopping/rest/python/scrape_google_shopping.py new file mode 100644 index 0000000..b59c4db --- /dev/null +++ b/examples/scrape-google-shopping/rest/python/scrape_google_shopping.py @@ -0,0 +1,42 @@ +# Scrapes Google Shopping results using BQL with stealth mode. +# +# Install: pip install requests +# Run: python scrape_google_shopping.py + +import requests + +query = """ +mutation ScrapeGoogleShopping { + goto(url: "https://www.google.com/search?q=wireless+headphones&tbm=shop", waitUntil: networkIdle) { + status + } + products: mapSelector(selector: ".sh-dgr__grid-result") { + title: mapSelector(selector: "h3.tAxDx") { + innerText + } + price: mapSelector(selector: ".a8Pemb") { + innerText + } + store: mapSelector(selector: ".aULzUe") { + innerText + } + rating: mapSelector(selector: ".Rsc7Yb") { + innerText + } + } +} +""" + +response = requests.post( + 'https://production-sfo.browserless.io/stealth/bql', + params={'token': 'YOUR_API_TOKEN_HERE'}, + json={'query': query, 'variables': {}, 'operationName': 'ScrapeGoogleShopping'}, +) + +data = response.json()['data'] +for product in data['products']: + title = product['title'][0]['innerText'] if product['title'] else '' + price = product['price'][0]['innerText'] if product['price'] else '' + store = product['store'][0]['innerText'] if product['store'] else '' + rating = product['rating'][0]['innerText'] if product['rating'] else '' + print(f'{title} — {price} — {store} — {rating}') diff --git a/examples/scrape-reddit/bql/scrape-reddit.graphql b/examples/scrape-reddit/bql/scrape-reddit.graphql new file mode 100644 index 0000000..501d7e5 --- /dev/null +++ b/examples/scrape-reddit/bql/scrape-reddit.graphql @@ -0,0 +1,22 @@ +# Scrapes Reddit posts from r/programming. Run against /stealth/bql. +mutation ScrapeReddit { + goto(url: "https://www.reddit.com/r/programming/", waitUntil: networkIdle) { + status + } + + waitForTimeout(time: 2000) { + time + } + + posts: mapSelector(selector: "article") { + title: mapSelector(selector: "[id*='post-title']") { + innerText + } + score: mapSelector(selector: "[id*='vote-arrows']") { + innerText + } + comments: mapSelector(selector: "a[data-click-id='comments']") { + innerText + } + } +} diff --git a/examples/scrape-reddit/frameworks/playwright/nodejs/scrape-reddit.mjs b/examples/scrape-reddit/frameworks/playwright/nodejs/scrape-reddit.mjs new file mode 100644 index 0000000..f406962 --- /dev/null +++ b/examples/scrape-reddit/frameworks/playwright/nodejs/scrape-reddit.mjs @@ -0,0 +1,32 @@ +// Scrapes Reddit posts from a subreddit using Playwright with stealth mode. +// +// Install: npm install playwright-core +// Run: node scrape-reddit.mjs + +import { chromium } from 'playwright-core'; + +const browser = await chromium.connectOverCDP( + 'wss://production-sfo.browserless.io?token=YOUR_API_TOKEN_HERE&stealth' +); + +try { + const context = browser.contexts()[0]; + const page = await context.newPage(); + await page.goto('https://www.reddit.com/r/programming/', { + waitUntil: 'networkidle', + }); + + await page.waitForTimeout(2000); + + const posts = await page.evaluate(() => + Array.from(document.querySelectorAll('article')).map((article) => ({ + title: article.querySelector('[id*="post-title"]')?.innerText?.trim() ?? '', + score: article.querySelector('[id*="vote-arrows"]')?.innerText?.trim() ?? '', + comments: article.querySelector('a[data-click-id="comments"]')?.innerText?.trim() ?? '', + })) + ); + + console.log(JSON.stringify(posts, null, 2)); +} finally { + await browser.close(); +} diff --git a/examples/scrape-reddit/frameworks/puppeteer/scrape-reddit.mjs b/examples/scrape-reddit/frameworks/puppeteer/scrape-reddit.mjs new file mode 100644 index 0000000..d632f7f --- /dev/null +++ b/examples/scrape-reddit/frameworks/puppeteer/scrape-reddit.mjs @@ -0,0 +1,32 @@ +// Scrapes Reddit posts from a subreddit using Puppeteer with stealth mode. +// +// Install: npm install puppeteer-core +// Run: node scrape-reddit.mjs + +import puppeteer from 'puppeteer-core'; + +const browser = await puppeteer.connect({ + browserWSEndpoint: + 'wss://production-sfo.browserless.io/stealth?token=YOUR_API_TOKEN_HERE', +}); + +try { + const page = await browser.newPage(); + await page.goto('https://www.reddit.com/r/programming/', { + waitUntil: 'networkidle2', + }); + + await new Promise((r) => setTimeout(r, 2000)); + + const posts = await page.evaluate(() => + Array.from(document.querySelectorAll('article')).map((article) => ({ + title: article.querySelector('[id*="post-title"]')?.innerText?.trim() ?? '', + score: article.querySelector('[id*="vote-arrows"]')?.innerText?.trim() ?? '', + comments: article.querySelector('a[data-click-id="comments"]')?.innerText?.trim() ?? '', + })) + ); + + console.log(JSON.stringify(posts, null, 2)); +} finally { + await browser.close(); +} diff --git a/examples/scrape-reddit/rest/csharp/ScrapeReddit.cs b/examples/scrape-reddit/rest/csharp/ScrapeReddit.cs new file mode 100644 index 0000000..e1b8c63 --- /dev/null +++ b/examples/scrape-reddit/rest/csharp/ScrapeReddit.cs @@ -0,0 +1,34 @@ +// Scrapes Reddit posts from a subreddit using BQL with stealth mode. +// +// Run: dotnet run + +using System.Net.Http; +using System.Text; +using System.Text.Json; + +string token = "YOUR_API_TOKEN_HERE"; +string endpoint = $"https://production-sfo.browserless.io/stealth/bql?token={token}"; + +var payload = new +{ + query = @"mutation ScrapeReddit { + goto(url: ""https://www.reddit.com/r/programming/"", waitUntil: networkIdle) { status } + waitForTimeout(time: 2000) { time } + posts: mapSelector(selector: ""article"") { + title: mapSelector(selector: ""[id*='post-title']"") { innerText } + score: mapSelector(selector: ""[id*='vote-arrows']"") { innerText } + comments: mapSelector(selector: ""a[data-click-id='comments']"") { innerText } + } + }", + variables = new { }, + operationName = "ScrapeReddit", +}; + +using (HttpClient httpClient = new HttpClient()) +{ + var content = new StringContent( + JsonSerializer.Serialize(payload), Encoding.UTF8, "application/json"); + var response = await httpClient.PostAsync(endpoint, content); + string body = await response.Content.ReadAsStringAsync(); + Console.WriteLine(body); +} diff --git a/examples/scrape-reddit/rest/curl/scrape-reddit.sh b/examples/scrape-reddit/rest/curl/scrape-reddit.sh new file mode 100644 index 0000000..d1aa97c --- /dev/null +++ b/examples/scrape-reddit/rest/curl/scrape-reddit.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Scrapes Reddit posts from a subreddit using BQL with stealth mode. +# Run: bash scrape-reddit.sh + +curl -X POST \ + "https://production-sfo.browserless.io/stealth/bql?token=YOUR_API_TOKEN_HERE" \ + -H "Content-Type: application/json" \ + -d '{ + "query": "mutation ScrapeReddit { goto(url: \"https://www.reddit.com/r/programming/\", waitUntil: networkIdle) { status } waitForTimeout(time: 2000) { time } posts: mapSelector(selector: \"article\") { title: mapSelector(selector: \"[id*='\''post-title'\'']\") { innerText } score: mapSelector(selector: \"[id*='\''vote-arrows'\'']\") { innerText } comments: mapSelector(selector: \"a[data-click-id='\''comments'\'']\") { innerText } } }", + "variables": {}, + "operationName": "ScrapeReddit" + }' diff --git a/examples/scrape-reddit/rest/java/ScrapeReddit.java b/examples/scrape-reddit/rest/java/ScrapeReddit.java new file mode 100644 index 0000000..0b1b52f --- /dev/null +++ b/examples/scrape-reddit/rest/java/ScrapeReddit.java @@ -0,0 +1,35 @@ +// Scrapes Reddit posts from a subreddit using BQL with stealth mode. +// +// Run: javac ScrapeReddit.java && java ScrapeReddit + +import java.net.URI; +import java.net.http.*; + +public class ScrapeReddit { + public static void main(String[] args) throws Exception { + String token = "YOUR_API_TOKEN_HERE"; + String endpoint = "https://production-sfo.browserless.io/stealth/bql?token=" + token; + + String query = "mutation ScrapeReddit {" + + " goto(url: \\\"https://www.reddit.com/r/programming/\\\", waitUntil: networkIdle) { status }" + + " waitForTimeout(time: 2000) { time }" + + " posts: mapSelector(selector: \\\"article\\\") {" + + " title: mapSelector(selector: \\\"[id*='post-title']\\\") { innerText }" + + " score: mapSelector(selector: \\\"[id*='vote-arrows']\\\") { innerText }" + + " comments: mapSelector(selector: \\\"a[data-click-id='comments']\\\") { innerText }" + + " }" + + " }"; + + String payload = "{\"query\": \"" + query + "\", \"variables\": {}, \"operationName\": \"ScrapeReddit\"}"; + + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(endpoint)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(payload)) + .build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + System.out.println(response.body()); + } +} diff --git a/examples/scrape-reddit/rest/nodejs/scrape-reddit.mjs b/examples/scrape-reddit/rest/nodejs/scrape-reddit.mjs new file mode 100644 index 0000000..bea314a --- /dev/null +++ b/examples/scrape-reddit/rest/nodejs/scrape-reddit.mjs @@ -0,0 +1,40 @@ +// Scrapes Reddit posts from a subreddit using BQL with stealth mode. +// +// Run: node scrape-reddit.mjs + +const query = `mutation ScrapeReddit { + goto(url: "https://www.reddit.com/r/programming/", waitUntil: networkIdle) { + status + } + waitForTimeout(time: 2000) { + time + } + posts: mapSelector(selector: "article") { + title: mapSelector(selector: "[id*='post-title']") { + innerText + } + score: mapSelector(selector: "[id*='vote-arrows']") { + innerText + } + comments: mapSelector(selector: "a[data-click-id='comments']") { + innerText + } + } +}`; + +const response = await fetch( + 'https://production-sfo.browserless.io/stealth/bql?token=YOUR_API_TOKEN_HERE', + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ query, variables: {}, operationName: 'ScrapeReddit' }), + } +); + +const { data } = await response.json(); +const posts = data.posts.map((p) => ({ + title: p.title?.[0]?.innerText ?? '', + score: p.score?.[0]?.innerText ?? '', + comments: p.comments?.[0]?.innerText ?? '', +})); +console.log(JSON.stringify(posts, null, 2)); diff --git a/examples/scrape-reddit/rest/python/scrape_reddit.py b/examples/scrape-reddit/rest/python/scrape_reddit.py new file mode 100644 index 0000000..6f122c7 --- /dev/null +++ b/examples/scrape-reddit/rest/python/scrape_reddit.py @@ -0,0 +1,41 @@ +# Scrapes Reddit posts from a subreddit using BQL with stealth mode. +# +# Install: pip install requests +# Run: python scrape_reddit.py + +import requests + +query = """ +mutation ScrapeReddit { + goto(url: "https://www.reddit.com/r/programming/", waitUntil: networkIdle) { + status + } + waitForTimeout(time: 2000) { + time + } + posts: mapSelector(selector: "article") { + title: mapSelector(selector: "[id*='post-title']") { + innerText + } + score: mapSelector(selector: "[id*='vote-arrows']") { + innerText + } + comments: mapSelector(selector: "a[data-click-id='comments']") { + innerText + } + } +} +""" + +response = requests.post( + 'https://production-sfo.browserless.io/stealth/bql', + params={'token': 'YOUR_API_TOKEN_HERE'}, + json={'query': query, 'variables': {}, 'operationName': 'ScrapeReddit'}, +) + +data = response.json()['data'] +for post in data['posts']: + title = post['title'][0]['innerText'] if post['title'] else '' + score = post['score'][0]['innerText'] if post['score'] else '' + comments = post['comments'][0]['innerText'] if post['comments'] else '' + print(f'{title}\n Score: {score} | {comments}\n') diff --git a/examples/scrape-walmart/bql/scrape-walmart.graphql b/examples/scrape-walmart/bql/scrape-walmart.graphql new file mode 100644 index 0000000..a3c7caf --- /dev/null +++ b/examples/scrape-walmart/bql/scrape-walmart.graphql @@ -0,0 +1,22 @@ +# Scrapes Walmart product listings. Run against /stealth/bql with residential proxy. +mutation ScrapeWalmart { + goto(url: "https://www.walmart.com/search?q=coffee+maker", waitUntil: networkIdle) { + status + } + + waitForTimeout(time: 2000) { + time + } + + products: mapSelector(selector: "[data-item-id]") { + title: mapSelector(selector: "[data-automation-id='product-title']") { + innerText + } + price: mapSelector(selector: "[itemprop='price']") { + innerText + } + rating: mapSelector(selector: "[data-testid='product-ratings']") { + innerText + } + } +} diff --git a/examples/scrape-walmart/frameworks/playwright/nodejs/scrape-walmart.mjs b/examples/scrape-walmart/frameworks/playwright/nodejs/scrape-walmart.mjs new file mode 100644 index 0000000..1a3244e --- /dev/null +++ b/examples/scrape-walmart/frameworks/playwright/nodejs/scrape-walmart.mjs @@ -0,0 +1,32 @@ +// Scrapes Walmart product listings using Playwright with stealth mode and residential proxy. +// +// Install: npm install playwright-core +// Run: node scrape-walmart.mjs + +import { chromium } from 'playwright-core'; + +const browser = await chromium.connectOverCDP( + 'wss://production-sfo.browserless.io?token=YOUR_API_TOKEN_HERE&stealth&proxy=residential&proxyCountry=us' +); + +try { + const context = browser.contexts()[0]; + const page = await context.newPage(); + await page.goto('https://www.walmart.com/search?q=coffee+maker', { + waitUntil: 'networkidle', + }); + + await page.waitForTimeout(2000); + + const products = await page.evaluate(() => + Array.from(document.querySelectorAll('[data-item-id]')).map((card) => ({ + title: card.querySelector('[data-automation-id="product-title"]')?.innerText?.trim() ?? '', + price: card.querySelector('[itemprop="price"]')?.innerText?.trim() ?? '', + rating: card.querySelector('[data-testid="product-ratings"]')?.innerText?.trim() ?? '', + })) + ); + + console.log(JSON.stringify(products, null, 2)); +} finally { + await browser.close(); +} diff --git a/examples/scrape-walmart/frameworks/puppeteer/scrape-walmart.mjs b/examples/scrape-walmart/frameworks/puppeteer/scrape-walmart.mjs new file mode 100644 index 0000000..c935c98 --- /dev/null +++ b/examples/scrape-walmart/frameworks/puppeteer/scrape-walmart.mjs @@ -0,0 +1,32 @@ +// Scrapes Walmart product listings using Puppeteer with stealth mode and residential proxy. +// +// Install: npm install puppeteer-core +// Run: node scrape-walmart.mjs + +import puppeteer from 'puppeteer-core'; + +const browser = await puppeteer.connect({ + browserWSEndpoint: + 'wss://production-sfo.browserless.io/stealth?token=YOUR_API_TOKEN_HERE&proxy=residential&proxyCountry=us', +}); + +try { + const page = await browser.newPage(); + await page.goto('https://www.walmart.com/search?q=coffee+maker', { + waitUntil: 'networkidle2', + }); + + await new Promise((r) => setTimeout(r, 2000)); + + const products = await page.evaluate(() => + Array.from(document.querySelectorAll('[data-item-id]')).map((card) => ({ + title: card.querySelector('[data-automation-id="product-title"]')?.innerText?.trim() ?? '', + price: card.querySelector('[itemprop="price"]')?.innerText?.trim() ?? '', + rating: card.querySelector('[data-testid="product-ratings"]')?.innerText?.trim() ?? '', + })) + ); + + console.log(JSON.stringify(products, null, 2)); +} finally { + await browser.close(); +} diff --git a/examples/scrape-walmart/rest/csharp/ScrapeWalmart.cs b/examples/scrape-walmart/rest/csharp/ScrapeWalmart.cs new file mode 100644 index 0000000..e4da99c --- /dev/null +++ b/examples/scrape-walmart/rest/csharp/ScrapeWalmart.cs @@ -0,0 +1,34 @@ +// Scrapes Walmart product listings using BQL with stealth mode and residential proxy. +// +// Run: dotnet run + +using System.Net.Http; +using System.Text; +using System.Text.Json; + +string token = "YOUR_API_TOKEN_HERE"; +string endpoint = $"https://production-sfo.browserless.io/stealth/bql?token={token}&proxy=residential&proxyCountry=us"; + +var payload = new +{ + query = @"mutation ScrapeWalmart { + goto(url: ""https://www.walmart.com/search?q=coffee+maker"", waitUntil: networkIdle) { status } + waitForTimeout(time: 2000) { time } + products: mapSelector(selector: ""[data-item-id]"") { + title: mapSelector(selector: ""[data-automation-id='product-title']"") { innerText } + price: mapSelector(selector: ""[itemprop='price']"") { innerText } + rating: mapSelector(selector: ""[data-testid='product-ratings']"") { innerText } + } + }", + variables = new { }, + operationName = "ScrapeWalmart", +}; + +using (HttpClient httpClient = new HttpClient()) +{ + var content = new StringContent( + JsonSerializer.Serialize(payload), Encoding.UTF8, "application/json"); + var response = await httpClient.PostAsync(endpoint, content); + string body = await response.Content.ReadAsStringAsync(); + Console.WriteLine(body); +} diff --git a/examples/scrape-walmart/rest/curl/scrape-walmart.sh b/examples/scrape-walmart/rest/curl/scrape-walmart.sh new file mode 100644 index 0000000..c34e8c2 --- /dev/null +++ b/examples/scrape-walmart/rest/curl/scrape-walmart.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Scrapes Walmart product listings using BQL with stealth mode and residential proxy. +# Run: bash scrape-walmart.sh + +curl -X POST \ + "https://production-sfo.browserless.io/stealth/bql?token=YOUR_API_TOKEN_HERE&proxy=residential&proxyCountry=us" \ + -H "Content-Type: application/json" \ + -d '{ + "query": "mutation ScrapeWalmart { goto(url: \"https://www.walmart.com/search?q=coffee+maker\", waitUntil: networkIdle) { status } waitForTimeout(time: 2000) { time } products: mapSelector(selector: \"[data-item-id]\") { title: mapSelector(selector: \"[data-automation-id='\''product-title'\'']\") { innerText } price: mapSelector(selector: \"[itemprop='\''price'\'']\") { innerText } rating: mapSelector(selector: \"[data-testid='\''product-ratings'\'']\") { innerText } } }", + "variables": {}, + "operationName": "ScrapeWalmart" + }' diff --git a/examples/scrape-walmart/rest/java/ScrapeWalmart.java b/examples/scrape-walmart/rest/java/ScrapeWalmart.java new file mode 100644 index 0000000..6023b51 --- /dev/null +++ b/examples/scrape-walmart/rest/java/ScrapeWalmart.java @@ -0,0 +1,36 @@ +// Scrapes Walmart product listings using BQL with stealth mode and residential proxy. +// +// Run: javac ScrapeWalmart.java && java ScrapeWalmart + +import java.net.URI; +import java.net.http.*; + +public class ScrapeWalmart { + public static void main(String[] args) throws Exception { + String token = "YOUR_API_TOKEN_HERE"; + String endpoint = "https://production-sfo.browserless.io/stealth/bql?token=" + + token + "&proxy=residential&proxyCountry=us"; + + String query = "mutation ScrapeWalmart {" + + " goto(url: \\\"https://www.walmart.com/search?q=coffee+maker\\\", waitUntil: networkIdle) { status }" + + " waitForTimeout(time: 2000) { time }" + + " products: mapSelector(selector: \\\"[data-item-id]\\\") {" + + " title: mapSelector(selector: \\\"[data-automation-id='product-title']\\\") { innerText }" + + " price: mapSelector(selector: \\\"[itemprop='price']\\\") { innerText }" + + " rating: mapSelector(selector: \\\"[data-testid='product-ratings']\\\") { innerText }" + + " }" + + " }"; + + String payload = "{\"query\": \"" + query + "\", \"variables\": {}, \"operationName\": \"ScrapeWalmart\"}"; + + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(endpoint)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(payload)) + .build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + System.out.println(response.body()); + } +} diff --git a/examples/scrape-walmart/rest/nodejs/scrape-walmart.mjs b/examples/scrape-walmart/rest/nodejs/scrape-walmart.mjs new file mode 100644 index 0000000..ef172b9 --- /dev/null +++ b/examples/scrape-walmart/rest/nodejs/scrape-walmart.mjs @@ -0,0 +1,40 @@ +// Scrapes Walmart product listings using BQL with stealth mode and residential proxy. +// +// Run: node scrape-walmart.mjs + +const query = `mutation ScrapeWalmart { + goto(url: "https://www.walmart.com/search?q=coffee+maker", waitUntil: networkIdle) { + status + } + waitForTimeout(time: 2000) { + time + } + products: mapSelector(selector: "[data-item-id]") { + title: mapSelector(selector: "[data-automation-id='product-title']") { + innerText + } + price: mapSelector(selector: "[itemprop='price']") { + innerText + } + rating: mapSelector(selector: "[data-testid='product-ratings']") { + innerText + } + } +}`; + +const response = await fetch( + 'https://production-sfo.browserless.io/stealth/bql?token=YOUR_API_TOKEN_HERE&proxy=residential&proxyCountry=us', + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ query, variables: {}, operationName: 'ScrapeWalmart' }), + } +); + +const { data } = await response.json(); +const products = data.products.map((p) => ({ + title: p.title?.[0]?.innerText ?? '', + price: p.price?.[0]?.innerText ?? '', + rating: p.rating?.[0]?.innerText ?? '', +})); +console.log(JSON.stringify(products, null, 2)); diff --git a/examples/scrape-walmart/rest/python/scrape_walmart.py b/examples/scrape-walmart/rest/python/scrape_walmart.py new file mode 100644 index 0000000..c129515 --- /dev/null +++ b/examples/scrape-walmart/rest/python/scrape_walmart.py @@ -0,0 +1,45 @@ +# Scrapes Walmart product listings using BQL with stealth mode and residential proxy. +# +# Install: pip install requests +# Run: python scrape_walmart.py + +import requests + +query = """ +mutation ScrapeWalmart { + goto(url: "https://www.walmart.com/search?q=coffee+maker", waitUntil: networkIdle) { + status + } + waitForTimeout(time: 2000) { + time + } + products: mapSelector(selector: "[data-item-id]") { + title: mapSelector(selector: "[data-automation-id='product-title']") { + innerText + } + price: mapSelector(selector: "[itemprop='price']") { + innerText + } + rating: mapSelector(selector: "[data-testid='product-ratings']") { + innerText + } + } +} +""" + +response = requests.post( + 'https://production-sfo.browserless.io/stealth/bql', + params={ + 'token': 'YOUR_API_TOKEN_HERE', + 'proxy': 'residential', + 'proxyCountry': 'us', + }, + json={'query': query, 'variables': {}, 'operationName': 'ScrapeWalmart'}, +) + +data = response.json()['data'] +for product in data['products']: + title = product['title'][0]['innerText'] if product['title'] else '' + price = product['price'][0]['innerText'] if product['price'] else '' + rating = product['rating'][0]['innerText'] if product['rating'] else '' + print(f'{title} — {price} — {rating}') diff --git a/examples/scrape-youtube/bql/scrape-youtube.graphql b/examples/scrape-youtube/bql/scrape-youtube.graphql new file mode 100644 index 0000000..2cfb6c5 --- /dev/null +++ b/examples/scrape-youtube/bql/scrape-youtube.graphql @@ -0,0 +1,25 @@ +# Scrapes YouTube search results. Run against /stealth/bql. +mutation ScrapeYouTube { + goto( + url: "https://www.youtube.com/results?search_query=javascript+tutorial" + waitUntil: networkIdle + ) { + status + } + + waitForTimeout(time: 2000) { + time + } + + videos: mapSelector(selector: "ytd-video-renderer") { + title: mapSelector(selector: "#video-title") { + innerText + } + channel: mapSelector(selector: "[id='channel-name']") { + innerText + } + views: mapSelector(selector: "span.inline-metadata-item") { + innerText + } + } +} diff --git a/examples/scrape-youtube/frameworks/playwright/nodejs/scrape-youtube.mjs b/examples/scrape-youtube/frameworks/playwright/nodejs/scrape-youtube.mjs new file mode 100644 index 0000000..fcb8b24 --- /dev/null +++ b/examples/scrape-youtube/frameworks/playwright/nodejs/scrape-youtube.mjs @@ -0,0 +1,34 @@ +// Scrapes YouTube search results using Playwright with stealth mode. +// +// Install: npm install playwright-core +// Run: node scrape-youtube.mjs + +import { chromium } from 'playwright-core'; + +const browser = await chromium.connectOverCDP( + 'wss://production-sfo.browserless.io?token=YOUR_API_TOKEN_HERE&stealth' +); + +try { + const context = browser.contexts()[0]; + const page = await context.newPage(); + await page.goto( + 'https://www.youtube.com/results?search_query=javascript+tutorial', + { waitUntil: 'networkidle' } + ); + + await page.waitForTimeout(2000); + + const videos = await page.evaluate(() => + Array.from(document.querySelectorAll('ytd-video-renderer')).map((card) => ({ + title: card.querySelector('#video-title')?.innerText?.trim() ?? '', + channel: card.querySelector('[id="channel-name"]')?.innerText?.trim() ?? '', + views: card.querySelector('span.inline-metadata-item')?.innerText?.trim() ?? '', + url: card.querySelector('a#video-title')?.href ?? '', + })) + ); + + console.log(JSON.stringify(videos, null, 2)); +} finally { + await browser.close(); +} diff --git a/examples/scrape-youtube/frameworks/puppeteer/scrape-youtube.mjs b/examples/scrape-youtube/frameworks/puppeteer/scrape-youtube.mjs new file mode 100644 index 0000000..2b986a3 --- /dev/null +++ b/examples/scrape-youtube/frameworks/puppeteer/scrape-youtube.mjs @@ -0,0 +1,34 @@ +// Scrapes YouTube search results using Puppeteer with stealth mode. +// +// Install: npm install puppeteer-core +// Run: node scrape-youtube.mjs + +import puppeteer from 'puppeteer-core'; + +const browser = await puppeteer.connect({ + browserWSEndpoint: + 'wss://production-sfo.browserless.io/stealth?token=YOUR_API_TOKEN_HERE', +}); + +try { + const page = await browser.newPage(); + await page.goto( + 'https://www.youtube.com/results?search_query=javascript+tutorial', + { waitUntil: 'networkidle2' } + ); + + await new Promise((r) => setTimeout(r, 2000)); + + const videos = await page.evaluate(() => + Array.from(document.querySelectorAll('ytd-video-renderer')).map((card) => ({ + title: card.querySelector('#video-title')?.innerText?.trim() ?? '', + channel: card.querySelector('[id="channel-name"]')?.innerText?.trim() ?? '', + views: card.querySelector('span.inline-metadata-item')?.innerText?.trim() ?? '', + url: card.querySelector('a#video-title')?.href ?? '', + })) + ); + + console.log(JSON.stringify(videos, null, 2)); +} finally { + await browser.close(); +} diff --git a/examples/scrape-youtube/rest/csharp/ScrapeYouTube.cs b/examples/scrape-youtube/rest/csharp/ScrapeYouTube.cs new file mode 100644 index 0000000..4e5b1fd --- /dev/null +++ b/examples/scrape-youtube/rest/csharp/ScrapeYouTube.cs @@ -0,0 +1,34 @@ +// Scrapes YouTube search results using BQL with stealth mode. +// +// Run: dotnet run + +using System.Net.Http; +using System.Text; +using System.Text.Json; + +string token = "YOUR_API_TOKEN_HERE"; +string endpoint = $"https://production-sfo.browserless.io/stealth/bql?token={token}"; + +var payload = new +{ + query = @"mutation ScrapeYouTube { + goto(url: ""https://www.youtube.com/results?search_query=javascript+tutorial"", waitUntil: networkIdle) { status } + waitForTimeout(time: 2000) { time } + videos: mapSelector(selector: ""ytd-video-renderer"") { + title: mapSelector(selector: ""#video-title"") { innerText } + channel: mapSelector(selector: ""[id='channel-name']"") { innerText } + views: mapSelector(selector: ""span.inline-metadata-item"") { innerText } + } + }", + variables = new { }, + operationName = "ScrapeYouTube", +}; + +using (HttpClient httpClient = new HttpClient()) +{ + var content = new StringContent( + JsonSerializer.Serialize(payload), Encoding.UTF8, "application/json"); + var response = await httpClient.PostAsync(endpoint, content); + string body = await response.Content.ReadAsStringAsync(); + Console.WriteLine(body); +} diff --git a/examples/scrape-youtube/rest/curl/scrape-youtube.sh b/examples/scrape-youtube/rest/curl/scrape-youtube.sh new file mode 100644 index 0000000..678ee63 --- /dev/null +++ b/examples/scrape-youtube/rest/curl/scrape-youtube.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Scrapes YouTube search results using BQL with stealth mode. +# Run: bash scrape-youtube.sh + +curl -X POST \ + "https://production-sfo.browserless.io/stealth/bql?token=YOUR_API_TOKEN_HERE" \ + -H "Content-Type: application/json" \ + -d '{ + "query": "mutation ScrapeYouTube { goto(url: \"https://www.youtube.com/results?search_query=javascript+tutorial\", waitUntil: networkIdle) { status } waitForTimeout(time: 2000) { time } videos: mapSelector(selector: \"ytd-video-renderer\") { title: mapSelector(selector: \"#video-title\") { innerText } channel: mapSelector(selector: \"[id='\''channel-name'\'']\") { innerText } views: mapSelector(selector: \"span.inline-metadata-item\") { innerText } } }", + "variables": {}, + "operationName": "ScrapeYouTube" + }' diff --git a/examples/scrape-youtube/rest/java/ScrapeYouTube.java b/examples/scrape-youtube/rest/java/ScrapeYouTube.java new file mode 100644 index 0000000..c8d69cd --- /dev/null +++ b/examples/scrape-youtube/rest/java/ScrapeYouTube.java @@ -0,0 +1,35 @@ +// Scrapes YouTube search results using BQL with stealth mode. +// +// Run: javac ScrapeYouTube.java && java ScrapeYouTube + +import java.net.URI; +import java.net.http.*; + +public class ScrapeYouTube { + public static void main(String[] args) throws Exception { + String token = "YOUR_API_TOKEN_HERE"; + String endpoint = "https://production-sfo.browserless.io/stealth/bql?token=" + token; + + String query = "mutation ScrapeYouTube {" + + " goto(url: \\\"https://www.youtube.com/results?search_query=javascript+tutorial\\\", waitUntil: networkIdle) { status }" + + " waitForTimeout(time: 2000) { time }" + + " videos: mapSelector(selector: \\\"ytd-video-renderer\\\") {" + + " title: mapSelector(selector: \\\"#video-title\\\") { innerText }" + + " channel: mapSelector(selector: \\\"[id='channel-name']\\\") { innerText }" + + " views: mapSelector(selector: \\\"span.inline-metadata-item\\\") { innerText }" + + " }" + + " }"; + + String payload = "{\"query\": \"" + query + "\", \"variables\": {}, \"operationName\": \"ScrapeYouTube\"}"; + + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(endpoint)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(payload)) + .build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + System.out.println(response.body()); + } +} diff --git a/examples/scrape-youtube/rest/nodejs/scrape-youtube.mjs b/examples/scrape-youtube/rest/nodejs/scrape-youtube.mjs new file mode 100644 index 0000000..30bbbc0 --- /dev/null +++ b/examples/scrape-youtube/rest/nodejs/scrape-youtube.mjs @@ -0,0 +1,40 @@ +// Scrapes YouTube search results using BQL with stealth mode. +// +// Run: node scrape-youtube.mjs + +const query = `mutation ScrapeYouTube { + goto(url: "https://www.youtube.com/results?search_query=javascript+tutorial", waitUntil: networkIdle) { + status + } + waitForTimeout(time: 2000) { + time + } + videos: mapSelector(selector: "ytd-video-renderer") { + title: mapSelector(selector: "#video-title") { + innerText + } + channel: mapSelector(selector: "[id='channel-name']") { + innerText + } + views: mapSelector(selector: "span.inline-metadata-item") { + innerText + } + } +}`; + +const response = await fetch( + 'https://production-sfo.browserless.io/stealth/bql?token=YOUR_API_TOKEN_HERE', + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ query, variables: {}, operationName: 'ScrapeYouTube' }), + } +); + +const { data } = await response.json(); +const videos = data.videos.map((v) => ({ + title: v.title?.[0]?.innerText ?? '', + channel: v.channel?.[0]?.innerText ?? '', + views: v.views?.[0]?.innerText ?? '', +})); +console.log(JSON.stringify(videos, null, 2)); diff --git a/examples/scrape-youtube/rest/python/scrape_youtube.py b/examples/scrape-youtube/rest/python/scrape_youtube.py new file mode 100644 index 0000000..9690416 --- /dev/null +++ b/examples/scrape-youtube/rest/python/scrape_youtube.py @@ -0,0 +1,41 @@ +# Scrapes YouTube search results using BQL with stealth mode. +# +# Install: pip install requests +# Run: python scrape_youtube.py + +import requests + +query = """ +mutation ScrapeYouTube { + goto(url: "https://www.youtube.com/results?search_query=javascript+tutorial", waitUntil: networkIdle) { + status + } + waitForTimeout(time: 2000) { + time + } + videos: mapSelector(selector: "ytd-video-renderer") { + title: mapSelector(selector: "#video-title") { + innerText + } + channel: mapSelector(selector: "[id='channel-name']") { + innerText + } + views: mapSelector(selector: "span.inline-metadata-item") { + innerText + } + } +} +""" + +response = requests.post( + 'https://production-sfo.browserless.io/stealth/bql', + params={'token': 'YOUR_API_TOKEN_HERE'}, + json={'query': query, 'variables': {}, 'operationName': 'ScrapeYouTube'}, +) + +data = response.json()['data'] +for video in data['videos']: + title = video['title'][0]['innerText'] if video['title'] else '' + channel = video['channel'][0]['innerText'] if video['channel'] else '' + views = video['views'][0]['innerText'] if video['views'] else '' + print(f'{title}\n {channel} — {views}\n') diff --git a/examples/scrape-zillow/bql/scrape-zillow.graphql b/examples/scrape-zillow/bql/scrape-zillow.graphql new file mode 100644 index 0000000..7a9fc40 --- /dev/null +++ b/examples/scrape-zillow/bql/scrape-zillow.graphql @@ -0,0 +1,22 @@ +# Scrapes Zillow property listings. Run against /stealth/bql with residential proxy. +mutation ScrapeZillow { + goto(url: "https://www.zillow.com/new-york-ny/", waitUntil: networkIdle) { + status + } + + waitForTimeout(time: 3000) { + time + } + + listings: mapSelector(selector: "[data-test='property-card']") { + address: mapSelector(selector: "[data-test='property-card-addr']") { + innerText + } + price: mapSelector(selector: "[data-test='property-card-price']") { + innerText + } + details: mapSelector(selector: ".StyledPropertyCardHomeDetails") { + innerText + } + } +} diff --git a/examples/scrape-zillow/frameworks/playwright/nodejs/scrape-zillow.mjs b/examples/scrape-zillow/frameworks/playwright/nodejs/scrape-zillow.mjs new file mode 100644 index 0000000..6e8d161 --- /dev/null +++ b/examples/scrape-zillow/frameworks/playwright/nodejs/scrape-zillow.mjs @@ -0,0 +1,32 @@ +// Scrapes Zillow property listings using Playwright with stealth mode and residential proxy. +// +// Install: npm install playwright-core +// Run: node scrape-zillow.mjs + +import { chromium } from 'playwright-core'; + +const browser = await chromium.connectOverCDP( + 'wss://production-sfo.browserless.io?token=YOUR_API_TOKEN_HERE&stealth&proxy=residential&proxyCountry=us' +); + +try { + const context = browser.contexts()[0]; + const page = await context.newPage(); + await page.goto('https://www.zillow.com/new-york-ny/', { + waitUntil: 'networkidle', + }); + + await page.waitForTimeout(3000); + + const listings = await page.evaluate(() => + Array.from(document.querySelectorAll('[data-test="property-card"]')).map((card) => ({ + address: card.querySelector('[data-test="property-card-addr"]')?.innerText?.trim() ?? '', + price: card.querySelector('[data-test="property-card-price"]')?.innerText?.trim() ?? '', + details: card.querySelector('.StyledPropertyCardHomeDetails')?.innerText?.trim() ?? '', + })) + ); + + console.log(JSON.stringify(listings, null, 2)); +} finally { + await browser.close(); +} diff --git a/examples/scrape-zillow/frameworks/puppeteer/scrape-zillow.mjs b/examples/scrape-zillow/frameworks/puppeteer/scrape-zillow.mjs new file mode 100644 index 0000000..1cd11f3 --- /dev/null +++ b/examples/scrape-zillow/frameworks/puppeteer/scrape-zillow.mjs @@ -0,0 +1,32 @@ +// Scrapes Zillow property listings using Puppeteer with stealth mode and residential proxy. +// +// Install: npm install puppeteer-core +// Run: node scrape-zillow.mjs + +import puppeteer from 'puppeteer-core'; + +const browser = await puppeteer.connect({ + browserWSEndpoint: + 'wss://production-sfo.browserless.io/stealth?token=YOUR_API_TOKEN_HERE&proxy=residential&proxyCountry=us', +}); + +try { + const page = await browser.newPage(); + await page.goto('https://www.zillow.com/new-york-ny/', { + waitUntil: 'networkidle2', + }); + + await new Promise((r) => setTimeout(r, 3000)); + + const listings = await page.evaluate(() => + Array.from(document.querySelectorAll('[data-test="property-card"]')).map((card) => ({ + address: card.querySelector('[data-test="property-card-addr"]')?.innerText?.trim() ?? '', + price: card.querySelector('[data-test="property-card-price"]')?.innerText?.trim() ?? '', + details: card.querySelector('.StyledPropertyCardHomeDetails')?.innerText?.trim() ?? '', + })) + ); + + console.log(JSON.stringify(listings, null, 2)); +} finally { + await browser.close(); +} diff --git a/examples/scrape-zillow/rest/csharp/ScrapeZillow.cs b/examples/scrape-zillow/rest/csharp/ScrapeZillow.cs new file mode 100644 index 0000000..660bb21 --- /dev/null +++ b/examples/scrape-zillow/rest/csharp/ScrapeZillow.cs @@ -0,0 +1,34 @@ +// Scrapes Zillow property listings using BQL with stealth mode and residential proxy. +// +// Run: dotnet run + +using System.Net.Http; +using System.Text; +using System.Text.Json; + +string token = "YOUR_API_TOKEN_HERE"; +string endpoint = $"https://production-sfo.browserless.io/stealth/bql?token={token}&proxy=residential&proxyCountry=us"; + +var payload = new +{ + query = @"mutation ScrapeZillow { + goto(url: ""https://www.zillow.com/new-york-ny/"", waitUntil: networkIdle) { status } + waitForTimeout(time: 3000) { time } + listings: mapSelector(selector: ""[data-test='property-card']"") { + address: mapSelector(selector: ""[data-test='property-card-addr']"") { innerText } + price: mapSelector(selector: ""[data-test='property-card-price']"") { innerText } + details: mapSelector(selector: "".StyledPropertyCardHomeDetails"") { innerText } + } + }", + variables = new { }, + operationName = "ScrapeZillow", +}; + +using (HttpClient httpClient = new HttpClient()) +{ + var content = new StringContent( + JsonSerializer.Serialize(payload), Encoding.UTF8, "application/json"); + var response = await httpClient.PostAsync(endpoint, content); + string body = await response.Content.ReadAsStringAsync(); + Console.WriteLine(body); +} diff --git a/examples/scrape-zillow/rest/curl/scrape-zillow.sh b/examples/scrape-zillow/rest/curl/scrape-zillow.sh new file mode 100644 index 0000000..acd2114 --- /dev/null +++ b/examples/scrape-zillow/rest/curl/scrape-zillow.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Scrapes Zillow property listings using BQL with stealth mode and residential proxy. +# Run: bash scrape-zillow.sh + +curl -X POST \ + "https://production-sfo.browserless.io/stealth/bql?token=YOUR_API_TOKEN_HERE&proxy=residential&proxyCountry=us" \ + -H "Content-Type: application/json" \ + -d '{ + "query": "mutation ScrapeZillow { goto(url: \"https://www.zillow.com/new-york-ny/\", waitUntil: networkIdle) { status } waitForTimeout(time: 3000) { time } listings: mapSelector(selector: \"[data-test='\''property-card'\'']\") { address: mapSelector(selector: \"[data-test='\''property-card-addr'\'']\") { innerText } price: mapSelector(selector: \"[data-test='\''property-card-price'\'']\") { innerText } details: mapSelector(selector: \".StyledPropertyCardHomeDetails\") { innerText } } }", + "variables": {}, + "operationName": "ScrapeZillow" + }' diff --git a/examples/scrape-zillow/rest/java/ScrapeZillow.java b/examples/scrape-zillow/rest/java/ScrapeZillow.java new file mode 100644 index 0000000..3e4ac02 --- /dev/null +++ b/examples/scrape-zillow/rest/java/ScrapeZillow.java @@ -0,0 +1,36 @@ +// Scrapes Zillow property listings using BQL with stealth mode and residential proxy. +// +// Run: javac ScrapeZillow.java && java ScrapeZillow + +import java.net.URI; +import java.net.http.*; + +public class ScrapeZillow { + public static void main(String[] args) throws Exception { + String token = "YOUR_API_TOKEN_HERE"; + String endpoint = "https://production-sfo.browserless.io/stealth/bql?token=" + + token + "&proxy=residential&proxyCountry=us"; + + String query = "mutation ScrapeZillow {" + + " goto(url: \\\"https://www.zillow.com/new-york-ny/\\\", waitUntil: networkIdle) { status }" + + " waitForTimeout(time: 3000) { time }" + + " listings: mapSelector(selector: \\\"[data-test='property-card']\\\") {" + + " address: mapSelector(selector: \\\"[data-test='property-card-addr']\\\") { innerText }" + + " price: mapSelector(selector: \\\"[data-test='property-card-price']\\\") { innerText }" + + " details: mapSelector(selector: \\\".StyledPropertyCardHomeDetails\\\") { innerText }" + + " }" + + " }"; + + String payload = "{\"query\": \"" + query + "\", \"variables\": {}, \"operationName\": \"ScrapeZillow\"}"; + + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(endpoint)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(payload)) + .build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + System.out.println(response.body()); + } +} diff --git a/examples/scrape-zillow/rest/nodejs/scrape-zillow.mjs b/examples/scrape-zillow/rest/nodejs/scrape-zillow.mjs new file mode 100644 index 0000000..d85cb67 --- /dev/null +++ b/examples/scrape-zillow/rest/nodejs/scrape-zillow.mjs @@ -0,0 +1,40 @@ +// Scrapes Zillow property listings using BQL with stealth mode and residential proxy. +// +// Run: node scrape-zillow.mjs + +const query = `mutation ScrapeZillow { + goto(url: "https://www.zillow.com/new-york-ny/", waitUntil: networkIdle) { + status + } + waitForTimeout(time: 3000) { + time + } + listings: mapSelector(selector: "[data-test='property-card']") { + address: mapSelector(selector: "[data-test='property-card-addr']") { + innerText + } + price: mapSelector(selector: "[data-test='property-card-price']") { + innerText + } + details: mapSelector(selector: ".StyledPropertyCardHomeDetails") { + innerText + } + } +}`; + +const response = await fetch( + 'https://production-sfo.browserless.io/stealth/bql?token=YOUR_API_TOKEN_HERE&proxy=residential&proxyCountry=us', + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ query, variables: {}, operationName: 'ScrapeZillow' }), + } +); + +const { data } = await response.json(); +const listings = data.listings.map((l) => ({ + address: l.address?.[0]?.innerText ?? '', + price: l.price?.[0]?.innerText ?? '', + details: l.details?.[0]?.innerText ?? '', +})); +console.log(JSON.stringify(listings, null, 2)); diff --git a/examples/scrape-zillow/rest/python/scrape_zillow.py b/examples/scrape-zillow/rest/python/scrape_zillow.py new file mode 100644 index 0000000..f6e4ac7 --- /dev/null +++ b/examples/scrape-zillow/rest/python/scrape_zillow.py @@ -0,0 +1,45 @@ +# Scrapes Zillow property listings using BQL with stealth mode and residential proxy. +# +# Install: pip install requests +# Run: python scrape_zillow.py + +import requests + +query = """ +mutation ScrapeZillow { + goto(url: "https://www.zillow.com/new-york-ny/", waitUntil: networkIdle) { + status + } + waitForTimeout(time: 3000) { + time + } + listings: mapSelector(selector: "[data-test='property-card']") { + address: mapSelector(selector: "[data-test='property-card-addr']") { + innerText + } + price: mapSelector(selector: "[data-test='property-card-price']") { + innerText + } + details: mapSelector(selector: ".StyledPropertyCardHomeDetails") { + innerText + } + } +} +""" + +response = requests.post( + 'https://production-sfo.browserless.io/stealth/bql', + params={ + 'token': 'YOUR_API_TOKEN_HERE', + 'proxy': 'residential', + 'proxyCountry': 'us', + }, + json={'query': query, 'variables': {}, 'operationName': 'ScrapeZillow'}, +) + +data = response.json()['data'] +for listing in data['listings']: + address = listing['address'][0]['innerText'] if listing['address'] else '' + price = listing['price'][0]['innerText'] if listing['price'] else '' + details = listing['details'][0]['innerText'] if listing['details'] else '' + print(f'{address} — {price}\n {details}\n')