diff --git a/.env.example b/.env.example index 93b47a3..cf10219 100644 --- a/.env.example +++ b/.env.example @@ -3,11 +3,37 @@ POSTHOG_API_KEY=your_posthog_api_key_here POSTHOG_HOST=https://app.posthog.com # or http://localhost:8010 for local development POSTHOG_DISTINCT_ID=example-user -# AI Provider API Keys +# AI Provider API Keys (core) ANTHROPIC_API_KEY=your_anthropic_api_key_here GEMINI_API_KEY=your_gemini_api_key_here OPENAI_API_KEY=your_openai_api_key_here +# AI Provider API Keys (OpenAI-compatible providers) +# Uncomment the ones you want to use: +# GROQ_API_KEY=your_groq_api_key_here +# DEEPSEEK_API_KEY=your_deepseek_api_key_here +# MISTRAL_API_KEY=your_mistral_api_key_here +# XAI_API_KEY=your_xai_api_key_here +# TOGETHER_API_KEY=your_together_api_key_here +# COHERE_API_KEY=your_cohere_api_key_here +# HUGGINGFACE_API_KEY=your_huggingface_api_key_here +# PERPLEXITY_API_KEY=your_perplexity_api_key_here +# CEREBRAS_API_KEY=your_cerebras_api_key_here +# FIREWORKS_API_KEY=your_fireworks_api_key_here +# OPENROUTER_API_KEY=your_openrouter_api_key_here +# HELICONE_API_KEY=your_helicone_api_key_here +# PORTKEY_API_KEY=your_portkey_api_key_here +# VERCEL_AI_GATEWAY_API_KEY=your_vercel_ai_gateway_api_key_here + +# Azure OpenAI +# AZURE_OPENAI_API_KEY=your_azure_api_key_here +# AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com + +# AWS Bedrock (uses AWS credentials) +# AWS_REGION=us-east-1 +# AWS_ACCESS_KEY_ID=your_access_key_here +# AWS_SECRET_ACCESS_KEY=your_secret_key_here + # Local SDK Development (optional) # Uncomment and adjust paths to your local PostHog repositories: # POSTHOG_PYTHON_PATH=../posthog-python diff --git a/.gitignore b/.gitignore index a2fc5d1..a422e76 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,7 @@ __pycache__/ example.json # Node.js -node_modules/ \ No newline at end of file +node_modules/ + +# Example runner results cache +.results/ \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index 7527bce..4a8660d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,9 +20,9 @@ What remains here: make setup # install deps via uv # Run SDK examples from sibling repos -make examples-list # list all available examples -make examples-parallel # run all in parallel via mprocs -./run-examples.sh anthropic # run by name filter +./run-examples.sh --list # list all available examples +./run-examples.sh --parallel # run all in parallel via phrocs +./run-examples.sh anthropic # run by name filter # Generate demo data make demo-data # 5 conversations, random providers diff --git a/Makefile b/Makefile index 37234fe..717bba7 100644 --- a/Makefile +++ b/Makefile @@ -1,30 +1,10 @@ -.PHONY: setup examples examples-list examples-all examples-parallel examples-install run-trace-generator run-trace-generator-debug demo-data demo-data-quick demo-data-tools demo-data-negative +.PHONY: setup run-trace-generator run-trace-generator-debug demo-data demo-data-quick demo-data-tools demo-data-negative ## Install all dependencies setup: @uv sync @pnpm install -## Run the interactive example picker (sources .env, discovers examples from sibling SDK repos) -examples: - @./run-examples.sh - -## List all available examples -examples-list: - @./run-examples.sh --list - -## Run all examples sequentially -examples-all: - @./run-examples.sh --all - -## Run all examples in parallel via mprocs (or filtered: make examples-parallel F=anthropic) -examples-parallel: - @./run-examples.sh --parallel $(F) - -## Install dependencies for all examples -examples-install: - @./run-examples.sh --install - ## Run the trace generator (mock trace data, no LLM calls) run-trace-generator: @uv run trace-generator/trace_generator.py diff --git a/README.md b/README.md index a7927b1..ef529ca 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,10 @@ For copy-paste-able provider integration examples, see the `examples/example-ai- ## Setup -Requires [uv](https://docs.astral.sh/uv/) and [pnpm](https://pnpm.io/). +Requires: +- [uv](https://docs.astral.sh/uv/) +- [pnpm](https://pnpm.io/) +- [phrocs](https://github.com/PostHog/posthog/tre/master/tools/phrocs) ```bash cp .env.example .env @@ -21,18 +24,24 @@ make setup Discovers and runs all `example-ai-*` examples from sibling `posthog-python` and `posthog-js` repos. ```bash -# List all available examples -make examples-list +# List all available examples (with cache status) +./run-examples.sh --list # Run a specific example or group by name ./run-examples.sh anthropic # all anthropic examples ./run-examples.sh python/openai # python openai examples only -# Run all examples in parallel via mprocs -make examples-parallel +# Run all examples in parallel via phrocs +./run-examples.sh --parallel + +# Run all sequentially +./run-examples.sh --all + +# Force re-run (ignore cache) +./run-examples.sh --rerun --all # Install dependencies for all examples -make examples-install +./run-examples.sh --install ``` ### Demo Data Generator diff --git a/run-examples.sh b/run-examples.sh index a5a51f6..93bbf1f 100755 --- a/run-examples.sh +++ b/run-examples.sh @@ -4,19 +4,25 @@ set -euo pipefail # Run AI provider examples from posthog-python and posthog-js repos. # Uses the .env file from this repo for API keys. # +# Results are cached in .results/ — examples that passed with the same file +# content are skipped on subsequent runs to save API costs. +# # Usage: # ./run-examples.sh Interactive menu -# ./run-examples.sh --list List all examples +# ./run-examples.sh --list List all examples (with cache status) # ./run-examples.sh --all Run all examples sequentially -# ./run-examples.sh --parallel Run all examples in parallel (mprocs) +# ./run-examples.sh --parallel Run all examples in parallel (phrocs) # ./run-examples.sh --parallel anthropic Run matching examples in parallel # ./run-examples.sh --install Install deps for all examples +# ./run-examples.sh --rerun Force re-run (ignore cache), combinable with other flags +# ./run-examples.sh --reset Clear the results cache # ./run-examples.sh openai/embeddings Run a specific example (fuzzy match) # ./run-examples.sh anthropic Run all examples matching "anthropic" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" PYTHON_REPO="${POSTHOG_PYTHON_PATH:-$SCRIPT_DIR/../posthog-python}" JS_REPO="${POSTHOG_JS_PATH:-$SCRIPT_DIR/../posthog-js}" +RESULTS_DIR="$SCRIPT_DIR/.results" # Load .env if [[ -f "$SCRIPT_DIR/.env" ]]; then @@ -25,6 +31,21 @@ if [[ -f "$SCRIPT_DIR/.env" ]]; then set +a fi +# --------------------------------------------------------------------------- +# Parse --rerun flag from anywhere in argv +# --------------------------------------------------------------------------- + +RERUN=0 +ARGS=() +for arg in "$@"; do + if [[ "$arg" == "--rerun" ]]; then + RERUN=1 + else + ARGS+=("$arg") + fi +done +set -- ${ARGS[@]+"${ARGS[@]}"} + # Find Python: prefer the posthog-python venv, fall back to system python3 if [[ -x "$PYTHON_REPO/.venv/bin/python" ]]; then PYTHON="$PYTHON_REPO/.venv/bin/python" @@ -39,32 +60,23 @@ fi # --------------------------------------------------------------------------- install_python_deps() { - echo "Installing Python example dependencies into $PYTHON_REPO/.venv..." - - local all_deps=() - for req in "$PYTHON_REPO"/examples/example-ai-*/requirements.txt; do - [[ -f "$req" ]] || continue - while IFS= read -r line; do - [[ -z "$line" || "$line" == \#* ]] && continue - all_deps+=("$line") - done < "$req" - done + echo "Installing Python example dependencies..." - if [[ ${#all_deps[@]} -eq 0 ]]; then - echo " No Python requirements found." - return + if ! command -v uv &>/dev/null; then + echo " uv is required to install Python example dependencies." + echo " Install it: https://docs.astral.sh/uv/getting-started/installation/" + return 1 fi - local unique_deps - unique_deps=$(printf '%s\n' "${all_deps[@]}" | sort -u) - - echo " Dependencies: $(echo "$unique_deps" | tr '\n' ' ')" - - if command -v uv &>/dev/null; then - echo "$unique_deps" | xargs uv pip install --python "$PYTHON" 2>&1 | tail -3 - else - echo "$unique_deps" | xargs "$PYTHON" -m pip install 2>&1 | tail -3 - fi + for dir in "$PYTHON_REPO"/examples/example-ai-*/; do + [[ -d "$dir" ]] || continue + local name + name=$(basename "$dir") + if [[ -f "$dir/pyproject.toml" ]]; then + echo " $name..." + (cd "$dir" && uv sync 2>&1 | tail -1) + fi + done echo " Done." } @@ -76,7 +88,7 @@ install_node_deps() { name=$(basename "$dir") if [[ -f "$dir/package.json" ]]; then echo " $name..." - (cd "$dir" && pnpm install --no-frozen-lockfile 2>&1 | tail -1) + (cd "$dir" && pnpm install 2>&1 | tail -1) fi done echo " Done." @@ -123,6 +135,63 @@ discover_node() { done } +# --------------------------------------------------------------------------- +# Results cache +# --------------------------------------------------------------------------- + +cache_key() { + local name="$1" + echo "${name//\//__}" +} + +# Associative array of file path → hash, populated by precompute_hashes +declare -A FILE_HASHES=() + +file_hash() { + local file="$1" + if [[ -n "${FILE_HASHES[$file]+x}" ]]; then + echo "${FILE_HASHES[$file]}" + else + shasum -a 256 "$file" | cut -d' ' -f1 + fi +} + +# Compute all example file hashes in a single shasum call +precompute_hashes() { + [[ ${#FILES[@]} -eq 0 ]] && return + while IFS=' ' read -r hash filepath; do + FILE_HASHES["$filepath"]="$hash" + done < <(shasum -a 256 "${FILES[@]}") +} + +is_cached() { + local idx="$1" + [[ "$RERUN" == "0" ]] || return 1 + local key + key=$(cache_key "${NAMES[$idx]}") + local cache_file="$RESULTS_DIR/$key.hash" + [[ -f "$cache_file" ]] || return 1 + local cached_hash current_hash + cached_hash=$(cat "$cache_file") + current_hash=$(file_hash "${FILES[$idx]}") + [[ "$cached_hash" == "$current_hash" ]] +} + +mark_passed() { + local idx="$1" + mkdir -p "$RESULTS_DIR" + local key + key=$(cache_key "${NAMES[$idx]}") + file_hash "${FILES[$idx]}" > "$RESULTS_DIR/$key.hash" +} + +mark_failed() { + local idx="$1" + local key + key=$(cache_key "${NAMES[$idx]}") + rm -f "$RESULTS_DIR/$key.hash" +} + # --------------------------------------------------------------------------- # Run an example # --------------------------------------------------------------------------- @@ -135,20 +204,37 @@ run_example() { local dir dir=$(dirname "$file") + if is_cached "$idx"; then + echo " ✓ $name (cached)" + return 0 + fi + echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo " $name" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" + local rc=0 if [[ "$lang" == "py" ]]; then - "$PYTHON" "$file" + if [[ -f "$dir/pyproject.toml" ]] && command -v uv &>/dev/null; then + (cd "$dir" && uv run python "$(basename "$file")") || rc=$? + else + "$PYTHON" "$file" || rc=$? + fi else - (cd "$dir" && npx tsx "$(basename "$file")") + (cd "$dir" && npx tsx "$(basename "$file")") || rc=$? fi + + if [[ $rc -eq 0 ]]; then + mark_passed "$idx" + else + mark_failed "$idx" + fi + return $rc } -# Build the shell command string for a single example (used by mprocs) +# Build the shell command string for a single example (used by phrocs) example_cmd() { local idx="$1" local file="${FILES[$idx]}" @@ -157,7 +243,11 @@ example_cmd() { dir=$(dirname "$file") if [[ "$lang" == "py" ]]; then - echo "$PYTHON $file" + if [[ -f "$dir/pyproject.toml" ]] && command -v uv &>/dev/null; then + echo "cd $dir && uv run python $(basename "$file")" + else + echo "$PYTHON $file" + fi else echo "cd $dir && npx tsx $(basename "$file")" fi @@ -175,35 +265,95 @@ find_matches() { } # --------------------------------------------------------------------------- -# Parallel execution with mprocs +# Parallel execution with phrocs # --------------------------------------------------------------------------- run_parallel() { local indices=("$@") - if ! command -v mprocs &>/dev/null; then - echo "mprocs is not installed. Install it with: brew install mprocs" + if ! command -v phrocs &>/dev/null; then + echo "phrocs is not installed. Install it with: brew tap posthog/tap && brew install phrocs" exit 1 fi - # Build mprocs config - local config - config=$(mktemp /tmp/mprocs-examples-XXXXXX.yaml) - trap "rm -f $config" EXIT + # Filter out cached examples + local filtered=() + local skipped=0 + for i in "${indices[@]}"; do + if is_cached "$i"; then + (( skipped++ )) || true + else + filtered+=("$i") + fi + done + + if [[ ${#filtered[@]} -eq 0 ]]; then + echo "All ${#indices[@]} examples cached. Use --rerun to force re-running." + return 0 + fi + + mkdir -p "$RESULTS_DIR" + + # Build phrocs config + local config info_script + config=$(mktemp /tmp/phrocs-examples-XXXXXX.yaml) + info_script=$(mktemp /tmp/phrocs-info-XXXXXX.sh) + trap "rm -f $config $info_script" EXIT + + local total=${#indices[@]} + local cached=$skipped + local pending=${#filtered[@]} + + # Info tab script with PostHog brand colors + cat > "$info_script" <<'INFOEOF' +#!/usr/bin/env bash +o='\033[38;2;245;78;0m' # orange #F54E00 +b='\033[38;2;29;74;255m' # blue #1D4AFF +g='\033[38;5;245m' # gray +B='\033[1m' # bold +r='\033[0m' # reset +INFOEOF + cat >> "$info_script" < "$config" - for i in "${indices[@]}"; do + echo " info:" >> "$config" + echo " shell: \"bash $info_script\"" >> "$config" + + for i in "${filtered[@]}"; do local name="${NAMES[$i]}" local cmd cmd=$(example_cmd "$i") - # Wrap in env-loading shell so each proc has the API keys - local full_cmd="set -a; source $SCRIPT_DIR/.env 2>/dev/null; set +a; $cmd" + local key + key=$(cache_key "$name") + local hash + hash=$(file_hash "${FILES[$i]}") + local cache_file="$RESULTS_DIR/$key.hash" + # Wrap command to record pass/fail in the results cache + local full_cmd="set -a; source $SCRIPT_DIR/.env 2>/dev/null; set +a; ($cmd) && printf '%s' '$hash' > '$cache_file' || { rm -f '$cache_file'; exit 1; }" echo " \"$name\":" >> "$config" echo " shell: \"$full_cmd\"" >> "$config" done - echo "Running ${#indices[@]} examples in parallel..." - mprocs --config "$config" + phrocs --config "$config" } # --------------------------------------------------------------------------- @@ -218,8 +368,20 @@ if [[ "${1:-}" == "--install" ]]; then exit 0 fi +# Handle --reset before discovering examples +if [[ "${1:-}" == "--reset" ]]; then + if [[ -d "$RESULTS_DIR" ]]; then + rm -rf "$RESULTS_DIR" + echo "Results cache cleared." + else + echo "No results cache to clear." + fi + exit 0 +fi + discover_python discover_node +precompute_hashes if [[ ${#NAMES[@]} -eq 0 ]]; then echo "No examples found." @@ -251,12 +413,23 @@ if [[ "$MODE" == "--parallel" ]]; then fi elif [[ "$MODE" == "--all" ]]; then - echo "Running all ${#NAMES[@]} examples..." + echo "Running ${#NAMES[@]} examples..." + if [[ "$RERUN" == "0" ]]; then + echo "(use --rerun to ignore cache)" + fi + echo "" FAILED=0 PASSED=0 + SKIPPED=0 for i in "${!NAMES[@]}"; do + was_cached=0 + is_cached "$i" && was_cached=1 if run_example "$i"; then - (( PASSED++ )) || true + if [[ $was_cached -eq 1 ]]; then + (( SKIPPED++ )) || true + else + (( PASSED++ )) || true + fi else (( FAILED++ )) || true echo "FAILED: ${NAMES[$i]}" @@ -264,18 +437,32 @@ elif [[ "$MODE" == "--all" ]]; then done echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo "Results: $PASSED passed, $FAILED failed (${#NAMES[@]} total)" + summary="Results: $PASSED passed, $FAILED failed" + if [[ $SKIPPED -gt 0 ]]; then + summary="$summary, $SKIPPED cached" + fi + echo "$summary (${#NAMES[@]} total)" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" exit $FAILED elif [[ "$MODE" == "--list" ]]; then echo "Available examples:" echo "" - for name in "${NAMES[@]}"; do - echo " $name" + cached_count=0 + for i in "${!NAMES[@]}"; do + if is_cached "$i"; then + echo " ✓ ${NAMES[$i]}" + (( cached_count++ )) || true + else + echo " ${NAMES[$i]}" + fi done echo "" - echo "${#NAMES[@]} examples found." + if [[ $cached_count -gt 0 ]]; then + echo "${#NAMES[@]} examples found ($cached_count cached, $((${#NAMES[@]} - cached_count)) pending)." + else + echo "${#NAMES[@]} examples found." + fi elif [[ -n "$MODE" && "$MODE" != --* ]]; then # Name-based matching @@ -303,13 +490,18 @@ else echo "AI Provider Examples" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" - for name in "${NAMES[@]}"; do - echo " $name" + for i in "${!NAMES[@]}"; do + if is_cached "$i"; then + echo " ✓ ${NAMES[$i]}" + else + echo " ${NAMES[$i]}" + fi done echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" echo "Type a name (or partial match) to run, 'a' for all, or 'q' to quit." + echo "✓ = cached (will be skipped). Use --rerun to force." echo "" while true; do diff --git a/uv.lock b/uv.lock index 20444c8..7857b4f 100644 --- a/uv.lock +++ b/uv.lock @@ -1381,7 +1381,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.82.4" +version = "1.81.13" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -1397,9 +1397,9 @@ dependencies = [ { name = "tiktoken" }, { name = "tokenizers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e6/79/b492be13542aebd62aafc0490e4d5d6e8e00ce54240bcabf5c3e46b1a49b/litellm-1.82.4.tar.gz", hash = "sha256:9c52b1c0762cb0593cdc97b26a8e05004e19b03f394ccd0f42fac82eff0d4980", size = 17378196, upload-time = "2026-03-18T01:18:05.378Z" } +sdist = { url = "https://files.pythonhosted.org/packages/72/80/b6cb799e7100953d848e106d0575db34c75bc3b57f31f2eefdfb1e23655f/litellm-1.81.13.tar.gz", hash = "sha256:083788d9c94e3371ff1c42e40e0e8198c497772643292a65b1bc91a3b3b537ea", size = 16562861, upload-time = "2026-02-17T02:00:47.466Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/ad/7eaa1121c6b191f2f5f2e8c7379823ece6ec83741a4b3c81b82fe2832401/litellm-1.82.4-py3-none-any.whl", hash = "sha256:d37c34a847e7952a146ed0e2888a24d3edec7787955c6826337395e755ad5c4b", size = 15559801, upload-time = "2026-03-18T01:18:02.026Z" }, + { url = "https://files.pythonhosted.org/packages/be/f3/fffb7932870163cea7addc392165647a9a8a5489967de486c854226f1141/litellm-1.81.13-py3-none-any.whl", hash = "sha256:ae4aea2a55e85993f5f6dd36d036519422d24812a1a3e8540d9e987f2d7a4304", size = 14587505, upload-time = "2026-02-17T02:00:44.22Z" }, ] [[package]] @@ -1434,7 +1434,7 @@ requires-dist = [ { name = "langchain-anthropic" }, { name = "langchain-core" }, { name = "langchain-openai" }, - { name = "litellm" }, + { name = "litellm", specifier = "==1.81.13" }, { name = "openai" }, { name = "openai-agents" }, { name = "opentelemetry-api" },