Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/benchmark.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,8 @@ jobs:
env:
CODEARTIFACT_AUTH_TOKEN: ${{ steps.aws.outputs.token }}
VLT_REGISTRY_AUTH_TOKEN: ${{ secrets.VLT_REGISTRY_AUTH_TOKEN }}
CLOUDSMITH_REGISTRY: ${{ secrets.CLOUDSMITH_REGISTRY }}
CLOUDSMITH_AUTH_TOKEN: ${{ secrets.CLOUDSMITH_AUTH_TOKEN }}
run: |
./bench run \
--fixtures="${{ matrix.fixture }}" \
Expand Down
14 changes: 14 additions & 0 deletions app/src/components/icons/cloudsmith.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
import { createLucideIcon } from "lucide-react";

export const Cloudsmith = createLucideIcon("Cloudsmith", [
[
"path",
{
d: "m23.99 10.67v2.67l-10.66 10.66h-2.67l-10.66-10.66v-2.67l10.66-10.66h2.67zm-5.74 1.33c0-3.46-2.80-6.26-6.26-6.26-3.46 0-6.26 2.80-6.26 6.26 0 3.46 2.80 6.26 6.26 6.26 3.46 0 6.26-2.80 6.26-6.26z",
fill: "currentColor",
fillRule: "evenodd",
key: "1",
strokeWidth: "0",
},
],
]);
1 change: 1 addition & 0 deletions app/src/components/icons/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ export * from "./astro.tsx";
export * from "./aws.tsx";
export * from "./berry.tsx";
export * from "./bun.tsx";
export * from "./cloudsmith.tsx";
export * from "./deno.tsx";
export * from "./next.tsx";
export * from "./node.tsx";
Expand Down
1 change: 1 addition & 0 deletions app/src/hooks/use-history-data.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ const PACKAGE_MANAGERS = [
"turbo",
"node",
"aws",
"cloudsmith",
];

interface UseHistoryDataReturn {
Expand Down
2 changes: 2 additions & 0 deletions app/src/lib/get-icons.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import {
Aws,
Berry,
Bun,
Cloudsmith,
Deno,
Next,
Node,
Expand All @@ -24,6 +25,7 @@ import type { Fixture, PackageManager } from "@/types/chart-data";
const packageManagerMap: Partial<Record<PackageManager, LucideIcon>> = {
aws: Aws,
bun: Bun,
cloudsmith: Cloudsmith,
deno: Deno,
node: Node,
npm: Npm,
Expand Down
2 changes: 2 additions & 0 deletions app/src/lib/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -465,11 +465,13 @@ export function getPackageManagerDisplayName(
if (packageManager === "vlt") return "registry.vlt.io";
if (packageManager === "aws")
return "codeartifact.us-east-1.amazonaws.com";
if (packageManager === "cloudsmith") return "npm.cloudsmith.io";
}
if (packageManager === "berry") return "yarn (berry)";
if (packageManager === "zpm") return "yarn (zpm)";
if (packageManager === "turbo") return "turborepo";
if (packageManager === "aws") return "AWS CodeArtifact";
if (packageManager === "cloudsmith") return "Cloudsmith";
return packageManager;
}

Expand Down
10 changes: 9 additions & 1 deletion app/src/types/chart-data.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ export type PackageManager =
| "nx"
| "turbo"
| "node"
| "aws";
| "aws"
| "cloudsmith";

export type Fixture =
| "next"
Expand Down Expand Up @@ -50,6 +51,7 @@ export interface PackageManagerVersions {
turbo?: string;
node?: string;
aws?: string;
cloudsmith?: string;
}

export interface BaseFixtureResult {
Expand All @@ -69,6 +71,7 @@ export interface PackageManagerData {
turbo?: number;
node?: number;
aws?: number;
cloudsmith?: number;

npm_stddev?: number;
yarn_stddev?: number;
Expand All @@ -82,6 +85,7 @@ export interface PackageManagerData {
turbo_stddev?: number;
node_stddev?: number;
aws_stddev?: number;
cloudsmith_stddev?: number;

npm_fill?: string;
yarn_fill?: string;
Expand All @@ -95,6 +99,7 @@ export interface PackageManagerData {
turbo_fill?: string;
node_fill?: string;
aws_fill?: string;
cloudsmith_fill?: string;

npm_count?: number;
yarn_count?: number;
Expand All @@ -108,6 +113,7 @@ export interface PackageManagerData {
turbo_count?: number;
node_count?: number;
aws_count?: number;
cloudsmith_count?: number;

npm_dnf?: boolean;
yarn_dnf?: boolean;
Expand All @@ -121,6 +127,7 @@ export interface PackageManagerData {
turbo_dnf?: boolean;
node_dnf?: boolean;
aws_dnf?: boolean;
cloudsmith_dnf?: boolean;
}

export type FixtureResult = BaseFixtureResult & PackageManagerData;
Expand Down Expand Up @@ -159,6 +166,7 @@ export interface PackageCountData {
turbo?: PackageCountEntry;
node?: PackageCountEntry;
aws?: PackageCountEntry;
cloudsmith?: PackageCountEntry;
}

export interface PackageCountTableRow {
Expand Down
6 changes: 4 additions & 2 deletions bench
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ AVAILABLE_REGISTRIES=(
npm
vlt
aws
cloudsmith
)

usage() {
Expand All @@ -55,7 +56,7 @@ Usage:
Options:
--fixtures Comma or space-separated fixture names (default: next)
--pms Comma or space-separated package managers (default: all)
--registries Comma-separated registries for registry-* variations (default: npm,vlt,aws)
--registries Comma-separated registries for registry-* variations (default: npm,vlt,aws,cloudsmith)
--variation Comma or space-separated benchmark variations (default: clean)
--runs Hyperfine runs (default: scripts/variations/common.sh default)
--warmup Hyperfine warmup runs (default: scripts/variations/common.sh default)
Expand All @@ -76,6 +77,7 @@ Examples:
./bench run --variation=registry-clean --fixtures=next
./bench run --variation=registry-lockfile --registries=npm,vlt
CODEARTIFACT_AUTH_TOKEN=<token> ./bench run --variation=registry-clean --fixtures=next --registries=aws
CLOUDSMITH_REGISTRY=<url> CLOUDSMITH_AUTH_TOKEN=<token> ./bench run --variation=registry-clean --fixtures=next --registries=cloudsmith
./bench chart --fixtures=next --variation=clean
./bench process
EOF
Expand Down Expand Up @@ -446,7 +448,7 @@ run_bench() {
if [[ -n "$registries_env" ]]; then
echo " registries: $registries_env"
else
echo " registries: npm,vlt,aws (default)"
echo " registries: npm,vlt,aws,cloudsmith (default)"
fi
fi

Expand Down
1 change: 1 addition & 0 deletions scripts/generate-chart.js
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ const REGISTRY_COLORS = {
npm: "#cb0606",
vlt: "#000000",
aws: "#ff9900",
cloudsmith: "#2a6fe1",
};

const parseNumeric = (value) => {
Expand Down
22 changes: 21 additions & 1 deletion scripts/registry/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,11 @@ BENCH_REGISTRY_VLT_URL="https://registry.vlt.io/npm/"
BENCH_REGISTRY_VLT_URL_NPMRC_KEY="${BENCH_REGISTRY_VLT_URL#http*://}"
BENCH_REGISTRY_AWS_URL="https://vlt-451504312483.d.codeartifact.us-east-1.amazonaws.com/npm/code-artifact-benchmark-test/"
BENCH_REGISTRY_AWS_NPMRC_KEY="${BENCH_REGISTRY_AWS_URL#http*://}"
# Cloudsmith registry URL is injected without the protocol prefix
# (e.g. "//example.com/..."), so we prepend "https:" for the registry config.
# The auth .npmrc key uses the URL as-is (without protocol).
BENCH_REGISTRY_CLOUDSMITH_URL="https:${CLOUDSMITH_REGISTRY:-}"
BENCH_REGISTRY_CLOUDSMITH_NPMRC_KEY="${CLOUDSMITH_REGISTRY#//}"

# Registry setup commands run in hyperfine --prepare (untimed, before each run).
# Auth token is written as a literal placeholder so npm resolves it from env.
Expand All @@ -75,6 +80,7 @@ BENCH_SETUP_REGISTRY_NPM="npm config set registry \"$BENCH_REGISTRY_NPM_URL\" --
# will allow sharing a public cache regardless of the authorization header.
BENCH_SETUP_REGISTRY_VLT="npm config set registry \"$BENCH_REGISTRY_VLT_URL\" --location=project && npm config set \"//${BENCH_REGISTRY_VLT_URL_NPMRC_KEY}:_authToken=\\\${VLT_REGISTRY_AUTH_TOKEN}:$(head -c 16 /dev/urandom | xxd -p)_\\\${HYPERFINE_ITERATION}\" --location=project"
BENCH_SETUP_REGISTRY_AWS="npm config set registry \"$BENCH_REGISTRY_AWS_URL\" --location=project && npm config set \"//${BENCH_REGISTRY_AWS_NPMRC_KEY}:_authToken=\\\${CODEARTIFACT_AUTH_TOKEN}\" --location=project"
BENCH_SETUP_REGISTRY_CLOUDSMITH="npm config set registry \"$BENCH_REGISTRY_CLOUDSMITH_URL\" --location=project && npm config set \"//${BENCH_REGISTRY_CLOUDSMITH_NPMRC_KEY}:_authToken=\\\${CLOUDSMITH_AUTH_TOKEN}\" --location=project"

# Registry verification helper runs in hyperfine --conclude (untimed, after each run).
BENCH_VERIFY_REGISTRY="npm config get registry && ((grep -m3 '\"resolved\"' package-lock.json 2>/dev/null | sed 's/^[[:space:]]*//') || echo 'no lockfile yet') && echo ''"
Expand All @@ -83,28 +89,32 @@ BENCH_VERIFY_REGISTRY="npm config get registry && ((grep -m3 '\"resolved\"' pack
BENCH_CONCLUDE_NPM="{ $BENCH_VERIFY_REGISTRY; } >> $BENCH_OUTPUT_FOLDER/npm-verify.log 2>&1"
BENCH_CONCLUDE_VLT_REG="{ $BENCH_VERIFY_REGISTRY; } >> $BENCH_OUTPUT_FOLDER/vlt-verify.log 2>&1"
BENCH_CONCLUDE_AWS="{ $BENCH_VERIFY_REGISTRY; } >> $BENCH_OUTPUT_FOLDER/aws-verify.log 2>&1"
BENCH_CONCLUDE_CLOUDSMITH="{ $BENCH_VERIFY_REGISTRY; } >> $BENCH_OUTPUT_FOLDER/cloudsmith-verify.log 2>&1"

# Registry commands are timed and should only run installs.
BENCH_COMMAND_NPM="$BENCH_NPM_INSTALL >> $BENCH_OUTPUT_FOLDER/npm-output-\${HYPERFINE_ITERATION}.log 2>&1"
BENCH_COMMAND_VLT_REG="$BENCH_NPM_INSTALL >> $BENCH_OUTPUT_FOLDER/vlt-output-\${HYPERFINE_ITERATION}.log 2>&1"
BENCH_COMMAND_AWS="$BENCH_NPM_INSTALL >> $BENCH_OUTPUT_FOLDER/aws-output-\${HYPERFINE_ITERATION}.log 2>&1"
BENCH_COMMAND_CLOUDSMITH="$BENCH_NPM_INSTALL >> $BENCH_OUTPUT_FOLDER/cloudsmith-output-\${HYPERFINE_ITERATION}.log 2>&1"

# Registry include flags
# If BENCH_INCLUDE_REGISTRY is not set, default to running all registries.
if [ -z "${BENCH_INCLUDE_REGISTRY:-}" ]; then
BENCH_INCLUDE_REGISTRY="npm,vlt,aws"
BENCH_INCLUDE_REGISTRY="npm,vlt,aws,cloudsmith"
fi

BENCH_INCLUDE_REG_NPM=""
BENCH_INCLUDE_REG_VLT=""
BENCH_INCLUDE_REG_AWS=""
BENCH_INCLUDE_REG_CLOUDSMITH=""

for entry in $(echo "$BENCH_INCLUDE_REGISTRY" | tr ',' '\n'); do
case "$entry" in
"") continue ;;
npm) BENCH_INCLUDE_REG_NPM=1 ;;
vlt) BENCH_INCLUDE_REG_VLT=1 ;;
aws) BENCH_INCLUDE_REG_AWS=1 ;;
cloudsmith) BENCH_INCLUDE_REG_CLOUDSMITH=1 ;;
*)
echo "Error: Unknown registry '$entry' in BENCH_INCLUDE_REGISTRY"
exit 1
Expand All @@ -122,6 +132,16 @@ if [ -n "$BENCH_INCLUDE_REG_AWS" ] && [ -z "${CODEARTIFACT_AUTH_TOKEN:-}" ]; the
exit 1
fi

if [ -n "$BENCH_INCLUDE_REG_CLOUDSMITH" ] && [ -z "${CLOUDSMITH_AUTH_TOKEN:-}" ]; then
echo "Error: 'cloudsmith' registry was requested, but CLOUDSMITH_AUTH_TOKEN is not set"
exit 1
fi

if [ -n "$BENCH_INCLUDE_REG_CLOUDSMITH" ] && [ -z "${CLOUDSMITH_REGISTRY:-}" ]; then
echo "Error: 'cloudsmith' registry was requested, but CLOUDSMITH_REGISTRY is not set"
exit 1
fi

echo "Registry benchmarks will run: $BENCH_INCLUDE_REGISTRY"

# Clean up & create the results directory
Expand Down
5 changes: 4 additions & 1 deletion scripts/variations/registry-clean.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,4 +25,7 @@ hyperfine --ignore-failure \
${BENCH_INCLUDE_REG_VLT:+--conclude="$BENCH_CONCLUDE_VLT_REG"} \
${BENCH_INCLUDE_REG_AWS:+--prepare="$BENCH_PREPARE_BASE && $BENCH_SETUP_REGISTRY_AWS"} \
${BENCH_INCLUDE_REG_AWS:+--command-name="aws" "$BENCH_COMMAND_AWS"} \
${BENCH_INCLUDE_REG_AWS:+--conclude="$BENCH_CONCLUDE_AWS"}
${BENCH_INCLUDE_REG_AWS:+--conclude="$BENCH_CONCLUDE_AWS"} \
${BENCH_INCLUDE_REG_CLOUDSMITH:+--prepare="$BENCH_PREPARE_BASE && $BENCH_SETUP_REGISTRY_CLOUDSMITH"} \
${BENCH_INCLUDE_REG_CLOUDSMITH:+--command-name="cloudsmith" "$BENCH_COMMAND_CLOUDSMITH"} \
${BENCH_INCLUDE_REG_CLOUDSMITH:+--conclude="$BENCH_CONCLUDE_CLOUDSMITH"}
5 changes: 4 additions & 1 deletion scripts/variations/registry-lockfile.sh
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,7 @@ hyperfine --ignore-failure \
${BENCH_INCLUDE_REG_VLT:+--conclude="$BENCH_CONCLUDE_VLT_REG"} \
${BENCH_INCLUDE_REG_AWS:+--prepare="$BENCH_PREPARE_BASE && $BENCH_SETUP_REGISTRY_AWS"} \
${BENCH_INCLUDE_REG_AWS:+--command-name="aws" "$BENCH_COMMAND_AWS"} \
${BENCH_INCLUDE_REG_AWS:+--conclude="$BENCH_CONCLUDE_AWS"}
${BENCH_INCLUDE_REG_AWS:+--conclude="$BENCH_CONCLUDE_AWS"} \
${BENCH_INCLUDE_REG_CLOUDSMITH:+--prepare="$BENCH_PREPARE_BASE && $BENCH_SETUP_REGISTRY_CLOUDSMITH"} \
${BENCH_INCLUDE_REG_CLOUDSMITH:+--command-name="cloudsmith" "$BENCH_COMMAND_CLOUDSMITH"} \
${BENCH_INCLUDE_REG_CLOUDSMITH:+--conclude="$BENCH_CONCLUDE_CLOUDSMITH"}
Loading