From 2437d16effe1cb28b55236320fd13dca7dbc442e Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Mon, 15 Jun 2026 21:11:04 +0530 Subject: [PATCH 01/30] feat: sleep v2, step goal, queue fan-out scaling, event-driven biometrics - Sleep v2: sleep_periods table + /sleep/v2 + /day/v2/sleep (multi-period; additive, v1 untouched) - Step goal: users.step_goal + PATCH/GET /profile + /today - /day/strain now a PURE READ (cron precomputes strain_curve + HR stats) - Queues fan-out: cron enqueues per-(user) sweep + per-(user,day) heavy jobs; consumer runs one bounded unit/invocation (biometrics/resp/steps_full) - Incremental steps (settled-minute cursor) + nightly full true-up - Event-driven biometrics: sweep fires biometrics on freshly-detected sleep - Ingest marks dirty only; the 30-min cron is the single deduped enqueue point - Minute retention 90 -> 10 days - Admin: /admin/enqueue, /admin/run-steps; 14-day caps on R2 re-run routes - Migrations v6-v10 --- src/analytics.ts | 99 +++++++++++-- src/biometrics.ts | 20 ++- src/daydetail.ts | 92 +++++++----- src/db/migrate_v10_bio_trigger.sql | 4 + src/db/migrate_v6_sleep_periods.sql | 15 ++ src/db/migrate_v7_step_goal.sql | 3 + src/db/migrate_v8_strain_curve.sql | 7 + src/db/migrate_v9_steps_cursor.sql | 6 + src/db/schema.sql | 23 ++- src/index.ts | 210 ++++++++++++++++++++-------- src/ingest.ts | 19 ++- src/query.ts | 50 +++++++ src/queue.ts | 88 ++++++++++-- src/resp.ts | 8 +- src/steps_imu.ts | 111 +++++++++++++-- wrangler.toml | 41 +++--- 16 files changed, 622 insertions(+), 174 deletions(-) create mode 100644 src/db/migrate_v10_bio_trigger.sql create mode 100644 src/db/migrate_v6_sleep_periods.sql create mode 100644 src/db/migrate_v7_step_goal.sql create mode 100644 src/db/migrate_v8_strain_curve.sql create mode 100644 src/db/migrate_v9_steps_cursor.sql diff --git a/src/analytics.ts b/src/analytics.ts index 2774487..f453605 100644 --- a/src/analytics.ts +++ b/src/analytics.ts @@ -7,7 +7,7 @@ // Missing input → null + confidence 0; never fabricated. import { - calcRestingHR, calcStrain, calcHrZones, calcCalories, calcSleep, + calcRestingHR, calcStrain, calcHrZones, calcCalories, calcSleep, calcSleepPeriods, calcSleepRegularity, detectSessions, calcLoad, calcFitnessTrend, calcVo2Max, calcFitnessModel, calcMonotony, calcAnomaly, calcBaselines, buildCoach, @@ -92,6 +92,45 @@ function sleepSearchWindow(dateDayStart: number): { from: number; to: number } { return { from: dateDayStart - 6 * 3600, to: dateDayStart + 12 * 3600 } } +// Precompute the intra-day cumulative-strain curve + day HR stats so /day/strain +// is a PURE READ (no live recompute on the endpoint). Uses the SAME resolved max +// HR + sex coefficients calcStrain used, so the curve's final point equals the +// stored daily.strain. Curve is downsampled to keep the JSON small. +function dayStrainDetail( + dayMin: Minute[], rhr: number, maxHr: number, sex?: 'm' | 'f', +): { curve: string; hrMax: number | null; hrMin: number | null; hrAvg: number | null } { + const [k, b] = sex === 'f' ? [0.86, 1.67] : [0.64, 1.92] + const denom = Math.max(1, maxHr - rhr) + const scale = (trimp: number) => Math.min(21, Math.log(trimp + 1) / Math.log(1.5)) + const sorted = [...dayMin].sort((a, c) => a.ts - c.ts) + let trimp = 0 + let hrMax = 0, hrMin = 0, hrSum = 0, hrN = 0 + const pts: { t: number; v: number }[] = [] + for (const m of sorted) { + const hr = m.hr_avg + if (m.wrist_on && hr > 0) { + const ratio = Math.max(0, Math.min(1, (hr - rhr) / denom)) + trimp += ratio * k * Math.exp(b * ratio) + hrMax = Math.max(hrMax, hr) + hrMin = hrMin === 0 ? hr : Math.min(hrMin, hr) + hrSum += hr; hrN++ + } + pts.push({ t: m.ts, v: Math.round(scale(trimp) * 10) / 10 }) + } + const MAXP = 120 + let curvePts = pts + if (pts.length > MAXP) { + const step = pts.length / MAXP + curvePts = Array.from({ length: MAXP }, (_, i) => pts[Math.min(pts.length - 1, Math.floor(i * step))]) + } + return { + curve: JSON.stringify(curvePts), + hrMax: hrMax || null, + hrMin: hrMin || null, + hrAvg: hrN ? Math.round(hrSum / hrN) : null, + } +} + interface DayBuf { date: string dayStart: number @@ -102,12 +141,15 @@ interface DayBuf { calories: ReturnType sleep: Metric wearMin: number - steps: number sleepStress: string // calcSleepStress JSON (nocturnal arousal) nocturnal: string // calcNocturnalHeart JSON sleepingHr: number | null // this night's sleeping-HR avg (for baseline) nocturnalElevated: boolean // overnight HR notably above baseline mainDrivers: Record // strain/rhr/sleep/zones driver graph + strainCurve: string // precomputed cumulative-strain curve (JSON) for /day/strain + hrMax: number | null + hrMin: number | null + hrAvg: number | null } /** @@ -246,6 +288,8 @@ export async function processUser( const zones = calcHrZones(dayMin, baseline, profile) const calories = calcCalories(dayMin, profile, baseline.resting_hr, zones.max_hr_used) dailyStrains.push({ ts: dayStart, strain: strain.score }) + // Precompute the strain curve + HR stats here (cron) so /day/strain just reads. + const strainDetail = dayStrainDetail(dayMin, baseline.resting_hr, strain.max_hr_used, profile?.sex) // -- Sessions for this day. -- const sessions = detectSessions(dayMin, baseline, profile) @@ -274,9 +318,9 @@ export async function processUser( s.strain, s.kcal, s.hrr60 == null ? null : Math.round(s.hrr60), JSON.stringify(s.zones), s.confidence)) } - // -- Wear time (worn minutes) + daily steps (sum of detected per-minute). -- + // -- Wear time (worn minutes). Steps are owned by steps_imu.ts (AN-2554 over + // the raw IMU), written separately — processUser no longer computes them. -- const wearMin = dayMin.filter((m) => m.wrist_on).length - const steps = Math.round(dayMin.reduce((s, m) => s + (m.steps || 0), 0)) // -- Nocturnal heart (sleeping-HR dynamics over the main sleep period). -- const sleepWorn = (sleep.onset_ts && sleep.wake_ts) @@ -323,13 +367,39 @@ export async function processUser( regularityForSleep, sleep.confidence, sleepFlags, now)) sleepN++ + // -- Sleep v2 (multi-period; naps = shorter sleeps). ADDITIVE — the v1 write + // above is untouched. The window extends to THIS day's evening (18:00) so + // daytime naps are captured, without pulling in the NEXT night's onset + // (which belongs to date+1, same attribution convention as the night). -- + const periodsWin = sleepMinutes(dayStart, sw.from, dayStart + 18 * 3600) + const sleepV2 = calcSleepPeriods(periodsWin, baseline) + statements.push( + db.prepare('DELETE FROM sleep_periods WHERE user_id = ? AND date = ?').bind(userId, date), + ) + for (const p of sleepV2.periods) { + const pid = `${userId}:${p.onset_ts}` + statements.push(db.prepare( + 'INSERT INTO sleep_periods (user_id, id, date, onset_ts, wake_ts, duration_min, in_bed_min, efficiency, light_min, deep_min, rem_min, is_main, confidence, updated_at) ' + + 'VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(user_id, id) DO UPDATE SET ' + + 'date=excluded.date, wake_ts=excluded.wake_ts, duration_min=excluded.duration_min, in_bed_min=excluded.in_bed_min, ' + + 'efficiency=excluded.efficiency, light_min=excluded.light_min, deep_min=excluded.deep_min, rem_min=excluded.rem_min, ' + + 'is_main=excluded.is_main, confidence=excluded.confidence, updated_at=excluded.updated_at', + ).bind(userId, pid, date, p.onset_ts, p.wake_ts, p.duration_min, p.in_bed_min, p.efficiency, + p.stages?.light_min ?? null, p.stages?.deep_min ?? null, p.stages?.rem_min ?? null, + p.is_main ? 1 : 0, p.confidence, now)) + } + dayBuffer.push({ - date, dayStart, idx: dayBuffer.length, rhr, strain, zones, calories, sleep, wearMin, steps, + date, dayStart, idx: dayBuffer.length, rhr, strain, zones, calories, sleep, wearMin, sleepStress: JSON.stringify(sleepStress), nocturnal: JSON.stringify(nocturnal), sleepingHr: nocturnal.sleeping_hr_avg, nocturnalElevated: nocturnal.elevated, mainDrivers, + strainCurve: strainDetail.curve, + hrMax: strainDetail.hrMax, + hrMin: strainDetail.hrMin, + hrAvg: strainDetail.hrAvg, }) } @@ -337,8 +407,9 @@ export async function processUser( // trailing windows that end at that day (so ACWR/fitness/SRI/anomaly vary // across the history instead of collapsing to a single value). ── for (const buf of dayBuffer) { - const { date, idx, rhr, strain, zones, calories, sleep, wearMin, steps, - sleepStress, nocturnal, nocturnalElevated, mainDrivers } = buf + const { date, idx, rhr, strain, zones, calories, sleep, wearMin, + sleepStress, nocturnal, nocturnalElevated, mainDrivers, + strainCurve, hrMax, hrMin, hrAvg } = buf // Trailing windows ending at this day (inclusive). const sriNights = nightSummaries.slice(Math.max(0, idx - 13), idx + 1) // ~2 weeks @@ -385,7 +456,7 @@ export async function processUser( recovery: { c: recovery != null ? 0.8 : 0, tier: 'HIGH', label: 'Recovery (HRV)' }, calories: flag(calories, calories.label), zones: flag(zones, zones.max_hr_source === 'age' ? 'HR zones (estimated max HR)' : 'HR zones'), - steps: { c: steps > 0 ? 0.5 : 0, tier: 'ESTIMATE', label: 'Steps (est.)' }, + steps: { c: wearMin > 0 ? 0.5 : 0, tier: 'ESTIMATE', label: 'Steps (est.)' }, load: flag(load, `Load — ${load.band}`), fitness: flag(fitness, `Fitness trend — ${fitness.direction}`), anomaly: flag(anomaly, anomaly.note), @@ -461,21 +532,25 @@ export async function processUser( // It writes `drivers` (main metrics) via COALESCE-free set but biometrics // read-merges, and on the hourly path (no biometrics) main drivers stand alone. statements.push(db.prepare( - 'INSERT INTO daily (user_id, date, strain, resting_hr, calories, wear_min, steps, hr_zones, acwr, fitness_trend, anomaly, coach, nocturnal, sleep_stress, drivers, vo2max, fitness, fatigue, form, monotony, nocturnal_dip_pct, confidence, flags, updated_at) ' + - 'VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(user_id, date) DO UPDATE SET ' + + // NOTE: `steps` is intentionally NOT written here — steps_imu.ts is the sole, + // authoritative writer (AN-2554 over the raw IMU). processUser must not clobber it. + 'INSERT INTO daily (user_id, date, strain, resting_hr, calories, wear_min, hr_zones, acwr, fitness_trend, anomaly, coach, nocturnal, sleep_stress, drivers, vo2max, fitness, fatigue, form, monotony, nocturnal_dip_pct, strain_curve, hr_max, hr_min, hr_avg, confidence, flags, updated_at) ' + + 'VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(user_id, date) DO UPDATE SET ' + 'strain=excluded.strain, resting_hr=excluded.resting_hr, ' + - 'calories=excluded.calories, wear_min=excluded.wear_min, steps=excluded.steps, hr_zones=excluded.hr_zones, ' + + 'calories=excluded.calories, wear_min=excluded.wear_min, hr_zones=excluded.hr_zones, ' + 'acwr=excluded.acwr, fitness_trend=excluded.fitness_trend, anomaly=excluded.anomaly, coach=excluded.coach, ' + 'nocturnal=excluded.nocturnal, sleep_stress=excluded.sleep_stress, ' + 'drivers=json_patch(COALESCE(daily.drivers,\'{}\'), excluded.drivers), ' + 'vo2max=excluded.vo2max, fitness=excluded.fitness, fatigue=excluded.fatigue, form=excluded.form, ' + 'monotony=excluded.monotony, nocturnal_dip_pct=excluded.nocturnal_dip_pct, ' + + 'strain_curve=excluded.strain_curve, hr_max=excluded.hr_max, hr_min=excluded.hr_min, hr_avg=excluded.hr_avg, ' + 'confidence=excluded.confidence, flags=excluded.flags, updated_at=excluded.updated_at', ).bind(userId, date, strain.score, rhr.resting_hr == null ? null : Math.round(rhr.resting_hr), calories.kcal, - wearMin, steps, JSON.stringify(zones), load.acwr, fitness.direction, + wearMin, JSON.stringify(zones), load.acwr, fitness.direction, bodyAlert, JSON.stringify(coach), nocturnal, sleepStress, driversJson, vo2.vo2max, fitModel.fitness, fitModel.fatigue, fitModel.form, monotony.monotony, nocDip, + strainCurve, hrMax, hrMin, hrAvg, confidence, flags, now)) dailyN++ diff --git a/src/biometrics.ts b/src/biometrics.ts index c46886b..fe67776 100644 --- a/src/biometrics.ts +++ b/src/biometrics.ts @@ -144,10 +144,12 @@ interface DailyHistRow { date: string; resting_hr: number | null; hrv_rmssd: num * (Mahalanobis). Writes those + their drivers to the daily row. Heavy (R2 reads) → * nightly cron / admin only, never inline ingest. */ -export async function runBiometrics(env: BioEnv, userId: string, days = 3): Promise<{ days: number; computed: number }> { +export async function runBiometrics(env: BioEnv, userId: string, days = 3, onlyDate?: string): Promise<{ days: number; computed: number }> { const now = Math.floor(Date.now() / 1000) - const since = new Date((now - days * DAY) * 1000).toISOString().slice(0, 10) + // Per-(user,day) fan-out: when onlyDate is set, process exactly that one UTC day + // (a single bounded unit). Otherwise the trailing `days`. + const since = onlyDate ?? new Date((now - days * DAY) * 1000).toISOString().slice(0, 10) const { results: sleeps } = await env.DB.prepare( 'SELECT date, onset_ts, wake_ts, duration_min FROM sleep WHERE user_id = ? AND date >= ? AND onset_ts IS NOT NULL AND wake_ts IS NOT NULL', ).bind(userId, since).all<{ date: string; onset_ts: number; wake_ts: number; duration_min: number | null }>() @@ -174,9 +176,17 @@ export async function runBiometrics(env: BioEnv, userId: string, days = 3): Prom const TOTAL_OBJECTS = 24 let spent = 0 const out: NightBio[] = [] - for (let d = 0; d < days; d++) { - const dayStart = Math.floor((now - d * DAY) / DAY) * DAY - const date = new Date(dayStart * 1000).toISOString().slice(0, 10) + // Build the explicit list of (dayStart, date) to process. + const dayList: { dayStart: number; date: string }[] = [] + if (onlyDate) { + dayList.push({ dayStart: Math.floor(Date.parse(`${onlyDate}T00:00:00Z`) / 1000), date: onlyDate }) + } else { + for (let d = 0; d < days; d++) { + const ds = Math.floor((now - d * DAY) / DAY) * DAY + dayList.push({ dayStart: ds, date: new Date(ds * 1000).toISOString().slice(0, 10) }) + } + } + for (const { dayStart, date } of dayList) { const budget = Math.min(PER_DAY, TOTAL_OBJECTS - spent) if (budget <= 0) { out.push({ date, rmssd: null, sdnn: null, lfhf: null, si: null, resp: null, respConf: 0, nBeats: 0, temp: null, spo2: null, rr: [] }); continue } const sl = sleepByDate.get(date) diff --git a/src/daydetail.ts b/src/daydetail.ts index 0d68ab0..2d0d822 100644 --- a/src/daydetail.ts +++ b/src/daydetail.ts @@ -44,56 +44,40 @@ function downsample(arr: T[], cap = 300): T[] { } // ── /day/strain ────────────────────────────────────────────────────────────── +// PURE READ: serve the snapshot the analytics cron precomputed (daily row + +// sessions). No live recompute on the endpoint — strain, curve, zones and HR +// stats all come from `daily`, so /day/strain can never diverge from /today and +// the read is fast. Freshness is the cron's job (ingest → dirty → cron). export async function getDayStrain(c: Ctx) { const date = (c.req.query('date') || '').trim() if (!/^\d{4}-\d{2}-\d{2}$/.test(date)) return c.json({ error: 'date=YYYY-MM-DD required' }, 400) const start = dayStartOf(date) - const mins = await loadMinutes(c, start, start + DAY) - const { rhr, maxHr } = await loadHr(c) - const denom = Math.max(1, maxHr - rhr) - - let trimp = 0 - const curve: { t: number; v: number }[] = [] - const zones = [0, 0, 0, 0, 0] - let hrMax = 0, hrMinNonZero = 0, hrSum = 0, hrN = 0 - for (const m of mins) { - const hr = m.hr_avg ?? 0 - if (hr <= 0) { curve.push({ t: m.ts_min, v: round1(strainScale(trimp)) }); continue } - const ratio = clamp((hr - rhr) / denom, 0, 1) - trimp += ratio * 0.64 * Math.exp(1.92 * ratio) - curve.push({ t: m.ts_min, v: round1(strainScale(trimp)) }) - const pct = (hr / maxHr) * 100 - if (pct >= 90) zones[4]++ - else if (pct >= 80) zones[3]++ - else if (pct >= 70) zones[2]++ - else if (pct >= 60) zones[1]++ - else if (pct >= 50) zones[0]++ - hrMax = Math.max(hrMax, hr) - hrMinNonZero = hrMinNonZero === 0 ? hr : Math.min(hrMinNonZero, hr) - hrSum += hr; hrN++ - } - const { results: sessions } = await c.env.DB.prepare( - 'SELECT id, start_ts, end_ts, type, avg_hr, max_hr, strain, calories, hrr60, zones FROM sessions ' + - "WHERE user_id = ? AND start_ts >= ? AND start_ts < ? AND status != 'deleted' ORDER BY start_ts ASC", - ).bind(c.get('userId'), start, start + DAY).all() - - // Training-load + day totals from the derived `daily` row (acwr/fitness/cals/steps + drivers). const dr = await c.env.DB.prepare( - 'SELECT acwr, fitness_trend, calories, steps, drivers, vo2max, fitness, fatigue, form, monotony FROM daily WHERE user_id = ? AND date = ?', + 'SELECT strain, hr_zones, wear_min, strain_curve, hr_max, hr_min, hr_avg, acwr, fitness_trend, ' + + 'calories, steps, drivers, vo2max, fitness, fatigue, form, monotony FROM daily WHERE user_id = ? AND date = ?', ).bind(c.get('userId'), date).first() + + const z = dr?.hr_zones ? safe(dr.hr_zones) : null const acwr = dr?.acwr ?? null const band = acwr == null ? null : acwr < 0.8 ? 'detraining' : acwr <= 1.3 ? 'optimal' : acwr <= 1.5 ? 'caution' : 'high-risk' + const { results: sessions } = await c.env.DB.prepare( + 'SELECT id, start_ts, end_ts, type, avg_hr, max_hr, strain, calories, hrr60, zones FROM sessions ' + + "WHERE user_id = ? AND start_ts >= ? AND start_ts < ? AND status != 'deleted' ORDER BY start_ts ASC", + ).bind(c.get('userId'), start, start + DAY).all() + return c.json({ date, - strain: round1(strainScale(trimp)), - curve: downsample(curve), - zones: { z1: zones[0], z2: zones[1], z3: zones[2], z4: zones[3], z5: zones[4] }, - hr: { max: hrMax || null, min: hrMinNonZero || null, avg: hrN ? Math.round(hrSum / hrN) : null }, - max_hr_used: maxHr, - worn_min: mins.filter((m) => m.wrist_on).length, + strain: dr?.strain ?? 0, + curve: dr?.strain_curve ? safe(dr.strain_curve) : [], + zones: z + ? { z1: z.zone1_min ?? 0, z2: z.zone2_min ?? 0, z3: z.zone3_min ?? 0, z4: z.zone4_min ?? 0, z5: z.zone5_min ?? 0 } + : { z1: 0, z2: 0, z3: 0, z4: 0, z5: 0 }, + hr: { max: dr?.hr_max ?? null, min: dr?.hr_min ?? null, avg: dr?.hr_avg ?? null }, + max_hr_used: z?.max_hr_used ?? null, + worn_min: dr?.wear_min ?? 0, // Training load (ACWR band) + fitness trend + day energy/steps. load: acwr == null ? null : { acwr: Math.round(acwr * 100) / 100, band }, fitness_trend: dr?.fitness_trend ?? null, @@ -180,6 +164,40 @@ export async function getDayWear(c: Ctx) { } // ── /day/sleep ─────────────────────────────────────────────────────────────── +// GET /day/v2/sleep?date= — every sleep period for the date (one card each; +// naps = shorter sleeps). Additive companion to GET /day/sleep (single-period). +export async function getDaySleepV2(c: Ctx) { + const date = (c.req.query('date') || '').trim() + if (!/^\d{4}-\d{2}-\d{2}$/.test(date)) return c.json({ error: 'date=YYYY-MM-DD required' }, 400) + const userId = c.get('userId') + const { results } = await c.env.DB.prepare( + 'SELECT * FROM sleep_periods WHERE user_id = ? AND date = ? ORDER BY onset_ts ASC', + ).bind(userId, date).all() + const baseline = await c.env.DB.prepare('SELECT sleep_need_min FROM baselines WHERE user_id = ?') + .bind(userId).first() + const need = (baseline?.sleep_need_min && baseline.sleep_need_min >= 180) ? baseline.sleep_need_min : 480 + const periods = (results ?? []).map((r: any) => ({ + id: r.id, + onset_ts: r.onset_ts, + wake_ts: r.wake_ts, + duration_min: r.duration_min, + in_bed_min: r.in_bed_min, + efficiency: r.efficiency, + stages: (r.light_min != null || r.deep_min != null || r.rem_min != null) + ? { light_min: r.light_min, deep_min: r.deep_min, rem_min: r.rem_min } : null, + is_main: !!r.is_main, + confidence: r.confidence, + })) + return c.json({ + date, + has_sleep: periods.length > 0, + need_min: need, + total_asleep_min: periods.reduce((a: number, p: any) => a + (p.duration_min || 0), 0), + periods, + stages_beta: true, + }) +} + export async function getDaySleep(c: Ctx) { const date = (c.req.query('date') || '').trim() if (!/^\d{4}-\d{2}-\d{2}$/.test(date)) return c.json({ error: 'date=YYYY-MM-DD required' }, 400) diff --git a/src/db/migrate_v10_bio_trigger.sql b/src/db/migrate_v10_bio_trigger.sql new file mode 100644 index 0000000..0bc2559 --- /dev/null +++ b/src/db/migrate_v10_bio_trigger.sql @@ -0,0 +1,4 @@ +-- v10 — event-driven biometrics. Track the last sleep-date for which the sweep +-- already fired biometrics, so "a fresh night just finished" triggers recovery/HRV +-- once (not every sweep). The nightly cron stays as the catch-all. Additive column. +ALTER TABLE analytics_cursor ADD COLUMN bio_last_date TEXT; diff --git a/src/db/migrate_v6_sleep_periods.sql b/src/db/migrate_v6_sleep_periods.sql new file mode 100644 index 0000000..b423fc0 --- /dev/null +++ b/src/db/migrate_v6_sleep_periods.sql @@ -0,0 +1,15 @@ +-- v6 — Sleep v2 (multi-period). Naps are not a special case: every consolidated +-- sleep period is a first-class row with its own breakdown. Purely ADDITIVE — the +-- v1 `sleep` table and its endpoints are untouched. id = `${user_id}:${onset_ts}`. +CREATE TABLE IF NOT EXISTS sleep_periods( + user_id TEXT, id TEXT, + date TEXT, -- the sleep "day" this period was derived under + onset_ts INTEGER, wake_ts INTEGER, + duration_min REAL, in_bed_min REAL, efficiency REAL, + light_min REAL, deep_min REAL, rem_min REAL, + is_main INTEGER, -- 1 = longest period of the day (UI hint only) + confidence REAL, updated_at INTEGER, + PRIMARY KEY(user_id, id) +); +CREATE INDEX IF NOT EXISTS idx_sleep_periods_date ON sleep_periods(user_id, date); +CREATE INDEX IF NOT EXISTS idx_sleep_periods_onset ON sleep_periods(user_id, onset_ts); diff --git a/src/db/migrate_v7_step_goal.sql b/src/db/migrate_v7_step_goal.sql new file mode 100644 index 0000000..fe50614 --- /dev/null +++ b/src/db/migrate_v7_step_goal.sql @@ -0,0 +1,3 @@ +-- v7 — user-settable daily step goal. Additive nullable column; NULL means the +-- client uses its own default (8000). Non-breaking: existing rows read as NULL. +ALTER TABLE users ADD COLUMN step_goal INTEGER; diff --git a/src/db/migrate_v8_strain_curve.sql b/src/db/migrate_v8_strain_curve.sql new file mode 100644 index 0000000..3122082 --- /dev/null +++ b/src/db/migrate_v8_strain_curve.sql @@ -0,0 +1,7 @@ +-- v8 — precompute the intra-day strain curve + day HR stats in the cron so +-- /day/strain becomes a PURE READ (no live recompute on the endpoint). Additive +-- nullable columns; existing rows read as NULL until the next analytics run. +ALTER TABLE daily ADD COLUMN strain_curve TEXT; +ALTER TABLE daily ADD COLUMN hr_max INTEGER; +ALTER TABLE daily ADD COLUMN hr_min INTEGER; +ALTER TABLE daily ADD COLUMN hr_avg INTEGER; diff --git a/src/db/migrate_v9_steps_cursor.sql b/src/db/migrate_v9_steps_cursor.sql new file mode 100644 index 0000000..3505ee2 --- /dev/null +++ b/src/db/migrate_v9_steps_cursor.sql @@ -0,0 +1,6 @@ +-- v9 — incremental steps cursor. Lets the 30-min sweep recompute steps over only +-- the newly-SETTLED minutes since the last run (a few R2 objects) instead of +-- re-reading 2 days every time — keeping AN-2554 accuracy at ~free R2 cost. +-- Additive nullable columns on the existing analytics_cursor row. +ALTER TABLE analytics_cursor ADD COLUMN steps_cursor_ts INTEGER; -- last settled minute counted (unix s) +ALTER TABLE analytics_cursor ADD COLUMN steps_cursor_day TEXT; -- UTC day the accumulator is for diff --git a/src/db/schema.sql b/src/db/schema.sql index 9d5b48e..5fb8edb 100644 --- a/src/db/schema.sql +++ b/src/db/schema.sql @@ -11,6 +11,7 @@ CREATE TABLE IF NOT EXISTS users( height_cm REAL, weight_kg REAL, sex TEXT, -- 'm' | 'f' | NULL (NULL → sex-neutral calories) + step_goal INTEGER, -- user's daily step goal; NULL → client default (8000) created_at INTEGER NOT NULL ); @@ -92,6 +93,8 @@ CREATE TABLE IF NOT EXISTS daily( hrv_cv REAL, -- coefficient of variation of nightly RMSSD (%) nocturnal_dip_pct REAL, -- nocturnal HR dip (fraction) — trendable copy irregular TEXT, -- irregular-rhythm SCREEN (JSON, Poincaré) — not a diagnosis + strain_curve TEXT, -- precomputed intra-day cumulative-strain curve (JSON) so /day/strain is a pure read + hr_max INTEGER, hr_min INTEGER, hr_avg INTEGER, -- precomputed day HR stats (pure-read /day/strain) -- NOTE: `readiness` (above) is REPURPOSED for the composite Readiness index -- (HRV ∩ sleep ∩ dip ∩ arousal), written by biometrics.ts. confidence REAL, flags TEXT, updated_at INTEGER, @@ -106,6 +109,21 @@ CREATE TABLE IF NOT EXISTS sleep( PRIMARY KEY(user_id, date) ); +-- Sleep v2 (multi-period; naps = shorter sleeps). Additive; v1 `sleep` above is +-- the single main-period table and stays as-is. See migrate_v6_sleep_periods.sql. +CREATE TABLE IF NOT EXISTS sleep_periods( + user_id TEXT, id TEXT, + date TEXT, + onset_ts INTEGER, wake_ts INTEGER, + duration_min REAL, in_bed_min REAL, efficiency REAL, + light_min REAL, deep_min REAL, rem_min REAL, + is_main INTEGER, + confidence REAL, updated_at INTEGER, + PRIMARY KEY(user_id, id) +); +CREATE INDEX IF NOT EXISTS idx_sleep_periods_date ON sleep_periods(user_id, date); +CREATE INDEX IF NOT EXISTS idx_sleep_periods_onset ON sleep_periods(user_id, onset_ts); + CREATE TABLE IF NOT EXISTS sessions( user_id TEXT, id TEXT, start_ts INTEGER, end_ts INTEGER, type TEXT, @@ -145,7 +163,10 @@ CREATE TABLE IF NOT EXISTS analytics_cursor( user_id TEXT PRIMARY KEY, last_min_ts INTEGER DEFAULT 0, dirty INTEGER DEFAULT 1, - last_run INTEGER DEFAULT 0 + last_run INTEGER DEFAULT 0, + steps_cursor_ts INTEGER, -- incremental steps: last settled minute counted (unix s) + steps_cursor_day TEXT, -- UTC day the steps accumulator is for + bio_last_date TEXT -- event-driven biometrics: last sleep-date already triggered ); -- Per-user ingest rate-limit token bucket (RESILIENCE §7). diff --git a/src/index.ts b/src/index.ts index b91a68a..8d3c227 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,18 +4,18 @@ import { } from './auth' import { runAnalytics, processUser } from './analytics' import { ingestBatch, ingestEvents } from './ingest' -import { handleQueueBatch, type AnalyticsMessage } from './queue' -import { getToday, getSleep, getStrain, getSessions, getTrends, getChart } from './query' +import { handleQueueBatch, type AnalyticsMessage, type AnalyticsJob } from './queue' +import { getToday, getSleep, getSleepV2, getStrain, getSessions, getTrends, getChart } from './query' import { getHistory } from './history' import { postJournal, getJournal, getJournalInsights } from './journal' -import { getDayStrain, getDaySleep, getDayTimeline, getDayStress, getDayHeart, getDayLungs, getDayWear } from './daydetail' +import { getDayStrain, getDaySleep, getDaySleepV2, getDayTimeline, getDayStress, getDayHeart, getDayLungs, getDayWear } from './daydetail' import { getTrend } from './trend' +import { runStepsImu, runStepsIncremental } from './steps_imu' import { workoutStart, workoutEnd, listWorkouts, getWorkout, deleteWorkout, autoCloseStaleWorkouts } from './workouts' import { getRecords } from './records' import { getNotifications, markNotificationsRead } from './notifications' import { runRespRate } from './resp' import { runBiometrics } from './biometrics' -import { runStepsImu } from './steps_imu' import { getAppStatus, adminGetConfig, adminSetConfig } from './appconfig' import { seedInit, seedMinutes, seedAnalytics } from './seed' @@ -39,6 +39,11 @@ const REFRESH_TTL = 30 * 24 * 60 * 60 // 30d const OTP_TTL = 10 * 60 // 10m const OTP_MAX_ATTEMPTS = 5 const DAY = 86400 +// Per-minute rollups are kept this long. The app shows minute-level detail +// (hypnogram, 24h timelines, wear, workout HR curve) for the last 7 days; this +3 +// buffer avoids a boundary race at the edge of that window. Derived metrics +// (daily/sleep/sessions) are permanent — pruning minutes never affects them. +const MINUTE_RETENTION_DAYS = 10 const app = new Hono<{ Bindings: Bindings; Variables: Vars }>() @@ -64,6 +69,7 @@ app.use('/ingest/*', requireJwt) app.use('/profile', requireJwt) app.use('/today', requireJwt) app.use('/sleep', requireJwt) +app.use('/sleep/v2', requireJwt) app.use('/strain', requireJwt) app.use('/sessions', requireJwt) app.use('/trends', requireJwt) @@ -191,22 +197,25 @@ app.post('/auth/refresh', async (c) => { app.get('/profile', async (c) => { const user = await c.env.DB.prepare( - 'SELECT id, email, name, age, height_cm, weight_kg, sex, created_at FROM users WHERE id = ?', + 'SELECT id, email, name, age, height_cm, weight_kg, sex, step_goal, created_at FROM users WHERE id = ?', ).bind(c.get('userId')).first() if (!user) return c.json({ error: 'Not found' }, 404) return c.json(user) }) app.patch('/profile', async (c) => { - const { name, age, height_cm, weight_kg, sex } = - await c.req.json<{ name?: string; age?: number; height_cm?: number; weight_kg?: number; sex?: string }>() + const { name, age, height_cm, weight_kg, sex, step_goal } = + await c.req.json<{ name?: string; age?: number; height_cm?: number; weight_kg?: number; sex?: string; step_goal?: number }>() const sexVal = sex === 'm' || sex === 'f' ? sex : null + // step_goal: clamp to a sane range when provided; null leaves it unchanged. + const goalVal = (typeof step_goal === 'number' && isFinite(step_goal)) + ? Math.max(1000, Math.min(50000, Math.round(step_goal))) : null await c.env.DB.prepare( 'UPDATE users SET name=COALESCE(?,name), age=COALESCE(?,age), height_cm=COALESCE(?,height_cm), ' + - 'weight_kg=COALESCE(?,weight_kg), sex=COALESCE(?,sex) WHERE id = ?', - ).bind(name ?? null, age ?? null, height_cm ?? null, weight_kg ?? null, sexVal, c.get('userId')).run() + 'weight_kg=COALESCE(?,weight_kg), sex=COALESCE(?,sex), step_goal=COALESCE(?,step_goal) WHERE id = ?', + ).bind(name ?? null, age ?? null, height_cm ?? null, weight_kg ?? null, sexVal, goalVal, c.get('userId')).run() const user = await c.env.DB.prepare( - 'SELECT id, email, name, age, height_cm, weight_kg, sex, created_at FROM users WHERE id = ?', + 'SELECT id, email, name, age, height_cm, weight_kg, sex, step_goal, created_at FROM users WHERE id = ?', ).bind(c.get('userId')).first() return c.json(user) }) @@ -218,6 +227,7 @@ app.post('/ingest/events', ingestEvents) // ========================= QUERY ========================= app.get('/today', getToday) app.get('/sleep', getSleep) +app.get('/sleep/v2', getSleepV2) app.get('/strain', getStrain) app.get('/sessions', getSessions) app.get('/trends', getTrends) @@ -228,6 +238,7 @@ app.get('/journal', getJournal) app.get('/journal/insights', getJournalInsights) app.get('/day/strain', getDayStrain) app.get('/day/sleep', getDaySleep) +app.get('/day/v2/sleep', getDaySleepV2) app.get('/day/timeline', getDayTimeline) app.get('/day/stress', getDayStress) app.get('/day/heart', getDayHeart) @@ -249,21 +260,30 @@ app.post('/notifications/read', markNotificationsRead) app.get('/admin/config', adminGetConfig) app.post('/admin/config', adminSetConfig) +// Raw R2 objects auto-expire after this many days (bucket lifecycle rule +// `expire-raw-14d`). The R2-re-decode jobs (HRV/resp/IMU steps) can therefore +// only run over data this recent — older raw is gone, so we cap their `days` +// here. Keep this in sync with the R2 lifecycle rule. +const RAW_RETENTION_DAYS = 14 + app.post('/admin/run-analytics', async (c) => { const body = await c.req.json<{ user_id?: string; days?: number; bio?: boolean }>().catch(() => ({} as any)) const days = body.days ?? 3 + // Biometrics re-decodes from R2 → can only go back as far as raw is retained. + const bioDays = Math.min(days, RAW_RETENTION_DAYS) if (body.user_id) { // Full re-derive sequence so HRV recovery feeds the coach: // 1. processUser — minute metrics (strain/RHR/sleep/sessions...) + daily rows - // 2. runBiometrics — HRV recovery/stress/illness from RR (needs daily RHR) + // (reads the `minute` table, retained 90d — not R2 — so `days` is unbounded here) + // 2. runBiometrics — HRV recovery/stress/illness from RR (R2; capped to retention) // 3. processUser — coach picks up the recovery written in step 2 const r1 = await processUser(c.env.DB, body.user_id, { historyDays: days }) let bio: any = null if (body.bio !== false) { - bio = await runBiometrics(c.env, body.user_id, days) + bio = await runBiometrics(c.env, body.user_id, bioDays) await processUser(c.env.DB, body.user_id, { historyDays: days }) } - return c.json({ ok: true, ...r1, bio }) + return c.json({ ok: true, ...r1, bio, bio_days: bioDays, bio_capped: bioDays < days }) } const res = await runAnalytics(c.env.DB, { historyDays: days }) return c.json({ ok: true, ...res }) @@ -271,20 +291,42 @@ app.post('/admin/run-analytics', async (c) => { // Respiratory rate from PPG (R21 re-decoded from R2). Heavy (R2 reads) → admin / // cron only. Stores resp_rate/resp_conf on daily; gated (conf ≥ 0.5) before surfaced. +// `days` is capped to the R2 retention window (older raw no longer exists). app.post('/admin/run-resp', async (c) => { const body = await c.req.json<{ user_id: string; days?: number }>().catch(() => ({} as any)) if (!body.user_id) return c.json({ error: 'user_id required' }, 400) - const res = await runRespRate(c.env, body.user_id, body.days ?? 3) - return c.json({ ok: true, ...res }) + const reqDays = body.days ?? 3 + const days = Math.min(reqDays, RAW_RETENTION_DAYS) + const res = await runRespRate(c.env, body.user_id, days) + return c.json({ ok: true, ...res, capped: days < reqDays, max_days: RAW_RETENTION_DAYS }) }) // HRV (RMSSD) + relative skin-temp / SpO₂ from the V24 RR/ADC bytes, re-decoded // from R2. Heavy (R2 reads) → admin / cron only. Writes daily.hrv_rmssd etc. +// `days` is capped to the R2 retention window (older raw no longer exists). app.post('/admin/run-biometrics', async (c) => { const body = await c.req.json<{ user_id: string; days?: number }>().catch(() => ({} as any)) if (!body.user_id) return c.json({ error: 'user_id required' }, 400) - const res = await runBiometrics(c.env, body.user_id, body.days ?? 3) - return c.json({ ok: true, ...res }) + const reqDays = body.days ?? 3 + const days = Math.min(reqDays, RAW_RETENTION_DAYS) + const res = await runBiometrics(c.env, body.user_id, days) + return c.json({ ok: true, ...res, capped: days < reqDays, max_days: RAW_RETENTION_DAYS }) +}) + +// Steps from the wrist IMU (AN-2554 over raw R2). mode=incremental (default) counts +// only newly-settled minutes since the cursor; mode=full recomputes the last `days` +// (capped to R2 retention) and realigns the cursor. Heavy (R2) → admin / cron only. +app.post('/admin/run-steps', async (c) => { + const body = await c.req.json<{ user_id: string; mode?: 'incremental' | 'full'; days?: number }>().catch(() => ({} as any)) + if (!body.user_id) return c.json({ error: 'user_id required' }, 400) + if (body.mode === 'full') { + const reqDays = body.days ?? 1 + const days = Math.min(reqDays, RAW_RETENTION_DAYS) + const res = await runStepsImu(c.env, body.user_id, days) + return c.json({ ok: true, mode: 'full', ...res, capped: days < reqDays }) + } + const res = await runStepsIncremental(c.env, body.user_id) + return c.json({ ok: true, mode: 'incremental', ...res }) }) // Phased to stay under the free-plan per-request subrequest cap. The shell @@ -338,6 +380,17 @@ app.post('/admin/issue-token', async (c) => { return c.json({ access_jwt: access, refresh_token: refresh, user_id: user.id }) }) +// Enqueue an analytics job for a user (ops + verification). The consumer runs it +// in its own bounded invocation. 404 if Queues isn't bound. +app.post('/admin/enqueue', async (c) => { + const body = await c.req.json<{ user_id: string; job?: AnalyticsJob; day?: string }>().catch(() => ({} as any)) + if (!body.user_id) return c.json({ error: 'user_id required' }, 400) + if (!c.env.ANALYTICS_Q) return c.json({ error: 'queue not bound' }, 404) + const msg = { user_id: body.user_id, job: body.job ?? 'sweep', ...(body.day ? { day: body.day } : {}) } + await c.env.ANALYTICS_Q.send(msg) + return c.json({ ok: true, enqueued: msg }) +}) + app.post('/admin/wipe-raw', async (c) => { let deleted = 0 let cursor: string | undefined @@ -355,68 +408,105 @@ app.post('/admin/wipe-raw', async (c) => { // Prune minute rows older than 90 days (raw stays in R2). app.post('/admin/prune', async (c) => { - const cutoff = Math.floor(Date.now() / 1000) - 90 * DAY + const cutoff = Math.floor(Date.now() / 1000) - MINUTE_RETENTION_DAYS * DAY const res = await c.env.DB.prepare('DELETE FROM minute WHERE ts_min < ?').bind(cutoff).run() return c.json({ ok: true, deleted: res.meta?.changes ?? 0, cutoff }) }) +// Send messages in sendBatch chunks of 100 (the Queues per-call max), so the cron +// itself never blows its own subrequest budget no matter how many users. +async function sendChunked(q: Queue, msgs: { body: AnalyticsMessage }[]): Promise { + for (let i = 0; i < msgs.length; i += 100) { + await q.sendBatch(msgs.slice(i, i + 100)) + } +} + +// One (user, job) message per user (day-less — for 'sweep'). +async function enqueueJobs(q: Queue, userIds: string[], job: AnalyticsJob): Promise { + await sendChunked(q, userIds.map((user_id) => ({ body: { user_id, job } }))) +} + +// One (user, job, day) message per user × day — heavy R2 jobs fanned out so each +// consumer invocation processes exactly ONE bounded day (works for any user, no +// matter how much they wore the band). +async function enqueueJobDays(q: Queue, userIds: string[], job: AnalyticsJob, dates: string[]): Promise { + const msgs: { body: AnalyticsMessage }[] = [] + for (const user_id of userIds) for (const day of dates) msgs.push({ body: { user_id, job, day } }) + await sendChunked(q, msgs) +} + +// The last `n` UTC dates (today first) as YYYY-MM-DD. +function lastNDates(n: number): string[] { + const now = Math.floor(Date.now() / 1000) + return Array.from({ length: n }, (_, d) => new Date((Math.floor((now - d * DAY) / DAY) * DAY) * 1000).toISOString().slice(0, 10)) +} + export default { fetch: app.fetch, - // Queue consumer (no-op if Queues disabled / binding absent — export is safe). + // Queue consumer — one bounded (user, job) unit per invocation. async queue(batch: MessageBatch, env: Bindings): Promise { await handleQueueBatch(batch, env) }, - // Crons: "7 * * * *" hourly safety sweep; "30 3 * * *" nightly re-derive+prune. + // Crons: "*/30 * * * *" sweep (enqueue dirty users); "30 3 * * *" nightly full + // re-derive + R2 jobs + prune. With Queues bound, the cron only ENQUEUES; the + // consumer does the bounded per-user work. Inline fallback if the binding is absent. async scheduled(event: ScheduledEvent, env: Bindings, ctx: ExecutionContext): Promise { if (event.cron === '30 3 * * *') { ctx.waitUntil((async () => { - // Full re-derive yesterday for all users that have any minute data, then prune. - await runAnalytics(env.DB, { historyDays: 2 }) - // Respiratory rate from PPG for users with a recent sleep row (R2 re-decode; - // null on nights without live PPG — gate handles it). Bounded per night. - try { - const since = new Date(Date.now() - 2 * DAY * 1000).toISOString().slice(0, 10) - const { results: users } = await env.DB.prepare( - 'SELECT DISTINCT user_id FROM sleep WHERE date >= ? AND onset_ts IS NOT NULL', - ).bind(since).all<{ user_id: string }>() - for (const u of users ?? []) await runRespRate(env, u.user_id, 2) - } catch (e) { console.error('resp cron failed', e) } - // HRV + relative skin-temp/SpO₂ from the V24 RR/ADC bytes (R2 re-decode). - try { - const since = new Date(Date.now() - 3 * DAY * 1000).toISOString().slice(0, 10) - const { results: users } = await env.DB.prepare( - 'SELECT DISTINCT user_id FROM daily WHERE date >= ?', - ).bind(since).all<{ user_id: string }>() - for (const u of users ?? []) await runBiometrics(env, u.user_id, 3) - } catch (e) { console.error('biometrics cron failed', e) } - // Steps from the wrist IMU (R10 + 0x33 re-decode → AN-2554 pedometer). - try { - const since = new Date(Date.now() - 2 * DAY * 1000).toISOString().slice(0, 10) - const { results: users } = await env.DB.prepare( - 'SELECT DISTINCT user_id FROM daily WHERE date >= ?', - ).bind(since).all<{ user_id: string }>() - for (const u of users ?? []) await runStepsImu(env, u.user_id, 2) - } catch (e) { console.error('steps cron failed', e) } - const cutoff = Math.floor(Date.now() / 1000) - 90 * DAY + const since2 = new Date(Date.now() - 2 * DAY * 1000).toISOString().slice(0, 10) + const since3 = new Date(Date.now() - 3 * DAY * 1000).toISOString().slice(0, 10) + if (env.ANALYTICS_Q) { + // Fan out per (user, job): each consumer invocation = one bounded unit. + const dailyU = (await env.DB.prepare('SELECT DISTINCT user_id FROM daily WHERE date >= ?') + .bind(since3).all<{ user_id: string }>()).results ?? [] + const sleepU = (await env.DB.prepare('SELECT DISTINCT user_id FROM sleep WHERE date >= ? AND onset_ts IS NOT NULL') + .bind(since2).all<{ user_id: string }>()).results ?? [] + const ids = dailyU.map((u) => u.user_id) + // 'sweep' = one per user (D1 + incremental steps, light). Heavy R2 jobs + // fan out per (user, day) so each unit reads ≤ one day — bounded for ANY user. + await enqueueJobs(env.ANALYTICS_Q, ids, 'sweep') + await enqueueJobDays(env.ANALYTICS_Q, ids, 'biometrics', lastNDates(3)) + await enqueueJobDays(env.ANALYTICS_Q, ids, 'steps_full', lastNDates(2)) + await enqueueJobDays(env.ANALYTICS_Q, sleepU.map((u) => u.user_id), 'resp', lastNDates(2)) + } else { + // Fallback (no queue): inline. Only viable at small scale. + await runAnalytics(env.DB, { historyDays: 2 }) + try { + const u = (await env.DB.prepare('SELECT DISTINCT user_id FROM sleep WHERE date >= ? AND onset_ts IS NOT NULL').bind(since2).all<{ user_id: string }>()).results ?? [] + for (const x of u) await runRespRate(env, x.user_id, 2) + } catch (e) { console.error('resp cron failed', e) } + try { + const u = (await env.DB.prepare('SELECT DISTINCT user_id FROM daily WHERE date >= ?').bind(since3).all<{ user_id: string }>()).results ?? [] + for (const x of u) await runBiometrics(env, x.user_id, 3) + } catch (e) { console.error('biometrics cron failed', e) } + try { + const u = (await env.DB.prepare('SELECT DISTINCT user_id FROM daily WHERE date >= ?').bind(since2).all<{ user_id: string }>()).results ?? [] + for (const x of u) await runStepsImu(env, x.user_id, 2) + } catch (e) { console.error('steps cron failed', e) } + } + // Prune is light D1 — always inline. + const cutoff = Math.floor(Date.now() / 1000) - MINUTE_RETENTION_DAYS * DAY await env.DB.prepare('DELETE FROM minute WHERE ts_min < ?').bind(cutoff).run() })()) } else { ctx.waitUntil((async () => { - // Hourly safety net: process any dirty users, then close stale workouts. - await runAnalytics(env.DB, { historyDays: 3 }) await autoCloseStaleWorkouts(env.DB) - // Steps LAST so the IMU-derived value (AN-2554) is authoritative over the - // minute-based one analytics writes. Refresh today + yesterday for users - // active in the last 2h (bounded R2 reads). - try { - const since = Math.floor(Date.now() / 1000) - 2 * 3600 - const { results: users } = await env.DB.prepare( - 'SELECT DISTINCT user_id FROM minute WHERE ts_min >= ?', - ).bind(since).all<{ user_id: string }>() - for (const u of users ?? []) await runStepsImu(env, u.user_id, 2) - } catch (e) { console.error('steps hourly cron failed', e) } + if (env.ANALYTICS_Q) { + // Enqueue a 'sweep' (processUser + incremental steps) per dirty user. + const dirty = (await env.DB.prepare('SELECT user_id FROM analytics_cursor WHERE dirty = 1') + .all<{ user_id: string }>()).results ?? [] + await enqueueJobs(env.ANALYTICS_Q, dirty.map((u) => u.user_id), 'sweep') + } else { + // Fallback (no queue): inline. + await runAnalytics(env.DB, { historyDays: 3 }) + try { + const since = Math.floor(Date.now() / 1000) - 2 * 3600 + const u = (await env.DB.prepare('SELECT DISTINCT user_id FROM minute WHERE ts_min >= ?').bind(since).all<{ user_id: string }>()).results ?? [] + for (const x of u) await runStepsIncremental(env, x.user_id) + } catch (e) { console.error('steps incremental cron failed', e) } + } })()) } }, diff --git a/src/ingest.ts b/src/ingest.ts index 3df7204..bcd5acb 100644 --- a/src/ingest.ts +++ b/src/ingest.ts @@ -43,16 +43,13 @@ interface IngestEnv { ANALYTICS_Q?: Queue } -async function enqueueOrMarkDirty(env: IngestEnv, userId: string, upto: number) { - if (env.ANALYTICS_Q) { - try { - await env.ANALYTICS_Q.send({ user_id: userId, upto }) - return - } catch (e) { - console.error('queue send failed; falling back to dirty flag', e) - } - } - // Fallback: mark dirty for the cron sweep. +// Mark the user dirty so the NEXT 30-min cron sweep enqueues exactly ONE analytics +// job for them (deduped). We deliberately do NOT enqueue per ingest batch: at scale +// that storms the queue (~240 msgs/user/day) and re-derives the same day hundreds of +// times. A cheap dirty flag here + the cron as the single enqueue point makes it one +// sweep per user per 30 min. Freshness is bounded by the sweep interval (fine — and +// recovery still fires promptly on wake via the sleep-detection trigger). +async function markUserDirty(env: IngestEnv, userId: string) { await env.DB.prepare( 'INSERT INTO analytics_cursor (user_id, last_min_ts, dirty) VALUES (?,0,1) ' + 'ON CONFLICT(user_id) DO UPDATE SET dirty = 1', @@ -130,7 +127,7 @@ export async function ingestBatch(c: Context<{ Bindings: IngestEnv; Variables: { // /ingest/events route handles event-only payloads. Here we skip. // 6. Enqueue analytics (or mark dirty). Never run inline. - if (minutesWritten > 0) await enqueueOrMarkDirty(c.env, userId, maxTsMin || Math.floor(Date.now() / 1000)) + if (minutesWritten > 0) await markUserDirty(c.env, userId) // 7. Respond. `received` = records persisted raw to R2 (the re-decodable system of // record); `decoded` = records that yielded a surfaceable sample; `minutes_written` diff --git a/src/query.ts b/src/query.ts index a989583..37e13a4 100644 --- a/src/query.ts +++ b/src/query.ts @@ -38,6 +38,10 @@ export async function getToday(c: Ctx) { .bind(userId).first() const baseline = await c.env.DB.prepare('SELECT * FROM baselines WHERE user_id = ?') .bind(userId).first() + // User's daily step goal (null → client default). Carried at top level so the + // Today step ring can render progress without a second call. + const u = await c.env.DB.prepare('SELECT step_goal FROM users WHERE id = ?') + .bind(userId).first() // Live status: most recent minute (HR + worn) in the last 10 minutes. const live = await c.env.DB.prepare( @@ -60,6 +64,8 @@ export async function getToday(c: Ctx) { return c.json({ date, + // User's daily step goal (null → client falls back to its own default). + step_goal: u?.step_goal ?? null, // Deterministic coach plan + strain target + readiness contributors + summary. coach: daily?.coach ? safeParse(daily.coach) : null, // HRV stress (Baevsky SI + LF/HF, personal-relative) — includes its drivers. @@ -191,6 +197,50 @@ export async function getSleep(c: Ctx) { return c.json(rows) } +// GET /sleep/v2?from&to — multi-period sleep (naps = shorter sleeps), grouped by +// date. Additive companion to GET /sleep (which stays the single-period shape). +export async function getSleepV2(c: Ctx) { + const userId = c.get('userId') + const from = c.req.query('from'), to = c.req.query('to') + let sql = 'SELECT * FROM sleep_periods WHERE user_id = ?' + const binds: any[] = [userId] + if (from) { sql += ' AND date >= ?'; binds.push(from) } + if (to) { sql += ' AND date <= ?'; binds.push(to) } + sql += ' ORDER BY date DESC, onset_ts ASC LIMIT 400' + const { results } = await c.env.DB.prepare(sql).bind(...binds).all() + const baseline = await c.env.DB.prepare('SELECT sleep_need_min FROM baselines WHERE user_id = ?') + .bind(userId).first() + const needMin = (baseline?.sleep_need_min && baseline.sleep_need_min >= 180) ? baseline.sleep_need_min : 480 + const byDate = new Map() + for (const r of results ?? []) { + const p = sleepPeriodRow(r) + const arr = byDate.get(r.date); if (arr) arr.push(p); else byDate.set(r.date, [p]) + } + const days = [...byDate.entries()].map(([date, periods]) => ({ + date, + periods, + period_count: periods.length, + total_asleep_min: periods.reduce((a: number, p: any) => a + (p.duration_min || 0), 0), + })) + return c.json({ days, need_min: needMin, stages_beta: true }) +} + +// Shape one sleep_periods row into the API period object. +function sleepPeriodRow(r: any) { + return { + id: r.id, + onset_ts: r.onset_ts, + wake_ts: r.wake_ts, + duration_min: r.duration_min, + in_bed_min: r.in_bed_min, + efficiency: r.efficiency, + stages: (r.light_min != null || r.deep_min != null || r.rem_min != null) + ? { light_min: r.light_min, deep_min: r.deep_min, rem_min: r.rem_min } : null, + is_main: !!r.is_main, + confidence: r.confidence, + } +} + export const getStrain = rangeRows('daily') // GET /sessions?from&to — auto-detected workouts (by start_ts unix range). diff --git a/src/queue.ts b/src/queue.ts index ffb6ced..4ac47a0 100644 --- a/src/queue.ts +++ b/src/queue.ts @@ -1,38 +1,106 @@ -// queue.ts — analytics queue consumer. Batched (≤20), dedup user_ids, per-user -// isolation (one user's failure never blocks others; retried → DLQ). +// queue.ts — analytics queue consumer. Each message is one (user, job) UNIT of +// work, so every consumer invocation does a single bounded job for a single user +// and stays well under the per-invocation subrequest cap (1000 on Paid). This is +// what lets the heavy R2-re-decode jobs (biometrics/resp/steps) scale: the cron +// just enqueues; the consumer fans them out, one bounded unit per invocation. import { processUser } from './analytics' +import { runBiometrics } from './biometrics' +import { runRespRate } from './resp' +import { runStepsImu, runStepsIncremental } from './steps_imu' + +export type AnalyticsJob = 'sweep' | 'biometrics' | 'resp' | 'steps_full' export interface AnalyticsMessage { user_id: string upto?: number + job?: AnalyticsJob // default 'sweep' + day?: string // YYYY-MM-DD — per-(user,day) fan-out for heavy R2 jobs } interface QueueEnv { DB: D1Database + RAW_BUCKET: R2Bucket + ANALYTICS_Q?: Queue +} + +// After a sweep, if a fresh night just finished (sleep with wake_ts, settled, for a +// date we haven't scored yet), enqueue biometrics + resp for it. This makes +// recovery/HRV appear shortly after each user WAKES — no timezone needed — instead +// of waiting for the fixed nightly run. Fires once per night (bio_last_date guard). +async function maybeTriggerBiometrics(env: QueueEnv, userId: string): Promise { + if (!env.ANALYTICS_Q) return + const sleep = await env.DB.prepare( + 'SELECT date, wake_ts FROM sleep WHERE user_id = ? AND onset_ts IS NOT NULL AND wake_ts IS NOT NULL ORDER BY date DESC LIMIT 1', + ).bind(userId).first<{ date: string; wake_ts: number }>() + if (!sleep) return + const now = Math.floor(Date.now() / 1000) + if (sleep.wake_ts > now - 600) return // woke <10 min ago → let the night settle; next sweep gets it + const cur = await env.DB.prepare('SELECT bio_last_date FROM analytics_cursor WHERE user_id = ?') + .bind(userId).first<{ bio_last_date: string | null }>() + if (cur?.bio_last_date && cur.bio_last_date >= sleep.date) return // already fired for this night + await env.ANALYTICS_Q.send({ user_id: userId, job: 'biometrics', day: sleep.date }) + await env.ANALYTICS_Q.send({ user_id: userId, job: 'resp', day: sleep.date }) + await env.DB.prepare( + 'INSERT INTO analytics_cursor (user_id, bio_last_date) VALUES (?,?) ' + + 'ON CONFLICT(user_id) DO UPDATE SET bio_last_date = excluded.bio_last_date', + ).bind(userId, sleep.date).run() +} + +// Run one bounded unit of work. Each branch is sized to fit in a single +// invocation's budget (one user, one job, a few days of R2 at most). +async function runJob(env: QueueEnv, userId: string, job: AnalyticsJob, day?: string): Promise { + switch (job) { + case 'biometrics': + // HRV/temp/SpO₂ from RR (R2) for ONE day (day set) or the trailing window. + await runBiometrics(env, userId, 3, day) + // Legacy multi-day mode re-runs analytics so the coach picks up recovery; in + // per-day mode the next 'sweep' does that (recovery is written either way). + if (!day) await processUser(env.DB, userId, { historyDays: 2 }) + break + case 'resp': + await runRespRate(env, userId, 2, day) + break + case 'steps_full': + // Full AN-2554 true-up for late-arriving frames (realigns the incremental cursor). + await runStepsImu(env, userId, 2, day) + break + case 'sweep': + default: + // The frequent path: derive daily/sleep/strain (D1) + incremental steps (bounded R2). + await processUser(env.DB, userId, { historyDays: 3 }) + await runStepsIncremental(env, userId) + // Event-driven: if this sweep just finished a night, fire biometrics for it. + await maybeTriggerBiometrics(env, userId) + break + } } export async function handleQueueBatch( batch: MessageBatch, env: QueueEnv, ): Promise { - // Dedup user_ids within the batch (multiple ingests for one user → one run). - const byUser = new Map[]>() + // Dedup identical (user, job) units within the batch. + const groups = new Map[]>() for (const msg of batch.messages) { const uid = msg.body.user_id if (!uid) { msg.ack(); continue } - const arr = byUser.get(uid) ?? [] + const key = `${uid}::${msg.body.job ?? 'sweep'}::${msg.body.day ?? ''}` + const arr = groups.get(key) ?? [] arr.push(msg) - byUser.set(uid, arr) + groups.set(key, arr) } - for (const [userId, msgs] of byUser) { + for (const [, msgs] of groups) { + const uid = msgs[0].body.user_id + const job = msgs[0].body.job ?? 'sweep' + const day = msgs[0].body.day try { - await processUser(env.DB, userId) + await runJob(env, uid, job, day) for (const m of msgs) m.ack() } catch (e) { - console.error('queue: processUser failed for', userId, e) - // Retry the whole group (Cloudflare re-delivers; max_retries → DLQ). + console.error('queue: job failed', uid, job, day, e) + // Retry the group (Cloudflare re-delivers; max_retries → DLQ). for (const m of msgs) m.retry() } } diff --git a/src/resp.ts b/src/resp.ts index 2cc2a3f..8ee66e8 100644 --- a/src/resp.ts +++ b/src/resp.ts @@ -140,10 +140,12 @@ async function computeNight(env: RespEnv, userId: string, date: string, from: nu * baseline (median of nights with conf ≥ 0.5). Heavy (R2 reads) → nightly cron / * admin only, NEVER inline ingest. */ -export async function runRespRate(env: RespEnv, userId: string, days = 3): Promise<{ nights: number; computed: number }> { - const since = new Date(Date.now() - days * DAY * 1000).toISOString().slice(0, 10) +export async function runRespRate(env: RespEnv, userId: string, days = 3, onlyDate?: string): Promise<{ nights: number; computed: number }> { + // Per-(user,day) fan-out: onlyDate → just that night; else the trailing `days`. + const since = onlyDate ?? new Date(Date.now() - days * DAY * 1000).toISOString().slice(0, 10) + const dateClause = onlyDate ? 'date = ?' : 'date >= ?' const { results: nights } = await env.DB.prepare( - 'SELECT date, onset_ts, wake_ts FROM sleep WHERE user_id = ? AND date >= ? AND onset_ts IS NOT NULL AND wake_ts IS NOT NULL ORDER BY date DESC', + `SELECT date, onset_ts, wake_ts FROM sleep WHERE user_id = ? AND ${dateClause} AND onset_ts IS NOT NULL AND wake_ts IS NOT NULL ORDER BY date DESC`, ).bind(userId, since).all<{ date: string; onset_ts: number; wake_ts: number }>() let computed = 0 diff --git a/src/steps_imu.ts b/src/steps_imu.ts index 4e6f3bc..fd14a35 100644 --- a/src/steps_imu.ts +++ b/src/steps_imu.ts @@ -32,10 +32,12 @@ async function rawKeysInWindow(bucket: R2Bucket, userId: string, from: number, t return out } -// Steps for one UTC day: assemble per-minute accel (ordered by ts,idx), then run -// the pure AN-2554 pedometer (analytics calcSteps) over the per-minute signals. -async function computeDaySteps(env: StepsEnv, userId: string, dayStart: number): Promise { - const from = dayStart, to = dayStart + DAY +// Steps over an arbitrary [from, to) window: assemble per-minute accel (ordered by +// ts,idx), then run the pure AN-2554 pedometer (analytics calcSteps). calcSteps is +// per-minute-independent (it sums pedometer(minute) × GAIN), so the count over a +// window is exactly the sum of the counts over any partition of it into whole +// minutes — which is what makes incremental accumulation exact. +async function computeWindowSteps(env: StepsEnv, userId: string, from: number, to: number): Promise { const keys = await rawKeysInWindow(env.RAW_BUCKET, userId, from, to) // DEDUP by (ts, idx): R2 upload windows overlap, so the same frame appears in // multiple objects — without this, samples (and steps) are double-counted. @@ -67,21 +69,100 @@ async function computeDaySteps(env: StepsEnv, userId: string, dayStart: number): return calcSteps(minuteSignals) } +// Minutes are considered "settled" (safe to count incrementally) once they're this +// old, giving late/bursty band syncs time to land. Anything later is caught by the +// nightly full recompute. 30 min matches the sweep cadence. +const SETTLE_GRACE = 1800 + +function ymd(dayStart: number): string { + return new Date(dayStart * 1000).toISOString().slice(0, 10) +} + /** - * runStepsImu — recompute steps for the last `days` UTC days from R2 IMU and store - * on daily.steps. Authoritative source of steps (analytics no longer needs to be). + * runStepsIncremental — the 30-min-sweep path. Counts only the minutes that have + * newly SETTLED since the last run (a few R2 objects) and accumulates into + * daily.steps via a per-user cursor. Exact for settled data (per-minute pedometer); + * the nightly full recompute trues up any late-arriving frames. Cheap: bounded R2 + * reads per run regardless of how long the day is or how many users. */ -export async function runStepsImu(env: StepsEnv, userId: string, days = 1): Promise<{ days: number; total: number }> { +export async function runStepsIncremental(env: StepsEnv, userId: string): Promise<{ added: number }> { const now = Math.floor(Date.now() / 1000) + const todayStart = Math.floor(now / DAY) * DAY + const date = ymd(todayStart) + const settledCutoff = Math.floor((now - SETTLE_GRACE) / 60) * 60 // last fully-settled minute boundary + + const cur = await env.DB.prepare( + 'SELECT steps_cursor_ts, steps_cursor_day FROM analytics_cursor WHERE user_id = ?', + ).bind(userId).first<{ steps_cursor_ts: number | null; steps_cursor_day: string | null }>() + const sameDay = !!cur && cur.steps_cursor_day === date && cur.steps_cursor_ts != null + const cursor = sameDay ? cur!.steps_cursor_ts! : todayStart + + const setCursor = env.DB.prepare( + 'INSERT INTO analytics_cursor (user_id, steps_cursor_ts, steps_cursor_day) VALUES (?,?,?) ' + + 'ON CONFLICT(user_id) DO UPDATE SET steps_cursor_ts=excluded.steps_cursor_ts, steps_cursor_day=excluded.steps_cursor_day', + ) + + if (settledCutoff <= cursor) { + // Nothing newly settled. On a fresh day, still seed the row at 0 + set cursor. + if (!sameDay) { + await env.DB.batch([ + env.DB.prepare('INSERT OR IGNORE INTO daily(user_id, date) VALUES(?,?)').bind(userId, date), + env.DB.prepare('UPDATE daily SET steps = 0 WHERE user_id = ? AND date = ?').bind(userId, date), + setCursor.bind(userId, todayStart, date), + ]) + } + return { added: 0 } + } + + const chunk = await computeWindowSteps(env, userId, cursor, settledCutoff) + await env.DB.batch([ + env.DB.prepare('INSERT OR IGNORE INTO daily(user_id, date) VALUES(?,?)').bind(userId, date), + sameDay + ? env.DB.prepare('UPDATE daily SET steps = COALESCE(steps,0) + ? WHERE user_id = ? AND date = ?').bind(chunk, userId, date) + : env.DB.prepare('UPDATE daily SET steps = ? WHERE user_id = ? AND date = ?').bind(chunk, userId, date), + setCursor.bind(userId, settledCutoff, date), + ]) + return { added: chunk } +} + +/** + * runStepsImu — FULL recompute for the last `days` UTC days from R2 IMU. The + * authoritative true-up: catches late-arriving frames the incremental path may + * have missed. For TODAY it also realigns the incremental cursor so the sweep + * continues from the trued-up baseline without double-counting. + */ +export async function runStepsImu(env: StepsEnv, userId: string, days = 1, onlyDate?: string): Promise<{ days: number; total: number }> { + const now = Math.floor(Date.now() / 1000) + const todayStart = Math.floor(now / DAY) * DAY + const settledCutoff = Math.floor((now - SETTLE_GRACE) / 60) * 60 let grand = 0 - for (let d = 0; d < days; d++) { - const dayStart = Math.floor((now - d * DAY) / DAY) * DAY - const date = new Date(dayStart * 1000).toISOString().slice(0, 10) - const steps = await computeDaySteps(env, userId, dayStart) - await env.DB.prepare('INSERT OR IGNORE INTO daily(user_id, date) VALUES(?,?)').bind(userId, date).run() - await env.DB.prepare('UPDATE daily SET steps = ? WHERE user_id = ? AND date = ?') - .bind(steps, userId, date).run() - grand += steps + // Per-(user,day) fan-out: onlyDate → just that day; else the trailing `days`. + const dayStarts = onlyDate + ? [Math.floor(Date.parse(`${onlyDate}T00:00:00Z`) / 1000)] + : Array.from({ length: days }, (_, d) => Math.floor((now - d * DAY) / DAY) * DAY) + for (const dayStart of dayStarts) { + const date = ymd(dayStart) + if (dayStart === todayStart) { + // Today: count only settled minutes (consistent with the incremental path) + // and realign the cursor so the next sweep appends from here. + const steps = await computeWindowSteps(env, userId, todayStart, settledCutoff) + await env.DB.batch([ + env.DB.prepare('INSERT OR IGNORE INTO daily(user_id, date) VALUES(?,?)').bind(userId, date), + env.DB.prepare('UPDATE daily SET steps = ? WHERE user_id = ? AND date = ?').bind(steps, userId, date), + env.DB.prepare( + 'INSERT INTO analytics_cursor (user_id, steps_cursor_ts, steps_cursor_day) VALUES (?,?,?) ' + + 'ON CONFLICT(user_id) DO UPDATE SET steps_cursor_ts=excluded.steps_cursor_ts, steps_cursor_day=excluded.steps_cursor_day', + ).bind(userId, settledCutoff, date), + ]) + grand += steps + } else { + // Past days are fully settled → count the whole day. + const steps = await computeWindowSteps(env, userId, dayStart, dayStart + DAY) + await env.DB.prepare('INSERT OR IGNORE INTO daily(user_id, date) VALUES(?,?)').bind(userId, date).run() + await env.DB.prepare('UPDATE daily SET steps = ? WHERE user_id = ? AND date = ?') + .bind(steps, userId, date).run() + grand += steps + } } return { days, total: grand } } diff --git a/wrangler.toml b/wrangler.toml index 2c17360..1adf2f9 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -12,28 +12,29 @@ database_id = "REPLACE_WITH_YOUR_D1_DATABASE_ID" binding = "RAW_BUCKET" bucket_name = "openstrap-raw" -# Analytics queue (producer + consumer). DISABLED on this account: Cloudflare -# Queues require the Workers Paid plan, which isn't enabled here (queue create -# returns "specified queue settings are invalid"). The code path is fully -# preserved — ingest.ts falls back to setting analytics_cursor.dirty=1 and the -# hourly cron ("7 * * * *") + /admin/run-analytics process the work. To enable: -# upgrade to Workers Paid, `wrangler queues create openstrap-analytics` + -# `... -dlq`, then uncomment these blocks. The queue() consumer export in -# index.ts is harmless to keep exported while the binding is absent. -# [[queues.producers]] -# binding = "ANALYTICS_Q" -# queue = "openstrap-analytics" -# -# [[queues.consumers]] -# queue = "openstrap-analytics" -# max_batch_size = 20 -# max_batch_timeout = 30 -# max_retries = 3 -# dead_letter_queue = "openstrap-analytics-dlq" +# Analytics queue (producer + consumer). ENABLED on Workers Paid. The cron just +# ENQUEUES one (user, job) message per unit of work; the consumer processes one +# bounded unit per invocation, so heavy R2 jobs (biometrics/resp/steps) never hit +# the 1000-subrequest per-invocation cap. ingest.ts also enqueues a 'sweep' on +# upload (falls back to analytics_cursor.dirty=1 if the binding is absent). +# max_batch_size = 1 → exactly one (user, job) per invocation (guaranteed bounded). +[[queues.producers]] +binding = "ANALYTICS_Q" +queue = "openstrap-analytics" -# Crons: hourly safety sweep (enqueue dirty users) + nightly re-derive/prune. +[[queues.consumers]] +queue = "openstrap-analytics" +max_batch_size = 1 +max_batch_timeout = 10 +max_retries = 3 +dead_letter_queue = "openstrap-analytics-dlq" + +# Crons: 30-min sweep (re-derive dirty users → keeps "today" fresh; pure-read +# endpoints serve this snapshot) + nightly full re-derive/prune. Any non-nightly +# cron string falls through to the sweep branch in scheduled(); only "30 3 * * *" +# triggers the nightly path. (Future: tighten the sweep to */15 if needed.) [triggers] -crons = ["7 * * * *", "30 3 * * *"] +crons = ["*/30 * * * *", "30 3 * * *"] [vars] EMAIL_FROM_NAME = "OpenStrap" From 480a985506b5608b095979bdcfad7f14cad699d8 Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Sahil <127765312+abdulsaheel@users.noreply.github.com> Date: Tue, 16 Jun 2026 12:30:13 +0530 Subject: [PATCH 02/30] fix: dedup biometrics/resp re-decode, self-heal stuck HRV, tame dirty storm (#6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cost + correctness on the deployed analytics pipeline: - Nightly cron now enqueues biometrics/resp ONLY for (user,day) where hrv_rmssd/resp_rate IS NULL (last 2 days). Already-scored nights are skipped entirely (no redundant R2 re-decode vs the event-driven fire) — the biggest R2 cost cut — and this doubles as the retry for nights the event fire left null, fixing HRV that got permanently stuck pending. - queue.ts: event-driven fire skips if HRV already exists for that night, and stays capped at one heavy decode per night via bio_last_date. - 30-min sweep clears `dirty` optimistically for enqueued users so a persistently-failing sweep isn't re-enqueued every 30 min. - sleep_efficiency: a real 0% is no longer coerced to null ("no data"). Co-authored-by: abdulsaheel --- src/analytics.ts | 4 ++-- src/index.ts | 36 +++++++++++++++++++++++++++++------- src/queue.ts | 9 +++++++++ 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/src/analytics.ts b/src/analytics.ts index f453605..ef091fe 100644 --- a/src/analytics.ts +++ b/src/analytics.ts @@ -446,7 +446,7 @@ export async function processUser( const anomaly = calcAnomaly({ recent_rhr: recentRhr, skin_temp: baseline.skin_temp ?? null, - sleep_efficiency: sleep.efficiency || null, + sleep_efficiency: sleep.efficiency != null ? sleep.efficiency : null, // keep a real 0% (not coerced to "no data") baseline_sleep_efficiency: null, }, baseline) @@ -514,7 +514,7 @@ export async function processUser( sleep_last_min: sleep.duration_min || null, sleep_need_min: needFloored, sleep_debt_min: sleepDebt, - sleep_efficiency: sleep.efficiency || null, + sleep_efficiency: sleep.efficiency != null ? sleep.efficiency : null, // keep a real 0% (not coerced to "no data") sri: sri.sri, fitness_direction: fitness.direction, anomaly: bodyAlert ? JSON.parse(bodyAlert) : null, diff --git a/src/index.ts b/src/index.ts index 8d3c227..b798c6c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -461,15 +461,27 @@ export default { // Fan out per (user, job): each consumer invocation = one bounded unit. const dailyU = (await env.DB.prepare('SELECT DISTINCT user_id FROM daily WHERE date >= ?') .bind(since3).all<{ user_id: string }>()).results ?? [] - const sleepU = (await env.DB.prepare('SELECT DISTINCT user_id FROM sleep WHERE date >= ? AND onset_ts IS NOT NULL') - .bind(since2).all<{ user_id: string }>()).results ?? [] const ids = dailyU.map((u) => u.user_id) - // 'sweep' = one per user (D1 + incremental steps, light). Heavy R2 jobs - // fan out per (user, day) so each unit reads ≤ one day — bounded for ANY user. + // 'sweep' = one per user (D1 + incremental steps, light). 'steps_full' = + // late-frame true-up, per (user, day). await enqueueJobs(env.ANALYTICS_Q, ids, 'sweep') - await enqueueJobDays(env.ANALYTICS_Q, ids, 'biometrics', lastNDates(3)) await enqueueJobDays(env.ANALYTICS_Q, ids, 'steps_full', lastNDates(2)) - await enqueueJobDays(env.ANALYTICS_Q, sleepU.map((u) => u.user_id), 'resp', lastNDates(2)) + // Biometrics + resp are the expensive R2 re-decodes. Enqueue ONLY the + // (user, day) nights still MISSING the metric in the last 2 days — this + // skips every already-scored night (no redundant decode vs the + // event-driven fire) AND retries nights the event fire left null + // (self-healing). The single biggest R2 cost saver. + const bioNeed = (await env.DB.prepare( + 'SELECT user_id, date FROM daily WHERE date >= ? AND hrv_rmssd IS NULL', + ).bind(since2).all<{ user_id: string; date: string }>()).results ?? [] + await sendChunked(env.ANALYTICS_Q, + bioNeed.map((r) => ({ body: { user_id: r.user_id, job: 'biometrics', day: r.date } as AnalyticsMessage }))) + const respNeed = (await env.DB.prepare( + 'SELECT d.user_id, d.date FROM daily d WHERE d.date >= ? AND d.resp_rate IS NULL ' + + 'AND EXISTS (SELECT 1 FROM sleep s WHERE s.user_id = d.user_id AND s.date = d.date AND s.onset_ts IS NOT NULL)', + ).bind(since2).all<{ user_id: string; date: string }>()).results ?? [] + await sendChunked(env.ANALYTICS_Q, + respNeed.map((r) => ({ body: { user_id: r.user_id, job: 'resp', day: r.date } as AnalyticsMessage }))) } else { // Fallback (no queue): inline. Only viable at small scale. await runAnalytics(env.DB, { historyDays: 2 }) @@ -497,7 +509,17 @@ export default { // Enqueue a 'sweep' (processUser + incremental steps) per dirty user. const dirty = (await env.DB.prepare('SELECT user_id FROM analytics_cursor WHERE dirty = 1') .all<{ user_id: string }>()).results ?? [] - await enqueueJobs(env.ANALYTICS_Q, dirty.map((u) => u.user_id), 'sweep') + const dids = dirty.map((u) => u.user_id) + await enqueueJobs(env.ANALYTICS_Q, dids, 'sweep') + // Optimistically clear dirty for exactly the users we enqueued, so a + // persistently-failing sweep isn't re-enqueued every 30 min (a user + // re-dirtied after this SELECT keeps dirty=1 and is caught next tick). + for (let i = 0; i < dids.length; i += 100) { + const chunk = dids.slice(i, i + 100) + await env.DB.prepare( + `UPDATE analytics_cursor SET dirty = 0 WHERE user_id IN (${chunk.map(() => '?').join(',')})`, + ).bind(...chunk).run() + } } else { // Fallback (no queue): inline. await runAnalytics(env.DB, { historyDays: 3 }) diff --git a/src/queue.ts b/src/queue.ts index 4ac47a0..ccf42e4 100644 --- a/src/queue.ts +++ b/src/queue.ts @@ -36,8 +36,17 @@ async function maybeTriggerBiometrics(env: QueueEnv, userId: string): Promise now - 600) return // woke <10 min ago → let the night settle; next sweep gets it + // Already have HRV for this night (nightly backstop or an earlier fire computed + // it)? Nothing to do — never re-decode R2 for a night we've already scored. + const have = await env.DB.prepare( + 'SELECT 1 FROM daily WHERE user_id = ? AND date = ? AND hrv_rmssd IS NOT NULL', + ).bind(userId, sleep.date).first() + if (have) return const cur = await env.DB.prepare('SELECT bio_last_date FROM analytics_cursor WHERE user_id = ?') .bind(userId).first<{ bio_last_date: string | null }>() + // Cost cap: fire the heavy R2 decode at most ONCE per night from the event path. + // If that fire yields null (partial/late R2), the nightly cron retries missing + // nights — so we never spam the decode every 30-min sweep. if (cur?.bio_last_date && cur.bio_last_date >= sleep.date) return // already fired for this night await env.ANALYTICS_Q.send({ user_id: userId, job: 'biometrics', day: sleep.date }) await env.ANALYTICS_Q.send({ user_id: userId, job: 'resp', day: sleep.date }) From de3b97ea373fe7b5cd6422e5b40243a1a88ac933 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Tue, 16 Jun 2026 12:39:20 +0530 Subject: [PATCH 03/30] =?UTF-8?q?docs:=20README=20=E2=80=94=20correct=20cr?= =?UTF-8?q?on=20schedule=20+=20that=20HRV=20is=20now=20computed=20from=20R?= =?UTF-8?q?-R?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index affb2da..232fc00 100644 --- a/README.md +++ b/README.md @@ -52,13 +52,21 @@ the request path. ## Where the metrics actually get computed -`analytics.ts` is the brain, and it runs on a cron . Two schedules, -both in `wrangler.toml`: - -- **Every hour** (`runAnalytics` with a 3-day window): re-derive every dirty user. This - is the safety net, it catches anyone the queue missed. -- **Every night at 3:30** (2-day window): a fuller pass, plus respiratory rate for anyone - who slept recently, plus pruning minute rows older than 90 days. +`analytics.ts` is the brain, and it runs on a cron. Two schedules, both in +`wrangler.toml`: + +- **Every 30 minutes** (`*/30`): a light sweep — re-derive every dirty user (their daily + numbers, sleep, incremental steps), and the moment a night actually finishes, kick off + that night's HRV. Cheap work only; the heavy R2 re-decodes are fanned out onto a queue, + one bounded `(user, day)` unit per consumer invocation. +- **Every night at 3:30** (`30 3`): the backstop — re-decode HRV and respiratory rate + only for the recent nights still *missing* them (so a night is never decoded twice, and + a night the wake-time run left empty gets retried), true up steps, and prune minute rows + past their retention. Anything already computed is skipped. + +HRV is real now: the beat-to-beat R-R intervals live in the 1 Hz (V24) records, so +`biometrics.ts` re-decodes them from the raw bytes in R2 (off the request path) to drive +recovery, readiness, and HRV-based stress. `processUser` is where it happens for one person. It reads their minutes, pulls their baselines, and calls into the [analytics package](https://github.com/OpenStrap/analytics) @@ -84,6 +92,9 @@ moment it starts inventing numbers, so it doesn't. | `decode.ts` | Hex frames into decoded samples | | `rollup.ts` | Decoded samples into per-minute buckets | | `analytics.ts` | The cron brain, `processUser` / `runAnalytics` | +| `queue.ts` | The queue consumer: one bounded `(user, job, day)` unit per invocation, and the wake-time HRV trigger | +| `biometrics.ts` | HRV (RMSSD/SDNN/LF-HF), recovery, stress, relative temp/SpO₂ — re-decoded from the R-R intervals in R2 | +| `steps_imu.ts` | Step counting from the wrist accelerometer (incremental + nightly true-up) | | `query.ts` | The read endpoints: today, sleep, strain, trends, chart, history | | `daydetail.ts` | Single-day drill-downs (the strain curve, the hypnogram, the stress band) | | `history.ts` | Range aggregation and the calendar heatmap | From 57b91100a49e602639ad4ea968ddd224d924efd3 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Tue, 16 Jun 2026 17:18:58 +0530 Subject: [PATCH 04/30] feat: prune events table on the nightly cron (same 10-day window as minute) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit events is only read by /day/timeline (single day) and the app only requests the last 7 days, backed by minute data that's pruned at 10 days. So events older than the minute cutoff are never read — age them out together. Also mirrored in /admin/prune. --- src/index.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/index.ts b/src/index.ts index b798c6c..47b5942 100644 --- a/src/index.ts +++ b/src/index.ts @@ -410,7 +410,8 @@ app.post('/admin/wipe-raw', async (c) => { app.post('/admin/prune', async (c) => { const cutoff = Math.floor(Date.now() / 1000) - MINUTE_RETENTION_DAYS * DAY const res = await c.env.DB.prepare('DELETE FROM minute WHERE ts_min < ?').bind(cutoff).run() - return c.json({ ok: true, deleted: res.meta?.changes ?? 0, cutoff }) + const ev = await c.env.DB.prepare('DELETE FROM events WHERE ts < ?').bind(cutoff).run() + return c.json({ ok: true, deleted: res.meta?.changes ?? 0, events_deleted: ev.meta?.changes ?? 0, cutoff }) }) // Send messages in sendBatch chunks of 100 (the Queues per-call max), so the cron @@ -498,9 +499,11 @@ export default { for (const x of u) await runStepsImu(env, x.user_id, 2) } catch (e) { console.error('steps cron failed', e) } } - // Prune is light D1 — always inline. + // Prune is light D1 — always inline. Minute + events share the same + // drill-down window (/day/timeline), so they age out together. const cutoff = Math.floor(Date.now() / 1000) - MINUTE_RETENTION_DAYS * DAY await env.DB.prepare('DELETE FROM minute WHERE ts_min < ?').bind(cutoff).run() + await env.DB.prepare('DELETE FROM events WHERE ts < ?').bind(cutoff).run() })()) } else { ctx.waitUntil((async () => { From 2307f7070681157731ef4bf1641ead8851bd08f5 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Tue, 16 Jun 2026 21:34:36 +0530 Subject: [PATCH 05/30] feat: seed trailing-window metrics from durable daily/sleep history MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ACWR / monotony / Banister fitness model / fitness-trend / SRI were fed only the ~3-day recompute window, so 7/28/30-day windows were degenerate (ACWR≈null/1.0). Now seed those series from the permanent daily/sleep tables (fetch 90 days, reuse the rows already loaded for baselines) and offset idx so each day's trailing window includes real history. Intra-span gaps = rest days (strain 0); pre-account days are never fabricated. Forward-only, no re-decode. D1 reads ≈free at scale. --- src/analytics.ts | 57 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 52 insertions(+), 5 deletions(-) diff --git a/src/analytics.ts b/src/analytics.ts index ef091fe..0836517 100644 --- a/src/analytics.ts +++ b/src/analytics.ts @@ -188,12 +188,18 @@ export async function processUser( ).bind(userId).all<{ date: string; recovery: number }>() const recoveryByDate = new Map((recRows ?? []).map((r) => [r.date, r.recovery])) - // ── Pass 1: recompute baselines from prior derived history (last 30 days). ── + // ── Pass 1: recompute baselines from prior derived history. ── + // 90 days (not 30): baselines use the most-recent 30, but the trailing-window + // metrics (ACWR/monotony/RHR-trend, and SRI from sleep) need a longer seed — + // EWMA chronic (τ=28) wants ~3× that to converge. These are permanent derived + // rows and D1 reads are effectively free at this scale, so the longer read is + // ~$0. We NEVER re-decode old days (minute data is pruned at 10d); we just + // reuse what was already computed. Forward-only, no backfill. const { results: histRows } = await db.prepare( - 'SELECT date, resting_hr, strain, hr_zones FROM daily WHERE user_id = ? ORDER BY date DESC LIMIT 30', + 'SELECT date, resting_hr, strain, hr_zones FROM daily WHERE user_id = ? ORDER BY date DESC LIMIT 90', ).bind(userId).all() const { results: sleepHistRows } = await db.prepare( - 'SELECT date, duration_min FROM sleep WHERE user_id = ? ORDER BY date DESC LIMIT 30', + 'SELECT date, duration_min, onset_ts, wake_ts FROM sleep WHERE user_id = ? ORDER BY date DESC LIMIT 90', ).bind(userId).all() const sleepByDate = new Map() for (const s of sleepHistRows ?? []) if (s.duration_min != null) sleepByDate.set(s.date, s.duration_min) @@ -213,7 +219,8 @@ export async function processUser( }) if (history.length > 0) { - const bl = calcBaselines(history, profile) + // Baselines stay a 30-day metric even though we now fetch 90 for the windows. + const bl = calcBaselines(history.slice(-30), profile) baseline = { resting_hr: bl.resting_hr ?? baseline.resting_hr, max_hr: bl.max_hr ?? baseline.max_hr, @@ -263,6 +270,44 @@ export async function processUser( return out } + // ── Seed the trailing-window series from DURABLE derived history, so the + // cross-day metrics in Pass 3 (ACWR/monotony/RHR-trend/SRI) see their true + // 28/14/7-day windows instead of just the ~3-day recompute window. We reuse + // the daily/sleep rows already fetched above (no extra query, no re-decode). + // Gaps INSIDE the active span are genuine rest days → strain 0; days before + // the user's earliest derived row are NOT fabricated (else a young account + // gets a tiny chronic and a bogus high ACWR). + const SEED_DAYS = 84 // ≈3× the 28-day chronic time constant + let seedLen = 0 + { + const dayStrain = new Map() + const dayRhr = new Map() + for (const d of histRows ?? []) { + if (d.strain != null) dayStrain.set(d.date, d.strain) + if (d.resting_hr != null) dayRhr.set(d.date, d.resting_hr) + } + const dayWake = new Map() + for (const s of sleepHistRows ?? []) dayWake.set(s.date, { onset: s.onset_ts ?? null, wake: s.wake_ts ?? null }) + + // Earliest derived day within the seed window — start the continuous series + // there (don't 0-fill pre-account days). + const windowStart = firstDayStart - SEED_DAYS * DAY + let seedStart = firstDayStart + for (let d = windowStart; d < firstDayStart; d += DAY) { + const k = dayKey(d) + if (dayStrain.has(k) || dayRhr.has(k) || dayWake.has(k)) { seedStart = d; break } + } + for (let d = seedStart; d < firstDayStart; d += DAY) { + const k = dayKey(d) + dailyStrains.push({ ts: d, strain: dayStrain.get(k) ?? 0 }) // intra-span gap = rest day + rhrSeries.push(dayRhr.get(k) ?? null) + hrrSeries.push(null) // HRR isn't stored per-day → fitness-trend HRR stays sparse + const w = dayWake.get(k) + nightSummaries.push({ onset_ts: w?.onset ?? null, wake_ts: w?.wake ?? null }) + seedLen++ + } + } + for (let dayStart = firstDayStart; dayStart <= lastDayStart; dayStart += DAY) { const date = dayKey(dayStart) const dayMin = dayMinutes(dayStart) @@ -390,7 +435,9 @@ export async function processUser( } dayBuffer.push({ - date, dayStart, idx: dayBuffer.length, rhr, strain, zones, calories, sleep, wearMin, + // idx points into the parallel arrays, which now START with `seedLen` + // seeded history days — so trailing slices in Pass 3 reach into real history. + date, dayStart, idx: seedLen + dayBuffer.length, rhr, strain, zones, calories, sleep, wearMin, sleepStress: JSON.stringify(sleepStress), nocturnal: JSON.stringify(nocturnal), sleepingHr: nocturnal.sleeping_hr_avg, From 803fdaddc8c7b649e9fad4cadc28b43950a208e0 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Tue, 16 Jun 2026 21:37:27 +0530 Subject: [PATCH 06/30] =?UTF-8?q?docs:=20README=20=E2=80=94=20retention=20?= =?UTF-8?q?layers=20(raw=2014d=20/=20minute+events=2010d=20/=20derived=20p?= =?UTF-8?q?ermanent)=20+=20trailing-window=20metrics=20seeded=20from=20dur?= =?UTF-8?q?able=20history?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 232fc00..7c66f91 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,12 @@ When a batch lands, `ingest.ts` does four things in order: borrowed from the protocol package) and the IMU-bearing R10 are the ones that carry real signal. 3. **Saves the raw bytes to R2** under `raw/{you}/{device}/{when}-{first}-{last}.txt`, - one frame per line. This is the bit I never throw away. Minute rollups get pruned after - 90 days; the raw frames stay so they can be re-decoded forever. + one frame per line — the system of record for re-decoding. Retention is short and + layered: raw R2 objects expire at **14 days** (an R2 bucket lifecycle rule, matching + `RAW_RETENTION_DAYS` — the re-decode horizon), the per-minute `minute` table and the + device `events` table are pruned at **10 days**, and the *derived* tables (`daily`, + `sleep`, `baselines`) are **kept permanently** — that's what the long-window trend + metrics read from, so nothing needs re-decoding past 10 days. 4. **Rolls everything into minutes.** `rollup.ts` buckets the decoded samples by `floor(ts/60)*60` and writes them to the `minute` table. @@ -77,6 +81,12 @@ signal, the coach plan, stress, nocturnal heart. The results land in `daily`, `s actually move day to day instead of collapsing into one flat value, which was a real bug early on. +The cross-day metrics that need long windows — training load (EWMA acute:chronic over +7/28 days), Banister fitness/fatigue/form, Foster monotony, sleep regularity — are +**seeded from the permanent `daily`/`sleep` tables**, not just the few days being +recomputed. We never re-decode old days (raw is gone after 14 days); we reuse the +already-derived rows, so a real 28-day ACWR or 14-night regularity actually exists. + Every number comes back wrapped: a value, a unit, a confidence between 0 and 1, a tier, and a label. If the inputs aren't there, the value is `null` and the confidence is `0`. I would rather show you a dash than make something up. The whole project falls apart the @@ -130,7 +140,7 @@ Admin stuff for when you run your own: `/admin/run-analytics`, `/admin/run-resp` ## The database Thirteen tables. The ones you'll care about: `minute` (the running-sum rollups, pruned at -90 days), `daily` and `sleep` and `sessions` (the derived output, mostly JSON columns for +10 days), `daily` and `sleep` and `sessions` (the derived output, mostly JSON columns for the structured bits like coach plans and HR zones), `baselines` (your resting HR, max HR, sleep need, the anchors everything else is measured against), and the auth trio (`users`, `otps`, `refresh_tokens`). Full DDL is in `src/db/schema.sql`, it's commented, go read it. From 90560a0258ad1b8d156e1f37dfab012eb1e293fa Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Wed, 17 Jun 2026 00:30:11 +0530 Subject: [PATCH 07/30] Wire measured maxHR into baselines; harden drivers/steps coupling - analytics history query now selects hr_max and feeds it as session_hr_max, so calcBaselines can use a real measured max instead of always falling to the Tanaka age formula (the measured-max path was dead). - biometrics: merge drivers via SQL json_patch of only the bio-owned keys, replacing the read-modify-write. Removes the TOCTOU clobber against a concurrent processUser sweep and drops a column from the read. - steps: compare-and-swap on the steps cursor so a concurrent nightly full recompute can't cause the incremental sweep to double-count the day. - trend: relabel sleep regularity "Sleep consistency" (not the Phillips SRI). - README: correct the minute-table idempotency claim (additive upsert is order/partition-independent but relies on the client's hex-PK frame dedup). --- README.md | 17 +++++++++++++---- src/analytics.ts | 7 ++++++- src/biometrics.ts | 26 ++++++++++++++++---------- src/steps_imu.ts | 37 ++++++++++++++++++++++++++++++------- src/trend.ts | 2 +- 5 files changed, 66 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 7c66f91..883ca2d 100644 --- a/README.md +++ b/README.md @@ -45,10 +45,19 @@ When a batch lands, `ingest.ts` does four things in order: The `minute` table is the one clever bit worth understanding before you touch anything. It doesn't store an average heart rate, it stores the running pieces: `hr_sum`, `hr_n`, `act_sum`, `act_n`, plus min/max. The upsert adds the new pieces onto whatever's already -there. The reason is that uploads aren't clean. The phone retries, batches overlap, the -same frame shows up twice. If I stored averages I'd corrupt them on every double-send. -Storing sums means re-uploading the exact same data converges to the exact same answer. -Idempotency for free. Don't break this. +there. The reason is that uploads aren't clean. The phone retries and batches overlap, so +a day's frames arrive split across many POSTs in no particular order. Storing running sums +makes the minute totals **order- and partition-independent**: chop the same frames into any +set of batches, in any order, and the minute rollups come out identical. If I stored +averages instead, the split would corrupt them. + +One honest caveat, because it's easy to misread this as "duplicate-proof": the upsert is +*additive*, so the **same frame delivered in two separate POSTs is counted twice.** This is +not idempotent against duplicate sends on its own — it relies on the client deduping frames +before they ever hit here. The app does exactly that: `raw_records` has the frame hex as a +primary key (`INSERT OR IGNORE`) and only deletes a record after a 200, so each frame is +POSTed once. Keep that contract; if you ever ingest from a client without it, dedup frames +server-side before the rollup. Don't break this. Once minutes are written, the user gets flagged dirty (or pushed onto a queue if you've got the paid plan), and that's where ingest stops. The heavy math is deliberately not on diff --git a/src/analytics.ts b/src/analytics.ts index 0836517..c538825 100644 --- a/src/analytics.ts +++ b/src/analytics.ts @@ -196,7 +196,7 @@ export async function processUser( // ~$0. We NEVER re-decode old days (minute data is pruned at 10d); we just // reuse what was already computed. Forward-only, no backfill. const { results: histRows } = await db.prepare( - 'SELECT date, resting_hr, strain, hr_zones FROM daily WHERE user_id = ? ORDER BY date DESC LIMIT 90', + 'SELECT date, resting_hr, strain, hr_zones, hr_max FROM daily WHERE user_id = ? ORDER BY date DESC LIMIT 90', ).bind(userId).all() const { results: sleepHistRows } = await db.prepare( 'SELECT date, duration_min, onset_ts, wake_ts FROM sleep WHERE user_id = ? ORDER BY date DESC LIMIT 90', @@ -214,6 +214,11 @@ export async function processUser( resting_hr: d.resting_hr ?? undefined, daily_strain: d.strain ?? undefined, sleep_duration_min: sleepByDate.get(d.date), + // The day's observed HR peak. calcBaselines only trusts it as a measured + // HRmax when it clears the age-predicted floor (a genuine effort); otherwise + // it falls back to Tanaka. Without this, the measured-max path was dead and + // everyone ran on the age formula. Forward-only, reuses the daily row. + session_hr_max: d.hr_max ?? undefined, zone_min, } as DayHistory }) diff --git a/src/biometrics.ts b/src/biometrics.ts index fe67776..78b2888 100644 --- a/src/biometrics.ts +++ b/src/biometrics.ts @@ -232,13 +232,18 @@ export async function runBiometrics(env: BioEnv, userId: string, days = 3, onlyD // Composite Readiness — recovery (HRV) ∩ sleep vs need ∩ nocturnal dip ∩ arousal. // Read this day's dip + sleep-stress (written by processUser) alongside drivers. await env.DB.prepare('INSERT OR IGNORE INTO daily(user_id, date) VALUES(?, ?)').bind(userId, o.date).run() - const existing = await env.DB.prepare('SELECT drivers, sleep_stress, nocturnal_dip_pct FROM daily WHERE user_id = ? AND date = ?') - .bind(userId, o.date).first<{ drivers: string | null; sleep_stress: string | null; nocturnal_dip_pct: number | null }>() - let drivers: Record = {} - try { drivers = existing?.drivers ? JSON.parse(existing.drivers) : {} } catch { drivers = {} } - if (recovery.drivers) drivers.recovery = recovery.drivers - if (stress.drivers) drivers.stress = stress.drivers - if (illness.drivers) drivers.illness = illness.drivers + const existing = await env.DB.prepare('SELECT sleep_stress, nocturnal_dip_pct FROM daily WHERE user_id = ? AND date = ?') + .bind(userId, o.date).first<{ sleep_stress: string | null; nocturnal_dip_pct: number | null }>() + // Only the driver keys biometrics OWNS. We merge these into daily.drivers via + // SQL json_patch at write time (below) rather than read-modify-write, so a + // concurrent processUser sweep writing the MAIN drivers can't be clobbered by a + // stale snapshot (the previous read→merge→overwrite had a TOCTOU window: a + // 'sweep' and a 'biometrics' for the same user are distinct queue jobs and run + // concurrently). json_patch is additive + order-independent for disjoint keys. + const bioDrivers: Record = {} + if (recovery.drivers) bioDrivers.recovery = recovery.drivers + if (stress.drivers) bioDrivers.stress = stress.drivers + if (illness.drivers) bioDrivers.illness = illness.drivers const sleepStressScore = (() => { try { return existing?.sleep_stress ? JSON.parse(existing.sleep_stress)?.score ?? null : null } catch { return null } })() const readiness = calcReadinessIndex({ @@ -248,19 +253,20 @@ export async function runBiometrics(env: BioEnv, userId: string, days = 3, onlyD dipPct: existing?.nocturnal_dip_pct ?? null, sleepStress: sleepStressScore, }) - if (readiness.drivers) drivers.readiness = readiness.drivers + if (readiness.drivers) bioDrivers.readiness = readiness.drivers await env.DB.prepare( 'UPDATE daily SET recovery = ?, hrv_rmssd = ?, hrv_conf = ?, hrv_sdnn = ?, hrv_lfhf = ?, hrv_si = ?, ' + 'hrv_cv = ?, irregular = ?, readiness = ?, ' + 'resp_rate = COALESCE(?, resp_rate), resp_conf = COALESCE(?, resp_conf), ' + - 'skin_temp_idx = ?, spo2_idx = ?, stress = ?, illness = ?, drivers = ?, updated_at = ? ' + + 'skin_temp_idx = ?, spo2_idx = ?, stress = ?, illness = ?, ' + + 'drivers = json_patch(COALESCE(drivers, \'{}\'), ?), updated_at = ? ' + 'WHERE user_id = ? AND date = ?', ).bind( recovery.score, o.rmssd, conf, o.sdnn, o.lfhf, o.si, hrvStab.cv, JSON.stringify(irregular), readiness.score, o.respConf >= 0.3 ? o.resp : null, o.respConf >= 0.3 ? o.respConf : null, - tempIdx, spo2Idx, JSON.stringify(stress), JSON.stringify(illness), JSON.stringify(drivers), now, + tempIdx, spo2Idx, JSON.stringify(stress), JSON.stringify(illness), JSON.stringify(bioDrivers), now, userId, o.date, ).run() if (o.rmssd != null) computed++ diff --git a/src/steps_imu.ts b/src/steps_imu.ts index fd14a35..746c824 100644 --- a/src/steps_imu.ts +++ b/src/steps_imu.ts @@ -115,13 +115,36 @@ export async function runStepsIncremental(env: StepsEnv, userId: string): Promis } const chunk = await computeWindowSteps(env, userId, cursor, settledCutoff) - await env.DB.batch([ - env.DB.prepare('INSERT OR IGNORE INTO daily(user_id, date) VALUES(?,?)').bind(userId, date), - sameDay - ? env.DB.prepare('UPDATE daily SET steps = COALESCE(steps,0) + ? WHERE user_id = ? AND date = ?').bind(chunk, userId, date) - : env.DB.prepare('UPDATE daily SET steps = ? WHERE user_id = ? AND date = ?').bind(chunk, userId, date), - setCursor.bind(userId, settledCutoff, date), - ]) + if (sameDay) { + // Compare-and-swap on the cursor: apply this chunk AND advance the cursor only + // if no concurrent writer moved the cursor since we read it (line above). A + // nightly full recompute (runStepsImu, the 'steps_full' job) SETs today's steps + // and the cursor to settledCutoff; it and this incremental 'sweep' are distinct + // queue jobs that aren't deduped against each other, so they CAN run together + // (both are enqueued by the 3:30 cron). Without the guard, a full landing + // between our cursor read and our slow R2 compute would let us add a stale chunk + // on top of the full count → double-count for the day. If the cursor moved, our + // chunk is stale: both statements no-op and the next sweep recomputes from the + // new cursor. D1 batch is one serialized transaction, so the guard is reliable. + await env.DB.batch([ + env.DB.prepare('INSERT OR IGNORE INTO daily(user_id, date) VALUES(?,?)').bind(userId, date), + env.DB.prepare( + 'UPDATE daily SET steps = COALESCE(steps,0) + ? WHERE user_id = ? AND date = ? ' + + 'AND (SELECT steps_cursor_ts FROM analytics_cursor WHERE user_id = ?) = ?', + ).bind(chunk, userId, date, userId, cursor), + env.DB.prepare( + 'UPDATE analytics_cursor SET steps_cursor_ts = ? WHERE user_id = ? AND steps_cursor_day = ? AND steps_cursor_ts = ?', + ).bind(settledCutoff, userId, date, cursor), + ]) + } else { + // Fresh day: REPLACE (not accumulate) with the full settled count, which is + // idempotent with a concurrent full recompute (both write the same window). + await env.DB.batch([ + env.DB.prepare('INSERT OR IGNORE INTO daily(user_id, date) VALUES(?,?)').bind(userId, date), + env.DB.prepare('UPDATE daily SET steps = ? WHERE user_id = ? AND date = ?').bind(chunk, userId, date), + setCursor.bind(userId, settledCutoff, date), + ]) + } return { added: chunk } } diff --git a/src/trend.ts b/src/trend.ts index 161d394..e4c80ca 100644 --- a/src/trend.ts +++ b/src/trend.ts @@ -68,7 +68,7 @@ const REGISTRY: Record = { deep: { pull: (r) => r.deep_min, unit: 'min', label: 'Deep sleep' }, rem: { pull: (r) => r.rem_min, unit: 'min', label: 'REM sleep' }, light: { pull: (r) => r.light_min, unit: 'min', label: 'Light sleep' }, - regularity: { pull: (r) => r.regularity, unit: '', label: 'Sleep regularity (SRI)' }, + regularity: { pull: (r) => r.regularity, unit: '', label: 'Sleep consistency' }, resp: { pull: (r) => r.resp_rate, unit: 'brpm', label: 'Respiratory rate' }, sleep: { pull: (r) => r.duration_min, unit: 'min', label: 'Sleep', target: (_r, need) => need, goalDir: 'min' }, From 20ae36e4f3dde1a76625defea6c697eadff481e4 Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Sahil <127765312+abdulsaheel@users.noreply.github.com> Date: Wed, 17 Jun 2026 17:33:13 +0530 Subject: [PATCH 08/30] refactor: move frame decoding to openstrap-protocol (#7) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backend should hold no domain knowledge — byte layouts belong in openstrap-protocol, sports-science math in openstrap-analytics, leaving the worker as pure orchestration (bindings, queues, cursors, row->Minute mapping). - Deletes src/decode.ts; its decoders now live in openstrap-protocol/ts/live.ts. - Repoints imports in ingest.ts, rollup.ts, steps_imu.ts to 'openstrap-protocol/ts/live'. steps_imu.ts stays in the backend by design: it is I/O orchestration (R2 list/get, D1 cursor CAS, daily.steps write); the pure pedometer math (calcSteps) already lives in openstrap-analytics. No behavior change. --- src/decode.ts | 206 ----------------------------------------------- src/ingest.ts | 2 +- src/rollup.ts | 2 +- src/steps_imu.ts | 7 +- 4 files changed, 6 insertions(+), 211 deletions(-) delete mode 100644 src/decode.ts diff --git a/src/decode.ts b/src/decode.ts deleted file mode 100644 index 77f5dae..0000000 --- a/src/decode.ts +++ /dev/null @@ -1,206 +0,0 @@ -// decode.ts — verified WHOOP record decoders, mirroring the reference client / PROTOCOL.md. -// -// Each decoded record emits a DecodedSample: -// { ts, hr, activity, steps_inc, wrist_on, rec_type } -// where `activity` is the actigraphy signal = stddev of |accel(g)| over the -// 100-sample IMU window (R10 only; 0 for HR-only records). -// -// Offsets (PROTOCOL.md): -// R10 (rec_type 10, pkt 0x2F live/0x2B): ts@7, hr@17, counter@3, -// accel arrays @85/285/485, gyro @688/888/1088, scale ÷4096 (4096 LSB/g). -// 0x28 (live compact HR): ts@2 (u32 LE), hr@8 (u8), wrist via hr>0. -// 0x2B: same layout as R10 (live R10). -// R24 (rec_type 24): header ts@7, counter@3; spo2@72, skin_temp@70/4, resting_hr@88 (RELATIVE-only, not surfaced here). -// 0x33: live IMU stream — RAW-ONLY, no sample emitted (low decode confidence). -// -// NEVER decode HRV / RR-intervals. (R17 is BANNED.) - -import { parse_r24 } from 'openstrap-protocol/ts/records' - -export interface DecodedSample { - ts: number // unix seconds - hr: number // bpm (0 = off-wrist / no reading) - activity: number // motion magnitude (stddev of |accel(g)|), 0 if no IMU - steps_inc: number // steps detected in this record's IMU window (R10 only) - wrist_on: boolean // worn proxy (hr>0; authoritative wear is WRIST_ON/OFF events) - rec_type: number // 10 | 24 | 28 -} - -export const hexToBytes = (hex: string): Uint8Array => - new Uint8Array(hex.trim().match(/.{1,2}/g)!.map((b) => parseInt(b, 16))) - -/** One IMU frame's accel as ordered magnitude samples (g) + its time + sub-order. */ -export interface ImuFrame { ts: number; idx: number; mags: number[] } - -// frameAccel — decode one IMU frame's accelerometer into ordered |accel|(g) -// samples. Handles BOTH channels the strap streams: -// • 0x33 live IMU stream — ts@4, sub-frame idx@14, 10 accel samples -// (X[0:10],Y[10:20],Z[20:30]) from offset 24, scale 1/4096. -// • R10 (rec 0x0A) — ts@7, 100 accel samples @85/285/485, scale 1/4096. -// Returns null if it isn't an accel-bearing frame. Used by the backend steps -// runner (steps_imu.ts) to rebuild the signal for the AN-2554 pedometer that -// now lives in openstrap-analytics (calcSteps). Kept here with the other IMU -// decoders (see r10Motion) so all byte-offset knowledge stays in one place. -export function frameAccel(hex: string): ImuFrame | null { - let b: Uint8Array - try { b = hexToBytes(hex) } catch { return null } - if (b.length < 32) return null - const view = new DataView(b.buffer, b.byteOffset, b.byteLength) - const pkt = b[0], rec = b[1] - // 0x33 IMU stream: 10 accel samples (X,Y,Z) from offset 24. - if (pkt === 0x33 && b.length >= 84) { - const ts = view.getUint32(4, true) - const idx = view.getUint16(14, true) - const mags: number[] = [] - for (let i = 0; i < 10; i++) { - const x = view.getInt16(24 + 2 * i, true) - const y = view.getInt16(24 + 2 * (10 + i), true) - const z = view.getInt16(24 + 2 * (20 + i), true) - mags.push(Math.sqrt(x * x + y * y + z * z) / 4096) - } - return ts > 0 ? { ts, idx, mags } : null - } - // R10: rec 0x0A, ts@7, accel X@85/Y@285/Z@485 (100 int16 each). - if (rec === 0x0a && b.length >= 685) { - const ts = view.getUint32(7, true) - const mags: number[] = [] - for (let i = 0; i < 100; i++) { - const x = view.getInt16(85 + 2 * i, true) - const y = view.getInt16(285 + 2 * i, true) - const z = view.getInt16(485 + 2 * i, true) - mags.push(Math.sqrt(x * x + y * y + z * z) / 4096) - } - return ts > 0 ? { ts, idx: 0, mags } : null - } - return null -} - -// Decode the R10 IMU arrays into (activity, steps) over the 100-sample window. -// activity = stddev of per-sample |accel|(g) — actigraphy intensity. -// steps = count of GAIT CYCLES only when the window is genuinely rhythmic. -// -// The band has no pedometer field — steps are estimated from wrist IMU (ESTIMATE -// tier). The naive "count every peak" approach over-counts badly: any arm gesture, -// typing, or vehicle bump clears a fixed threshold. Instead we require RHYTHM: -// walking produces a periodic acceleration signal, so we (1) detrend the magnitude -// to remove gravity + slow drift, (2) measure how periodic it is via normalized -// autocorrelation over plausible step lags, and (3) only count steps when that -// periodicity is strong AND the limb is actually moving. Steps in the window = -// number of gait cycles = n / (dominant lag). Non-rhythmic motion → 0 steps. -// (Same autocorrelation idea we use for respiratory rate; standard wrist pedometry.) -function r10Motion(view: DataView, len: number): { activity: number; steps: number } { - if (len < 685) return { activity: 0, steps: 0 } - const ACC = 1 / 4096 - const arr = (off: number): number[] => { - const out: number[] = [] - for (let i = 0; i < 100; i++) { - const o = off + 2 * i - if (o + 2 <= len) out.push(view.getInt16(o, true)) - } - return out - } - const ax = arr(85), ay = arr(285), az = arr(485) // accel X/Y/Z - const n = Math.min(ax.length, ay.length, az.length) - if (n === 0) return { activity: 0, steps: 0 } - const mags: number[] = [] - for (let i = 0; i < n; i++) { - mags.push(Math.hypot(ax[i] * ACC, ay[i] * ACC, az[i] * ACC)) - } - const mean = mags.reduce((s, v) => s + v, 0) / n - const variance = mags.reduce((s, v) => s + (v - mean) ** 2, 0) / n - const std = Math.sqrt(variance) - const activity = Math.round(std * 1000) / 1000 - - // Limb must actually be oscillating — quiet wrist (typing/holding) → no steps. - const ACTIVITY_FLOOR = 0.05 // g RMS of the detrended signal - if (std < ACTIVITY_FLOOR || n < 24) return { activity, steps: 0 } - - // Detrend: remove a centered moving average (gravity + slow drift), leaving the - // gait oscillation around 0. - const W = 9 - const x: number[] = new Array(n); - for (let i = 0; i < n; i++) { - let s = 0, c = 0 - for (let j = Math.max(0, i - W); j <= Math.min(n - 1, i + W); j++) { s += mags[j]; c++ } - x[i] = mags[i] - s / c - } - const x0 = x.reduce((s, v) => s + v, 0) / n - let denom = 0 - for (let i = 0; i < n; i++) denom += (x[i] - x0) ** 2 - if (denom <= 1e-9) return { activity, steps: 0 } - - // Normalized autocorrelation over plausible step lags. Cadence ~1.4–3.0 steps/s; - // with a ~25 Hz IMU window that's ~8–18 samples/step. We scan a generous band - // and rely on the periodicity strength to confirm gait. - const MIN_LAG = 7, MAX_LAG = 40 - let bestLag = 0, bestR = 0 - for (let lag = MIN_LAG; lag <= Math.min(MAX_LAG, n - 1); lag++) { - let num = 0 - for (let i = 0; i < n - lag; i++) num += (x[i] - x0) * (x[i + lag] - x0) - const r = num / denom - if (r > bestR) { bestR = r; bestLag = lag } - } - - // Strong, sustained periodicity ⇒ walking/running; count the gait cycles. - const RHYTHM_THRESH = 0.45 - if (bestLag === 0 || bestR < RHYTHM_THRESH) return { activity, steps: 0 } - const steps = Math.round(n / bestLag) - return { activity, steps } -} - -/** - * Decode one hex record into a DecodedSample, or null if it carries no - * surfaceable sample (0x33 IMU stream, malformed, or unknown type). - */ -export function decodeRecord(hex: string): DecodedSample | null { - let b: Uint8Array - try { - b = hexToBytes(hex) - } catch { - return null - } - if (b.length < 4) return null - const view = new DataView(b.buffer, b.byteOffset, b.byteLength) - const pktType = b[0] - const recType = b[1] - - // 0x28 — live compact HR: ts@2 (u32 LE), hr@8 (u8). NO RR-intervals. - if (pktType === 0x28) { - if (b.length < 9) return null - const ts = view.getUint32(2, true) - const hr = b[8] - return { ts, hr, activity: 0, steps_inc: 0, wrist_on: hr > 0, rec_type: 28 } - } - - // 0x33 — live IMU stream: raw-only (kept in R2, no sample emitted). - if (pktType === 0x33) return null - - if (b.length < 18) return null - - // R24 — type-24 historical telemetry. - if (recType === 24) { - const d = parse_r24(b) - if (!d) return null - return { ts: d.ts_epoch, hr: d.hr, activity: 0, steps_inc: 0, wrist_on: d.hr > 0, rec_type: 24 } - } - - // R10 / 0x2B — ts@7, hr@17, IMU arrays → activity. - if (recType === 10) { - const ts = view.getUint32(7, true) - const hr = b[17] - const m = r10Motion(view, b.length) - return { ts, hr, activity: m.activity, steps_inc: m.steps, wrist_on: hr > 0, rec_type: 10 } - } - - return null -} - -/** Decode a batch of hex records, returning all surfaceable samples. */ -export function decodeBatch(records: string[]): DecodedSample[] { - const out: DecodedSample[] = [] - for (const hex of records) { - const s = decodeRecord(hex) - if (s) out.push(s) - } - return out -} diff --git a/src/ingest.ts b/src/ingest.ts index bcd5acb..7dfdae7 100644 --- a/src/ingest.ts +++ b/src/ingest.ts @@ -2,7 +2,7 @@ // Target <50ms CPU; NEVER runs analytics inline. import type { Context } from 'hono' -import { decodeBatch, hexToBytes } from './decode' +import { decodeBatch, hexToBytes } from 'openstrap-protocol/ts/live' import { rollupMinutes } from './rollup' // ── Rate limiting: per-user token bucket in a D1 row (RESILIENCE §7). ── diff --git a/src/rollup.ts b/src/rollup.ts index 33a17a5..f606bbb 100644 --- a/src/rollup.ts +++ b/src/rollup.ts @@ -9,7 +9,7 @@ // ingest). We store enough to recompute averages exactly: the upsert SQL // folds new (sum,n) into the stored running aggregate — see ingest.ts. -import type { DecodedSample } from './decode' +import type { DecodedSample } from 'openstrap-protocol/ts/live' export interface MinuteBucket { ts_min: number diff --git a/src/steps_imu.ts b/src/steps_imu.ts index 746c824..4347b33 100644 --- a/src/steps_imu.ts +++ b/src/steps_imu.ts @@ -4,14 +4,15 @@ // group per minute — then hands the per-minute magnitude signals to the PURE // AN-2554 pedometer in openstrap-analytics (calcSteps) and persists the result. // -// Mirrors the runBiometrics / runRespRate pattern: decode (decode.ts) + math -// (analytics) live elsewhere; this owns only R2 reads and the daily.steps write. +// Mirrors the runBiometrics / runRespRate pattern: decode (openstrap-protocol) + +// math (openstrap-analytics) live elsewhere; this owns only R2 reads and the +// daily.steps write. // // Heavy (R2 reads) → cron / admin only, NEVER inline ingest. Owns daily.steps // (written AFTER analytics in the cron so it's authoritative). import { calcSteps } from 'openstrap-analytics' -import { frameAccel, type ImuFrame } from './decode' +import { frameAccel, type ImuFrame } from 'openstrap-protocol/ts/live' const DAY = 86400 From 998be28ca5599658f45570963060f428dae9ce74 Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Sahil <127765312+abdulsaheel@users.noreply.github.com> Date: Wed, 17 Jun 2026 18:05:24 +0530 Subject: [PATCH 09/30] fix: stop R2-derived jobs from erasing good HRV/resp with null (#8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two instances of the same data-integrity bug: a job that re-derives a metric from R2 raw (14d retention, per-run object budget) writes the result into the PERMANENT daily row unconditionally — so a null/partial pass (budget exhausted, sparse data, or source expired) overwrites a previously-computed good value. biometrics.ts (HRV/recovery): - Budget-starved days no longer emit a null row (was: pushed null -> clobber). With TOTAL_OBJECTS=24/PER_DAY=12, multi-day runs funded only 2 days and nulled the rest, erasing real HRV. Now: skip (day keeps stored value; per-day onlyDate job recomputes with full budget). - Skip the daily write entirely when rmssd is null (nothing to write that wouldn't risk regressing a good night). - COALESCE(?, col) on the measured indices (hrv_rmssd/sdnn/lfhf/si/cv/temp/spo2) as defense-in-depth, matching the resp_rate guard already present. resp.ts (respiratory rate): - Only persist resp_rate/resp_conf when resp_rate != null. The band emits R21 only during live streaming, so most nights are null -> the unconditional write erased prior good values. COALESCE can't protect resp_conf (it's 0, not null, when absent), so the write is skipped instead. Audit of all D1 writes confirms these are the only two instances: sleep/strain/ steps read the minute table within a window shorter than minute retention, so the source always outlives the write and they can't hit this clobber. Verified: tsc --noEmit clean. Restored the affected dev account's 7 nights of HRV via the per-day path. Co-authored-by: abdulsaheel --- src/biometrics.ts | 29 +++++++++++++++++++++++------ src/resp.ts | 15 +++++++++++---- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/src/biometrics.ts b/src/biometrics.ts index 78b2888..3d4f6fe 100644 --- a/src/biometrics.ts +++ b/src/biometrics.ts @@ -188,7 +188,13 @@ export async function runBiometrics(env: BioEnv, userId: string, days = 3, onlyD } for (const { dayStart, date } of dayList) { const budget = Math.min(PER_DAY, TOTAL_OBJECTS - spent) - if (budget <= 0) { out.push({ date, rmssd: null, sdnn: null, lfhf: null, si: null, resp: null, respConf: 0, nBeats: 0, temp: null, spo2: null, rr: [] }); continue } + // Budget exhausted for this run: do NOT emit a null row. A null here would flow + // into the write loop and overwrite this day's already-computed HRV with null + // (the multi-day clobber bug). Skip it entirely — the day keeps its stored value + // and the next run (or a per-day onlyDate job, which always has full budget) + // recomputes it. computeWindowSteps/restingSamples are bounded per day, so + // multi-day runs starve the tail; skipping is the only non-destructive choice. + if (budget <= 0) continue const sl = sleepByDate.get(date) const from = sl ? sl.onset_ts : dayStart const to = sl ? sl.wake_ts + 60 : dayStart + DAY @@ -213,7 +219,12 @@ export async function runBiometrics(env: BioEnv, userId: string, days = 3, onlyD let computed = 0 for (const o of out) { - const conf = o.rmssd == null ? 0 : Math.round(Math.min(1, o.nBeats / 500) * 1000) / 1000 + // No usable RMSSD this night → there is nothing to write that wouldn't risk + // erasing a previously-computed value with null. Leave the stored row untouched; + // a thin/empty pass must never regress a good night. (processUser owns the + // non-HRV daily fields, so skipping here loses nothing.) + if (o.rmssd == null) continue + const conf = Math.round(Math.min(1, o.nBeats / 500) * 1000) / 1000 const tempIdx = (o.temp != null && blTemp != null) ? Math.round((o.temp - blTemp) * 10) / 10 : null const spo2Idx = (o.spo2 != null && blSpo2 != null) ? Math.round((o.spo2 - blSpo2) * 10) / 10 : null @@ -255,11 +266,17 @@ export async function runBiometrics(env: BioEnv, userId: string, days = 3, onlyD }) if (readiness.drivers) bioDrivers.readiness = readiness.drivers + // Measured HRV indices use COALESCE(?, col): a value computed this run wins, but + // a null (e.g. enough beats for RMSSD but not for LF/HF) never erases a richer + // prior night. Mirrors the resp_rate guard. Derived scores (recovery/readiness/ + // stress/illness/irregular) are recomputed fresh whenever we have an RMSSD (we + // continue'd out of the loop otherwise), so they write directly. await env.DB.prepare( - 'UPDATE daily SET recovery = ?, hrv_rmssd = ?, hrv_conf = ?, hrv_sdnn = ?, hrv_lfhf = ?, hrv_si = ?, ' + - 'hrv_cv = ?, irregular = ?, readiness = ?, ' + + 'UPDATE daily SET recovery = ?, hrv_rmssd = COALESCE(?, hrv_rmssd), hrv_conf = ?, ' + + 'hrv_sdnn = COALESCE(?, hrv_sdnn), hrv_lfhf = COALESCE(?, hrv_lfhf), hrv_si = COALESCE(?, hrv_si), ' + + 'hrv_cv = COALESCE(?, hrv_cv), irregular = ?, readiness = ?, ' + 'resp_rate = COALESCE(?, resp_rate), resp_conf = COALESCE(?, resp_conf), ' + - 'skin_temp_idx = ?, spo2_idx = ?, stress = ?, illness = ?, ' + + 'skin_temp_idx = COALESCE(?, skin_temp_idx), spo2_idx = COALESCE(?, spo2_idx), stress = ?, illness = ?, ' + 'drivers = json_patch(COALESCE(drivers, \'{}\'), ?), updated_at = ? ' + 'WHERE user_id = ? AND date = ?', ).bind( @@ -269,7 +286,7 @@ export async function runBiometrics(env: BioEnv, userId: string, days = 3, onlyD tempIdx, spo2Idx, JSON.stringify(stress), JSON.stringify(illness), JSON.stringify(bioDrivers), now, userId, o.date, ).run() - if (o.rmssd != null) computed++ + computed++ // reached only when o.rmssd != null (we continue'd above otherwise) } return { days, computed } } diff --git a/src/resp.ts b/src/resp.ts index 8ee66e8..d27678e 100644 --- a/src/resp.ts +++ b/src/resp.ts @@ -127,10 +127,17 @@ async function computeNight(env: RespEnv, userId: string, date: string, from: nu } } const { resp_rate, confidence } = estimateResp(records) - // Always write (null + 0 conf when absent) so the gate is authoritative. - await env.DB.prepare( - 'UPDATE daily SET resp_rate = ?, resp_conf = ? WHERE user_id = ? AND date = ?', - ).bind(resp_rate, confidence, userId, date).run() + // Only persist a real reading. A null pass (absent/sparse R21 — the band only + // emits R21 during live streaming, so most nights have none) must NOT erase a + // prior good night: the night is over, its resp doesn't change after the fact. + // A never-written night stays null by default, so the display gate + // (resp_conf >= 0.5) remains authoritative either way. COALESCE can't be used + // here because confidence is 0 (not null) when absent, which would still clobber. + if (resp_rate != null) { + await env.DB.prepare( + 'UPDATE daily SET resp_rate = ?, resp_conf = ? WHERE user_id = ? AND date = ?', + ).bind(resp_rate, confidence, userId, date).run() + } return { resp_rate, confidence, r21: records.length } } From c1a66005035969b5d4b5db1076bda14274e78040 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Fri, 19 Jun 2026 23:03:50 +0530 Subject: [PATCH 10/30] feat(backend): RR-from-minute HRV stack (ingest_signals, biometrics_minute, D1-only loadDayRr) + TTL read-cache + v11/v12 migrations --- src/biometrics_minute.ts | 114 ++++++++++++++++++++++++++++++ src/cache.ts | 37 ++++++++++ src/dayseries.ts | 27 +++++++ src/db/migrate_v11_minute_rr.sql | 3 + src/db/migrate_v12_read_cache.sql | 8 +++ src/ingest_signals.ts | 104 +++++++++++++++++++++++++++ 6 files changed, 293 insertions(+) create mode 100644 src/biometrics_minute.ts create mode 100644 src/cache.ts create mode 100644 src/dayseries.ts create mode 100644 src/db/migrate_v11_minute_rr.sql create mode 100644 src/db/migrate_v12_read_cache.sql create mode 100644 src/ingest_signals.ts diff --git a/src/biometrics_minute.ts b/src/biometrics_minute.ts new file mode 100644 index 0000000..ff4df5c --- /dev/null +++ b/src/biometrics_minute.ts @@ -0,0 +1,114 @@ +// biometrics_minute.ts — [v2] HRV + recovery from minute.rr (D1), NOT R2. +// +// The demand-driven engine stores per-minute RR at ingest (minute.rr), so the +// headline HRV stack (RMSSD/SDNN/LF-HF, Plews recovery, Baevsky stress, illness, +// irregular screen) is computed from D1 — no R2 list/get, no re-decode. Skin-temp +// and SpO₂ (raw ADCs @64/@68) are NOT in the minute rollup, so they stay R2-only +// (rare admin re-derive); we pass null → COALESCE keeps any prior value. +// +// Mirrors runBiometrics' write contract exactly (same daily columns, same +// null-safe COALESCE / json_patch drivers merge) so /today and the trend caches +// see an identical shape. + +import { + timeDomainHrv, freqDomainHrv, baevskyStressIndex, + calcRecovery, calcStress, calcIllness, calcHrvStability, calcIrregular, calcReadinessIndex, +} from 'openstrap-analytics' +import { loadDayRr } from './dayseries' + +// [v3] RAW_BUCKET + MINUTE_SOURCE: RR comes via loadDayRr (D1 minute.rr by default, +// R2-decoded series when MINUTE_SOURCE='r2'). +interface BioEnv { DB: D1Database; RAW_BUCKET?: R2Bucket; MINUTE_SOURCE?: string } + +interface DailyHistRow { date: string; resting_hr: number | null; hrv_rmssd: number | null; hrv_si: number | null; skin_temp_idx: number | null } + +/** + * Compute + persist HRV/recovery for ONE physiological night, sourced from D1 + * minute.rr over [from,to] (sleep window). Idempotent & null-safe: a night with + * no usable RR leaves the stored row untouched (never regresses a good night). + */ +export async function runBiometricsMinute( + env: BioEnv, userId: string, date: string, from: number, to: number, +): Promise<{ computed: boolean }> { + const now = Math.floor(Date.now() / 1000) + + // 1. RR stream over the sleep window, time-ordered (minute.rr / R2 series). + const rrByMin = await loadDayRr(env, userId, from, to) + const rr: number[] = [] + for (const ts of [...rrByMin.keys()].sort((a, b) => a - b)) { + for (const v of rrByMin.get(ts)!) if (v >= 300 && v <= 2000) rr.push(v) + } + + const td = timeDomainHrv(rr) + if (td.rmssd == null) return { computed: false } // no usable HRV → never write null + + const fd = freqDomainHrv(rr) + const si = baevskyStressIndex(rr) + + // 2. Prior history for Plews baseline / personal SI / illness covariance + sleep-need. + const { results: hist } = await env.DB.prepare( + 'SELECT date, resting_hr, hrv_rmssd, hrv_si, skin_temp_idx FROM daily WHERE user_id = ? ORDER BY date DESC LIMIT 60', + ).bind(userId).all() + const histRows = hist ?? [] + const rmssdHist = histRows.map((h) => h.hrv_rmssd).filter((x): x is number => x != null) + const siHist = histRows.map((h) => h.hrv_si).filter((x): x is number => x != null) + const rhrHist = histRows.map((h) => h.resting_hr).filter((x): x is number => x != null) + const tempHist = histRows.map((h) => h.skin_temp_idx).filter((x): x is number => x != null) + const rhrByDate = new Map(histRows.map((h) => [h.date, h.resting_hr])) + + const baseRow = await env.DB.prepare( + 'SELECT sleep_need_min FROM baselines WHERE user_id = ?', + ).bind(userId).first<{ sleep_need_min: number | null }>() + const sleepNeedMin = baseRow?.sleep_need_min ?? null + const sleepRow = await env.DB.prepare( + 'SELECT duration_min FROM sleep WHERE user_id = ? AND date = ?', + ).bind(userId, date).first<{ duration_min: number | null }>() + + // 3. Derived HRV metrics (published; temp/spo2 omitted — not in minute.rr). + const conf = Math.round(Math.min(1, td.n_beats / 500) * 1000) / 1000 + const recovery = calcRecovery(td.rmssd, rmssdHist, { date }) + const stress = calcStress(rr, siHist, { date }) + const illness = calcIllness( + { resting_hr: rhrByDate.get(date) ?? null, rmssd: td.rmssd, skin_temp: null }, + { resting_hr: rhrHist, rmssd: rmssdHist, skin_temp: tempHist }, + ) + const hrvStab = calcHrvStability([td.rmssd, ...rmssdHist].filter((x): x is number => x != null).slice(0, 14)) + const irregular = calcIrregular(rr) + + // 4. Composite Readiness (reads this night's dip + sleep-stress written by processUser). + await env.DB.prepare('INSERT OR IGNORE INTO daily(user_id, date) VALUES(?, ?)').bind(userId, date).run() + const existing = await env.DB.prepare( + 'SELECT sleep_stress, nocturnal_dip_pct FROM daily WHERE user_id = ? AND date = ?', + ).bind(userId, date).first<{ sleep_stress: string | null; nocturnal_dip_pct: number | null }>() + const sleepStressScore = (() => { try { return existing?.sleep_stress ? JSON.parse(existing.sleep_stress)?.score ?? null : null } catch { return null } })() + const readiness = calcReadinessIndex({ + recovery: recovery.score, + sleepDurationMin: sleepRow?.duration_min ?? null, + sleepNeedMin, + dipPct: existing?.nocturnal_dip_pct ?? null, + sleepStress: sleepStressScore, + }) + + const bioDrivers: Record = {} + if (recovery.drivers) bioDrivers.recovery = recovery.drivers + if (stress.drivers) bioDrivers.stress = stress.drivers + if (illness.drivers) bioDrivers.illness = illness.drivers + if (readiness.drivers) bioDrivers.readiness = readiness.drivers + + // 5. Persist (same null-safe COALESCE + json_patch contract as runBiometrics; temp/spo2 → null). + await env.DB.prepare( + 'UPDATE daily SET recovery = ?, hrv_rmssd = COALESCE(?, hrv_rmssd), hrv_conf = ?, ' + + 'hrv_sdnn = COALESCE(?, hrv_sdnn), hrv_lfhf = COALESCE(?, hrv_lfhf), hrv_si = COALESCE(?, hrv_si), ' + + 'hrv_cv = COALESCE(?, hrv_cv), irregular = ?, readiness = ?, ' + + 'resp_rate = COALESCE(?, resp_rate), resp_conf = COALESCE(?, resp_conf), ' + + 'stress = ?, illness = ?, drivers = json_patch(COALESCE(drivers, \'{}\'), ?), updated_at = ? ' + + 'WHERE user_id = ? AND date = ?', + ).bind( + recovery.score, td.rmssd, conf, td.sdnn, fd.lf_hf, si.si, + hrvStab.cv, JSON.stringify(irregular), readiness.score, + fd.resp_conf >= 0.3 ? fd.resp_rate : null, fd.resp_conf >= 0.3 ? fd.resp_conf : null, + JSON.stringify(stress), JSON.stringify(illness), JSON.stringify(bioDrivers), now, + userId, date, + ).run() + return { computed: true } +} diff --git a/src/cache.ts b/src/cache.ts new file mode 100644 index 0000000..18ccbff --- /dev/null +++ b/src/cache.ts @@ -0,0 +1,37 @@ +// cache.ts — [feat/wake-trigger] simple TTL read-cache for Tier 1/2 on-read metrics. +// +// NO watermark. Invalidation is purely time-based: "today" gets a short TTL (~60s, +// the 1-min rollup granularity, so it stays near-live without recomputing on every +// request), a completed past day is effectively immutable until its minutes prune. +// This sidesteps the v2 day_version watermark that re-fired every minute as 1 Hz +// data rolled in. Tier 3/4 are NOT cached here — they're served from `daily`. + +/** Serve `key` from read_cache if fresh (< ttlSec old); else compute, store, return. */ +export async function cached( + db: D1Database, userId: string, key: string, ttlSec: number, compute: () => Promise, +): Promise { + const now = Math.floor(Date.now() / 1000) + const row = await db.prepare( + 'SELECT payload, computed_at FROM read_cache WHERE user_id = ? AND key = ?', + ).bind(userId, key).first<{ payload: string; computed_at: number }>() + if (row && now - row.computed_at < ttlSec) { + try { return JSON.parse(row.payload) as T } catch { /* corrupt → recompute */ } + } + const val = await compute() + await db.prepare( + 'INSERT INTO read_cache (user_id, key, payload, computed_at) VALUES (?,?,?,?) ' + + 'ON CONFLICT(user_id, key) DO UPDATE SET payload = excluded.payload, computed_at = excluded.computed_at', + ).bind(userId, key, JSON.stringify(val), now).run() + return val +} + +/** TTL by date: today (UTC) is live (60s); a past day is immutable-until-prune. */ +export function ttlForDate(ymd: string, nowSec = Math.floor(Date.now() / 1000)): number { + const today = new Date(nowSec * 1000).toISOString().slice(0, 10) + return ymd >= today ? 60 : 86400 +} + +/** Invalidate a day's cached Tier-2 entries (called by the day-close). */ +export async function invalidateDay(db: D1Database, userId: string, ymd: string): Promise { + await db.prepare("DELETE FROM read_cache WHERE user_id = ? AND key LIKE ?").bind(userId, `%:${ymd}`).run() +} diff --git a/src/dayseries.ts b/src/dayseries.ts new file mode 100644 index 0000000..22ba27e --- /dev/null +++ b/src/dayseries.ts @@ -0,0 +1,27 @@ +// dayseries.ts — minimal D1-only RR loader for the HRV-from-minute path. +// +// [feat/wake-trigger] Deliberately the *minimal* version: reads beat-to-beat RR +// straight from the D1 `minute.rr` BLOB column (populated at ingest by +// ingest_signals). NO R2 series cache / re-decode (that v3 machinery is left +// behind on purpose). HRV/recovery therefore costs zero R2 ops — it folds into the +// wake-triggered day-close. Returns a per-minute map keyed by ts_min. + +import { decodeRr } from './ingest_signals' + +interface DSEnv { DB: D1Database; RAW_BUCKET?: R2Bucket; MINUTE_SOURCE?: string } + +/** RR (ms) per minute over [from, to] from D1 minute.rr. Empty map if none. */ +export async function loadDayRr( + env: DSEnv, userId: string, from: number, to: number, +): Promise> { + const fromMin = Math.floor(from / 60) * 60 + const { results } = await env.DB.prepare( + 'SELECT ts_min, rr FROM minute WHERE user_id = ? AND ts_min >= ? AND ts_min <= ? AND rr IS NOT NULL ORDER BY ts_min', + ).bind(userId, fromMin, to).all<{ ts_min: number; rr: ArrayBuffer | null }>() + const map = new Map() + for (const r of results ?? []) { + const rr = decodeRr(r.rr) + if (rr.length) map.set(r.ts_min, rr) + } + return map +} diff --git a/src/db/migrate_v11_minute_rr.sql b/src/db/migrate_v11_minute_rr.sql new file mode 100644 index 0000000..4115e6d --- /dev/null +++ b/src/db/migrate_v11_minute_rr.sql @@ -0,0 +1,3 @@ +-- [feat/wake-trigger] v11 — beat-to-beat RR per minute, so HRV/recovery (Tier 4) +-- computes from D1 at the wake-close with ZERO R2 re-decode. Additive & safe. +ALTER TABLE minute ADD COLUMN rr BLOB; diff --git a/src/db/migrate_v12_read_cache.sql b/src/db/migrate_v12_read_cache.sql new file mode 100644 index 0000000..a73d3a0 --- /dev/null +++ b/src/db/migrate_v12_read_cache.sql @@ -0,0 +1,8 @@ +-- [feat/wake-trigger] v12 — TTL read-cache for Tier 1/2 on-read metrics (no watermark). +CREATE TABLE IF NOT EXISTS read_cache ( + user_id TEXT NOT NULL, + key TEXT NOT NULL, -- e.g. 'today:strain:2026-06-19' + payload TEXT NOT NULL, -- JSON + computed_at INTEGER NOT NULL, + PRIMARY KEY(user_id, key) +) WITHOUT ROWID; diff --git a/src/ingest_signals.ts b/src/ingest_signals.ts new file mode 100644 index 0000000..32313de --- /dev/null +++ b/src/ingest_signals.ts @@ -0,0 +1,104 @@ +// ingest_signals.ts — [v2] per-minute STEPS (AN-2554 pedometer) + RR, computed at +// ingest from the batch's raw frames, so the heavy jobs never re-read R2. +// +// For each minute the batch touches: +// • steps = calcSteps over that minute's accelerometer magnitude signal. +// • rr = beat-to-beat intervals (ms) from the minute's R24 (historical) AND +// live (0x28 compact HR + R10) records — all hard-gated to 300–2000 ms. +// +// R24 DETECTION mirrors decodeRecord EXACTLY: recType = b[1] === 24 (NOT gated on +// packet type) — so historical R24 RR is never silently dropped. +// +// IDEMPOTENCY: both merge into the minute row with "keep the fuller value" (steps = +// MAX, rr = longer blob), so a re-uploaded batch can't double-count and a fuller +// batch wins. + +import { calcSteps } from 'openstrap-analytics' +import { frameAccel, hexToBytes, realtimeRr } from 'openstrap-protocol/ts/live' +import { parse_r24 } from 'openstrap-protocol/ts/records' + +export interface MinuteSignal { + steps: number + rr: number[] +} + +interface AccelFrame { idx: number; ts: number; mags: number[] } + +/** Build per-minute {steps, rr} from a batch of hex records. Pure; no I/O. */ +export function perMinuteSignals(records: string[]): Map { + const accelByMin = new Map>() + const rrByMin = new Map() + + for (const hex of records) { + let b: Uint8Array + try { b = hexToBytes(hex) } catch { continue } + if (b.length < 2) continue + + // R24 (recType @ b[1] === 24): RR intervals via the protocol decoder. + if (b[1] === 24 && b.length >= 89) { + const r = parse_r24(b) + if (r && r.ts_epoch > 0) { + const m = Math.floor(r.ts_epoch / 60) * 60 + const arr = rrByMin.get(m) ?? [] + for (const v of r.rr_intervals_ms) if (v >= 300 && v <= 2000) arr.push(v) + rrByMin.set(m, arr) + } + continue + } + + // [v2] LIVE RR — un-banned: RR unit confirmed ms (cross-validated vs noop/Strand). + // 0x28 compact HR + R10 carry beat-to-beat intervals; collect them with the SAME + // 300–2000 ms physiological gate as R24, so an unvalidated 0x28 offset can only + // drop values, never store a bogus interval. (R10 also feeds accel below.) + const rr = realtimeRr(hex) + if (rr) { + const m = Math.floor(rr.ts / 60) * 60 + const arr = rrByMin.get(m) ?? [] + for (const v of rr.rr_ms) if (v >= 300 && v <= 2000) arr.push(v) + rrByMin.set(m, arr) + } + + // Accel-bearing frames (0x33 IMU stream, R10) → magnitude samples for the pedometer. + const f = frameAccel(hex) + if (f && f.ts > 0) { + const m = Math.floor(f.ts / 60) * 60 + let mm = accelByMin.get(m) + if (!mm) { mm = new Map(); accelByMin.set(m, mm) } + mm.set(`${f.ts}:${f.idx}`, { ts: f.ts, idx: f.idx, mags: f.mags }) + } + } + + const out = new Map() + const minutes = new Set([...accelByMin.keys(), ...rrByMin.keys()]) + for (const m of minutes) { + let steps = 0 + const frames = accelByMin.get(m) + if (frames && frames.size > 0) { + const ordered = [...frames.values()].sort((a, b) => a.ts - b.ts || a.idx - b.idx) + const sig: number[] = [] + for (const fr of ordered) for (const v of fr.mags) sig.push(v) + steps = calcSteps([sig]) + } + out.set(m, { steps, rr: rrByMin.get(m) ?? [] }) + } + return out +} + +/** Encode an RR list (ms) to a compact little-endian int16 blob. */ +export function encodeRr(rr: number[]): Uint8Array | null { + if (!rr.length) return null + const buf = new Uint8Array(rr.length * 2) + const view = new DataView(buf.buffer) + for (let i = 0; i < rr.length; i++) view.setInt16(i * 2, Math.max(0, Math.min(32767, Math.round(rr[i]))), true) + return buf +} + +/** Decode a minute.rr blob back to an RR list (ms). */ +export function decodeRr(blob: ArrayBuffer | Uint8Array | null | undefined): number[] { + if (!blob) return [] + const u8 = blob instanceof Uint8Array ? blob : new Uint8Array(blob) + const view = new DataView(u8.buffer, u8.byteOffset, u8.byteLength) + const out: number[] = [] + for (let i = 0; i + 2 <= u8.byteLength; i += 2) out.push(view.getInt16(i, true)) + return out +} From 006393b219ad968908462c2ac8f2a5bcadb90f35 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Fri, 19 Jun 2026 23:14:23 +0530 Subject: [PATCH 11/30] =?UTF-8?q?feat(backend):=20wake-triggered=20close?= =?UTF-8?q?=20=E2=80=94=20ingest=20RR=20wiring=20(cleanRr=20in=20analytics?= =?UTF-8?q?),=20cron=20isUserAwake=20ladder=20(wake=5Fcron),=20close=5Fday?= =?UTF-8?q?=20queue=20job=20(processUser=20+=20HRV-from-minute=20+=20cache?= =?UTF-8?q?=20invalidate),=20demote=20nightly=20to=20maintenance=20+=20ret?= =?UTF-8?q?ry-net,=20v13=20cursor=20migration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/analytics.ts | 4 +- src/db/migrate_v13_cursor_wake.sql | 6 ++ src/index.ts | 96 +++++------------------------- src/ingest.ts | 15 ++++- src/ingest_signals.ts | 17 +++--- src/queue.ts | 26 +++++++- src/wake_cron.ts | 92 ++++++++++++++++++++++++++++ 7 files changed, 160 insertions(+), 96 deletions(-) create mode 100644 src/db/migrate_v13_cursor_wake.sql create mode 100644 src/wake_cron.ts diff --git a/src/analytics.ts b/src/analytics.ts index c538825..deab3f3 100644 --- a/src/analytics.ts +++ b/src/analytics.ts @@ -63,7 +63,7 @@ async function loadProfile(db: D1Database, userId: string): Promise { type LoadedBaseline = Baseline & { sleeping_hr: number | null; resp_rate: number | null } -async function loadBaseline(db: D1Database, userId: string): Promise { +export async function loadBaseline(db: D1Database, userId: string): Promise { const b = await db.prepare( 'SELECT resting_hr, max_hr, sleep_need_min, skin_temp, chronic_strain, sleeping_hr, resp_rate FROM baselines WHERE user_id = ?', ).bind(userId).first() @@ -78,7 +78,7 @@ async function loadBaseline(db: D1Database, userId: string): Promise { +export async function loadMinutes(db: D1Database, userId: string, from: number, to: number): Promise { const { results } = await db.prepare( 'SELECT ts_min, hr_avg, hr_min, hr_max, hr_n, activity, steps, wrist_on FROM minute ' + 'WHERE user_id = ? AND ts_min >= ? AND ts_min < ? ORDER BY ts_min ASC', diff --git a/src/db/migrate_v13_cursor_wake.sql b/src/db/migrate_v13_cursor_wake.sql new file mode 100644 index 0000000..e114336 --- /dev/null +++ b/src/db/migrate_v13_cursor_wake.sql @@ -0,0 +1,6 @@ +-- [feat/wake-trigger] v13 — incremental sleep/wake state machine columns on the cursor. +-- The */N cron's ONLY job reads these: skip awake-and-closed users (cursor-only), +-- peek recent minutes for the asleep ones, fire close_day once per physiological day. +ALTER TABLE analytics_cursor ADD COLUMN sleep_phase TEXT; -- 'awake' | 'asleep' | NULL +ALTER TABLE analytics_cursor ADD COLUMN phase_since INTEGER; -- unix s of last transition +ALTER TABLE analytics_cursor ADD COLUMN last_close_date TEXT; -- YYYY-MM-DD of last day-close diff --git a/src/index.ts b/src/index.ts index 47b5942..9b4f81a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,6 +5,7 @@ import { import { runAnalytics, processUser } from './analytics' import { ingestBatch, ingestEvents } from './ingest' import { handleQueueBatch, type AnalyticsMessage, type AnalyticsJob } from './queue' +import { runWakeLadder, retryStaleCloses } from './wake_cron' import { getToday, getSleep, getSleepV2, getStrain, getSessions, getTrends, getChart } from './query' import { getHistory } from './history' import { postJournal, getJournal, getJournalInsights } from './journal' @@ -450,89 +451,24 @@ export default { await handleQueueBatch(batch, env) }, - // Crons: "*/30 * * * *" sweep (enqueue dirty users); "30 3 * * *" nightly full - // re-derive + R2 jobs + prune. With Queues bound, the cron only ENQUEUES; the - // consumer does the bounded per-user work. Inline fallback if the binding is absent. + // [feat/wake-trigger] The frequent cron does ONE thing: the isUserAwake ladder + // (wake_cron.runWakeLadder) — detect each user's real wake and fire ONE close_day + // per physiological day. Cheap enough for */10 (awake-and-closed users = a cursor + // read; asleep = a 30-min peek; the heavy ensemble runs once, at the wake). All + // derivation lives in the close_day QUEUE job, never here. A SEPARATE nightly tick + // does maintenance only: prune aged minute/events + a retry-net for missed closes. async scheduled(event: ScheduledEvent, env: Bindings, ctx: ExecutionContext): Promise { - if (event.cron === '30 3 * * *') { - ctx.waitUntil((async () => { - const since2 = new Date(Date.now() - 2 * DAY * 1000).toISOString().slice(0, 10) - const since3 = new Date(Date.now() - 3 * DAY * 1000).toISOString().slice(0, 10) - if (env.ANALYTICS_Q) { - // Fan out per (user, job): each consumer invocation = one bounded unit. - const dailyU = (await env.DB.prepare('SELECT DISTINCT user_id FROM daily WHERE date >= ?') - .bind(since3).all<{ user_id: string }>()).results ?? [] - const ids = dailyU.map((u) => u.user_id) - // 'sweep' = one per user (D1 + incremental steps, light). 'steps_full' = - // late-frame true-up, per (user, day). - await enqueueJobs(env.ANALYTICS_Q, ids, 'sweep') - await enqueueJobDays(env.ANALYTICS_Q, ids, 'steps_full', lastNDates(2)) - // Biometrics + resp are the expensive R2 re-decodes. Enqueue ONLY the - // (user, day) nights still MISSING the metric in the last 2 days — this - // skips every already-scored night (no redundant decode vs the - // event-driven fire) AND retries nights the event fire left null - // (self-healing). The single biggest R2 cost saver. - const bioNeed = (await env.DB.prepare( - 'SELECT user_id, date FROM daily WHERE date >= ? AND hrv_rmssd IS NULL', - ).bind(since2).all<{ user_id: string; date: string }>()).results ?? [] - await sendChunked(env.ANALYTICS_Q, - bioNeed.map((r) => ({ body: { user_id: r.user_id, job: 'biometrics', day: r.date } as AnalyticsMessage }))) - const respNeed = (await env.DB.prepare( - 'SELECT d.user_id, d.date FROM daily d WHERE d.date >= ? AND d.resp_rate IS NULL ' + - 'AND EXISTS (SELECT 1 FROM sleep s WHERE s.user_id = d.user_id AND s.date = d.date AND s.onset_ts IS NOT NULL)', - ).bind(since2).all<{ user_id: string; date: string }>()).results ?? [] - await sendChunked(env.ANALYTICS_Q, - respNeed.map((r) => ({ body: { user_id: r.user_id, job: 'resp', day: r.date } as AnalyticsMessage }))) - } else { - // Fallback (no queue): inline. Only viable at small scale. - await runAnalytics(env.DB, { historyDays: 2 }) - try { - const u = (await env.DB.prepare('SELECT DISTINCT user_id FROM sleep WHERE date >= ? AND onset_ts IS NOT NULL').bind(since2).all<{ user_id: string }>()).results ?? [] - for (const x of u) await runRespRate(env, x.user_id, 2) - } catch (e) { console.error('resp cron failed', e) } - try { - const u = (await env.DB.prepare('SELECT DISTINCT user_id FROM daily WHERE date >= ?').bind(since3).all<{ user_id: string }>()).results ?? [] - for (const x of u) await runBiometrics(env, x.user_id, 3) - } catch (e) { console.error('biometrics cron failed', e) } - try { - const u = (await env.DB.prepare('SELECT DISTINCT user_id FROM daily WHERE date >= ?').bind(since2).all<{ user_id: string }>()).results ?? [] - for (const x of u) await runStepsImu(env, x.user_id, 2) - } catch (e) { console.error('steps cron failed', e) } - } - // Prune is light D1 — always inline. Minute + events share the same - // drill-down window (/day/timeline), so they age out together. + ctx.waitUntil((async () => { + try { await autoCloseStaleWorkouts(env.DB) } catch (e) { console.error('autoclose failed', e) } + // EVERY tick — wake detection only (the cron's sole job). + try { await runWakeLadder(env) } catch (e) { console.error('wake ladder failed', e) } + // Nightly maintenance ONLY (separate from detection): retention + retry-net. + if (event.cron === '30 3 * * *') { const cutoff = Math.floor(Date.now() / 1000) - MINUTE_RETENTION_DAYS * DAY await env.DB.prepare('DELETE FROM minute WHERE ts_min < ?').bind(cutoff).run() await env.DB.prepare('DELETE FROM events WHERE ts < ?').bind(cutoff).run() - })()) - } else { - ctx.waitUntil((async () => { - await autoCloseStaleWorkouts(env.DB) - if (env.ANALYTICS_Q) { - // Enqueue a 'sweep' (processUser + incremental steps) per dirty user. - const dirty = (await env.DB.prepare('SELECT user_id FROM analytics_cursor WHERE dirty = 1') - .all<{ user_id: string }>()).results ?? [] - const dids = dirty.map((u) => u.user_id) - await enqueueJobs(env.ANALYTICS_Q, dids, 'sweep') - // Optimistically clear dirty for exactly the users we enqueued, so a - // persistently-failing sweep isn't re-enqueued every 30 min (a user - // re-dirtied after this SELECT keeps dirty=1 and is caught next tick). - for (let i = 0; i < dids.length; i += 100) { - const chunk = dids.slice(i, i + 100) - await env.DB.prepare( - `UPDATE analytics_cursor SET dirty = 0 WHERE user_id IN (${chunk.map(() => '?').join(',')})`, - ).bind(...chunk).run() - } - } else { - // Fallback (no queue): inline. - await runAnalytics(env.DB, { historyDays: 3 }) - try { - const since = Math.floor(Date.now() / 1000) - 2 * 3600 - const u = (await env.DB.prepare('SELECT DISTINCT user_id FROM minute WHERE ts_min >= ?').bind(since).all<{ user_id: string }>()).results ?? [] - for (const x of u) await runStepsIncremental(env, x.user_id) - } catch (e) { console.error('steps incremental cron failed', e) } - } - })()) - } + try { await retryStaleCloses(env) } catch (e) { console.error('retry-net failed', e) } + } + })()) }, } diff --git a/src/ingest.ts b/src/ingest.ts index 7dfdae7..ae070fc 100644 --- a/src/ingest.ts +++ b/src/ingest.ts @@ -4,6 +4,7 @@ import type { Context } from 'hono' import { decodeBatch, hexToBytes } from 'openstrap-protocol/ts/live' import { rollupMinutes } from './rollup' +import { perMinuteSignals, encodeRr } from './ingest_signals' // ── Rate limiting: per-user token bucket in a D1 row (RESILIENCE §7). ── // Refill RATE tokens/sec, cap BURST. One ingest = one token. Over budget → 429. @@ -92,9 +93,13 @@ export async function ingestBatch(c: Context<{ Bindings: IngestEnv; Variables: { if (buckets.length > 0) { // The upsert folds new running sums into stored aggregates and recomputes // the display columns (hr_avg, activity, hr_min/max) from the merged totals. + // [wake-trigger] per-minute RR (ms) decoded from this batch's frames (R24 + live + // 0x28/R10), so HRV folds into the wake-close with zero R2. We use ONLY the `rr` + // here; steps stay on the existing rollup path (no step-semantics change this phase). + const sig = perMinuteSignals(records) const stmt = c.env.DB.prepare( - 'INSERT INTO minute (user_id, ts_min, hr_avg, hr_min, hr_max, hr_n, hr_sum, activity, act_sum, act_n, steps, wrist_on) ' + - 'VALUES (?,?,?,?,?,?,?,?,?,?,?,?) ' + + 'INSERT INTO minute (user_id, ts_min, hr_avg, hr_min, hr_max, hr_n, hr_sum, activity, act_sum, act_n, steps, wrist_on, rr) ' + + 'VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?) ' + 'ON CONFLICT(user_id, ts_min) DO UPDATE SET ' + 'hr_sum = minute.hr_sum + excluded.hr_sum, ' + 'hr_n = minute.hr_n + excluded.hr_n, ' + @@ -109,14 +114,18 @@ export async function ingestBatch(c: Context<{ Bindings: IngestEnv; Variables: { 'activity = CASE WHEN (minute.act_n + excluded.act_n) > 0 ' + ' THEN (minute.act_sum + excluded.act_sum) / (minute.act_n + excluded.act_n) ELSE 0 END, ' + 'steps = minute.steps + excluded.steps, ' + + // RR: keep the fuller blob (idempotent — a re-uploaded/fuller batch wins; never doubles). + "rr = CASE WHEN excluded.rr IS NOT NULL AND length(excluded.rr) >= length(COALESCE(minute.rr, x'')) " + + ' THEN excluded.rr ELSE minute.rr END, ' + 'wrist_on = MAX(minute.wrist_on, excluded.wrist_on)', ) const batch = buckets.map((b) => { if (b.ts_min > maxTsMin) maxTsMin = b.ts_min const hr_avg = b.hr_n > 0 ? Math.round(b.hr_sum / b.hr_n) : 0 const activity = b.act_n > 0 ? b.act_sum / b.act_n : 0 + const rrBlob = encodeRr(sig.get(b.ts_min)?.rr ?? []) return stmt.bind(userId, b.ts_min, hr_avg, b.hr_min, b.hr_max, b.hr_n, b.hr_sum, - activity, b.act_sum, b.act_n, b.steps, b.wrist_on) + activity, b.act_sum, b.act_n, b.steps, b.wrist_on, rrBlob) }) await c.env.DB.batch(batch) minutesWritten = buckets.length diff --git a/src/ingest_signals.ts b/src/ingest_signals.ts index 32313de..765a695 100644 --- a/src/ingest_signals.ts +++ b/src/ingest_signals.ts @@ -13,7 +13,7 @@ // MAX, rr = longer blob), so a re-uploaded batch can't double-count and a fuller // batch wins. -import { calcSteps } from 'openstrap-analytics' +import { calcSteps, cleanRr } from 'openstrap-analytics' import { frameAccel, hexToBytes, realtimeRr } from 'openstrap-protocol/ts/live' import { parse_r24 } from 'openstrap-protocol/ts/records' @@ -40,21 +40,20 @@ export function perMinuteSignals(records: string[]): Map { if (r && r.ts_epoch > 0) { const m = Math.floor(r.ts_epoch / 60) * 60 const arr = rrByMin.get(m) ?? [] - for (const v of r.rr_intervals_ms) if (v >= 300 && v <= 2000) arr.push(v) + for (const v of r.rr_intervals_ms) arr.push(v) // raw; gated by analytics cleanRr below rrByMin.set(m, arr) } continue } - // [v2] LIVE RR — un-banned: RR unit confirmed ms (cross-validated vs noop/Strand). - // 0x28 compact HR + R10 carry beat-to-beat intervals; collect them with the SAME - // 300–2000 ms physiological gate as R24, so an unvalidated 0x28 offset can only - // drop values, never store a bogus interval. (R10 also feeds accel below.) + // LIVE RR (un-banned: unit confirmed ms, cross-validated vs noop/Strand). 0x28 + // compact HR + R10 carry beat-to-beat intervals; gated by analytics cleanRr below, + // so an unvalidated 0x28 offset can only drop values, never store a bogus interval. const rr = realtimeRr(hex) if (rr) { const m = Math.floor(rr.ts / 60) * 60 const arr = rrByMin.get(m) ?? [] - for (const v of rr.rr_ms) if (v >= 300 && v <= 2000) arr.push(v) + for (const v of rr.rr_ms) arr.push(v) // raw; cleaned below rrByMin.set(m, arr) } @@ -79,7 +78,9 @@ export function perMinuteSignals(records: string[]): Map { for (const fr of ordered) for (const v of fr.mags) sig.push(v) steps = calcSteps([sig]) } - out.set(m, { steps, rr: rrByMin.get(m) ?? [] }) + // Single library gate (300–2000 ms + ectopic |Δ|>200ms drop) — logic lives in + // analytics, not duplicated here. + out.set(m, { steps, rr: cleanRr(rrByMin.get(m) ?? []) }) } return out } diff --git a/src/queue.ts b/src/queue.ts index ccf42e4..ee0595e 100644 --- a/src/queue.ts +++ b/src/queue.ts @@ -6,16 +6,20 @@ import { processUser } from './analytics' import { runBiometrics } from './biometrics' +import { runBiometricsMinute } from './biometrics_minute' import { runRespRate } from './resp' import { runStepsImu, runStepsIncremental } from './steps_imu' +import { invalidateDay } from './cache' -export type AnalyticsJob = 'sweep' | 'biometrics' | 'resp' | 'steps_full' +export type AnalyticsJob = 'sweep' | 'biometrics' | 'resp' | 'steps_full' | 'close_day' export interface AnalyticsMessage { user_id: string upto?: number job?: AnalyticsJob // default 'sweep' day?: string // YYYY-MM-DD — per-(user,day) fan-out for heavy R2 jobs + onset_ts?: number // [wake-trigger] close_day: sleep onset (RR window start) + wake_ts?: number // [wake-trigger] close_day: wake (RR window end) } interface QueueEnv { @@ -58,8 +62,23 @@ async function maybeTriggerBiometrics(env: QueueEnv, userId: string): Promise { +async function runJob(env: QueueEnv, userId: string, job: AnalyticsJob, day?: string, onset_ts?: number, wake_ts?: number): Promise { switch (job) { + case 'close_day': + // [wake-trigger] fired ONCE per physiological day when the user wakes. Derives + // the day (sleep/naps/strain/sessions/baselines/coach via processUser) and folds + // in HRV/recovery from D1 minute.rr — zero R2. Then invalidates the day's Tier-2 + // cache and clears the dirty flag (daytime ingests re-set it; the cron skips + // awake-and-closed users by last_close_date, so no churn). + await processUser(env.DB, userId, { historyDays: 3 }) + await runStepsIncremental(env, userId) + if (day && wake_ts) { + const from = onset_ts ?? (wake_ts - 8 * 3600) + try { await runBiometricsMinute({ DB: env.DB }, userId, day, from, wake_ts + 60) } catch (e) { console.error('biometrics_minute failed', userId, day, e) } + await invalidateDay(env.DB, userId, day) + } + await env.DB.prepare('UPDATE analytics_cursor SET dirty = 0 WHERE user_id = ?').bind(userId).run() + break case 'biometrics': // HRV/temp/SpO₂ from RR (R2) for ONE day (day set) or the trailing window. await runBiometrics(env, userId, 3, day) @@ -104,8 +123,9 @@ export async function handleQueueBatch( const uid = msgs[0].body.user_id const job = msgs[0].body.job ?? 'sweep' const day = msgs[0].body.day + const { onset_ts, wake_ts } = msgs[0].body try { - await runJob(env, uid, job, day) + await runJob(env, uid, job, day, onset_ts, wake_ts) for (const m of msgs) m.ack() } catch (e) { console.error('queue: job failed', uid, job, day, e) diff --git a/src/wake_cron.ts b/src/wake_cron.ts new file mode 100644 index 0000000..3fa9ab4 --- /dev/null +++ b/src/wake_cron.ts @@ -0,0 +1,92 @@ +// wake_cron.ts — [feat/wake-trigger] the frequent cron's ONLY job: detect each +// user's real wake and fire close_day once per physiological day. Designed to be +// O(cheap) per tick so it's ~free even at */10: +// +// ladder step 1 awake & already closed today → cursor-only skip (no minute read) +// ladder step 2 else → cheap peek of the last ~30 min +// ladder step 3 peek says awake → ONE full ensemble over 36h, enqueue +// +// The expensive ensemble + 36h read runs at most ONCE per user per day (at the wake); +// every other tick is a tiny cursor/30-min read. No analytics, no prune here. + +import { detectWakeState, peekRecentState } from 'openstrap-analytics' +import { loadMinutes, loadBaseline } from './analytics' +import { loadDayRr } from './dayseries' +import type { AnalyticsMessage } from './queue' + +interface WakeEnv { DB: D1Database; RAW_BUCKET?: R2Bucket; ANALYTICS_Q?: Queue } + +const DAY = 86400 +const ymd = (ts: number): string => new Date(ts * 1000).toISOString().slice(0, 10) + +export async function runWakeLadder( + env: WakeEnv, now = Math.floor(Date.now() / 1000), +): Promise<{ checked: number; closed: number }> { + const today = ymd(now) + // Candidates = users with fresh data (dirty). Awake-and-closed ones are filtered + // by step 1 below at ~zero cost; the close_day job clears dirty when it runs. + const { results } = await env.DB.prepare( + 'SELECT user_id, last_close_date FROM analytics_cursor WHERE dirty = 1 LIMIT 1000', + ).all<{ user_id: string; last_close_date: string | null }>() + + let checked = 0, closed = 0 + for (const u of results ?? []) { + // Step 1 — got this user's wake already today → skip (cursor-only, cheapest path). + if (u.last_close_date === today) continue + checked++ + + const baseline = await loadBaseline(env.DB, u.user_id) + + // Step 2 — cheap, liberal peek over the last ~30 min (high recall on purpose). + const recent = await loadMinutes(env.DB, u.user_id, now - 30 * 60, now + 60) + const peek = peekRecentState(recent, baseline) + if (peek !== 'awake') { + // still asleep/unknown → mark phase ONLY on transition (no-op write = 0 rows billed). + await env.DB.prepare( + "UPDATE analytics_cursor SET sleep_phase = 'asleep', phase_since = ? WHERE user_id = ? AND COALESCE(sleep_phase,'') <> 'asleep'", + ).bind(now, u.user_id).run() + continue + } + + // Step 3 — candidate wake confirmed by the cheap peek → run the FULL ensemble ONCE + // over a generous 36h window (so the circadian fit isn't starved). RR enables the + // cardiac arm; missing RR just drops that voter. + const win = await loadMinutes(env.DB, u.user_id, now - 36 * 3600, now + 60) + if (win.length < 60) continue + const rrByMin = await loadDayRr(env, u.user_id, now - 36 * 3600, now + 60) + const ws = detectWakeState({ minutes: win, baseline, rrByMin, now }) + if (!ws.wake_ts || ws.state !== 'awake') continue + + const dayLabel = ymd(ws.wake_ts) + if (dayLabel === u.last_close_date) continue // already closed this physiological day + + // Set last_close_date NOW to dedupe re-enqueue; close_day clears dirty when done. + await env.DB.prepare( + "INSERT INTO analytics_cursor (user_id, last_close_date, sleep_phase, phase_since) VALUES (?,?,'awake',?) " + + "ON CONFLICT(user_id) DO UPDATE SET last_close_date = excluded.last_close_date, sleep_phase = 'awake', phase_since = excluded.phase_since", + ).bind(u.user_id, dayLabel, ws.wake_ts).run() + + if (env.ANALYTICS_Q) { + await env.ANALYTICS_Q.send({ + user_id: u.user_id, job: 'close_day', day: dayLabel, + onset_ts: ws.onset_ts ?? undefined, wake_ts: ws.wake_ts, + }) + } + closed++ + } + return { checked, closed } +} + +// Retry-net for the nightly maintenance tick: users with fresh data whose last close +// is stale (>~28h) — re-attempt a close so a missed/failed wake self-heals. +export async function retryStaleCloses(env: WakeEnv, now = Math.floor(Date.now() / 1000)): Promise { + const cutoff = ymd(now - 1 * DAY) + const { results } = await env.DB.prepare( + "SELECT user_id FROM analytics_cursor WHERE dirty = 1 AND (last_close_date IS NULL OR last_close_date < ?) LIMIT 1000", + ).bind(cutoff).all<{ user_id: string }>() + let n = 0 + for (const u of results ?? []) { + if (env.ANALYTICS_Q) { await env.ANALYTICS_Q.send({ user_id: u.user_id, job: 'sweep' }); n++ } + } + return n +} From bdc80d5328f3486f0b73eb963c628e287e2f3d10 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Fri, 19 Jun 2026 23:20:41 +0530 Subject: [PATCH 12/30] feat(backend): TTL-cache heavy day endpoints (heart/stress/sleepV2) + per-nap hypnograms on /day/v2/sleep (on-read, no migration) --- src/daydetail.ts | 101 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 72 insertions(+), 29 deletions(-) diff --git a/src/daydetail.ts b/src/daydetail.ts index 2d0d822..65e99be 100644 --- a/src/daydetail.ts +++ b/src/daydetail.ts @@ -8,11 +8,29 @@ // All JWT, scoped by user_id. import type { Context } from 'hono' +import { cached, ttlForDate } from './cache' type Ctx = Context<{ Bindings: { DB: D1Database }; Variables: { userId: string } }> const DAY = 86400 const dayStartOf = (date: string) => Math.floor(Date.parse(`${date}T00:00:00Z`) / 1000) + +// Per-epoch hypnogram (BETA) from minute HR/activity vs resting HR — the same +// heuristic the main-sleep view uses, factored out so naps render identically. +function buildHypnogram(mins: { ts_min: number; hr_avg?: number | null; activity?: number | null }[], rhr: number): { t: number; stage: string }[] { + const out: { t: number; stage: string }[] = [] + for (const m of mins) { + const hr = m.hr_avg ?? 0 + const act = m.activity ?? 0 + let stage: string + if (hr <= 0 || hr > 1.15 * rhr || act > 0.25) stage = 'awake' + else if (act < 0.01 && hr < rhr + 4) stage = 'deep' + else if (hr > rhr + 5) stage = 'rem' + else stage = 'light' + out.push({ t: m.ts_min, stage }) + } + return out +} const clamp = (v: number, lo: number, hi: number) => Math.max(lo, Math.min(hi, v)) interface Min { ts_min: number; hr_avg: number | null; hr_min: number | null; hr_max: number | null; activity: number | null; wrist_on: number | null } @@ -170,32 +188,51 @@ export async function getDaySleepV2(c: Ctx) { const date = (c.req.query('date') || '').trim() if (!/^\d{4}-\d{2}-\d{2}$/.test(date)) return c.json({ error: 'date=YYYY-MM-DD required' }, 400) const userId = c.get('userId') - const { results } = await c.env.DB.prepare( - 'SELECT * FROM sleep_periods WHERE user_id = ? AND date = ? ORDER BY onset_ts ASC', - ).bind(userId, date).all() - const baseline = await c.env.DB.prepare('SELECT sleep_need_min FROM baselines WHERE user_id = ?') - .bind(userId).first() - const need = (baseline?.sleep_need_min && baseline.sleep_need_min >= 180) ? baseline.sleep_need_min : 480 - const periods = (results ?? []).map((r: any) => ({ - id: r.id, - onset_ts: r.onset_ts, - wake_ts: r.wake_ts, - duration_min: r.duration_min, - in_bed_min: r.in_bed_min, - efficiency: r.efficiency, - stages: (r.light_min != null || r.deep_min != null || r.rem_min != null) - ? { light_min: r.light_min, deep_min: r.deep_min, rem_min: r.rem_min } : null, - is_main: !!r.is_main, - confidence: r.confidence, - })) - return c.json({ - date, - has_sleep: periods.length > 0, - need_min: need, - total_asleep_min: periods.reduce((a: number, p: any) => a + (p.duration_min || 0), 0), - periods, - stages_beta: true, + const payload = await cached(c.env.DB, userId, `daysleepv2:${date}`, ttlForDate(date), async () => { + const { results } = await c.env.DB.prepare( + 'SELECT * FROM sleep_periods WHERE user_id = ? AND date = ? ORDER BY onset_ts ASC', + ).bind(userId, date).all() + const baseRow = await c.env.DB.prepare('SELECT sleep_need_min, resting_hr FROM baselines WHERE user_id = ?') + .bind(userId).first() + const need = (baseRow?.sleep_need_min && baseRow.sleep_need_min >= 180) ? baseRow.sleep_need_min : 480 + const rhr = baseRow?.resting_hr && baseRow.resting_hr > 0 ? baseRow.resting_hr : 55 + const rows = results ?? [] + // ONE minute read spanning all periods; slice per period for its own hypnogram so + // every nap renders the same banded timeline as the main sleep (computed on-read; + // no per-nap storage). Naps are already counted in total_asleep_min. + let mins: Min[] = [] + const onsets = rows.map((r: any) => r.onset_ts).filter((x: any) => x) + const wakes = rows.map((r: any) => r.wake_ts).filter((x: any) => x) + if (onsets.length && wakes.length) { + mins = await loadMinutes(c, Math.min(...onsets), Math.max(...wakes) + 60) + } + const periods = rows.map((r: any) => { + const pm = mins.filter((m) => m.ts_min >= r.onset_ts && m.ts_min <= (r.wake_ts ?? r.onset_ts)) + return { + id: r.id, + onset_ts: r.onset_ts, + wake_ts: r.wake_ts, + duration_min: r.duration_min, + in_bed_min: r.in_bed_min, + efficiency: r.efficiency, + stages: (r.light_min != null || r.deep_min != null || r.rem_min != null) + ? { light_min: r.light_min, deep_min: r.deep_min, rem_min: r.rem_min } : null, + is_main: !!r.is_main, + confidence: r.confidence, + hypnogram: pm.length ? downsample(buildHypnogram(pm, rhr), 240) : [], + } + }) + return { + date, + has_sleep: periods.length > 0, + need_min: need, + total_asleep_min: periods.reduce((a: number, p: any) => a + (p.duration_min || 0), 0), + naps: periods.filter((p: any) => !p.is_main).length, + periods, + stages_beta: true, + } }) + return c.json(payload) } export async function getDaySleep(c: Ctx) { @@ -276,10 +313,11 @@ export async function getDaySleep(c: Ctx) { export async function getDayStress(c: Ctx) { const date = (c.req.query('date') || '').trim() if (!/^\d{4}-\d{2}-\d{2}$/.test(date)) return c.json({ error: 'date=YYYY-MM-DD required' }, 400) + const userId = c.get('userId') as string + const payload = await cached(c.env.DB, userId, `daystress:${date}`, ttlForDate(date), async () => { const start = dayStartOf(date) const mins = await loadMinutes(c, start, start + DAY) - const userId = c.get('userId') as string const row = await c.env.DB.prepare( 'SELECT stress, sleep_stress, drivers FROM daily WHERE user_id = ? AND date = ?', ).bind(userId, date).first<{ stress: string | null; sleep_stress: string | null; drivers: string | null }>() @@ -291,13 +329,15 @@ export async function getDayStress(c: Ctx) { // Factual HR timeline (bpm) — context, not a stress band. const hr = mins.map((m) => ({ t: m.ts_min, v: m.hr_avg ?? 0 })) - return c.json({ + return { date, stress, // {score, si, lf_hf, rmssd, level, drivers} sleep_stress: sleepStress, // {score, arousal_events, restless_min, events[...]} drivers: drivers?.stress ?? null, hr: downsample(hr, 240), + } }) + return c.json(payload) } // ── /day/timeline ──────────────────────────────────────────────────────────── @@ -353,9 +393,10 @@ export async function getDayTimeline(c: Ctx) { export async function getDayHeart(c: Ctx) { const date = (c.req.query('date') || '').trim() if (!/^\d{4}-\d{2}-\d{2}$/.test(date)) return c.json({ error: 'date=YYYY-MM-DD required' }, 400) + const userId = c.get('userId') + const payload = await cached(c.env.DB, userId, `dayheart:${date}`, ttlForDate(date), async () => { const start = dayStartOf(date) const mins = await loadMinutes(c, start, start + DAY) - const userId = c.get('userId') const d = await c.env.DB.prepare( 'SELECT resting_hr, recovery, readiness, hrv_rmssd, hrv_sdnn, hrv_lfhf, hrv_conf, hrv_cv, irregular, hr_zones, nocturnal, stress, illness, drivers, resp_rate, resp_conf, spo2_idx FROM daily WHERE user_id = ? AND date = ?', ).bind(userId, date).first() @@ -364,7 +405,7 @@ export async function getDayHeart(c: Ctx) { const parse = (s: string | null) => { try { return s ? JSON.parse(s) : null } catch { return null } } const worn = mins.filter((m) => m.wrist_on && (m.hr_avg ?? 0) > 0) const hrs = worn.map((m) => m.hr_avg as number) - return c.json({ + return { date, hr: downsample(mins.map((m) => ({ t: m.ts_min, v: m.hr_avg ?? 0 })), 240), avg_hr: hrs.length ? Math.round(hrs.reduce((a, b) => a + b, 0) / hrs.length) : null, @@ -386,7 +427,9 @@ export async function getDayHeart(c: Ctx) { ? { value: d.resp_rate, confidence: d.resp_conf } : null, spo2: d?.spo2_idx != null ? { value: d.spo2_idx } : null, drivers: parse(d?.drivers ?? null), + } }) + return c.json(payload) } // ── /day/lungs ───────────────────────────────────────────────────────────── From 4d812a7d9d6a5bdaa49128672a132c5c047a6f3f Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Fri, 19 Jun 2026 23:30:57 +0530 Subject: [PATCH 13/30] =?UTF-8?q?feat(backend):=20TTL-cache=20remaining=20?= =?UTF-8?q?live=20day=20endpoints=20(wear,=20timeline)=20=E2=80=94=20all?= =?UTF-8?q?=20minute-reading=20day=20endpoints=20now=20cached?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/daydetail.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/daydetail.ts b/src/daydetail.ts index 65e99be..2da20f6 100644 --- a/src/daydetail.ts +++ b/src/daydetail.ts @@ -127,6 +127,8 @@ const safe = (s: any) => { try { return JSON.parse(s) } catch { return null } } export async function getDayWear(c: Ctx) { const date = (c.req.query('date') || '').trim() if (!/^\d{4}-\d{2}-\d{2}$/.test(date)) return c.json({ error: 'date=YYYY-MM-DD required' }, 400) + const userId = c.get('userId') + const payload = await cached(c.env.DB, userId, `daywear:${date}`, ttlForDate(date), async () => { const start = dayStartOf(date) const mins = await loadMinutes(c, start, start + DAY) @@ -165,10 +167,10 @@ export async function getDayWear(c: Ctx) { // Prefer the derived daily.wear_min when present (same source), else the live count. const dr = await c.env.DB.prepare( 'SELECT wear_min FROM daily WHERE user_id = ? AND date = ?', - ).bind(c.get('userId'), date).first<{ wear_min: number | null }>() + ).bind(userId, date).first<{ wear_min: number | null }>() const wearMin = dr?.wear_min != null ? Math.round(dr.wear_min) : wornMin - return c.json({ + return { date, worn_min: wearMin, coverage_pct: Math.round((wearMin / 1440) * 100), @@ -178,7 +180,9 @@ export async function getDayWear(c: Ctx) { segments, // number of separate on-wrist stretches longest_off_min: longestGap, // longest off-wrist gap inside the worn window tier: 'AUTH', // straight from the device's wrist sensor + } }) + return c.json(payload) } // ── /day/sleep ─────────────────────────────────────────────────────────────── @@ -344,6 +348,8 @@ export async function getDayStress(c: Ctx) { export async function getDayTimeline(c: Ctx) { const date = (c.req.query('date') || '').trim() if (!/^\d{4}-\d{2}-\d{2}$/.test(date)) return c.json({ error: 'date=YYYY-MM-DD required' }, 400) + const userId = c.get('userId') + const payload = await cached(c.env.DB, userId, `daytimeline:${date}`, ttlForDate(date), async () => { const start = dayStartOf(date) const end = start + DAY const mins = await loadMinutes(c, start, end) @@ -369,9 +375,9 @@ export async function getDayTimeline(c: Ctx) { ).bind(c.get('userId'), start, end).all() const { results: events } = await c.env.DB.prepare( 'SELECT event_id, ts FROM events WHERE user_id = ? AND ts >= ? AND ts < ? ORDER BY ts ASC', - ).bind(c.get('userId'), start, end).all() + ).bind(userId, start, end).all() - return c.json({ + return { date, day_start: start, hr: downsample(hr), @@ -383,7 +389,9 @@ export async function getDayTimeline(c: Ctx) { peak_hr: peak.v ? peak : null, low_hr: low.v ? low : null, }, + } }) + return c.json(payload) } // ── /day/heart ───────────────────────────────────────────────────────────── From 4888335f911e108b5577452505d568c8d590a712 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Fri, 19 Jun 2026 23:34:36 +0530 Subject: [PATCH 14/30] =?UTF-8?q?feat(backend):=20D1-hot/R2-sealed=20tiere?= =?UTF-8?q?d=20minute=20storage=20=E2=80=94=20sealOldDays=20packs=20days?= =?UTF-8?q?=20>3d=20into=20gzipped=20R2=20objects=20(put-before-delete),?= =?UTF-8?q?=20day-detail=20reads=20fall=20back=20to=20R2;=20cuts=20D1=20st?= =?UTF-8?q?orage=20+=20per-row=20prune-deletes=20+=20dodges=20the=2010GB?= =?UTF-8?q?=20cap.=20Ingest/processUser=20paths=20unchanged=20(hot=20windo?= =?UTF-8?q?w).?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/daydetail.ts | 11 ++-- src/index.ts | 7 ++- src/minute_store.ts | 124 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 135 insertions(+), 7 deletions(-) create mode 100644 src/minute_store.ts diff --git a/src/daydetail.ts b/src/daydetail.ts index 2da20f6..41b700d 100644 --- a/src/daydetail.ts +++ b/src/daydetail.ts @@ -9,8 +9,9 @@ import type { Context } from 'hono' import { cached, ttlForDate } from './cache' +import { readMinutes } from './minute_store' -type Ctx = Context<{ Bindings: { DB: D1Database }; Variables: { userId: string } }> +type Ctx = Context<{ Bindings: { DB: D1Database; RAW_BUCKET?: R2Bucket }; Variables: { userId: string } }> const DAY = 86400 const dayStartOf = (date: string) => Math.floor(Date.parse(`${date}T00:00:00Z`) / 1000) @@ -36,11 +37,9 @@ const clamp = (v: number, lo: number, hi: number) => Math.max(lo, Math.min(hi, v interface Min { ts_min: number; hr_avg: number | null; hr_min: number | null; hr_max: number | null; activity: number | null; wrist_on: number | null } async function loadMinutes(c: Ctx, from: number, to: number): Promise { - const { results } = await c.env.DB.prepare( - 'SELECT ts_min, hr_avg, hr_min, hr_max, activity, wrist_on FROM minute ' + - 'WHERE user_id = ? AND ts_min >= ? AND ts_min < ? ORDER BY ts_min ASC', - ).bind(c.get('userId'), from, to).all() - return results ?? [] + // Tiered read: D1 for hot days, R2 fallback for sealed days (see minute_store). + const rows = await readMinutes(c.env, c.get('userId'), from, to) + return rows.map((m) => ({ ts_min: m.ts_min, hr_avg: m.hr_avg, hr_min: m.hr_min, hr_max: m.hr_max, activity: m.activity, wrist_on: m.wrist_on })) } async function loadHr(c: Ctx): Promise<{ rhr: number; maxHr: number }> { diff --git a/src/index.ts b/src/index.ts index 9b4f81a..3ea078f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,6 +6,7 @@ import { runAnalytics, processUser } from './analytics' import { ingestBatch, ingestEvents } from './ingest' import { handleQueueBatch, type AnalyticsMessage, type AnalyticsJob } from './queue' import { runWakeLadder, retryStaleCloses } from './wake_cron' +import { sealOldDays } from './minute_store' import { getToday, getSleep, getSleepV2, getStrain, getSessions, getTrends, getChart } from './query' import { getHistory } from './history' import { postJournal, getJournal, getJournalInsights } from './journal' @@ -462,8 +463,12 @@ export default { try { await autoCloseStaleWorkouts(env.DB) } catch (e) { console.error('autoclose failed', e) } // EVERY tick — wake detection only (the cron's sole job). try { await runWakeLadder(env) } catch (e) { console.error('wake ladder failed', e) } - // Nightly maintenance ONLY (separate from detection): retention + retry-net. + // Nightly maintenance ONLY (separate from detection): seal + retention + retry-net. if (event.cron === '30 3 * * *') { + // Seal days older than the hot window to gzipped R2 objects and drop them from + // D1 (D1-hot / R2-sealed tiering — cuts D1 storage + per-row prune-deletes). + try { await sealOldDays(env) } catch (e) { console.error('seal failed', e) } + // Backstop: clear any minute/events still unsealed past retention (e.g. no bucket). const cutoff = Math.floor(Date.now() / 1000) - MINUTE_RETENTION_DAYS * DAY await env.DB.prepare('DELETE FROM minute WHERE ts_min < ?').bind(cutoff).run() await env.DB.prepare('DELETE FROM events WHERE ts < ?').bind(cutoff).run() diff --git a/src/minute_store.ts b/src/minute_store.ts new file mode 100644 index 0000000..dc9905a --- /dev/null +++ b/src/minute_store.ts @@ -0,0 +1,124 @@ +// minute_store.ts — [feat/wake-trigger] D1-hot / R2-sealed tiered minute storage. +// +// Hot days (≤ HOT_DAYS old) live row-per-minute in D1 — the ingest write path and +// processUser/wake read path are UNCHANGED (their windows are always within the hot +// days), so the corruption-sensitive paths never touch this module. Older "sealed" +// days are packed into ONE gzipped R2 object and removed from D1, which: +// • cuts D1 minute storage (only ~3 days resident) and dodges the 10 GB-per-DB cap, +// • replaces ~1440 per-row prune-DELETE writes/day with ONE R2 PUT (cheap), +// • extends drill-down history to the R2 window (14 d) instead of the D1 prune (10 d). +// Reads for older days fall back to R2 (only the day-detail endpoints request them). +// +// SAFETY: sealDay PUTs to R2 and only DELETEs the D1 rows after the put succeeds +// (raw-first / never-drop-before-persist). pack/unpack are pure JSON → lossless. + +import { decodeRr } from './ingest_signals' + +export const HOT_DAYS = 3 +const DAY = 86400 +const ymd = (ts: number): string => new Date(ts * 1000).toISOString().slice(0, 10) +const dayStart = (d: string): number => Math.floor(Date.parse(`${d}T00:00:00Z`) / 1000) +const objKey = (userId: string, date: string): string => `minute/${userId}/${date}.json.gz` + +export interface StoredMin { + ts_min: number + hr_avg: number | null; hr_min: number | null; hr_max: number | null; hr_n: number | null + hr_sum?: number | null + activity: number | null; act_sum?: number | null; act_n?: number | null + steps: number | null; wrist_on: number | null + rr?: number[] | null +} + +interface StoreEnv { DB: D1Database; RAW_BUCKET?: R2Bucket } + +const SELECT_COLS = 'ts_min, hr_avg, hr_min, hr_max, hr_n, hr_sum, activity, act_sum, act_n, steps, wrist_on, rr' + +// ── gzip via the Workers CompressionStream (no extra deps) ──────────────────── +async function gzip(s: string): Promise { + const cs = new CompressionStream('gzip') + const w = cs.writable.getWriter() + void w.write(new TextEncoder().encode(s)); void w.close() + return new Uint8Array(await new Response(cs.readable).arrayBuffer()) +} +async function gunzip(b: ArrayBuffer): Promise { + const ds = new DecompressionStream('gzip') + const w = ds.writable.getWriter() + void w.write(new Uint8Array(b)); void w.close() + return new TextDecoder().decode(await new Response(ds.readable).arrayBuffer()) +} + +// pack/unpack — pure, lossless, unit-testable. +export function packDay(mins: StoredMin[]): string { return JSON.stringify(mins) } +export function unpackDay(json: string): StoredMin[] { + const v = JSON.parse(json) + return Array.isArray(v) ? v as StoredMin[] : [] +} + +function rowToStored(r: any): StoredMin { + return { + ts_min: r.ts_min, hr_avg: r.hr_avg, hr_min: r.hr_min, hr_max: r.hr_max, hr_n: r.hr_n, + hr_sum: r.hr_sum, activity: r.activity, act_sum: r.act_sum, act_n: r.act_n, + steps: r.steps, wrist_on: r.wrist_on, rr: r.rr ? decodeRr(r.rr) : null, + } +} + +/** Seal ONE day: pack D1 rows → gzipped R2 object, then (only on success) drop the + * D1 rows. Idempotent: re-sealing an already-empty day is a no-op. */ +export async function sealDay(env: StoreEnv, userId: string, date: string): Promise<{ sealed: boolean; n: number }> { + if (!env.RAW_BUCKET) return { sealed: false, n: 0 } + const start = dayStart(date) + const { results } = await env.DB.prepare( + `SELECT ${SELECT_COLS} FROM minute WHERE user_id = ? AND ts_min >= ? AND ts_min < ? ORDER BY ts_min ASC`, + ).bind(userId, start, start + DAY).all() + const rows = results ?? [] + if (!rows.length) return { sealed: false, n: 0 } + const gz = await gzip(packDay(rows.map(rowToStored))) + await env.RAW_BUCKET.put(objKey(userId, date), gz, { httpMetadata: { contentEncoding: 'gzip' } }) + // R2 put confirmed → safe to drop the D1 rows. + await env.DB.prepare('DELETE FROM minute WHERE user_id = ? AND ts_min >= ? AND ts_min < ?') + .bind(userId, start, start + DAY).run() + return { sealed: true, n: rows.length } +} + +/** Read one sealed day from R2 (empty if not sealed / no bucket). */ +export async function readSealedDay(env: StoreEnv, userId: string, date: string): Promise { + if (!env.RAW_BUCKET) return [] + const obj = await env.RAW_BUCKET.get(objKey(userId, date)) + if (!obj) return [] + try { return unpackDay(await gunzip(await obj.arrayBuffer())) } catch { return [] } +} + +/** Read minutes over [from,to): D1 first, then R2 for any whole day in range that + * has no D1 rows (i.e. already sealed). Used by the day-detail read path. */ +export async function readMinutes(env: StoreEnv, userId: string, from: number, to: number): Promise { + const { results } = await env.DB.prepare( + `SELECT ${SELECT_COLS} FROM minute WHERE user_id = ? AND ts_min >= ? AND ts_min < ? ORDER BY ts_min ASC`, + ).bind(userId, from, to).all() + const out = (results ?? []).map(rowToStored) + const haveDays = new Set(out.map((m) => ymd(m.ts_min))) + for (let t = Math.floor(from / DAY) * DAY; t < to; t += DAY) { + const d = ymd(t) + if (haveDays.has(d)) continue + const sealed = await readSealedDay(env, userId, d) + for (const m of sealed) if (m.ts_min >= from && m.ts_min < to) out.push(m) + } + out.sort((a, b) => a.ts_min - b.ts_min) + return out +} + +/** Seal every day older than HOT_DAYS that still has D1 rows (bounded per call). + * Runs in the nightly maintenance cron — replaces the blind prune-delete. */ +export async function sealOldDays(env: StoreEnv, now = Math.floor(Date.now() / 1000), limit = 500): Promise<{ users: number; sealed: number }> { + if (!env.RAW_BUCKET) return { users: 0, sealed: 0 } + const cutoff = now - HOT_DAYS * DAY + const { results } = await env.DB.prepare( + "SELECT DISTINCT user_id, strftime('%Y-%m-%d', ts_min, 'unixepoch') AS d FROM minute WHERE ts_min < ? LIMIT ?", + ).bind(cutoff, limit).all<{ user_id: string; d: string }>() + let sealed = 0 + const users = new Set() + for (const r of results ?? []) { + try { const res = await sealDay(env, r.user_id, r.d); if (res.sealed) { sealed++; users.add(r.user_id) } } + catch (e) { console.error('sealDay failed', r.user_id, r.d, e) } + } + return { users: users.size, sealed } +} From 7fef5a6214292b9f7f1b5b3d8ad3935539b15d25 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 20 Jun 2026 00:27:36 +0530 Subject: [PATCH 15/30] =?UTF-8?q?feat(backend):=20day-pack=20minute=20stor?= =?UTF-8?q?age=20=E2=80=94=20minute=5Fday=20(1=20row/user/day,=20gzipped?= =?UTF-8?q?=20blob,=20RMW=20merge=20mirroring=20the=20old=20ON=20CONFLICT)?= =?UTF-8?q?=20replaces=20row-per-minute=20as=20the=20hot=20store;=20all=20?= =?UTF-8?q?readers=20(analytics/daydetail/query/workouts/biometrics/seed)?= =?UTF-8?q?=20+=20seal=20+=20prune=20go=20through=20minute=5Fstore.=20The?= =?UTF-8?q?=20D1=20write-cost=20lever=20(~1,440=E2=86=92~1=20write/day).?= =?UTF-8?q?=20v14=20migration.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/analytics.ts | 10 +- src/dayseries.ts | 28 ++-- src/db/migrate_v14_minute_day.sql | 11 ++ src/index.ts | 18 +-- src/ingest.ts | 49 ++----- src/minute_store.ts | 226 +++++++++++++++++++----------- src/query.ts | 15 +- src/queue.ts | 2 +- src/seed.ts | 25 ++-- src/workouts.ts | 13 +- 10 files changed, 214 insertions(+), 183 deletions(-) create mode 100644 src/db/migrate_v14_minute_day.sql diff --git a/src/analytics.ts b/src/analytics.ts index deab3f3..e7ecfc8 100644 --- a/src/analytics.ts +++ b/src/analytics.ts @@ -15,6 +15,7 @@ import { type Minute, type Profile, type Baseline, type DayHistory, type DailyStrain, type NightSummary, type SleepValue, type Metric, type Driver, } from 'openstrap-analytics' +import { readMinutes } from './minute_store' const DAY = 86400 @@ -79,11 +80,10 @@ export async function loadBaseline(db: D1Database, userId: string): Promise { - const { results } = await db.prepare( - 'SELECT ts_min, hr_avg, hr_min, hr_max, hr_n, activity, steps, wrist_on FROM minute ' + - 'WHERE user_id = ? AND ts_min >= ? AND ts_min < ? ORDER BY ts_min ASC', - ).bind(userId, from, to).all() - return (results ?? []).map(toMinute) + // Day-packed store. processUser/wake windows are within HOT_DAYS → all in D1 + // minute_day (RAW_BUCKET omitted = no R2 fallback needed for hot reads). + const recs = await readMinutes({ DB: db }, userId, from, to) + return recs.map(toMinute) } // Sleep search window for the night that WAKES on `dateDayStart`: diff --git a/src/dayseries.ts b/src/dayseries.ts index 22ba27e..f06b1cd 100644 --- a/src/dayseries.ts +++ b/src/dayseries.ts @@ -1,27 +1,19 @@ -// dayseries.ts — minimal D1-only RR loader for the HRV-from-minute path. +// dayseries.ts — RR loader for the HRV-from-minute path. // -// [feat/wake-trigger] Deliberately the *minimal* version: reads beat-to-beat RR -// straight from the D1 `minute.rr` BLOB column (populated at ingest by -// ingest_signals). NO R2 series cache / re-decode (that v3 machinery is left -// behind on purpose). HRV/recovery therefore costs zero R2 ops — it folds into the -// wake-triggered day-close. Returns a per-minute map keyed by ts_min. +// [feat/wake-trigger] RR now rides the day-packed minute blob (minute_store) as a +// plain number[] per minute — no separate column, no per-blob decode. Reads via the +// tiered store (D1 hot + R2 sealed), so HRV at the wake-close still costs zero extra +// R2 for hot nights. Returns RR (ms) per minute over [from, to]. -import { decodeRr } from './ingest_signals' +import { readMinutes, type StoreEnv } from './minute_store' -interface DSEnv { DB: D1Database; RAW_BUCKET?: R2Bucket; MINUTE_SOURCE?: string } - -/** RR (ms) per minute over [from, to] from D1 minute.rr. Empty map if none. */ +/** RR (ms) per minute over [from, to] from the day-packed store. Empty if none. */ export async function loadDayRr( - env: DSEnv, userId: string, from: number, to: number, + env: StoreEnv, userId: string, from: number, to: number, ): Promise> { - const fromMin = Math.floor(from / 60) * 60 - const { results } = await env.DB.prepare( - 'SELECT ts_min, rr FROM minute WHERE user_id = ? AND ts_min >= ? AND ts_min <= ? AND rr IS NOT NULL ORDER BY ts_min', - ).bind(userId, fromMin, to).all<{ ts_min: number; rr: ArrayBuffer | null }>() const map = new Map() - for (const r of results ?? []) { - const rr = decodeRr(r.rr) - if (rr.length) map.set(r.ts_min, rr) + for (const m of await readMinutes(env, userId, from, to)) { + if (m.rr && m.rr.length) map.set(m.ts_min, m.rr) } return map } diff --git a/src/db/migrate_v14_minute_day.sql b/src/db/migrate_v14_minute_day.sql new file mode 100644 index 0000000..2a328a4 --- /dev/null +++ b/src/db/migrate_v14_minute_day.sql @@ -0,0 +1,11 @@ +-- [feat/wake-trigger] v14 — day-packed minute store: ONE row per (user, ymd), value = +-- gzipped JSON array of per-minute records. Replaces the row-per-minute `minute` table +-- as the hot store (ingest RMWs one row/day instead of ~1,440). Sealed days move to R2. +-- (The legacy `minute` table is left in place but unused; it stays empty on fresh DBs.) +CREATE TABLE IF NOT EXISTS minute_day ( + user_id TEXT NOT NULL, + ymd TEXT NOT NULL, -- 'YYYY-MM-DD' (UTC day of the minute) + blob BLOB NOT NULL, -- gzipped JSON MinuteRec[] (bound param; << 2MB cap) + updated_at INTEGER NOT NULL, + PRIMARY KEY(user_id, ymd) +) WITHOUT ROWID; diff --git a/src/index.ts b/src/index.ts index 3ea078f..28accf5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,7 +6,7 @@ import { runAnalytics, processUser } from './analytics' import { ingestBatch, ingestEvents } from './ingest' import { handleQueueBatch, type AnalyticsMessage, type AnalyticsJob } from './queue' import { runWakeLadder, retryStaleCloses } from './wake_cron' -import { sealOldDays } from './minute_store' +import { sealOldDays, pruneMinuteDays } from './minute_store' import { getToday, getSleep, getSleepV2, getStrain, getSessions, getTrends, getChart } from './query' import { getHistory } from './history' import { postJournal, getJournal, getJournalInsights } from './journal' @@ -410,10 +410,10 @@ app.post('/admin/wipe-raw', async (c) => { // Prune minute rows older than 90 days (raw stays in R2). app.post('/admin/prune', async (c) => { - const cutoff = Math.floor(Date.now() / 1000) - MINUTE_RETENTION_DAYS * DAY - const res = await c.env.DB.prepare('DELETE FROM minute WHERE ts_min < ?').bind(cutoff).run() - const ev = await c.env.DB.prepare('DELETE FROM events WHERE ts < ?').bind(cutoff).run() - return c.json({ ok: true, deleted: res.meta?.changes ?? 0, events_deleted: ev.meta?.changes ?? 0, cutoff }) + const now = Math.floor(Date.now() / 1000) + await pruneMinuteDays(c.env, now, MINUTE_RETENTION_DAYS) // day-packed minute_day rows + const ev = await c.env.DB.prepare('DELETE FROM events WHERE ts < ?').bind(now - MINUTE_RETENTION_DAYS * DAY).run() + return c.json({ ok: true, events_deleted: ev.meta?.changes ?? 0 }) }) // Send messages in sendBatch chunks of 100 (the Queues per-call max), so the cron @@ -468,10 +468,10 @@ export default { // Seal days older than the hot window to gzipped R2 objects and drop them from // D1 (D1-hot / R2-sealed tiering — cuts D1 storage + per-row prune-deletes). try { await sealOldDays(env) } catch (e) { console.error('seal failed', e) } - // Backstop: clear any minute/events still unsealed past retention (e.g. no bucket). - const cutoff = Math.floor(Date.now() / 1000) - MINUTE_RETENTION_DAYS * DAY - await env.DB.prepare('DELETE FROM minute WHERE ts_min < ?').bind(cutoff).run() - await env.DB.prepare('DELETE FROM events WHERE ts < ?').bind(cutoff).run() + // Backstop: clear any minute_day/events still unsealed past retention (e.g. no bucket). + const nowS = Math.floor(Date.now() / 1000) + await pruneMinuteDays(env, nowS, MINUTE_RETENTION_DAYS) + await env.DB.prepare('DELETE FROM events WHERE ts < ?').bind(nowS - MINUTE_RETENTION_DAYS * DAY).run() try { await retryStaleCloses(env) } catch (e) { console.error('retry-net failed', e) } } })()) diff --git a/src/ingest.ts b/src/ingest.ts index ae070fc..a789342 100644 --- a/src/ingest.ts +++ b/src/ingest.ts @@ -4,7 +4,8 @@ import type { Context } from 'hono' import { decodeBatch, hexToBytes } from 'openstrap-protocol/ts/live' import { rollupMinutes } from './rollup' -import { perMinuteSignals, encodeRr } from './ingest_signals' +import { perMinuteSignals } from './ingest_signals' +import { writeBatch } from './minute_store' // ── Rate limiting: per-user token bucket in a D1 row (RESILIENCE §7). ── // Refill RATE tokens/sec, cap BURST. One ingest = one token. Over budget → 429. @@ -86,48 +87,18 @@ export async function ingestBatch(c: Context<{ Bindings: IngestEnv; Variables: { } } - // 4. Rollup → minute upsert (one batch). Merge deterministically with stored. + // 4. Rollup → day-packed minute store (ONE row per day, read-merge-write). This is + // the cost lever: ~1 row written/day instead of ~1,440. The merge inside writeBatch + // mirrors the old per-minute ON CONFLICT exactly (additive sums, CASE hr_min, MAX + // hr_max, steps add, rr longer-wins). Idempotent IFF the edge dedupes by hex (it does). + // RR (ms, R24 + live 0x28/R10) rides the blob as number[] — HRV folds into the + // wake-close with zero R2. (Storage: gzipped blob via bound param — well under D1's + // 2 MB value cap; serialized outside the 100 KB SQL-statement limit.) const buckets = rollupMinutes(samples) let minutesWritten = 0 - let maxTsMin = 0 if (buckets.length > 0) { - // The upsert folds new running sums into stored aggregates and recomputes - // the display columns (hr_avg, activity, hr_min/max) from the merged totals. - // [wake-trigger] per-minute RR (ms) decoded from this batch's frames (R24 + live - // 0x28/R10), so HRV folds into the wake-close with zero R2. We use ONLY the `rr` - // here; steps stay on the existing rollup path (no step-semantics change this phase). const sig = perMinuteSignals(records) - const stmt = c.env.DB.prepare( - 'INSERT INTO minute (user_id, ts_min, hr_avg, hr_min, hr_max, hr_n, hr_sum, activity, act_sum, act_n, steps, wrist_on, rr) ' + - 'VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?) ' + - 'ON CONFLICT(user_id, ts_min) DO UPDATE SET ' + - 'hr_sum = minute.hr_sum + excluded.hr_sum, ' + - 'hr_n = minute.hr_n + excluded.hr_n, ' + - 'hr_min = CASE WHEN minute.hr_min = 0 THEN excluded.hr_min ' + - ' WHEN excluded.hr_min = 0 THEN minute.hr_min ' + - ' ELSE MIN(minute.hr_min, excluded.hr_min) END, ' + - 'hr_max = MAX(minute.hr_max, excluded.hr_max), ' + - 'hr_avg = CASE WHEN (minute.hr_n + excluded.hr_n) > 0 ' + - ' THEN (minute.hr_sum + excluded.hr_sum) / (minute.hr_n + excluded.hr_n) ELSE 0 END, ' + - 'act_sum = minute.act_sum + excluded.act_sum, ' + - 'act_n = minute.act_n + excluded.act_n, ' + - 'activity = CASE WHEN (minute.act_n + excluded.act_n) > 0 ' + - ' THEN (minute.act_sum + excluded.act_sum) / (minute.act_n + excluded.act_n) ELSE 0 END, ' + - 'steps = minute.steps + excluded.steps, ' + - // RR: keep the fuller blob (idempotent — a re-uploaded/fuller batch wins; never doubles). - "rr = CASE WHEN excluded.rr IS NOT NULL AND length(excluded.rr) >= length(COALESCE(minute.rr, x'')) " + - ' THEN excluded.rr ELSE minute.rr END, ' + - 'wrist_on = MAX(minute.wrist_on, excluded.wrist_on)', - ) - const batch = buckets.map((b) => { - if (b.ts_min > maxTsMin) maxTsMin = b.ts_min - const hr_avg = b.hr_n > 0 ? Math.round(b.hr_sum / b.hr_n) : 0 - const activity = b.act_n > 0 ? b.act_sum / b.act_n : 0 - const rrBlob = encodeRr(sig.get(b.ts_min)?.rr ?? []) - return stmt.bind(userId, b.ts_min, hr_avg, b.hr_min, b.hr_max, b.hr_n, b.hr_sum, - activity, b.act_sum, b.act_n, b.steps, b.wrist_on, rrBlob) - }) - await c.env.DB.batch(batch) + await writeBatch(c.env, userId, buckets, sig, Math.floor(Date.now() / 1000)) minutesWritten = buckets.length } diff --git a/src/minute_store.ts b/src/minute_store.ts index dc9905a..2184a2e 100644 --- a/src/minute_store.ts +++ b/src/minute_store.ts @@ -1,18 +1,24 @@ -// minute_store.ts — [feat/wake-trigger] D1-hot / R2-sealed tiered minute storage. +// minute_store.ts — [feat/wake-trigger] THE single minute-storage layer. // -// Hot days (≤ HOT_DAYS old) live row-per-minute in D1 — the ingest write path and -// processUser/wake read path are UNCHANGED (their windows are always within the hot -// days), so the corruption-sensitive paths never touch this module. Older "sealed" -// days are packed into ONE gzipped R2 object and removed from D1, which: -// • cuts D1 minute storage (only ~3 days resident) and dodges the 10 GB-per-DB cap, -// • replaces ~1440 per-row prune-DELETE writes/day with ONE R2 PUT (cheap), -// • extends drill-down history to the R2 window (14 d) instead of the D1 prune (10 d). -// Reads for older days fall back to R2 (only the day-detail endpoints request them). +// Storage model: ONE row per (user, ymd) in `minute_day`, value = a gzipped JSON +// array of MinuteRec (one entry per touched minute, keyed by ts_min). This replaces +// the old row-per-minute `minute` table — so ingest writes ~1 row/day instead of +// ~1,440, and the 10-day prune deletes ~1 row/day instead of ~1,440 (the big D1 +// write-cost cut). Reads unpack the blob to the same MinuteRec[] every caller used. // -// SAFETY: sealDay PUTs to R2 and only DELETEs the D1 rows after the put succeeds -// (raw-first / never-drop-before-persist). pack/unpack are pure JSON → lossless. +// Tiering: hot days live in D1 `minute_day`; days older than HOT_DAYS are sealed — +// the SAME gzipped blob is moved to a gzipped R2 object and the D1 row dropped. +// readDay/readMinutes fall back to R2 for sealed days. +// +// D1 limits respected: blob is gzipped (~150–250 KB worst-case day, << the 2 MB +// value cap) and ALWAYS written via a bound parameter (the 100 KB SQL-statement +// limit is for SQL text, not bound-param values). +// +// Concurrency: writeBatch is read-merge-write, NOT atomic. Safe because a user has +// ONE band syncing serially (the edge uploader awaits each batch). Two concurrent +// writers for the same (user, ymd) could lose an update — not a real scenario here. -import { decodeRr } from './ingest_signals' +import type { MinuteBucket } from './rollup' export const HOT_DAYS = 3 const DAY = 86400 @@ -20,105 +26,161 @@ const ymd = (ts: number): string => new Date(ts * 1000).toISOString().slice(0, 1 const dayStart = (d: string): number => Math.floor(Date.parse(`${d}T00:00:00Z`) / 1000) const objKey = (userId: string, date: string): string => `minute/${userId}/${date}.json.gz` -export interface StoredMin { +/** One minute's stored aggregate (running sums kept so merges stay exact). */ +export interface MinuteRec { ts_min: number - hr_avg: number | null; hr_min: number | null; hr_max: number | null; hr_n: number | null - hr_sum?: number | null - activity: number | null; act_sum?: number | null; act_n?: number | null - steps: number | null; wrist_on: number | null - rr?: number[] | null + hr_avg: number; hr_min: number; hr_max: number; hr_n: number; hr_sum: number + activity: number; act_sum: number; act_n: number + steps: number; wrist_on: number + rr: number[] } -interface StoreEnv { DB: D1Database; RAW_BUCKET?: R2Bucket } - -const SELECT_COLS = 'ts_min, hr_avg, hr_min, hr_max, hr_n, hr_sum, activity, act_sum, act_n, steps, wrist_on, rr' +export interface StoreEnv { DB: D1Database; RAW_BUCKET?: R2Bucket } -// ── gzip via the Workers CompressionStream (no extra deps) ──────────────────── +// ── gzip via Workers CompressionStream ──────────────────────────────────────── async function gzip(s: string): Promise { const cs = new CompressionStream('gzip') - const w = cs.writable.getWriter() - void w.write(new TextEncoder().encode(s)); void w.close() + const w = cs.writable.getWriter(); void w.write(new TextEncoder().encode(s)); void w.close() return new Uint8Array(await new Response(cs.readable).arrayBuffer()) } async function gunzip(b: ArrayBuffer): Promise { const ds = new DecompressionStream('gzip') - const w = ds.writable.getWriter() - void w.write(new Uint8Array(b)); void w.close() + const w = ds.writable.getWriter(); void w.write(new Uint8Array(b)); void w.close() return new TextDecoder().decode(await new Response(ds.readable).arrayBuffer()) } -// pack/unpack — pure, lossless, unit-testable. -export function packDay(mins: StoredMin[]): string { return JSON.stringify(mins) } -export function unpackDay(json: string): StoredMin[] { - const v = JSON.parse(json) - return Array.isArray(v) ? v as StoredMin[] : [] +function zeroRec(ts_min: number): MinuteRec { + return { ts_min, hr_avg: 0, hr_min: 0, hr_max: 0, hr_n: 0, hr_sum: 0, activity: 0, act_sum: 0, act_n: 0, steps: 0, wrist_on: 0, rr: [] } } -function rowToStored(r: any): StoredMin { - return { - ts_min: r.ts_min, hr_avg: r.hr_avg, hr_min: r.hr_min, hr_max: r.hr_max, hr_n: r.hr_n, - hr_sum: r.hr_sum, activity: r.activity, act_sum: r.act_sum, act_n: r.act_n, - steps: r.steps, wrist_on: r.wrist_on, rr: r.rr ? decodeRr(r.rr) : null, - } +// pack/unpack — pure, lossless JSON. +export function packDay(recs: MinuteRec[]): MinuteRec[] { return recs } +function serialize(map: Map): string { + return JSON.stringify([...map.values()].sort((a, b) => a.ts_min - b.ts_min)) +} +function deserialize(json: string): Map { + const out = new Map() + try { const arr = JSON.parse(json); if (Array.isArray(arr)) for (const r of arr) if (r && typeof r.ts_min === 'number') out.set(r.ts_min, r as MinuteRec) } catch { /* corrupt → empty */ } + return out } -/** Seal ONE day: pack D1 rows → gzipped R2 object, then (only on success) drop the - * D1 rows. Idempotent: re-sealing an already-empty day is a no-op. */ -export async function sealDay(env: StoreEnv, userId: string, date: string): Promise<{ sealed: boolean; n: number }> { - if (!env.RAW_BUCKET) return { sealed: false, n: 0 } - const start = dayStart(date) - const { results } = await env.DB.prepare( - `SELECT ${SELECT_COLS} FROM minute WHERE user_id = ? AND ts_min >= ? AND ts_min < ? ORDER BY ts_min ASC`, - ).bind(userId, start, start + DAY).all() - const rows = results ?? [] - if (!rows.length) return { sealed: false, n: 0 } - const gz = await gzip(packDay(rows.map(rowToStored))) - await env.RAW_BUCKET.put(objKey(userId, date), gz, { httpMetadata: { contentEncoding: 'gzip' } }) - // R2 put confirmed → safe to drop the D1 rows. - await env.DB.prepare('DELETE FROM minute WHERE user_id = ? AND ts_min >= ? AND ts_min < ?') - .bind(userId, start, start + DAY).run() - return { sealed: true, n: rows.length } +/** Read ONE day's minutes as a map: D1 minute_day (hot) → else R2 sealed → empty. */ +export async function readDay(env: StoreEnv, userId: string, date: string): Promise> { + const row = await env.DB.prepare('SELECT blob FROM minute_day WHERE user_id = ? AND ymd = ?') + .bind(userId, date).first<{ blob: ArrayBuffer | null }>() + if (row?.blob) return deserialize(await gunzip(row.blob)) + if (env.RAW_BUCKET) { + const obj = await env.RAW_BUCKET.get(objKey(userId, date)) + if (obj) { try { return deserialize(await gunzip(await obj.arrayBuffer())) } catch { /* fall through */ } } + } + return new Map() } -/** Read one sealed day from R2 (empty if not sealed / no bucket). */ -export async function readSealedDay(env: StoreEnv, userId: string, date: string): Promise { - if (!env.RAW_BUCKET) return [] - const obj = await env.RAW_BUCKET.get(objKey(userId, date)) - if (!obj) return [] - try { return unpackDay(await gunzip(await obj.arrayBuffer())) } catch { return [] } +/** Write ONE day's map back to D1 minute_day (gzip + bound param). */ +async function writeDay(env: StoreEnv, userId: string, date: string, map: Map, now: number): Promise { + if (map.size === 0) return + const gz = await gzip(serialize(map)) + await env.DB.prepare( + 'INSERT INTO minute_day (user_id, ymd, blob, updated_at) VALUES (?,?,?,?) ' + + 'ON CONFLICT(user_id, ymd) DO UPDATE SET blob = excluded.blob, updated_at = excluded.updated_at', + ).bind(userId, date, gz, now).run() } -/** Read minutes over [from,to): D1 first, then R2 for any whole day in range that - * has no D1 rows (i.e. already sealed). Used by the day-detail read path. */ -export async function readMinutes(env: StoreEnv, userId: string, from: number, to: number): Promise { - const { results } = await env.DB.prepare( - `SELECT ${SELECT_COLS} FROM minute WHERE user_id = ? AND ts_min >= ? AND ts_min < ? ORDER BY ts_min ASC`, - ).bind(userId, from, to).all() - const out = (results ?? []).map(rowToStored) - const haveDays = new Set(out.map((m) => ymd(m.ts_min))) +/** Read minutes over [from, to): unpack each day in range (D1 hot + R2 sealed). */ +export async function readMinutes(env: StoreEnv, userId: string, from: number, to: number): Promise { + const out: MinuteRec[] = [] for (let t = Math.floor(from / DAY) * DAY; t < to; t += DAY) { - const d = ymd(t) - if (haveDays.has(d)) continue - const sealed = await readSealedDay(env, userId, d) - for (const m of sealed) if (m.ts_min >= from && m.ts_min < to) out.push(m) + const day = await readDay(env, userId, ymd(t)) + for (const r of day.values()) if (r.ts_min >= from && r.ts_min < to) out.push(r) } out.sort((a, b) => a.ts_min - b.ts_min) return out } -/** Seal every day older than HOT_DAYS that still has D1 rows (bounded per call). - * Runs in the nightly maintenance cron — replaces the blind prune-delete. */ -export async function sealOldDays(env: StoreEnv, now = Math.floor(Date.now() / 1000), limit = 500): Promise<{ users: number; sealed: number }> { - if (!env.RAW_BUCKET) return { users: 0, sealed: 0 } - const cutoff = now - HOT_DAYS * DAY +/** Latest minute at/after `sinceTs` (for "current HR"): scans today's + sinceTs's day. */ +export async function latestMinute(env: StoreEnv, userId: string, sinceTs: number, now: number): Promise { + let best: MinuteRec | null = null + for (let t = Math.floor(sinceTs / DAY) * DAY; t <= now; t += DAY) { + for (const r of (await readDay(env, userId, ymd(t))).values()) { + if (r.ts_min >= sinceTs && (!best || r.ts_min > best.ts_min)) best = r + } + } + return best +} + +/** + * Fold a batch's per-minute buckets (+ steps/rr signals) into the day blob(s), + * RMW per touched day. Merge mirrors the old `minute` ON CONFLICT EXACTLY: + * hr/act sums + counts add, hr_min CASE-guarded for 0, hr_max MAX, steps add, + * rr keeps the longer (fuller) array, wrist_on MAX; hr_avg/activity recomputed. + * Idempotent for re-uploaded identical batches IFF the edge dedupes by hex (it does). + */ +export async function writeBatch( + env: StoreEnv, userId: string, + buckets: MinuteBucket[], + signals: Map, + now: number, +): Promise { + if (buckets.length === 0) return 0 + // group buckets by ymd + const byDay = new Map() + for (const b of buckets) { const d = ymd(b.ts_min); (byDay.get(d) ?? byDay.set(d, []).get(d)!).push(b) } + let daysWritten = 0 + for (const [date, dayBuckets] of byDay) { + const map = await readDay(env, userId, date) // hot (D1) or, rare re-drain, sealed (R2) + for (const b of dayBuckets) { + const rec = map.get(b.ts_min) ?? zeroRec(b.ts_min) + rec.hr_sum += b.hr_sum + rec.hr_n += b.hr_n + rec.hr_min = rec.hr_min === 0 ? b.hr_min : (b.hr_min === 0 ? rec.hr_min : Math.min(rec.hr_min, b.hr_min)) + rec.hr_max = Math.max(rec.hr_max, b.hr_max) + rec.act_sum += b.act_sum + rec.act_n += b.act_n + rec.steps += b.steps + rec.wrist_on = Math.max(rec.wrist_on, b.wrist_on) + const newRr = signals.get(b.ts_min)?.rr ?? [] + if (newRr.length >= rec.rr.length) rec.rr = newRr + rec.hr_avg = rec.hr_n > 0 ? Math.round(rec.hr_sum / rec.hr_n) : 0 + rec.activity = rec.act_n > 0 ? rec.act_sum / rec.act_n : 0 + map.set(b.ts_min, rec) + } + await writeDay(env, userId, date, map, now) + daysWritten++ + } + return daysWritten +} + +/** Seal days older than HOT_DAYS: move the D1 day blob → gzipped R2 object, drop the + * D1 row (the blob is already in the seal format). Bounded per call. */ +export async function sealOldDays(env: StoreEnv, now = Math.floor(Date.now() / 1000), limit = 1000): Promise<{ sealed: number }> { + if (!env.RAW_BUCKET) return { sealed: 0 } + const cutoff = ymd(now - HOT_DAYS * DAY) const { results } = await env.DB.prepare( - "SELECT DISTINCT user_id, strftime('%Y-%m-%d', ts_min, 'unixepoch') AS d FROM minute WHERE ts_min < ? LIMIT ?", - ).bind(cutoff, limit).all<{ user_id: string; d: string }>() + 'SELECT user_id, ymd, blob FROM minute_day WHERE ymd < ? LIMIT ?', + ).bind(cutoff, limit).all<{ user_id: string; ymd: string; blob: ArrayBuffer }>() let sealed = 0 - const users = new Set() for (const r of results ?? []) { - try { const res = await sealDay(env, r.user_id, r.d); if (res.sealed) { sealed++; users.add(r.user_id) } } - catch (e) { console.error('sealDay failed', r.user_id, r.d, e) } + try { + await env.RAW_BUCKET.put(objKey(r.user_id, r.ymd), r.blob, { httpMetadata: { contentEncoding: 'gzip' } }) + await env.DB.prepare('DELETE FROM minute_day WHERE user_id = ? AND ymd = ?').bind(r.user_id, r.ymd).run() + sealed++ + } catch (e) { console.error('sealDay failed', r.user_id, r.ymd, e) } } - return { users: users.size, sealed } + return { sealed } +} + +/** Seed/admin: write MinuteRec[] straight into the day-packed store (groups by ymd). */ +export async function putDay(env: StoreEnv, userId: string, recs: MinuteRec[], now: number): Promise { + const byDay = new Map>() + for (const r of recs) { + const d = ymd(r.ts_min) + let m = byDay.get(d); if (!m) { m = new Map(); byDay.set(d, m) } + m.set(r.ts_min, r) + } + for (const [d, m] of byDay) await writeDay(env, userId, d, m, now) +} + +/** Backstop retention delete: drop any minute_day rows older than `days` still in D1. */ +export async function pruneMinuteDays(env: StoreEnv, now: number, days: number): Promise { + await env.DB.prepare('DELETE FROM minute_day WHERE ymd < ?').bind(ymd(now - days * DAY)).run() } diff --git a/src/query.ts b/src/query.ts index 37e13a4..e5494b7 100644 --- a/src/query.ts +++ b/src/query.ts @@ -4,8 +4,9 @@ import type { Context } from 'hono' import { calcFitnessTrend, type DayHistory } from 'openstrap-analytics' +import { readMinutes, latestMinute } from './minute_store' -type Ctx = Context<{ Bindings: { DB: D1Database }; Variables: { userId: string } }> +type Ctx = Context<{ Bindings: { DB: D1Database; RAW_BUCKET?: R2Bucket }; Variables: { userId: string } }> const nowSec = () => Math.floor(Date.now() / 1000) const todayDate = () => new Date().toISOString().slice(0, 10) @@ -43,10 +44,8 @@ export async function getToday(c: Ctx) { const u = await c.env.DB.prepare('SELECT step_goal FROM users WHERE id = ?') .bind(userId).first() - // Live status: most recent minute (HR + worn) in the last 10 minutes. - const live = await c.env.DB.prepare( - 'SELECT ts_min, hr_avg, wrist_on FROM minute WHERE user_id = ? AND ts_min >= ? ORDER BY ts_min DESC LIMIT 1', - ).bind(userId, nowSec() - 600).first() + // Live status: most recent minute (HR + worn) in the last 10 minutes (day-packed store). + const live = await latestMinute(c.env, userId, nowSec() - 600, nowSec()) const df = parseFlags(daily?.flags) const sf = parseFlags(sleep?.flags) @@ -354,10 +353,8 @@ export async function getChart(c: Ctx) { ? 'steps' : 'hr_avg' - const { results } = await c.env.DB.prepare( - `SELECT ts_min, ${col} AS v, wrist_on FROM minute WHERE user_id = ? AND ts_min >= ? AND ts_min <= ? ORDER BY ts_min ASC`, - ).bind(userId, from, to).all<{ ts_min: number; v: number | null; wrist_on: number }>() - const rows = results ?? [] + const recs = await readMinutes(c.env, userId, from, to) + const rows = recs.map((r) => ({ ts_min: r.ts_min, v: (r as any)[col] as number | null, wrist_on: r.wrist_on })) const MAX = 500 let points: { ts: number; v: number | null }[] diff --git a/src/queue.ts b/src/queue.ts index ee0595e..e274aa7 100644 --- a/src/queue.ts +++ b/src/queue.ts @@ -74,7 +74,7 @@ async function runJob(env: QueueEnv, userId: string, job: AnalyticsJob, day?: st await runStepsIncremental(env, userId) if (day && wake_ts) { const from = onset_ts ?? (wake_ts - 8 * 3600) - try { await runBiometricsMinute({ DB: env.DB }, userId, day, from, wake_ts + 60) } catch (e) { console.error('biometrics_minute failed', userId, day, e) } + try { await runBiometricsMinute({ DB: env.DB, RAW_BUCKET: env.RAW_BUCKET }, userId, day, from, wake_ts + 60) } catch (e) { console.error('biometrics_minute failed', userId, day, e) } await invalidateDay(env.DB, userId, day) } await env.DB.prepare('UPDATE analytics_cursor SET dirty = 0 WHERE user_id = ?').bind(userId).run() diff --git a/src/seed.ts b/src/seed.ts index 5c408c7..d8f3144 100644 --- a/src/seed.ts +++ b/src/seed.ts @@ -11,6 +11,7 @@ // orchestrates the minutes phases in chunks. Each phase is idempotent. import { processUser } from './analytics' +import { putDay } from './minute_store' import { uuid } from './auth' const DAY = 86400 @@ -319,7 +320,7 @@ async function findOrCreateUser(db: D1Database, email: string, now: number): Pro // phase=init — create/clear user. Returns user_id. export async function seedInit(db: D1Database, email: string, now: number): Promise<{ user_id: string }> { const userId = await findOrCreateUser(db, email, now) - for (const t of ['minute', 'daily', 'sleep', 'sessions', 'baselines', 'analytics_cursor']) { + for (const t of ['minute', 'minute_day', 'daily', 'sleep', 'sessions', 'baselines', 'analytics_cursor']) { await db.prepare(`DELETE FROM ${t} WHERE user_id = ?`).bind(userId).run() } return { user_id: userId } @@ -332,23 +333,21 @@ export async function seedMinutes( ): Promise<{ minutes: number }> { const todayStart = Math.floor(now / DAY) * DAY const firstDayStart = todayStart - (days - 1) * DAY - const insert = db.prepare( - 'INSERT OR REPLACE INTO minute (user_id, ts_min, hr_avg, hr_min, hr_max, hr_n, hr_sum, activity, act_sum, act_n, steps, wrist_on) ' + - 'VALUES (?,?,?,?,?,?,?,?,?,?,?,?)', - ) let minutes = 0 for (let d = dayFrom; d < dayTo && d < days; d++) { const dayStart = firstDayStart + d * DAY const rows = buildDay(dayStart, d, days) - const batch: D1PreparedStatement[] = [] - for (const m of rows) { - const hr_sum = m.hr_avg * m.hr_n + // Day-packed store (minute_day). Build MinuteRec[] (running sums for exact merges; rr empty). + const recs = rows.map((m) => { const act_n = 60 - const act_sum = m.activity * act_n - batch.push(insert.bind(userId, m.ts_min, m.hr_avg, m.hr_min, m.hr_max, m.hr_n, hr_sum, - m.activity, act_sum, act_n, m.steps, m.wrist_on)) - } - if (batch.length) { await db.batch(batch); minutes += batch.length } + return { + ts_min: m.ts_min, hr_avg: m.hr_avg, hr_min: m.hr_min, hr_max: m.hr_max, hr_n: m.hr_n, + hr_sum: m.hr_avg * m.hr_n, activity: m.activity, act_sum: m.activity * act_n, act_n, + steps: m.steps, wrist_on: m.wrist_on, rr: [] as number[], + } + }) + await putDay({ DB: db }, userId, recs, now) + minutes += recs.length } return { minutes } } diff --git a/src/workouts.ts b/src/workouts.ts index f7f014a..6e7f4b6 100644 --- a/src/workouts.ts +++ b/src/workouts.ts @@ -8,6 +8,7 @@ import { calcStrain, calcHrZones, calcCalories, calcHrRecovery, type Minute, type Profile, type Baseline, } from 'openstrap-analytics' +import { readMinutes } from './minute_store' type Ctx = Context<{ Bindings: { DB: D1Database }; Variables: { userId: string } }> const nowSec = () => Math.floor(Date.now() / 1000) @@ -34,13 +35,11 @@ async function loadBaseline(db: D1Database, userId: string): Promise { } } async function loadMinutes(db: D1Database, userId: string, from: number, to: number): Promise { - const { results } = await db.prepare( - 'SELECT ts_min, hr_avg, hr_min, hr_max, hr_n, activity, steps, wrist_on FROM minute ' + - 'WHERE user_id = ? AND ts_min >= ? AND ts_min <= ? ORDER BY ts_min ASC', - ).bind(userId, from, to).all() - return (results ?? []).map((r: any) => ({ - ts: r.ts_min, hr_avg: r.hr_avg ?? 0, hr_min: r.hr_min ?? 0, hr_max: r.hr_max ?? 0, - hr_n: r.hr_n ?? 0, activity: r.activity ?? 0, steps: r.steps ?? 0, wrist_on: !!r.wrist_on, + // Day-packed store. Only ever called at workout-end (recent/hot) → D1 minute_day. + const recs = await readMinutes({ DB: db }, userId, from, to) + return recs.map((r) => ({ + ts: r.ts_min, hr_avg: r.hr_avg, hr_min: r.hr_min, hr_max: r.hr_max, + hr_n: r.hr_n, activity: r.activity, steps: r.steps, wrist_on: !!r.wrist_on, })) } From 65104f9d96216d7edf3c77508d13534c3d4205fa Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 20 Jun 2026 10:58:01 +0530 Subject: [PATCH 16/30] daydetail: thread per-minute RR into stageNight for RR-driven REM staging loadMinutes/stageNight now carry MinuteRec.rr through to stageSleep so the autonomic deep/REM split (RMSSD) runs on /day/sleep and /day/v2/sleep. Pairs with the analytics stageSleep RR change. ops/ + wrangler.v*.toml stay local. --- src/daydetail.ts | 85 ++++++++++++++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 36 deletions(-) diff --git a/src/daydetail.ts b/src/daydetail.ts index 41b700d..8cd439c 100644 --- a/src/daydetail.ts +++ b/src/daydetail.ts @@ -10,36 +10,57 @@ import type { Context } from 'hono' import { cached, ttlForDate } from './cache' import { readMinutes } from './minute_store' +import { calcCircadian, stageSleep } from 'openstrap-analytics' type Ctx = Context<{ Bindings: { DB: D1Database; RAW_BUCKET?: R2Bucket }; Variables: { userId: string } }> const DAY = 86400 const dayStartOf = (date: string) => Math.floor(Date.parse(`${date}T00:00:00Z`) / 1000) -// Per-epoch hypnogram (BETA) from minute HR/activity vs resting HR — the same -// heuristic the main-sleep view uses, factored out so naps render identically. -function buildHypnogram(mins: { ts_min: number; hr_avg?: number | null; activity?: number | null }[], rhr: number): { t: number; stage: string }[] { - const out: { t: number; stage: string }[] = [] - for (const m of mins) { - const hr = m.hr_avg ?? 0 - const act = m.activity ?? 0 - let stage: string - if (hr <= 0 || hr > 1.15 * rhr || act > 0.25) stage = 'awake' - else if (act < 0.01 && hr < rhr + 4) stage = 'deep' - else if (hr > rhr + 5) stage = 'rem' - else stage = 'light' - out.push({ t: m.ts_min, stage }) +// Per-epoch hypnogram (BETA) via analytics `stageSleep` — the SAME smoothed classifier +// behind the stage totals (sustained-≥20min-awake reclassify + bout-merge), so the graph +// matches the breakdown and never flaps on a single-minute HR blip or a brief off-wrist +// gap. `mesor` from calcCircadian over the window; falls back to the simple per-minute +// threshold ONLY if circadian can't fit (flat/too-short window). +interface NightStaging { hypnogram: { t: number; stage: string }[]; totals: { light_min: number; deep_min: number; rem_min: number } | null } +function stageNight( + mins: { ts_min: number; hr_avg?: number | null; hr_min?: number | null; hr_max?: number | null; activity?: number | null; wrist_on?: number | null; rr?: number[] | null }[], + onset: number, wake: number, rhr: number, +): NightStaging { + const ms = mins.map((m) => ({ + ts: m.ts_min, hr_avg: m.hr_avg ?? 0, hr_min: m.hr_min ?? 0, hr_max: m.hr_max ?? 0, + hr_n: (m.hr_avg ?? 0) > 0 ? 60 : 0, activity: m.activity ?? 0, steps: 0, wrist_on: !!m.wrist_on, + rr: m.rr ?? [], // per-minute beat-to-beat RR → autonomic REM/deep split in stageSleep + })) + const circ = calcCircadian(ms) + const mesor = (circ.mesor != null && circ.mesor > rhr) ? circ.mesor : rhr + 25 + const s = stageSleep(ms, onset, wake, mesor) + if (s.hypnogram.length) { + // ONE classifier drives BOTH the hypnogram and the reconciled totals → graph and + // breakdown can't disagree, no flapping, REM detected (re-tuned band). + return { + hypnogram: s.hypnogram.map((h) => ({ t: h.t, stage: h.stage })), + totals: { light_min: s.light_min, deep_min: s.deep_min, rem_min: s.rem_min }, + } } - return out + // Fallback (circadian abstained): unsmoothed per-minute hypnogram; totals=null → caller + // keeps the stored breakdown. + const hyp = ms.filter((m) => m.ts >= onset && m.ts <= wake).map((m) => { + const hr = m.hr_avg, act = m.activity + const stage = (hr <= 0 || hr > 1.15 * rhr || act > 0.25) ? 'awake' + : (act < 0.01 && hr < rhr + 4) ? 'deep' : (hr > rhr + 5) ? 'rem' : 'light' + return { t: m.ts, stage } + }) + return { hypnogram: hyp, totals: null } } const clamp = (v: number, lo: number, hi: number) => Math.max(lo, Math.min(hi, v)) -interface Min { ts_min: number; hr_avg: number | null; hr_min: number | null; hr_max: number | null; activity: number | null; wrist_on: number | null } +interface Min { ts_min: number; hr_avg: number | null; hr_min: number | null; hr_max: number | null; activity: number | null; wrist_on: number | null; rr: number[] } async function loadMinutes(c: Ctx, from: number, to: number): Promise { // Tiered read: D1 for hot days, R2 fallback for sealed days (see minute_store). const rows = await readMinutes(c.env, c.get('userId'), from, to) - return rows.map((m) => ({ ts_min: m.ts_min, hr_avg: m.hr_avg, hr_min: m.hr_min, hr_max: m.hr_max, activity: m.activity, wrist_on: m.wrist_on })) + return rows.map((m) => ({ ts_min: m.ts_min, hr_avg: m.hr_avg, hr_min: m.hr_min, hr_max: m.hr_max, activity: m.activity, wrist_on: m.wrist_on, rr: m.rr ?? [] })) } async function loadHr(c: Ctx): Promise<{ rhr: number; maxHr: number }> { @@ -211,6 +232,9 @@ export async function getDaySleepV2(c: Ctx) { } const periods = rows.map((r: any) => { const pm = mins.filter((m) => m.ts_min >= r.onset_ts && m.ts_min <= (r.wake_ts ?? r.onset_ts)) + const ng = pm.length ? stageNight(pm, r.onset_ts, r.wake_ts ?? r.onset_ts, rhr) : { hypnogram: [], totals: null } + const storedStages = (r.light_min != null || r.deep_min != null || r.rem_min != null) + ? { light_min: r.light_min, deep_min: r.deep_min, rem_min: r.rem_min } : null return { id: r.id, onset_ts: r.onset_ts, @@ -218,11 +242,10 @@ export async function getDaySleepV2(c: Ctx) { duration_min: r.duration_min, in_bed_min: r.in_bed_min, efficiency: r.efficiency, - stages: (r.light_min != null || r.deep_min != null || r.rem_min != null) - ? { light_min: r.light_min, deep_min: r.deep_min, rem_min: r.rem_min } : null, + stages: ng.totals ?? storedStages, // same stageSleep source as the hypnogram → consistent is_main: !!r.is_main, confidence: r.confidence, - hypnogram: pm.length ? downsample(buildHypnogram(pm, rhr), 240) : [], + hypnogram: downsample(ng.hypnogram, 240), } }) return { @@ -251,22 +274,12 @@ export async function getDaySleep(c: Ctx) { return c.json({ date, has_sleep: false, need_min: need }) } const rhr = baseline?.resting_hr && baseline.resting_hr > 0 ? baseline.resting_hr : 55 - const mins = await loadMinutes(c, row.onset_ts, row.wake_ts + 60) + // WIDE window (prior evening → wake) so calcCircadian inside buildHypnogram sees the + // daytime HR peak and derives a real mesor; stageSleep itself only stages [onset, wake]. + const mins = await loadMinutes(c, row.onset_ts - 16 * 3600, row.wake_ts + 3600) - // Per-epoch hypnogram (BETA, consistent with the summary's beta stages): - // off-wrist / HR > 1.15*RHR → awake; very low activity + low HR → deep; - // low activity + higher HR (REM-ish) → rem; else light. - const hypnogram: { t: number; stage: string }[] = [] - for (const m of mins) { - const hr = m.hr_avg ?? 0 - const act = m.activity ?? 0 - let stage: string - if (hr <= 0 || hr > 1.15 * rhr || act > 0.25) stage = 'awake' - else if (act < 0.01 && hr < rhr + 4) stage = 'deep' - else if (hr > rhr + 5) stage = 'rem' - else stage = 'light' - hypnogram.push({ t: m.ts_min, stage }) - } + // ONE classifier (analytics stageSleep) → smoothed hypnogram + reconciled totals. + const ng = stageNight(mins, row.onset_ts, row.wake_ts, rhr) // Sleep-debt over the last 7 real nights (incl. this one). const { results: recent } = await c.env.DB.prepare( @@ -303,9 +316,9 @@ export async function getDaySleep(c: Ctx) { need_min: need, debt_min: Math.round(debt), regularity: row.regularity, - stages: { light_min: row.light_min, deep_min: row.deep_min, rem_min: row.rem_min }, + stages: ng.totals ?? { light_min: row.light_min, deep_min: row.deep_min, rem_min: row.rem_min }, stages_beta: true, - hypnogram: downsample(hypnogram, 240), + hypnogram: downsample(ng.hypnogram, 240), }) } From 660ef56f1a9f5d5bdd48e415b3943c722a0eeb22 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 20 Jun 2026 13:20:17 +0530 Subject: [PATCH 17/30] workouts: move auto-detect + stale-live-close on-read; */10 cron is wake-only detectSessions and autoCloseStaleWorkouts ran only inside the once-a-day close_day / every */10 tick, so an auto-detected workout didn't surface until the next wake and the frequent cron wasn't purely wake-detection. - new workouts.ensureTodayWorkouts(db,user): closes this user's forgotten LIVE workouts + (re)detects TODAY's auto sessions, mirroring processUser Pass-2's session block EXACTLY (delete auto-non-deleted, insert done/auto; manual/ edited/deleted untouched). THROTTLED via read_cache (~120s) so a burst of reads costs one detect, and only for users actively opening the app. - called on-read from listWorkouts, getSessions, and getDayStrain(today). - autoCloseStaleWorkouts now takes an optional userId scope (on-read = one user). - scheduled(): removed autoCloseStaleWorkouts from the every-*/10-tick path; the */10 cron now does ONLY runWakeLadder. Kept autoCloseStaleWorkouts on the nightly 30:3 tick as a safety net for users who never open the app. Validated on v2: /workouts + /sessions 200, detection runs + throttles (wkscan read_cache sentinel set), 0 false workouts on a non-workout night. --- src/daydetail.ts | 4 +++ src/index.ts | 7 +++-- src/query.ts | 2 ++ src/workouts.ts | 66 ++++++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 72 insertions(+), 7 deletions(-) diff --git a/src/daydetail.ts b/src/daydetail.ts index 8cd439c..85ce1e5 100644 --- a/src/daydetail.ts +++ b/src/daydetail.ts @@ -10,6 +10,7 @@ import type { Context } from 'hono' import { cached, ttlForDate } from './cache' import { readMinutes } from './minute_store' +import { ensureTodayWorkouts } from './workouts' import { calcCircadian, stageSleep } from 'openstrap-analytics' type Ctx = Context<{ Bindings: { DB: D1Database; RAW_BUCKET?: R2Bucket }; Variables: { userId: string } }> @@ -90,6 +91,9 @@ export async function getDayStrain(c: Ctx) { const date = (c.req.query('date') || '').trim() if (!/^\d{4}-\d{2}-\d{2}$/.test(date)) return c.json({ error: 'date=YYYY-MM-DD required' }, 400) const start = dayStartOf(date) + // For today, refresh auto-detected sessions on read (throttled) so the day's + // workout list isn't stale until the next wake-close. + if (date === new Date().toISOString().slice(0, 10)) await ensureTodayWorkouts(c.env.DB, c.get('userId')) const dr = await c.env.DB.prepare( 'SELECT strain, hr_zones, wear_min, strain_curve, hr_max, hr_min, hr_avg, acwr, fitness_trend, ' + diff --git a/src/index.ts b/src/index.ts index 28accf5..c81ef0e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -460,11 +460,14 @@ export default { // does maintenance only: prune aged minute/events + a retry-net for missed closes. async scheduled(event: ScheduledEvent, env: Bindings, ctx: ExecutionContext): Promise { ctx.waitUntil((async () => { - try { await autoCloseStaleWorkouts(env.DB) } catch (e) { console.error('autoclose failed', e) } - // EVERY tick — wake detection only (the cron's sole job). + // EVERY tick — wake detection ONLY (the cron's sole job). Auto-workout detection + // and forgotten-live-workout close moved ON-READ (workouts.ensureTodayWorkouts); + // the nightly tick keeps autoCloseStaleWorkouts as a safety net for users who + // never open the app. try { await runWakeLadder(env) } catch (e) { console.error('wake ladder failed', e) } // Nightly maintenance ONLY (separate from detection): seal + retention + retry-net. if (event.cron === '30 3 * * *') { + try { await autoCloseStaleWorkouts(env.DB) } catch (e) { console.error('autoclose failed', e) } // Seal days older than the hot window to gzipped R2 objects and drop them from // D1 (D1-hot / R2-sealed tiering — cuts D1 storage + per-row prune-deletes). try { await sealOldDays(env) } catch (e) { console.error('seal failed', e) } diff --git a/src/query.ts b/src/query.ts index e5494b7..13d68fe 100644 --- a/src/query.ts +++ b/src/query.ts @@ -5,6 +5,7 @@ import type { Context } from 'hono' import { calcFitnessTrend, type DayHistory } from 'openstrap-analytics' import { readMinutes, latestMinute } from './minute_store' +import { ensureTodayWorkouts } from './workouts' type Ctx = Context<{ Bindings: { DB: D1Database; RAW_BUCKET?: R2Bucket }; Variables: { userId: string } }> @@ -244,6 +245,7 @@ export const getStrain = rangeRows('daily') // GET /sessions?from&to — auto-detected workouts (by start_ts unix range). export async function getSessions(c: Ctx) { + await ensureTodayWorkouts(c.env.DB, c.get('userId')) // on-read auto-detect + stale close (throttled) const from = parseInt(c.req.query('from') || '0') const to = parseInt(c.req.query('to') || String(nowSec())) const { results } = await c.env.DB.prepare( diff --git a/src/workouts.ts b/src/workouts.ts index 6e7f4b6..4123b06 100644 --- a/src/workouts.ts +++ b/src/workouts.ts @@ -5,14 +5,17 @@ // blipped, the record is reconstructed from uploaded data on /workout/end. import type { Context } from 'hono' import { - calcStrain, calcHrZones, calcCalories, calcHrRecovery, + calcStrain, calcHrZones, calcCalories, calcHrRecovery, detectSessions, type Minute, type Profile, type Baseline, } from 'openstrap-analytics' import { readMinutes } from './minute_store' +import { cached } from './cache' type Ctx = Context<{ Bindings: { DB: D1Database }; Variables: { userId: string } }> const nowSec = () => Math.floor(Date.now() / 1000) const DAY = 86400 +const startOfDayUtc = (ts: number) => Math.floor(ts / DAY) * DAY +const ymd = (ts: number) => new Date(ts * 1000).toISOString().slice(0, 10) const VALID_TYPES = ['run', 'cycle', 'strength', 'walk', 'swim', 'cardio', 'yoga', 'other'] @@ -118,6 +121,7 @@ const RANGE_DAYS: Record = { week: 7, month: 35, quarter: 91, '7 // GET /workouts?range=week|month|quarter → list + training-volume summary. export async function listWorkouts(c: Ctx) { const userId = c.get('userId') + await ensureTodayWorkouts(c.env.DB, userId) // on-read auto-detect + stale-live close (throttled) const range = c.req.query('range') || 'month' const days = RANGE_DAYS[range] ?? 35 const from = nowSec() - days * DAY @@ -276,11 +280,13 @@ function deriveEffort( // so a forgotten "stop" never inflates duration with idle time. Returns # closed. const QUIET_MIN = 12 const MAX_LIVE_HOURS = 6 -export async function autoCloseStaleWorkouts(db: D1Database): Promise { +export async function autoCloseStaleWorkouts(db: D1Database, userId?: string): Promise { const now = nowSec() - const { results } = await db.prepare( - "SELECT user_id, id, start_ts FROM sessions WHERE status = 'live'", - ).all() + // Optionally scope to ONE user (on-demand read path); else all (nightly safety net). + const stmt = userId + ? db.prepare("SELECT user_id, id, start_ts FROM sessions WHERE status = 'live' AND user_id = ?").bind(userId) + : db.prepare("SELECT user_id, id, start_ts FROM sessions WHERE status = 'live'") + const { results } = await stmt.all() let closed = 0 for (const s of results ?? []) { const baseline = await loadBaseline(db, s.user_id) @@ -313,3 +319,53 @@ export async function autoCloseStaleWorkouts(db: D1Database): Promise { } function safeParse(s: string) { try { return JSON.parse(s) } catch { return null } } + +// ── On-demand auto-workout detection (replaces the cron sweep) ──────────────── +// detectSessions used to run only inside the once-a-day close_day (via processUser), +// so an auto-detected workout didn't surface until the next wake. We now run it +// ON READ for TODAY, mirroring processUser Pass-2's session block EXACTLY (delete +// auto-non-deleted, insert 'done'/'auto'; manual/edited/deleted sessions untouched), +// so the on-read path and the day-close produce identical rows. +async function detectStoreDay(db: D1Database, userId: string, dayStart: number, baseline: Baseline, profile: Profile): Promise { + const recs = await readMinutes({ DB: db }, userId, dayStart, dayStart + DAY) + if (!recs.length) return + const dayMin: Minute[] = recs.map((r) => ({ + ts: r.ts_min, hr_avg: r.hr_avg, hr_min: r.hr_min, hr_max: r.hr_max, + hr_n: r.hr_n, activity: r.activity, steps: r.steps, wrist_on: !!r.wrist_on, + })) + const sessions = detectSessions(dayMin, baseline, profile) + const stmts: D1PreparedStatement[] = [ + db.prepare("DELETE FROM sessions WHERE user_id = ? AND start_ts >= ? AND start_ts < ? AND (source IS NULL OR source = 'auto') AND status != 'deleted'") + .bind(userId, dayStart, dayStart + DAY), + ] + for (const s of sessions) { + const sid = `${userId}:${s.start_ts}` + stmts.push(db.prepare( + 'INSERT INTO sessions (user_id, id, start_ts, end_ts, type, avg_hr, max_hr, strain, calories, hrr60, zones, confidence, status, source) ' + + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,'done','auto') ON CONFLICT(user_id, id) DO UPDATE SET " + + 'end_ts=excluded.end_ts, type=excluded.type, avg_hr=excluded.avg_hr, max_hr=excluded.max_hr, ' + + 'strain=excluded.strain, calories=excluded.calories, hrr60=excluded.hrr60, zones=excluded.zones, confidence=excluded.confidence', + ).bind(userId, sid, s.start_ts, s.end_ts, s.type, Math.round(s.avg_hr), Math.round(s.max_hr), + s.strain, s.kcal, s.hrr60 == null ? null : Math.round(s.hrr60), JSON.stringify(s.zones), s.confidence)) + } + await db.batch(stmts) +} + +/** + * On-read freshness for workouts (the wake-trigger replacement for the cron sweep). + * Closes this user's forgotten LIVE workouts and (re)detects TODAY's auto sessions. + * THROTTLED via read_cache (~120s) so a burst of reads costs one detect, not N — and + * it only ever runs for users actively opening the app. Call from any workout- + * surfacing read endpoint. The nightly cron still runs autoCloseStaleWorkouts as a + * safety net for users who never open the app; close_day still re-derives at wake. + */ +export async function ensureTodayWorkouts(db: D1Database, userId: string): Promise { + const today = ymd(nowSec()) + await cached(db, userId, `wkscan:${today}`, 120, async () => { + await autoCloseStaleWorkouts(db, userId) + const baseline = await loadBaseline(db, userId) + const profile = await loadProfile(db, userId) + await detectStoreDay(db, userId, startOfDayUtc(nowSec()), baseline, profile) + return { scanned: nowSec() } + }) +} From c7088106dc90d1dff78a262d749a5aef1832ef03 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 20 Jun 2026 13:44:33 +0530 Subject: [PATCH 18/30] =?UTF-8?q?steps:=20AN-2554=20only=20=E2=80=94=20cou?= =?UTF-8?q?nt=20at=20ingest=20into=20minute.steps,=20sum=20on-read;=20drop?= =?UTF-8?q?=20steps=5Fimu?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Steps had three code paths: a per-record r10Motion heuristic (-> minute.steps, fed the chart), the full AN-2554 (calcSteps in ingest_signals -> computed then DISCARDED), and an R2 recompute (steps_imu -> daily.steps at close, lagged all day). Collapse to one: AN-2554, counted once at ingest, live on read. - minute_store.writeBatch: minute.steps += ingest_signals.steps (AN-2554), additive (idempotent via edge hex-dedup). rollup no longer sums the r10Motion s.steps_inc. - processUser folds daily.steps = SUM(this day's minute.steps) into the close (no R2), so past days keep a permanent total after minutes prune. - reads serve TODAY live: getToday + getDayStrain(today) = SUM(minute.steps) from the hot day blob (zero R2); past days use the stored daily.steps. - REMOVED steps_imu.ts entirely + the 'steps_full' queue job + runStepsIncremental from close_day/sweep + the /admin/run-steps endpoint. No step work in any cron now. Validated on v2: /today + /day/strain return live steps (104) from minute.steps; tsc clean. --- src/analytics.ts | 25 +++--- src/daydetail.ts | 12 ++- src/index.ts | 18 +---- src/minute_store.ts | 9 ++- src/query.ts | 9 ++- src/queue.ts | 15 ++-- src/rollup.ts | 3 +- src/steps_imu.ts | 192 -------------------------------------------- 8 files changed, 50 insertions(+), 233 deletions(-) delete mode 100644 src/steps_imu.ts diff --git a/src/analytics.ts b/src/analytics.ts index e7ecfc8..46c6cfa 100644 --- a/src/analytics.ts +++ b/src/analytics.ts @@ -141,6 +141,7 @@ interface DayBuf { calories: ReturnType sleep: Metric wearMin: number + daySteps: number // SUM of this day's AN-2554 minute.steps sleepStress: string // calcSleepStress JSON (nocturnal arousal) nocturnal: string // calcNocturnalHeart JSON sleepingHr: number | null // this night's sleeping-HR avg (for baseline) @@ -368,9 +369,14 @@ export async function processUser( s.strain, s.kcal, s.hrr60 == null ? null : Math.round(s.hrr60), JSON.stringify(s.zones), s.confidence)) } - // -- Wear time (worn minutes). Steps are owned by steps_imu.ts (AN-2554 over - // the raw IMU), written separately — processUser no longer computes them. -- + // -- Wear time (worn minutes). -- const wearMin = dayMin.filter((m) => m.wrist_on).length + // -- Steps = SUM of this day's per-minute AN-2554 counts (computed at ingest, + // stored in minute.steps). Folded into the daily row here at the close so past + // days have a permanent total after their minutes prune. NO R2, no recompute — + // steps_imu.ts is removed; AN-2554 is the only pedometer. Today's live total is + // served on-read by summing minute.steps (query.ts/daydetail.ts). -- + const daySteps = dayMin.reduce((a, m) => a + (m.steps ?? 0), 0) // -- Nocturnal heart (sleeping-HR dynamics over the main sleep period). -- const sleepWorn = (sleep.onset_ts && sleep.wake_ts) @@ -442,7 +448,7 @@ export async function processUser( dayBuffer.push({ // idx points into the parallel arrays, which now START with `seedLen` // seeded history days — so trailing slices in Pass 3 reach into real history. - date, dayStart, idx: seedLen + dayBuffer.length, rhr, strain, zones, calories, sleep, wearMin, + date, dayStart, idx: seedLen + dayBuffer.length, rhr, strain, zones, calories, sleep, wearMin, daySteps, sleepStress: JSON.stringify(sleepStress), nocturnal: JSON.stringify(nocturnal), sleepingHr: nocturnal.sleeping_hr_avg, @@ -459,7 +465,7 @@ export async function processUser( // trailing windows that end at that day (so ACWR/fitness/SRI/anomaly vary // across the history instead of collapsing to a single value). ── for (const buf of dayBuffer) { - const { date, idx, rhr, strain, zones, calories, sleep, wearMin, + const { date, idx, rhr, strain, zones, calories, sleep, wearMin, daySteps, sleepStress, nocturnal, nocturnalElevated, mainDrivers, strainCurve, hrMax, hrMin, hrAvg } = buf @@ -584,12 +590,11 @@ export async function processUser( // It writes `drivers` (main metrics) via COALESCE-free set but biometrics // read-merges, and on the hourly path (no biometrics) main drivers stand alone. statements.push(db.prepare( - // NOTE: `steps` is intentionally NOT written here — steps_imu.ts is the sole, - // authoritative writer (AN-2554 over the raw IMU). processUser must not clobber it. - 'INSERT INTO daily (user_id, date, strain, resting_hr, calories, wear_min, hr_zones, acwr, fitness_trend, anomaly, coach, nocturnal, sleep_stress, drivers, vo2max, fitness, fatigue, form, monotony, nocturnal_dip_pct, strain_curve, hr_max, hr_min, hr_avg, confidence, flags, updated_at) ' + - 'VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(user_id, date) DO UPDATE SET ' + + // `steps` = SUM of this day's AN-2554 minute counts (the only pedometer now). + 'INSERT INTO daily (user_id, date, strain, resting_hr, calories, wear_min, steps, hr_zones, acwr, fitness_trend, anomaly, coach, nocturnal, sleep_stress, drivers, vo2max, fitness, fatigue, form, monotony, nocturnal_dip_pct, strain_curve, hr_max, hr_min, hr_avg, confidence, flags, updated_at) ' + + 'VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(user_id, date) DO UPDATE SET ' + 'strain=excluded.strain, resting_hr=excluded.resting_hr, ' + - 'calories=excluded.calories, wear_min=excluded.wear_min, hr_zones=excluded.hr_zones, ' + + 'calories=excluded.calories, wear_min=excluded.wear_min, steps=excluded.steps, hr_zones=excluded.hr_zones, ' + 'acwr=excluded.acwr, fitness_trend=excluded.fitness_trend, anomaly=excluded.anomaly, coach=excluded.coach, ' + 'nocturnal=excluded.nocturnal, sleep_stress=excluded.sleep_stress, ' + 'drivers=json_patch(COALESCE(daily.drivers,\'{}\'), excluded.drivers), ' + @@ -599,7 +604,7 @@ export async function processUser( 'confidence=excluded.confidence, flags=excluded.flags, updated_at=excluded.updated_at', ).bind(userId, date, strain.score, rhr.resting_hr == null ? null : Math.round(rhr.resting_hr), calories.kcal, - wearMin, JSON.stringify(zones), load.acwr, fitness.direction, + wearMin, daySteps, JSON.stringify(zones), load.acwr, fitness.direction, bodyAlert, JSON.stringify(coach), nocturnal, sleepStress, driversJson, vo2.vo2max, fitModel.fitness, fitModel.fatigue, fitModel.form, monotony.monotony, nocDip, strainCurve, hrMax, hrMin, hrAvg, diff --git a/src/daydetail.ts b/src/daydetail.ts index 85ce1e5..46802fc 100644 --- a/src/daydetail.ts +++ b/src/daydetail.ts @@ -91,9 +91,17 @@ export async function getDayStrain(c: Ctx) { const date = (c.req.query('date') || '').trim() if (!/^\d{4}-\d{2}-\d{2}$/.test(date)) return c.json({ error: 'date=YYYY-MM-DD required' }, 400) const start = dayStartOf(date) + const isToday = date === new Date().toISOString().slice(0, 10) // For today, refresh auto-detected sessions on read (throttled) so the day's // workout list isn't stale until the next wake-close. - if (date === new Date().toISOString().slice(0, 10)) await ensureTodayWorkouts(c.env.DB, c.get('userId')) + if (isToday) await ensureTodayWorkouts(c.env.DB, c.get('userId')) + // Steps (AN-2554): live SUM of today's minute.steps; past days use the stored + // daily.steps (persisted by processUser at close). Zero R2 — hot day blob. + let liveSteps: number | null = null + if (isToday) { + const mins = await readMinutes(c.env, c.get('userId'), start, start + DAY) + liveSteps = mins.reduce((a, m) => a + (m.steps ?? 0), 0) + } const dr = await c.env.DB.prepare( 'SELECT strain, hr_zones, wear_min, strain_curve, hr_max, hr_min, hr_avg, acwr, fitness_trend, ' + @@ -129,7 +137,7 @@ export async function getDayStrain(c: Ctx) { ? { fitness: dr?.fitness ?? null, fatigue: dr?.fatigue ?? null, form: dr?.form ?? null } : null, monotony: dr?.monotony ?? null, calories: dr?.calories ?? null, - steps: dr?.steps ?? null, + steps: liveSteps ?? dr?.steps ?? null, drivers: dr?.drivers ? safe(dr.drivers) : null, sessions: (sessions ?? []).map((s: any) => ({ ...s, zones: s.zones ? safe(s.zones) : null, diff --git a/src/index.ts b/src/index.ts index c81ef0e..6ad2192 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,7 +12,6 @@ import { getHistory } from './history' import { postJournal, getJournal, getJournalInsights } from './journal' import { getDayStrain, getDaySleep, getDaySleepV2, getDayTimeline, getDayStress, getDayHeart, getDayLungs, getDayWear } from './daydetail' import { getTrend } from './trend' -import { runStepsImu, runStepsIncremental } from './steps_imu' import { workoutStart, workoutEnd, listWorkouts, getWorkout, deleteWorkout, autoCloseStaleWorkouts } from './workouts' import { getRecords } from './records' import { getNotifications, markNotificationsRead } from './notifications' @@ -315,21 +314,8 @@ app.post('/admin/run-biometrics', async (c) => { return c.json({ ok: true, ...res, capped: days < reqDays, max_days: RAW_RETENTION_DAYS }) }) -// Steps from the wrist IMU (AN-2554 over raw R2). mode=incremental (default) counts -// only newly-settled minutes since the cursor; mode=full recomputes the last `days` -// (capped to R2 retention) and realigns the cursor. Heavy (R2) → admin / cron only. -app.post('/admin/run-steps', async (c) => { - const body = await c.req.json<{ user_id: string; mode?: 'incremental' | 'full'; days?: number }>().catch(() => ({} as any)) - if (!body.user_id) return c.json({ error: 'user_id required' }, 400) - if (body.mode === 'full') { - const reqDays = body.days ?? 1 - const days = Math.min(reqDays, RAW_RETENTION_DAYS) - const res = await runStepsImu(c.env, body.user_id, days) - return c.json({ ok: true, mode: 'full', ...res, capped: days < reqDays }) - } - const res = await runStepsIncremental(c.env, body.user_id) - return c.json({ ok: true, mode: 'incremental', ...res }) -}) +// (Steps are AN-2554-only now: counted at ingest into minute.steps, summed into +// daily.steps by processUser, served live on-read. No R2 step job / admin endpoint.) // Phased to stay under the free-plan per-request subrequest cap. The shell // orchestrates: init → minutes (chunked) → analytics. `now` is pinned across diff --git a/src/minute_store.ts b/src/minute_store.ts index 2184a2e..e6145bf 100644 --- a/src/minute_store.ts +++ b/src/minute_store.ts @@ -136,9 +136,14 @@ export async function writeBatch( rec.hr_max = Math.max(rec.hr_max, b.hr_max) rec.act_sum += b.act_sum rec.act_n += b.act_n - rec.steps += b.steps rec.wrist_on = Math.max(rec.wrist_on, b.wrist_on) - const newRr = signals.get(b.ts_min)?.rr ?? [] + // Steps: accumulate the AN-2554 per-minute count from the ingest signals — the + // SINGLE step source now (the per-record r10Motion heuristic and the R2 steps_imu + // recompute are both gone). Additive across batches; idempotent because the edge + // dedupes by raw hex, so a minute's frames are never counted twice. + const sig = signals.get(b.ts_min) + rec.steps += sig?.steps ?? 0 + const newRr = sig?.rr ?? [] if (newRr.length >= rec.rr.length) rec.rr = newRr rec.hr_avg = rec.hr_n > 0 ? Math.round(rec.hr_sum / rec.hr_n) : 0 rec.activity = rec.act_n > 0 ? rec.act_sum / rec.act_n : 0 diff --git a/src/query.ts b/src/query.ts index 13d68fe..d30db67 100644 --- a/src/query.ts +++ b/src/query.ts @@ -48,6 +48,13 @@ export async function getToday(c: Ctx) { // Live status: most recent minute (HR + worn) in the last 10 minutes (day-packed store). const live = await latestMinute(c.env, userId, nowSec() - 600, nowSec()) + // Steps (AN-2554): live total = SUM of today's per-minute counts (counted at ingest + // into minute.steps). close_day persists the same SUM to daily.steps for past days; + // for TODAY we sum live so the ring updates during the day. Zero R2 (hot day blob). + const dayStart = Math.floor(Date.parse(`${date}T00:00:00Z`) / 1000) + const todayMins = await readMinutes(c.env, userId, dayStart, dayStart + 86400) + const liveSteps = todayMins.reduce((a, m) => a + (m.steps ?? 0), 0) + const df = parseFlags(daily?.flags) const sf = parseFlags(sleep?.flags) @@ -120,7 +127,7 @@ export async function getToday(c: Ctx) { fitness: { value: daily.fitness ?? null, unit: '', confidence: daily.fitness != null ? 0.6 : 0, tier: 'ESTIMATE', label: 'Fitness' }, form: { value: daily.form ?? null, unit: '', confidence: daily.form != null ? 0.6 : 0, tier: 'ESTIMATE', label: 'Form' }, calories: metric(daily.calories, 'kcal', 'Active calories (est.)', df, 'calories'), - steps: metric(daily.steps, 'steps', 'Steps (est.)', df, 'steps'), + steps: metric(liveSteps, 'steps', 'Steps (est.)', df, 'steps'), // Wear is a direct count of worn minutes — full confidence when present // (otherwise the UI hides any metric with null/0 confidence). wear_min: { ...metric(daily.wear_min, 'min', 'Worn', df, 'wear'), confidence: daily.wear_min != null ? 1 : 0, tier: 'AUTH' }, diff --git a/src/queue.ts b/src/queue.ts index e274aa7..01ffc2d 100644 --- a/src/queue.ts +++ b/src/queue.ts @@ -8,10 +8,11 @@ import { processUser } from './analytics' import { runBiometrics } from './biometrics' import { runBiometricsMinute } from './biometrics_minute' import { runRespRate } from './resp' -import { runStepsImu, runStepsIncremental } from './steps_imu' import { invalidateDay } from './cache' -export type AnalyticsJob = 'sweep' | 'biometrics' | 'resp' | 'steps_full' | 'close_day' +// Steps are AN-2554 only, counted at ingest into minute.steps and summed into +// daily.steps by processUser — no R2 step recompute job (steps_imu removed). +export type AnalyticsJob = 'sweep' | 'biometrics' | 'resp' | 'close_day' export interface AnalyticsMessage { user_id: string @@ -70,8 +71,9 @@ async function runJob(env: QueueEnv, userId: string, job: AnalyticsJob, day?: st // in HRV/recovery from D1 minute.rr — zero R2. Then invalidates the day's Tier-2 // cache and clears the dirty flag (daytime ingests re-set it; the cron skips // awake-and-closed users by last_close_date, so no churn). + // processUser derives the day AND folds daily.steps = SUM(minute.steps) (AN-2554 + // counted at ingest) — no separate step job. await processUser(env.DB, userId, { historyDays: 3 }) - await runStepsIncremental(env, userId) if (day && wake_ts) { const from = onset_ts ?? (wake_ts - 8 * 3600) try { await runBiometricsMinute({ DB: env.DB, RAW_BUCKET: env.RAW_BUCKET }, userId, day, from, wake_ts + 60) } catch (e) { console.error('biometrics_minute failed', userId, day, e) } @@ -89,15 +91,10 @@ async function runJob(env: QueueEnv, userId: string, job: AnalyticsJob, day?: st case 'resp': await runRespRate(env, userId, 2, day) break - case 'steps_full': - // Full AN-2554 true-up for late-arriving frames (realigns the incremental cursor). - await runStepsImu(env, userId, 2, day) - break case 'sweep': default: - // The frequent path: derive daily/sleep/strain (D1) + incremental steps (bounded R2). + // The frequent path: derive daily/sleep/strain (D1); steps folded in by processUser. await processUser(env.DB, userId, { historyDays: 3 }) - await runStepsIncremental(env, userId) // Event-driven: if this sweep just finished a night, fire biometrics for it. await maybeTriggerBiometrics(env, userId) break diff --git a/src/rollup.ts b/src/rollup.ts index f606bbb..6442a52 100644 --- a/src/rollup.ts +++ b/src/rollup.ts @@ -46,7 +46,8 @@ export function rollupMinutes(samples: DecodedSample[]): MinuteBucket[] { } bk.act_sum += s.activity bk.act_n += 1 - bk.steps += s.steps_inc + // steps no longer come from the per-record r10Motion heuristic (s.steps_inc) — + // ingest_signals' AN-2554 (calcSteps) is the sole step source, folded in writeBatch. if (s.wrist_on) bk.wrist_on = 1 } return [...buckets.values()].sort((a, b) => a.ts_min - b.ts_min) diff --git a/src/steps_imu.ts b/src/steps_imu.ts deleted file mode 100644 index 4347b33..0000000 --- a/src/steps_imu.ts +++ /dev/null @@ -1,192 +0,0 @@ -// steps_imu.ts — the STEPS RUNNER (I/O only). The band has no pedometer field, so -// steps are derived from the wrist accelerometer. This file does the heavy, -// binding-dependent work — list + read the raw IMU frames from R2, dedup, order, -// group per minute — then hands the per-minute magnitude signals to the PURE -// AN-2554 pedometer in openstrap-analytics (calcSteps) and persists the result. -// -// Mirrors the runBiometrics / runRespRate pattern: decode (openstrap-protocol) + -// math (openstrap-analytics) live elsewhere; this owns only R2 reads and the -// daily.steps write. -// -// Heavy (R2 reads) → cron / admin only, NEVER inline ingest. Owns daily.steps -// (written AFTER analytics in the cron so it's authoritative). - -import { calcSteps } from 'openstrap-analytics' -import { frameAccel, type ImuFrame } from 'openstrap-protocol/ts/live' - -const DAY = 86400 - -interface StepsEnv { DB: D1Database; RAW_BUCKET: R2Bucket } - -async function rawKeysInWindow(bucket: R2Bucket, userId: string, from: number, to: number): Promise { - const out: string[] = [] - let cursor: string | undefined - do { - const listing = await bucket.list({ prefix: `raw/${userId}/`, cursor, limit: 1000 }) - for (const o of listing.objects) { - const m = o.key.match(/-(\d+)-(\d+)\.txt$/) - if (!m) { out.push(o.key); continue } - if (parseInt(m[2]) >= from && parseInt(m[1]) <= to) out.push(o.key) - } - cursor = listing.truncated ? listing.cursor : undefined - } while (cursor) - return out -} - -// Steps over an arbitrary [from, to) window: assemble per-minute accel (ordered by -// ts,idx), then run the pure AN-2554 pedometer (analytics calcSteps). calcSteps is -// per-minute-independent (it sums pedometer(minute) × GAIN), so the count over a -// window is exactly the sum of the counts over any partition of it into whole -// minutes — which is what makes incremental accumulation exact. -async function computeWindowSteps(env: StepsEnv, userId: string, from: number, to: number): Promise { - const keys = await rawKeysInWindow(env.RAW_BUCKET, userId, from, to) - // DEDUP by (ts, idx): R2 upload windows overlap, so the same frame appears in - // multiple objects — without this, samples (and steps) are double-counted. - const seen = new Map() - for (const key of keys) { - const obj = await env.RAW_BUCKET.get(key) - if (!obj) continue - const text = await obj.text() - for (const line of text.split('\n')) { - if (!line) continue - const f = frameAccel(line) - if (!f || f.ts < from || f.ts >= to) continue - seen.set(`${f.ts}:${f.idx}`, f) - } - } - // Group frames per minute, ordered by (ts, idx), into one contiguous signal each. - const byMin = new Map() - for (const f of seen.values()) { - const m = Math.floor(f.ts / 60) * 60 - const arr = byMin.get(m); if (arr) arr.push(f); else byMin.set(m, [f]) - } - const minuteSignals: number[][] = [] - for (const frames of byMin.values()) { - frames.sort((a, b) => a.ts - b.ts || a.idx - b.idx) - const sig: number[] = [] - for (const f of frames) for (const v of f.mags) sig.push(v) - minuteSignals.push(sig) - } - return calcSteps(minuteSignals) -} - -// Minutes are considered "settled" (safe to count incrementally) once they're this -// old, giving late/bursty band syncs time to land. Anything later is caught by the -// nightly full recompute. 30 min matches the sweep cadence. -const SETTLE_GRACE = 1800 - -function ymd(dayStart: number): string { - return new Date(dayStart * 1000).toISOString().slice(0, 10) -} - -/** - * runStepsIncremental — the 30-min-sweep path. Counts only the minutes that have - * newly SETTLED since the last run (a few R2 objects) and accumulates into - * daily.steps via a per-user cursor. Exact for settled data (per-minute pedometer); - * the nightly full recompute trues up any late-arriving frames. Cheap: bounded R2 - * reads per run regardless of how long the day is or how many users. - */ -export async function runStepsIncremental(env: StepsEnv, userId: string): Promise<{ added: number }> { - const now = Math.floor(Date.now() / 1000) - const todayStart = Math.floor(now / DAY) * DAY - const date = ymd(todayStart) - const settledCutoff = Math.floor((now - SETTLE_GRACE) / 60) * 60 // last fully-settled minute boundary - - const cur = await env.DB.prepare( - 'SELECT steps_cursor_ts, steps_cursor_day FROM analytics_cursor WHERE user_id = ?', - ).bind(userId).first<{ steps_cursor_ts: number | null; steps_cursor_day: string | null }>() - const sameDay = !!cur && cur.steps_cursor_day === date && cur.steps_cursor_ts != null - const cursor = sameDay ? cur!.steps_cursor_ts! : todayStart - - const setCursor = env.DB.prepare( - 'INSERT INTO analytics_cursor (user_id, steps_cursor_ts, steps_cursor_day) VALUES (?,?,?) ' + - 'ON CONFLICT(user_id) DO UPDATE SET steps_cursor_ts=excluded.steps_cursor_ts, steps_cursor_day=excluded.steps_cursor_day', - ) - - if (settledCutoff <= cursor) { - // Nothing newly settled. On a fresh day, still seed the row at 0 + set cursor. - if (!sameDay) { - await env.DB.batch([ - env.DB.prepare('INSERT OR IGNORE INTO daily(user_id, date) VALUES(?,?)').bind(userId, date), - env.DB.prepare('UPDATE daily SET steps = 0 WHERE user_id = ? AND date = ?').bind(userId, date), - setCursor.bind(userId, todayStart, date), - ]) - } - return { added: 0 } - } - - const chunk = await computeWindowSteps(env, userId, cursor, settledCutoff) - if (sameDay) { - // Compare-and-swap on the cursor: apply this chunk AND advance the cursor only - // if no concurrent writer moved the cursor since we read it (line above). A - // nightly full recompute (runStepsImu, the 'steps_full' job) SETs today's steps - // and the cursor to settledCutoff; it and this incremental 'sweep' are distinct - // queue jobs that aren't deduped against each other, so they CAN run together - // (both are enqueued by the 3:30 cron). Without the guard, a full landing - // between our cursor read and our slow R2 compute would let us add a stale chunk - // on top of the full count → double-count for the day. If the cursor moved, our - // chunk is stale: both statements no-op and the next sweep recomputes from the - // new cursor. D1 batch is one serialized transaction, so the guard is reliable. - await env.DB.batch([ - env.DB.prepare('INSERT OR IGNORE INTO daily(user_id, date) VALUES(?,?)').bind(userId, date), - env.DB.prepare( - 'UPDATE daily SET steps = COALESCE(steps,0) + ? WHERE user_id = ? AND date = ? ' + - 'AND (SELECT steps_cursor_ts FROM analytics_cursor WHERE user_id = ?) = ?', - ).bind(chunk, userId, date, userId, cursor), - env.DB.prepare( - 'UPDATE analytics_cursor SET steps_cursor_ts = ? WHERE user_id = ? AND steps_cursor_day = ? AND steps_cursor_ts = ?', - ).bind(settledCutoff, userId, date, cursor), - ]) - } else { - // Fresh day: REPLACE (not accumulate) with the full settled count, which is - // idempotent with a concurrent full recompute (both write the same window). - await env.DB.batch([ - env.DB.prepare('INSERT OR IGNORE INTO daily(user_id, date) VALUES(?,?)').bind(userId, date), - env.DB.prepare('UPDATE daily SET steps = ? WHERE user_id = ? AND date = ?').bind(chunk, userId, date), - setCursor.bind(userId, settledCutoff, date), - ]) - } - return { added: chunk } -} - -/** - * runStepsImu — FULL recompute for the last `days` UTC days from R2 IMU. The - * authoritative true-up: catches late-arriving frames the incremental path may - * have missed. For TODAY it also realigns the incremental cursor so the sweep - * continues from the trued-up baseline without double-counting. - */ -export async function runStepsImu(env: StepsEnv, userId: string, days = 1, onlyDate?: string): Promise<{ days: number; total: number }> { - const now = Math.floor(Date.now() / 1000) - const todayStart = Math.floor(now / DAY) * DAY - const settledCutoff = Math.floor((now - SETTLE_GRACE) / 60) * 60 - let grand = 0 - // Per-(user,day) fan-out: onlyDate → just that day; else the trailing `days`. - const dayStarts = onlyDate - ? [Math.floor(Date.parse(`${onlyDate}T00:00:00Z`) / 1000)] - : Array.from({ length: days }, (_, d) => Math.floor((now - d * DAY) / DAY) * DAY) - for (const dayStart of dayStarts) { - const date = ymd(dayStart) - if (dayStart === todayStart) { - // Today: count only settled minutes (consistent with the incremental path) - // and realign the cursor so the next sweep appends from here. - const steps = await computeWindowSteps(env, userId, todayStart, settledCutoff) - await env.DB.batch([ - env.DB.prepare('INSERT OR IGNORE INTO daily(user_id, date) VALUES(?,?)').bind(userId, date), - env.DB.prepare('UPDATE daily SET steps = ? WHERE user_id = ? AND date = ?').bind(steps, userId, date), - env.DB.prepare( - 'INSERT INTO analytics_cursor (user_id, steps_cursor_ts, steps_cursor_day) VALUES (?,?,?) ' + - 'ON CONFLICT(user_id) DO UPDATE SET steps_cursor_ts=excluded.steps_cursor_ts, steps_cursor_day=excluded.steps_cursor_day', - ).bind(userId, settledCutoff, date), - ]) - grand += steps - } else { - // Past days are fully settled → count the whole day. - const steps = await computeWindowSteps(env, userId, dayStart, dayStart + DAY) - await env.DB.prepare('INSERT OR IGNORE INTO daily(user_id, date) VALUES(?,?)').bind(userId, date).run() - await env.DB.prepare('UPDATE daily SET steps = ? WHERE user_id = ? AND date = ?') - .bind(steps, userId, date).run() - grand += steps - } - } - return { days, total: grand } -} From 91b31f6e9fd879d85af126aeeb32d7927d5644ed Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 20 Jun 2026 16:18:34 +0530 Subject: [PATCH 19/30] feat(v2): SpO2 (red/IR) + skin-temp indices on the minute path Ingest aggregates per-minute red/IR/temp ADCs (wrist-on) into the minute blob; biometrics_minute derives a RELATIVE red/IR SpO2 index (confidence- gated) + skin-temp index vs rolling EWMA baselines, feeds temp into illness. Zero extra R2 (reads the D1 minute blob). admin/enqueue forwards onset/wake for close_day verification. Validated end-to-end on v2. --- src/biometrics_minute.ts | 68 +++++++++++++++++++++++++++++++++------- src/index.ts | 11 +++++-- src/ingest_signals.ts | 25 +++++++++++++-- src/minute_store.ts | 13 +++++++- 4 files changed, 99 insertions(+), 18 deletions(-) diff --git a/src/biometrics_minute.ts b/src/biometrics_minute.ts index ff4df5c..9223366 100644 --- a/src/biometrics_minute.ts +++ b/src/biometrics_minute.ts @@ -3,8 +3,9 @@ // The demand-driven engine stores per-minute RR at ingest (minute.rr), so the // headline HRV stack (RMSSD/SDNN/LF-HF, Plews recovery, Baevsky stress, illness, // irregular screen) is computed from D1 — no R2 list/get, no re-decode. Skin-temp -// and SpO₂ (raw ADCs @64/@68) are NOT in the minute rollup, so they stay R2-only -// (rare admin re-derive); we pass null → COALESCE keeps any prior value. +// and SpO₂ now ride the same minute blob: ingest aggregates per-minute red/IR/temp +// ADCs (wrist-on), and the close path derives a RELATIVE skin-temp index + a red/IR +// SpO₂ index (confidence-gated) vs rolling personal baselines — still zero extra R2. // // Mirrors runBiometrics' write contract exactly (same daily columns, same // null-safe COALESCE / json_patch drivers merge) so /today and the trend caches @@ -13,8 +14,9 @@ import { timeDomainHrv, freqDomainHrv, baevskyStressIndex, calcRecovery, calcStress, calcIllness, calcHrvStability, calcIrregular, calcReadinessIndex, + calcSpo2Index, median, } from 'openstrap-analytics' -import { loadDayRr } from './dayseries' +import { readMinutes } from './minute_store' // [v3] RAW_BUCKET + MINUTE_SOURCE: RR comes via loadDayRr (D1 minute.rr by default, // R2-decoded series when MINUTE_SOURCE='r2'). @@ -32,11 +34,19 @@ export async function runBiometricsMinute( ): Promise<{ computed: boolean }> { const now = Math.floor(Date.now() / 1000) - // 1. RR stream over the sleep window, time-ordered (minute.rr / R2 series). - const rrByMin = await loadDayRr(env, userId, from, to) + // 1. Read the sleep window's minutes ONCE (D1 hot + R2 sealed) → RR stream + optical + // aggregates. Single read keeps the close path cheap (no extra R2). + const recs = await readMinutes(env, userId, from, to) const rr: number[] = [] - for (const ts of [...rrByMin.keys()].sort((a, b) => a - b)) { - for (const v of rrByMin.get(ts)!) if (v >= 300 && v <= 2000) rr.push(v) + const spo2Ratios: number[] = [] + const tempVals: number[] = [] + for (const m of recs) { + for (const v of m.rr ?? []) if (v >= 300 && v <= 2000) rr.push(v) + // Optical: per-minute mean red/IR ratio + mean temp (wrist-on samples only at ingest). + if (m.opt_n && m.opt_n > 0 && m.ir_sum && m.ir_sum > 0) { + spo2Ratios.push((m.red_sum ?? 0) / m.ir_sum) + tempVals.push((m.temp_sum ?? 0) / m.opt_n) + } } const td = timeDomainHrv(rr) @@ -57,19 +67,36 @@ export async function runBiometricsMinute( const rhrByDate = new Map(histRows.map((h) => [h.date, h.resting_hr])) const baseRow = await env.DB.prepare( - 'SELECT sleep_need_min FROM baselines WHERE user_id = ?', - ).bind(userId).first<{ sleep_need_min: number | null }>() + // spo2_raw holds the rolling baseline red/IR RATIO (v2 uses the ratio, not raw red); + // skin_temp_raw holds the rolling baseline temp ADC. + 'SELECT sleep_need_min, spo2_raw, skin_temp_raw FROM baselines WHERE user_id = ?', + ).bind(userId).first<{ sleep_need_min: number | null; spo2_raw: number | null; skin_temp_raw: number | null }>() const sleepNeedMin = baseRow?.sleep_need_min ?? null const sleepRow = await env.DB.prepare( 'SELECT duration_min FROM sleep WHERE user_id = ? AND date = ?', ).bind(userId, date).first<{ duration_min: number | null }>() - // 3. Derived HRV metrics (published; temp/spo2 omitted — not in minute.rr). + // 3a. Optical metrics (RELATIVE): red/IR SpO₂ index + skin-temp index, vs rolling + // personal baselines. Confidence-gated in analytics (noisy nights → low conf/null). + const baseRatio = baseRow?.spo2_raw ?? null + const baseTemp = baseRow?.skin_temp_raw ?? null + const spo2 = calcSpo2Index(spo2Ratios, baseRatio) + const nightTemp = tempVals.length >= 30 ? median(tempVals) : null + const tempIdx = (nightTemp != null && baseTemp != null) ? Math.round((nightTemp - baseTemp) * 10) / 10 : null + // EWMA-roll the baselines (α=0.1) so they adapt slowly; seed on first valid night. + const newBaseRatio = spo2.night_ratio != null + ? (baseRatio != null ? Math.round((baseRatio * 0.9 + spo2.night_ratio * 0.1) * 10000) / 10000 : spo2.night_ratio) + : null + const newBaseTemp = nightTemp != null + ? (baseTemp != null ? Math.round((baseTemp * 0.9 + nightTemp * 0.1) * 10) / 10 : Math.round(nightTemp * 10) / 10) + : null + + // 3b. Derived HRV metrics (published; SpO₂/temp now sourced from minute optical aggregates). const conf = Math.round(Math.min(1, td.n_beats / 500) * 1000) / 1000 const recovery = calcRecovery(td.rmssd, rmssdHist, { date }) const stress = calcStress(rr, siHist, { date }) const illness = calcIllness( - { resting_hr: rhrByDate.get(date) ?? null, rmssd: td.rmssd, skin_temp: null }, + { resting_hr: rhrByDate.get(date) ?? null, rmssd: td.rmssd, skin_temp: tempIdx }, { resting_hr: rhrHist, rmssd: rmssdHist, skin_temp: tempHist }, ) const hrvStab = calcHrvStability([td.rmssd, ...rmssdHist].filter((x): x is number => x != null).slice(0, 14)) @@ -95,20 +122,37 @@ export async function runBiometricsMinute( if (illness.drivers) bioDrivers.illness = illness.drivers if (readiness.drivers) bioDrivers.readiness = readiness.drivers - // 5. Persist (same null-safe COALESCE + json_patch contract as runBiometrics; temp/spo2 → null). + if (spo2.drivers) bioDrivers.spo2 = spo2.drivers + + // 5. Persist (same null-safe COALESCE + json_patch contract as runBiometrics). SpO₂/temp + // now sourced from minute optical aggregates → spo2_idx / skin_temp_idx (RELATIVE). await env.DB.prepare( 'UPDATE daily SET recovery = ?, hrv_rmssd = COALESCE(?, hrv_rmssd), hrv_conf = ?, ' + 'hrv_sdnn = COALESCE(?, hrv_sdnn), hrv_lfhf = COALESCE(?, hrv_lfhf), hrv_si = COALESCE(?, hrv_si), ' + 'hrv_cv = COALESCE(?, hrv_cv), irregular = ?, readiness = ?, ' + 'resp_rate = COALESCE(?, resp_rate), resp_conf = COALESCE(?, resp_conf), ' + + 'spo2_idx = COALESCE(?, spo2_idx), skin_temp_idx = COALESCE(?, skin_temp_idx), ' + 'stress = ?, illness = ?, drivers = json_patch(COALESCE(drivers, \'{}\'), ?), updated_at = ? ' + 'WHERE user_id = ? AND date = ?', ).bind( recovery.score, td.rmssd, conf, td.sdnn, fd.lf_hf, si.si, hrvStab.cv, JSON.stringify(irregular), readiness.score, fd.resp_conf >= 0.3 ? fd.resp_rate : null, fd.resp_conf >= 0.3 ? fd.resp_conf : null, + // SpO₂ index only when confidence is meaningful (noisy nights → keep prior, don't regress). + spo2.index != null && spo2.confidence >= 0.3 ? spo2.index : null, tempIdx, JSON.stringify(stress), JSON.stringify(illness), JSON.stringify(bioDrivers), now, userId, date, ).run() + + // 5b. Roll the optical baselines (EWMA). Only when we got a fresh night value; null-safe. + if (newBaseRatio != null || newBaseTemp != null) { + await env.DB.prepare( + 'UPDATE baselines SET spo2_raw = COALESCE(?, spo2_raw), skin_temp_raw = COALESCE(?, skin_temp_raw), updated_at = ? WHERE user_id = ?', + ).bind(newBaseRatio, newBaseTemp, now, userId).run() + // Ensure a baselines row exists (first-ever night for a brand-new user). + await env.DB.prepare( + 'INSERT OR IGNORE INTO baselines(user_id, spo2_raw, skin_temp_raw, updated_at) VALUES(?,?,?,?)', + ).bind(userId, newBaseRatio, newBaseTemp, now).run() + } return { computed: true } } diff --git a/src/index.ts b/src/index.ts index 6ad2192..60aeb28 100644 --- a/src/index.ts +++ b/src/index.ts @@ -371,10 +371,17 @@ app.post('/admin/issue-token', async (c) => { // Enqueue an analytics job for a user (ops + verification). The consumer runs it // in its own bounded invocation. 404 if Queues isn't bound. app.post('/admin/enqueue', async (c) => { - const body = await c.req.json<{ user_id: string; job?: AnalyticsJob; day?: string }>().catch(() => ({} as any)) + const body = await c.req.json<{ user_id: string; job?: AnalyticsJob; day?: string; onset_ts?: number; wake_ts?: number }>().catch(() => ({} as any)) if (!body.user_id) return c.json({ error: 'user_id required' }, 400) if (!c.env.ANALYTICS_Q) return c.json({ error: 'queue not bound' }, 404) - const msg = { user_id: body.user_id, job: body.job ?? 'sweep', ...(body.day ? { day: body.day } : {}) } + const msg = { + user_id: body.user_id, job: body.job ?? 'sweep', + ...(body.day ? { day: body.day } : {}), + // Forward the sleep window so a close_day job can be fired for verification + // (the wake_cron supplies these in production). + ...(body.onset_ts ? { onset_ts: body.onset_ts } : {}), + ...(body.wake_ts ? { wake_ts: body.wake_ts } : {}), + } await c.env.ANALYTICS_Q.send(msg) return c.json({ ok: true, enqueued: msg }) }) diff --git a/src/ingest_signals.ts b/src/ingest_signals.ts index 765a695..ddebf80 100644 --- a/src/ingest_signals.ts +++ b/src/ingest_signals.ts @@ -20,14 +20,22 @@ import { parse_r24 } from 'openstrap-protocol/ts/records' export interface MinuteSignal { steps: number rr: number[] + // Optical aggregates from wrist-on R24 (running sums + count). RELATIVE raw ADCs; + // the close path turns red/IR → SpO₂ index and temp → skin-temp index. + opt_n?: number + red_sum?: number + ir_sum?: number + temp_sum?: number } interface AccelFrame { idx: number; ts: number; mags: number[] } +interface OpticalAcc { n: number; red: number; ir: number; temp: number } -/** Build per-minute {steps, rr} from a batch of hex records. Pure; no I/O. */ +/** Build per-minute {steps, rr, optical} from a batch of hex records. Pure; no I/O. */ export function perMinuteSignals(records: string[]): Map { const accelByMin = new Map>() const rrByMin = new Map() + const optByMin = new Map() for (const hex of records) { let b: Uint8Array @@ -42,6 +50,13 @@ export function perMinuteSignals(records: string[]): Map { const arr = rrByMin.get(m) ?? [] for (const v of r.rr_intervals_ms) arr.push(v) // raw; gated by analytics cleanRr below rrByMin.set(m, arr) + // Optical: wrist-on only (hr>0 — off-wrist optical is meaningless). Sum red/IR/temp + // raw ADCs + count → per-minute means at the close, baseline-relative index there. + if (r.hr > 0 && r.spo2_ir_raw > 0) { + const o = optByMin.get(m) ?? { n: 0, red: 0, ir: 0, temp: 0 } + o.n += 1; o.red += r.spo2_red_raw; o.ir += r.spo2_ir_raw; o.temp += r.skin_temp_raw + optByMin.set(m, o) + } } continue } @@ -68,7 +83,7 @@ export function perMinuteSignals(records: string[]): Map { } const out = new Map() - const minutes = new Set([...accelByMin.keys(), ...rrByMin.keys()]) + const minutes = new Set([...accelByMin.keys(), ...rrByMin.keys(), ...optByMin.keys()]) for (const m of minutes) { let steps = 0 const frames = accelByMin.get(m) @@ -80,7 +95,11 @@ export function perMinuteSignals(records: string[]): Map { } // Single library gate (300–2000 ms + ectopic |Δ|>200ms drop) — logic lives in // analytics, not duplicated here. - out.set(m, { steps, rr: cleanRr(rrByMin.get(m) ?? []) }) + const o = optByMin.get(m) + out.set(m, { + steps, rr: cleanRr(rrByMin.get(m) ?? []), + ...(o ? { opt_n: o.n, red_sum: o.red, ir_sum: o.ir, temp_sum: o.temp } : {}), + }) } return out } diff --git a/src/minute_store.ts b/src/minute_store.ts index e6145bf..0df912e 100644 --- a/src/minute_store.ts +++ b/src/minute_store.ts @@ -33,6 +33,10 @@ export interface MinuteRec { activity: number; act_sum: number; act_n: number steps: number; wrist_on: number rr: number[] + // Optical aggregates (wrist-on R24 only) — running sums + count so merges stay + // exact and re-uploads can't double-count (edge dedupes by hex). RELATIVE raw ADCs; + // SpO₂/°C are derived in the close path, never on-band. Optional (older blobs lack them). + opt_n?: number; red_sum?: number; ir_sum?: number; temp_sum?: number } export interface StoreEnv { DB: D1Database; RAW_BUCKET?: R2Bucket } @@ -118,7 +122,7 @@ export async function latestMinute(env: StoreEnv, userId: string, sinceTs: numbe export async function writeBatch( env: StoreEnv, userId: string, buckets: MinuteBucket[], - signals: Map, + signals: Map, now: number, ): Promise { if (buckets.length === 0) return 0 @@ -145,6 +149,13 @@ export async function writeBatch( rec.steps += sig?.steps ?? 0 const newRr = sig?.rr ?? [] if (newRr.length >= rec.rr.length) rec.rr = newRr + // Optical: additive sums + count (same idempotency basis as steps — edge hex-dedup). + if (sig?.opt_n) { + rec.opt_n = (rec.opt_n ?? 0) + sig.opt_n + rec.red_sum = (rec.red_sum ?? 0) + (sig.red_sum ?? 0) + rec.ir_sum = (rec.ir_sum ?? 0) + (sig.ir_sum ?? 0) + rec.temp_sum = (rec.temp_sum ?? 0) + (sig.temp_sum ?? 0) + } rec.hr_avg = rec.hr_n > 0 ? Math.round(rec.hr_sum / rec.hr_n) : 0 rec.activity = rec.act_n > 0 ? rec.act_sum / rec.act_n : 0 map.set(b.ts_min, rec) From 9fc4e58cf95bef4f3a4407f33bac9454816adb5a Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 20 Jun 2026 18:07:24 +0530 Subject: [PATCH 20/30] daydetail: single-source sleep staging (stageHypnogram) + RR tiebreaker + cache /day/sleep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drive the hypnogram, stage breakdown, duration, awake and efficiency from ONE source (analytics stageHypnogram — v1 Cole-Kripke method) so the graph and breakdown can't disagree; pass per-minute RR so the REM tiebreaker reclaims REM mislabeled as awake. Add the fractal sleep cycles (detectSleepCycles) to the response. Wrap getDaySleep in the TTL read-cache: compute the night ONCE, serve from read_cache (past day immutable, today <=60s, close clears it) — no more recomputing the hypnogram on every read. --- src/daydetail.ts | 147 +++++++++++++++++++++++++---------------------- 1 file changed, 77 insertions(+), 70 deletions(-) diff --git a/src/daydetail.ts b/src/daydetail.ts index 46802fc..761f6ea 100644 --- a/src/daydetail.ts +++ b/src/daydetail.ts @@ -11,19 +11,19 @@ import type { Context } from 'hono' import { cached, ttlForDate } from './cache' import { readMinutes } from './minute_store' import { ensureTodayWorkouts } from './workouts' -import { calcCircadian, stageSleep } from 'openstrap-analytics' +import { stageHypnogram, detectSleepCycles } from 'openstrap-analytics' type Ctx = Context<{ Bindings: { DB: D1Database; RAW_BUCKET?: R2Bucket }; Variables: { userId: string } }> const DAY = 86400 const dayStartOf = (date: string) => Math.floor(Date.parse(`${date}T00:00:00Z`) / 1000) -// Per-epoch hypnogram (BETA) via analytics `stageSleep` — the SAME smoothed classifier -// behind the stage totals (sustained-≥20min-awake reclassify + bout-merge), so the graph -// matches the breakdown and never flaps on a single-minute HR blip or a brief off-wrist -// gap. `mesor` from calcCircadian over the window; falls back to the simple per-minute -// threshold ONLY if circadian can't fit (flat/too-short window). -interface NightStaging { hypnogram: { t: number; stage: string }[]; totals: { light_min: number; deep_min: number; rem_min: number } | null } +// Per-minute hypnogram (BETA) via analytics `stageHypnogram` — the v1 method: +// calcSleep's Cole-Kripke + HR-dip mask owns asleep/awake, the SAME HR-percentile bands +// as estimateStages own deep/light/rem within sleep, bout-smoothed once. ONE source for +// both the graph and the totals (so they can't disagree), no second stager, no RR, no +// per-night knob — exactly what worked in v1. +interface NightStaging { hypnogram: { t: number; stage: string }[]; totals: { light_min: number; deep_min: number; rem_min: number } | null; awake_min?: number; asleep_min?: number } function stageNight( mins: { ts_min: number; hr_avg?: number | null; hr_min?: number | null; hr_max?: number | null; activity?: number | null; wrist_on?: number | null; rr?: number[] | null }[], onset: number, wake: number, rhr: number, @@ -31,28 +31,21 @@ function stageNight( const ms = mins.map((m) => ({ ts: m.ts_min, hr_avg: m.hr_avg ?? 0, hr_min: m.hr_min ?? 0, hr_max: m.hr_max ?? 0, hr_n: (m.hr_avg ?? 0) > 0 ? 60 : 0, activity: m.activity ?? 0, steps: 0, wrist_on: !!m.wrist_on, - rr: m.rr ?? [], // per-minute beat-to-beat RR → autonomic REM/deep split in stageSleep })) - const circ = calcCircadian(ms) - const mesor = (circ.mesor != null && circ.mesor > rhr) ? circ.mesor : rhr + 25 - const s = stageSleep(ms, onset, wake, mesor) - if (s.hypnogram.length) { - // ONE classifier drives BOTH the hypnogram and the reconciled totals → graph and - // breakdown can't disagree, no flapping, REM detected (re-tuned band). + // Per-minute RR → the REM tiebreaker in stageHypnogram (high HR + low RMSSD = REM, not awake). + const rrByMin = new Map() + for (const m of mins) if (m.rr && m.rr.length) rrByMin.set(m.ts_min, m.rr) + const h = stageHypnogram(ms, onset, wake, { resting_hr: rhr, max_hr: 0, sleep_need_min: 480 }, rrByMin) + if (h) { return { - hypnogram: s.hypnogram.map((h) => ({ t: h.t, stage: h.stage })), - totals: { light_min: s.light_min, deep_min: s.deep_min, rem_min: s.rem_min }, + hypnogram: h.hypnogram.map((x) => ({ t: x.t, stage: x.stage })), + totals: { light_min: h.light_min, deep_min: h.deep_min, rem_min: h.rem_min }, + awake_min: h.awake_min, + asleep_min: h.asleep_min, } } - // Fallback (circadian abstained): unsmoothed per-minute hypnogram; totals=null → caller - // keeps the stored breakdown. - const hyp = ms.filter((m) => m.ts >= onset && m.ts <= wake).map((m) => { - const hr = m.hr_avg, act = m.activity - const stage = (hr <= 0 || hr > 1.15 * rhr || act > 0.25) ? 'awake' - : (act < 0.01 && hr < rhr + 4) ? 'deep' : (hr > rhr + 5) ? 'rem' : 'light' - return { t: m.ts, stage } - }) - return { hypnogram: hyp, totals: null } + // No resting-HR baseline → staging can't run; caller keeps the stored breakdown. + return { hypnogram: [], totals: null } } const clamp = (v: number, lo: number, hi: number) => Math.max(lo, Math.min(hi, v)) @@ -286,52 +279,66 @@ export async function getDaySleep(c: Ctx) { return c.json({ date, has_sleep: false, need_min: need }) } const rhr = baseline?.resting_hr && baseline.resting_hr > 0 ? baseline.resting_hr : 55 - // WIDE window (prior evening → wake) so calcCircadian inside buildHypnogram sees the - // daytime HR peak and derives a real mesor; stageSleep itself only stages [onset, wake]. - const mins = await loadMinutes(c, row.onset_ts - 16 * 3600, row.wake_ts + 3600) - - // ONE classifier (analytics stageSleep) → smoothed hypnogram + reconciled totals. - const ng = stageNight(mins, row.onset_ts, row.wake_ts, rhr) - - // Sleep-debt over the last 7 real nights (incl. this one). - const { results: recent } = await c.env.DB.prepare( - 'SELECT duration_min FROM sleep WHERE user_id = ? AND date <= ? ORDER BY date DESC LIMIT 7', - ).bind(c.get('userId'), date).all<{ duration_min: number | null }>() - let debt = 0 - for (const r of recent ?? []) { - const d = r.duration_min ?? 0 - if (d >= 120) debt += Math.max(0, need - d) - } - - // Nocturnal-heart summary + gated respiratory rate for this date (stored on daily). - const dailyRow = await c.env.DB.prepare( - 'SELECT nocturnal, resp_rate, resp_conf FROM daily WHERE user_id = ? AND date = ?', - ).bind(c.get('userId'), date).first() - const nocturnal = dailyRow?.nocturnal ? safe(dailyRow.nocturnal) : null - const resp = (dailyRow?.resp_rate != null && (dailyRow?.resp_conf ?? 0) >= 0.5) - ? { value: Math.round(dailyRow.resp_rate * 10) / 10, confidence: dailyRow.resp_conf } - : null - - const inBed = Math.round((row.wake_ts - row.onset_ts) / 60) - const asleep = row.duration_min ?? 0 - return c.json({ - date, - has_sleep: true, - nocturnal, - resp, - onset_ts: row.onset_ts, - wake_ts: row.wake_ts, - in_bed_min: inBed, - duration_min: asleep, - awake_min: Math.max(0, inBed - asleep), - efficiency: row.efficiency, - need_min: need, - debt_min: Math.round(debt), - regularity: row.regularity, - stages: ng.totals ?? { light_min: row.light_min, deep_min: row.deep_min, rem_min: row.rem_min }, - stages_beta: true, - hypnogram: downsample(ng.hypnogram, 240), + const userId = c.get('userId') + // Compute ONCE, then serve from the TTL read-cache (no per-read recompute): a past + // day is immutable (cached until prune), "today" refreshes ≤60s, and close_day's + // invalidateDay clears it so the next read after a close rebuilds it once. + const payload = await cached(c.env.DB, userId, `daysleep:${date}`, ttlForDate(date), async () => { + // WIDE window (prior evening → wake) so the staging window has full context. + const mins = await loadMinutes(c, row.onset_ts - 16 * 3600, row.wake_ts + 3600) + // ONE source (analytics stageHypnogram — v1 Cole-Kripke method) → hypnogram + totals. + const ng = stageNight(mins, row.onset_ts, row.wake_ts, rhr) + // Ultradian sleep cycles (Rosenblum 2024 fractal-cycle method, on smoothed z-RMSSD). + const cyc = detectSleepCycles(mins.map((m) => ({ ts: m.ts_min, rr: m.rr })), row.onset_ts, row.wake_ts) + + // Sleep-debt over the last 7 real nights (incl. this one). + const { results: recent } = await c.env.DB.prepare( + 'SELECT duration_min FROM sleep WHERE user_id = ? AND date <= ? ORDER BY date DESC LIMIT 7', + ).bind(userId, date).all<{ duration_min: number | null }>() + let debt = 0 + for (const r of recent ?? []) { const d = r.duration_min ?? 0; if (d >= 120) debt += Math.max(0, need - d) } + + // Nocturnal-heart summary + gated respiratory rate for this date (stored on daily). + const dailyRow = await c.env.DB.prepare( + 'SELECT nocturnal, resp_rate, resp_conf FROM daily WHERE user_id = ? AND date = ?', + ).bind(userId, date).first() + const nocturnal = dailyRow?.nocturnal ? safe(dailyRow.nocturnal) : null + const resp = (dailyRow?.resp_rate != null && (dailyRow?.resp_conf ?? 0) >= 0.5) + ? { value: Math.round(dailyRow.resp_rate * 10) / 10, confidence: dailyRow.resp_conf } : null + + const inBed = Math.round((row.wake_ts - row.onset_ts) / 60) + // Single source: duration/awake/stages all come from the one hypnogram so the + // breakdown and the graph can't disagree. Fall back to the stored row only if + // staging abstained (no resting-HR baseline). + const asleep = ng.asleep_min ?? row.duration_min ?? 0 + const awakeMin = ng.awake_min ?? Math.max(0, inBed - asleep) + const inBedMin = ng.asleep_min != null ? asleep + awakeMin : inBed + const efficiency = inBedMin > 0 && ng.asleep_min != null ? Math.round((asleep / inBedMin) * 10000) / 10000 : row.efficiency + return { + date, + has_sleep: true, + nocturnal, + resp, + onset_ts: row.onset_ts, + wake_ts: row.wake_ts, + in_bed_min: inBedMin, + duration_min: asleep, + awake_min: awakeMin, + efficiency, + need_min: need, + debt_min: Math.round(debt), + regularity: row.regularity, + stages: ng.totals ?? { light_min: row.light_min, deep_min: row.deep_min, rem_min: row.rem_min }, + stages_beta: true, + hypnogram: downsample(ng.hypnogram, 240), + // Ultradian cycles (NREM↔REM), fractal-cycle method on HRV. Beta. + cycles: cyc.cycles, + cycles_mean_min: cyc.mean_duration_min, + cycle_series: downsample(cyc.series, 240), + cycles_beta: true, + } }) + return c.json(payload) } // ── /day/stress ────────────────────────────────────────────────────────────── From 736d57a2a11d61ffc5cc772b55e81f6915148963 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 20 Jun 2026 18:18:08 +0530 Subject: [PATCH 21/30] analytics: store RR-aware sleep at close so Today matches the Sleep screen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The RR REM-tiebreaker lived only in the on-read /day/sleep path, so the STORED sleep.duration_min (what /today reads) stayed HR-only — Today showed 4.9h while the Sleep screen showed 5h43m for the same night. processUser now runs the same stageHypnogram (RR tiebreaker) at close and stores its duration/efficiency/stages, so both screens read one consistent value. Verified: /today and /day/sleep both 343min. --- src/analytics.ts | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/analytics.ts b/src/analytics.ts index 46c6cfa..d18002a 100644 --- a/src/analytics.ts +++ b/src/analytics.ts @@ -8,6 +8,7 @@ import { calcRestingHR, calcStrain, calcHrZones, calcCalories, calcSleep, calcSleepPeriods, + stageHypnogram, calcSleepRegularity, detectSessions, calcLoad, calcFitnessTrend, calcVo2Max, calcFitnessModel, calcMonotony, calcAnomaly, calcBaselines, buildCoach, @@ -265,6 +266,12 @@ export async function processUser( const arr = byDay.get(k) if (arr) arr.push(m); else byDay.set(k, [m]) } + // Per-minute RR for the window → the REM tiebreaker in stageHypnogram, so the STORED + // sleep duration/efficiency/stages match what /day/sleep shows (Today == Sleep). + const rrByMin = new Map() + for (const rec of await readMinutes({ DB: db }, userId, firstDayStart - DAY, lastDayStart + DAY)) { + if (rec.rr && rec.rr.length) rrByMin.set(rec.ts_min, rec.rr) + } const dayMinutes = (dayStart: number) => byDay.get(dayStart) ?? [] // Sleep window spans the previous evening → this noon; gather the two days. const sleepMinutes = (dayStart: number, from: number, to: number) => { @@ -322,6 +329,21 @@ export async function processUser( const sw = sleepSearchWindow(dayStart) const sleepMin = sleepMinutes(dayStart, sw.from, sw.to) const sleep = calcSleep(sleepMin, baseline) + // RR-aware single-source staging: calcSleep finds the boundary, stageHypnogram + // (same as /day/sleep, incl. the REM tiebreaker) sets the STORED duration/awake/ + // efficiency/stages — so the Today summary can't disagree with the Sleep screen. + let sleepDuration = sleep.duration_min + let sleepEff = sleep.efficiency + let sleepStages = sleep.stages + if (sleep.onset_ts != null && sleep.wake_ts != null) { + const hyp = stageHypnogram(sleepMin, sleep.onset_ts, sleep.wake_ts, baseline, rrByMin) + if (hyp) { + const inBed = hyp.asleep_min + hyp.awake_min + sleepDuration = hyp.asleep_min + sleepEff = inBed > 0 ? Math.round((hyp.asleep_min / inBed) * 10000) / 10000 : sleepEff + sleepStages = { light_min: hyp.light_min, deep_min: hyp.deep_min, rem_min: hyp.rem_min } + } + } nightSummaries.push({ onset_ts: sleep.onset_ts, wake_ts: sleep.wake_ts }) // SRI for THIS night = regularity over a trailing ~2-week window ending here. // Needs ≥3 valid nights or it returns conf 0 → store null (matches §6). @@ -418,8 +440,8 @@ export async function processUser( 'efficiency=excluded.efficiency, light_min=excluded.light_min, deep_min=excluded.deep_min, ' + 'rem_min=excluded.rem_min, regularity=excluded.regularity, confidence=excluded.confidence, ' + 'flags=excluded.flags, updated_at=excluded.updated_at', - ).bind(userId, date, sleep.onset_ts, sleep.wake_ts, sleep.duration_min, sleep.efficiency, - sleep.stages?.light_min ?? null, sleep.stages?.deep_min ?? null, sleep.stages?.rem_min ?? null, + ).bind(userId, date, sleep.onset_ts, sleep.wake_ts, sleepDuration, sleepEff, + sleepStages?.light_min ?? null, sleepStages?.deep_min ?? null, sleepStages?.rem_min ?? null, regularityForSleep, sleep.confidence, sleepFlags, now)) sleepN++ From 183a261f96549282cb4fe7c0300486238c801b1b Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 20 Jun 2026 22:49:30 +0530 Subject: [PATCH 22/30] feat: menstrual cycle, live HRV spot-check, skin-temp/SpO2 trends, always-on irregular - Cycle tracking: cycle_log table (migrate_v15) + opt-in users.track_cycle (migrate_v16); GET/POST/DELETE /cycle gated on explicit consent; biometric overlay. - Live HRV spot-check: POST /spotcheck decodes RR from posted live frames (realtimeRr) + timeDomainHrv (stateless). - Skin-temp + SpO2 registered in /trend; skin_temp added to /day/heart. - Irregular-rhythm served for the always-on on-screen card. - Profile GET/PATCH carry track_cycle. --- src/cycle.ts | 83 ++++++++++++++++++++++++++++++ src/daydetail.ts | 3 +- src/db/migrate_v15_cycle.sql | 10 ++++ src/db/migrate_v16_track_cycle.sql | 4 ++ src/db/schema.sql | 11 ++++ src/index.ts | 23 ++++++--- src/spotcheck.ts | 46 +++++++++++++++++ src/trend.ts | 4 ++ 8 files changed, 177 insertions(+), 7 deletions(-) create mode 100644 src/cycle.ts create mode 100644 src/db/migrate_v15_cycle.sql create mode 100644 src/db/migrate_v16_track_cycle.sql create mode 100644 src/spotcheck.ts diff --git a/src/cycle.ts b/src/cycle.ts new file mode 100644 index 0000000..1d7661a --- /dev/null +++ b/src/cycle.ts @@ -0,0 +1,83 @@ +// cycle.ts — menstrual cycle tracking endpoints. +// +// POST /cycle/log {date:'YYYY-MM-DD', kind?:'start'|'end'|'spotting', note?} → upsert +// DELETE /cycle/log?date=YYYY-MM-DD → remove +// GET /cycle → prediction + logs + biometric overlay +// +// The prediction (calcCycle) is LOG-ANCHORED — driven by logged period starts, +// never inferred from biometrics. The biometric overlay (skin-temp / RHR / HRV +// across the current cycle) is descriptive enrichment pulled from stored `daily`. +// All JWT, scoped by user_id. Honest: an ESTIMATE, not medical guidance. + +import type { Context } from 'hono' +import { calcCycle } from 'openstrap-analytics' + +type Ctx = Context<{ Bindings: { DB: D1Database }; Variables: { userId: string } }> + +const isDate = (s: unknown): s is string => + typeof s === 'string' && /^\d{4}-\d{2}-\d{2}$/.test(s) +const today = () => new Date(Date.now()).toISOString().slice(0, 10) +const KINDS = new Set(['start', 'end', 'spotting']) + +/** Log (or update) a period event for a date. */ +export async function postCycleLog(c: Ctx) { + const userId = c.get('userId') + const body = await c.req.json<{ date?: string; kind?: string; note?: string }>() + if (!isDate(body.date)) return c.json({ error: 'date must be YYYY-MM-DD' }, 400) + const kind = KINDS.has(body.kind ?? '') ? (body.kind as string) : 'start' + const note = (body.note ?? '').toString().slice(0, 500) + await c.env.DB.prepare( + 'INSERT INTO cycle_log (user_id, date, kind, note, updated_at) VALUES (?,?,?,?,?) ' + + 'ON CONFLICT(user_id, date) DO UPDATE SET kind=excluded.kind, note=excluded.note, updated_at=excluded.updated_at', + ).bind(userId, body.date, kind, note, Math.floor(Date.now() / 1000)).run() + return c.json({ ok: true, date: body.date, kind, note }) +} + +/** Remove a logged event. */ +export async function deleteCycleLog(c: Ctx) { + const userId = c.get('userId') + const date = (c.req.query('date') || '').trim() + if (!isDate(date)) return c.json({ error: 'date=YYYY-MM-DD required' }, 400) + await c.env.DB.prepare('DELETE FROM cycle_log WHERE user_id = ? AND date = ?') + .bind(userId, date).run() + return c.json({ ok: true, deleted: true }) +} + +/** Current cycle position + prediction + logged events + biometric overlay. + * Only computed for users who explicitly opted in (users.track_cycle). */ +export async function getCycle(c: Ctx) { + const userId = c.get('userId') + + // Consent gate: no cycle computation unless the user turned tracking on. + const pref = await c.env.DB.prepare('SELECT track_cycle FROM users WHERE id = ?') + .bind(userId).first<{ track_cycle: number | null }>() + if (!pref || !pref.track_cycle) { + return c.json({ enabled: false, note: 'Cycle tracking is off. Enable it in your profile to start.' }) + } + + const logsRes = await c.env.DB.prepare( + 'SELECT date, kind, note FROM cycle_log WHERE user_id = ? ORDER BY date DESC LIMIT 60', + ).bind(userId).all<{ date: string; kind: string; note: string | null }>() + const logs = logsRes.results ?? [] + const starts = logs.filter((l) => l.kind === 'start').map((l) => l.date) + + const cycle = calcCycle(starts, today()) + + // Biometric overlay for the current cycle (descriptive). Pull stored daily + // signals from the cycle start onward — cheap point range on (user_id, date). + let overlay: Array<{ date: string; skin_temp_idx: number | null; resting_hr: number | null; hrv_rmssd: number | null }> = [] + if (cycle.last_start) { + const ov = await c.env.DB.prepare( + 'SELECT date, skin_temp_idx, resting_hr, hrv_rmssd FROM daily ' + + 'WHERE user_id = ? AND date >= ? ORDER BY date ASC', + ).bind(userId, cycle.last_start).all() + overlay = (ov.results ?? []).map((r: any) => ({ + date: r.date, + skin_temp_idx: r.skin_temp_idx ?? null, + resting_hr: r.resting_hr ?? null, + hrv_rmssd: r.hrv_rmssd ?? null, + })) + } + + return c.json({ ...cycle, logs, overlay }) +} diff --git a/src/daydetail.ts b/src/daydetail.ts index 761f6ea..792f561 100644 --- a/src/daydetail.ts +++ b/src/daydetail.ts @@ -437,7 +437,7 @@ export async function getDayHeart(c: Ctx) { const start = dayStartOf(date) const mins = await loadMinutes(c, start, start + DAY) const d = await c.env.DB.prepare( - 'SELECT resting_hr, recovery, readiness, hrv_rmssd, hrv_sdnn, hrv_lfhf, hrv_conf, hrv_cv, irregular, hr_zones, nocturnal, stress, illness, drivers, resp_rate, resp_conf, spo2_idx FROM daily WHERE user_id = ? AND date = ?', + 'SELECT resting_hr, recovery, readiness, hrv_rmssd, hrv_sdnn, hrv_lfhf, hrv_conf, hrv_cv, irregular, hr_zones, nocturnal, stress, illness, drivers, resp_rate, resp_conf, spo2_idx, skin_temp_idx FROM daily WHERE user_id = ? AND date = ?', ).bind(userId, date).first() const base = await c.env.DB.prepare('SELECT resting_hr, hrv_rmssd FROM baselines WHERE user_id = ?') .bind(userId).first() @@ -465,6 +465,7 @@ export async function getDayHeart(c: Ctx) { resp: (d?.resp_rate != null && (d?.resp_conf ?? 0) >= 0.3) ? { value: d.resp_rate, confidence: d.resp_conf } : null, spo2: d?.spo2_idx != null ? { value: d.spo2_idx } : null, + skin_temp: d?.skin_temp_idx != null ? { value: d.skin_temp_idx } : null, drivers: parse(d?.drivers ?? null), } }) diff --git a/src/db/migrate_v15_cycle.sql b/src/db/migrate_v15_cycle.sql new file mode 100644 index 0000000..36af5f0 --- /dev/null +++ b/src/db/migrate_v15_cycle.sql @@ -0,0 +1,10 @@ +-- v15 — menstrual cycle log (user-logged period events; anchor for calcCycle). +-- Additive & safe on existing DBs. +CREATE TABLE IF NOT EXISTS cycle_log( + user_id TEXT NOT NULL, + date TEXT NOT NULL, -- YYYY-MM-DD + kind TEXT NOT NULL, -- 'start' | 'end' | 'spotting' + note TEXT, + updated_at INTEGER, + PRIMARY KEY(user_id, date) +); diff --git a/src/db/migrate_v16_track_cycle.sql b/src/db/migrate_v16_track_cycle.sql new file mode 100644 index 0000000..83eecb3 --- /dev/null +++ b/src/db/migrate_v16_track_cycle.sql @@ -0,0 +1,4 @@ +-- v16 — explicit opt-in for menstrual cycle tracking. Cycle data is only computed +-- and shown when the user turns this on themselves (consent, not inferred from sex). +-- Additive & safe on existing DBs. +ALTER TABLE users ADD COLUMN track_cycle INTEGER DEFAULT 0; diff --git a/src/db/schema.sql b/src/db/schema.sql index 5fb8edb..45e067b 100644 --- a/src/db/schema.sql +++ b/src/db/schema.sql @@ -12,6 +12,7 @@ CREATE TABLE IF NOT EXISTS users( weight_kg REAL, sex TEXT, -- 'm' | 'f' | NULL (NULL → sex-neutral calories) step_goal INTEGER, -- user's daily step goal; NULL → client default (8000) + track_cycle INTEGER DEFAULT 0, -- explicit opt-in to menstrual cycle tracking (0/1) created_at INTEGER NOT NULL ); @@ -70,6 +71,16 @@ CREATE TABLE IF NOT EXISTS journal( PRIMARY KEY(user_id, date) ); +-- Menstrual cycle log — user-logged period events (the anchor for calcCycle). +CREATE TABLE IF NOT EXISTS cycle_log( + user_id TEXT NOT NULL, + date TEXT NOT NULL, -- YYYY-MM-DD + kind TEXT NOT NULL, -- 'start' | 'end' | 'spotting' + note TEXT, + updated_at INTEGER, + PRIMARY KEY(user_id, date) +); + -- ── DERIVED (permanent, tiny) ───────────────────────────────────────────────── CREATE TABLE IF NOT EXISTS daily( user_id TEXT, date TEXT, diff --git a/src/index.ts b/src/index.ts index 60aeb28..23bc016 100644 --- a/src/index.ts +++ b/src/index.ts @@ -10,6 +10,8 @@ import { sealOldDays, pruneMinuteDays } from './minute_store' import { getToday, getSleep, getSleepV2, getStrain, getSessions, getTrends, getChart } from './query' import { getHistory } from './history' import { postJournal, getJournal, getJournalInsights } from './journal' +import { postCycleLog, deleteCycleLog, getCycle } from './cycle' +import { postSpotCheck } from './spotcheck' import { getDayStrain, getDaySleep, getDaySleepV2, getDayTimeline, getDayStress, getDayHeart, getDayLungs, getDayWear } from './daydetail' import { getTrend } from './trend' import { workoutStart, workoutEnd, listWorkouts, getWorkout, deleteWorkout, autoCloseStaleWorkouts } from './workouts' @@ -78,6 +80,9 @@ app.use('/chart', requireJwt) app.use('/history', requireJwt) app.use('/journal', requireJwt) app.use('/journal/*', requireJwt) +app.use('/cycle', requireJwt) +app.use('/cycle/*', requireJwt) +app.use('/spotcheck', requireJwt) app.use('/day/*', requireJwt) app.use('/trend/*', requireJwt) app.use('/workout/*', requireJwt) @@ -198,25 +203,27 @@ app.post('/auth/refresh', async (c) => { app.get('/profile', async (c) => { const user = await c.env.DB.prepare( - 'SELECT id, email, name, age, height_cm, weight_kg, sex, step_goal, created_at FROM users WHERE id = ?', + 'SELECT id, email, name, age, height_cm, weight_kg, sex, step_goal, track_cycle, created_at FROM users WHERE id = ?', ).bind(c.get('userId')).first() if (!user) return c.json({ error: 'Not found' }, 404) return c.json(user) }) app.patch('/profile', async (c) => { - const { name, age, height_cm, weight_kg, sex, step_goal } = - await c.req.json<{ name?: string; age?: number; height_cm?: number; weight_kg?: number; sex?: string; step_goal?: number }>() + const { name, age, height_cm, weight_kg, sex, step_goal, track_cycle } = + await c.req.json<{ name?: string; age?: number; height_cm?: number; weight_kg?: number; sex?: string; step_goal?: number; track_cycle?: boolean | number }>() const sexVal = sex === 'm' || sex === 'f' ? sex : null // step_goal: clamp to a sane range when provided; null leaves it unchanged. const goalVal = (typeof step_goal === 'number' && isFinite(step_goal)) ? Math.max(1000, Math.min(50000, Math.round(step_goal))) : null + // track_cycle: explicit opt-in toggle (null leaves it unchanged). + const trackVal = track_cycle === undefined ? null : (track_cycle ? 1 : 0) await c.env.DB.prepare( 'UPDATE users SET name=COALESCE(?,name), age=COALESCE(?,age), height_cm=COALESCE(?,height_cm), ' + - 'weight_kg=COALESCE(?,weight_kg), sex=COALESCE(?,sex), step_goal=COALESCE(?,step_goal) WHERE id = ?', - ).bind(name ?? null, age ?? null, height_cm ?? null, weight_kg ?? null, sexVal, goalVal, c.get('userId')).run() + 'weight_kg=COALESCE(?,weight_kg), sex=COALESCE(?,sex), step_goal=COALESCE(?,step_goal), track_cycle=COALESCE(?,track_cycle) WHERE id = ?', + ).bind(name ?? null, age ?? null, height_cm ?? null, weight_kg ?? null, sexVal, goalVal, trackVal, c.get('userId')).run() const user = await c.env.DB.prepare( - 'SELECT id, email, name, age, height_cm, weight_kg, sex, step_goal, created_at FROM users WHERE id = ?', + 'SELECT id, email, name, age, height_cm, weight_kg, sex, step_goal, track_cycle, created_at FROM users WHERE id = ?', ).bind(c.get('userId')).first() return c.json(user) }) @@ -237,6 +244,10 @@ app.get('/history', getHistory) app.post('/journal', postJournal) app.get('/journal', getJournal) app.get('/journal/insights', getJournalInsights) +app.post('/cycle/log', postCycleLog) +app.delete('/cycle/log', deleteCycleLog) +app.get('/cycle', getCycle) +app.post('/spotcheck', postSpotCheck) app.get('/day/strain', getDayStrain) app.get('/day/sleep', getDaySleep) app.get('/day/v2/sleep', getDaySleepV2) diff --git a/src/spotcheck.ts b/src/spotcheck.ts new file mode 100644 index 0000000..ecccae1 --- /dev/null +++ b/src/spotcheck.ts @@ -0,0 +1,46 @@ +// spotcheck.ts — on-demand live HRV reading. +// +// POST /spotcheck { records: [hex, ...] } → { rmssd, sdnn, pnn50, mean_hr, n_beats, ok } +// +// The edge enables wrist-gated optical + realtime records for ~60s, collects the +// raw live frames, and posts them here. We decode beat-to-beat RR from each frame +// (protocol realtimeRr — the SAME decoder the ingest path uses) and compute HRV +// with the analytics time-domain method (cleanRr gate + RMSSD/SDNN/pNN50). +// +// Heavy lifting stays server-side; the client just collects bytes and renders. +// Stateless: nothing is written to D1/R2 (a spot check is ephemeral by design). + +import type { Context } from 'hono' +import { realtimeRr } from 'openstrap-protocol/ts/live' +import { timeDomainHrv } from 'openstrap-analytics' + +type Ctx = Context<{ Bindings: { DB: D1Database }; Variables: { userId: string } }> + +export async function postSpotCheck(c: Ctx) { + const body = await c.req.json<{ records?: unknown }>().catch(() => ({ records: [] })) + const recs = Array.isArray(body.records) + ? body.records.filter((h): h is string => typeof h === 'string').slice(0, 5000) + : [] + if (recs.length === 0) return c.json({ error: 'records[] required' }, 400) + + // Decode RR (ms) from every live frame that carries it (0x28 / R10). + const rr: number[] = [] + for (const hex of recs) { + try { + const r = realtimeRr(hex) + if (r && Array.isArray(r.rr_ms) && r.rr_ms.length) rr.push(...r.rr_ms) + } catch { /* skip undecodable frame */ } + } + + const hrv = timeDomainHrv(rr) + return c.json({ + n_frames: recs.length, + n_beats: hrv.n_beats, + rmssd: hrv.rmssd, + sdnn: hrv.sdnn, + pnn50: hrv.pnn50, + mean_hr: hrv.mean_hr, + // honest gate: timeDomainHrv needs ≥20 clean beats, else everything is null + ok: hrv.rmssd != null, + }) +} diff --git a/src/trend.ts b/src/trend.ts index e4c80ca..18765af 100644 --- a/src/trend.ts +++ b/src/trend.ts @@ -40,6 +40,7 @@ interface DayRow { nocturnal_dip_pct: number | null; acwr: number | null efficiency: number | null; deep_min: number | null; rem_min: number | null light_min: number | null; regularity: number | null // from sleep + skin_temp_idx: number | null; spo2_idx: number | null // RELATIVE deviations } const REGISTRY: Record = { @@ -74,6 +75,8 @@ const REGISTRY: Record = { target: (_r, need) => need, goalDir: 'min' }, stress: { pull: (r) => { try { return r.stress ? (JSON.parse(r.stress).score ?? null) : null } catch { return null } }, unit: '', label: 'Stress' }, + skin_temp: { pull: (r) => r.skin_temp_idx, unit: 'Δ', label: 'Skin temp vs baseline' }, + spo2: { pull: (r) => r.spo2_idx, unit: 'Δ', label: 'Blood-oxygen vs baseline' }, } interface Bucket { @@ -104,6 +107,7 @@ export async function getTrend(c: Ctx) { 'SELECT d.date AS date, d.strain, d.recovery, d.resting_hr, d.calories, d.steps, d.wear_min, ' + 'd.hrv_rmssd, d.resp_rate, d.stress, d.readiness, d.vo2max, d.fitness, d.fatigue, d.form, ' + 'd.monotony, d.hrv_sdnn, d.hrv_lfhf, d.hrv_si, d.hrv_cv, d.nocturnal_dip_pct, d.acwr, ' + + 'd.skin_temp_idx, d.spo2_idx, ' + 's.duration_min AS duration_min, s.efficiency AS efficiency, s.deep_min AS deep_min, ' + 's.rem_min AS rem_min, s.light_min AS light_min, s.regularity AS regularity ' + 'FROM daily d LEFT JOIN sleep s ON s.user_id = d.user_id AND s.date = d.date ' + From d9eb8c147a4294ce126d4a2ab5fceda50b542600 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sun, 21 Jun 2026 14:36:34 +0530 Subject: [PATCH 23/30] chore: complete schema.sql bootstrap + remove dead code schema.sql: add minute_day + read_cache tables and analytics_cursor wake columns (sleep_phase/phase_since/last_close_date) so a fresh DB bootstraps fully from schema.sql alone; index minute_day(ymd) for the seal/prune scans; mark legacy minute table deprecated. Remove dead code: index.ts cron-sweep helpers (sendChunked/enqueueJobs/ enqueueJobDays/lastNDates) and ingest_signals encodeRr/decodeRr. --- src/db/schema.sql | 43 ++++++++++++++++++++++++++++++++++++++----- src/index.ts | 28 ---------------------------- src/ingest_signals.ts | 19 ------------------- 3 files changed, 38 insertions(+), 52 deletions(-) diff --git a/src/db/schema.sql b/src/db/schema.sql index 45e067b..387f125 100644 --- a/src/db/schema.sql +++ b/src/db/schema.sql @@ -30,10 +30,37 @@ CREATE TABLE IF NOT EXISTS refresh_tokens( ); CREATE INDEX IF NOT EXISTS idx_refresh_user ON refresh_tokens(user_id); --- ── TIMESERIES ROLLUP (≤1440 rows/day/user; pruned to R2 after 90 days) ─────── --- hr_sum / act_sum / act_n are running aggregates kept so the ingest upsert can --- merge new samples into the stored minute EXACTLY (deterministic idempotency). --- hr_avg / activity are the derived display values (kept in sync on every write). +-- ── TIMESERIES (day-packed) ─────────────────────────────────────────────────── +-- [feat/wake-trigger] minute_day is the HOT minute store: ONE row per (user, ymd), +-- value = gzipped JSON MinuteRec[] (one entry per touched minute; RR rides the blob +-- as number[]). Ingest read-merge-writes ~1 row/day instead of ~1,440. Days older +-- than the hot window are sealed to gzipped R2 objects and the D1 row dropped; the +-- 10-day prune drops ~1 row/day. See minute_store.ts. (migrate_v14) +CREATE TABLE IF NOT EXISTS minute_day( + user_id TEXT NOT NULL, + ymd TEXT NOT NULL, -- 'YYYY-MM-DD' (UTC day of the minute) + blob BLOB NOT NULL, -- gzipped JSON MinuteRec[] (bound param; << 2MB cap) + updated_at INTEGER NOT NULL, + PRIMARY KEY(user_id, ymd) +) WITHOUT ROWID; +-- seal/prune scan by day across all users (WHERE ymd < cutoff) — index the bare ymd. +CREATE INDEX IF NOT EXISTS idx_minute_day_ymd ON minute_day(ymd); + +-- TTL read-cache for Tier 1/2 on-read metrics (no watermark; time-based only). +-- key e.g. 'today:strain:2026-06-19'; today→60s, past days immutable-until-prune. +-- See cache.ts. (migrate_v12) +CREATE TABLE IF NOT EXISTS read_cache( + user_id TEXT NOT NULL, + key TEXT NOT NULL, + payload TEXT NOT NULL, -- JSON + computed_at INTEGER NOT NULL, + PRIMARY KEY(user_id, key) +) WITHOUT ROWID; + +-- ── LEGACY ROLLUP (deprecated; empty on fresh DBs) ──────────────────────────── +-- The pre-v14 row-per-minute store. Superseded by minute_day above; left in place so +-- old deployments keep working, but ingest no longer writes here. Safe to drop once +-- no instance predates v14. CREATE TABLE IF NOT EXISTS minute( user_id TEXT NOT NULL, ts_min INTEGER NOT NULL, -- unix sec floored to minute @@ -177,7 +204,13 @@ CREATE TABLE IF NOT EXISTS analytics_cursor( last_run INTEGER DEFAULT 0, steps_cursor_ts INTEGER, -- incremental steps: last settled minute counted (unix s) steps_cursor_day TEXT, -- UTC day the steps accumulator is for - bio_last_date TEXT -- event-driven biometrics: last sleep-date already triggered + bio_last_date TEXT, -- event-driven biometrics: last sleep-date already triggered + -- [feat/wake-trigger] incremental sleep/wake state machine (migrate_v13). The */N + -- cron reads these: skip awake-and-closed users, peek the asleep ones, fire close_day + -- once per physiological day. + sleep_phase TEXT, -- 'awake' | 'asleep' | NULL + phase_since INTEGER, -- unix s of last transition + last_close_date TEXT -- YYYY-MM-DD of last day-close ); -- Per-user ingest rate-limit token bucket (RESILIENCE §7). diff --git a/src/index.ts b/src/index.ts index 23bc016..79e561a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -420,34 +420,6 @@ app.post('/admin/prune', async (c) => { return c.json({ ok: true, events_deleted: ev.meta?.changes ?? 0 }) }) -// Send messages in sendBatch chunks of 100 (the Queues per-call max), so the cron -// itself never blows its own subrequest budget no matter how many users. -async function sendChunked(q: Queue, msgs: { body: AnalyticsMessage }[]): Promise { - for (let i = 0; i < msgs.length; i += 100) { - await q.sendBatch(msgs.slice(i, i + 100)) - } -} - -// One (user, job) message per user (day-less — for 'sweep'). -async function enqueueJobs(q: Queue, userIds: string[], job: AnalyticsJob): Promise { - await sendChunked(q, userIds.map((user_id) => ({ body: { user_id, job } }))) -} - -// One (user, job, day) message per user × day — heavy R2 jobs fanned out so each -// consumer invocation processes exactly ONE bounded day (works for any user, no -// matter how much they wore the band). -async function enqueueJobDays(q: Queue, userIds: string[], job: AnalyticsJob, dates: string[]): Promise { - const msgs: { body: AnalyticsMessage }[] = [] - for (const user_id of userIds) for (const day of dates) msgs.push({ body: { user_id, job, day } }) - await sendChunked(q, msgs) -} - -// The last `n` UTC dates (today first) as YYYY-MM-DD. -function lastNDates(n: number): string[] { - const now = Math.floor(Date.now() / 1000) - return Array.from({ length: n }, (_, d) => new Date((Math.floor((now - d * DAY) / DAY) * DAY) * 1000).toISOString().slice(0, 10)) -} - export default { fetch: app.fetch, diff --git a/src/ingest_signals.ts b/src/ingest_signals.ts index ddebf80..5d58ebd 100644 --- a/src/ingest_signals.ts +++ b/src/ingest_signals.ts @@ -103,22 +103,3 @@ export function perMinuteSignals(records: string[]): Map { } return out } - -/** Encode an RR list (ms) to a compact little-endian int16 blob. */ -export function encodeRr(rr: number[]): Uint8Array | null { - if (!rr.length) return null - const buf = new Uint8Array(rr.length * 2) - const view = new DataView(buf.buffer) - for (let i = 0; i < rr.length; i++) view.setInt16(i * 2, Math.max(0, Math.min(32767, Math.round(rr[i]))), true) - return buf -} - -/** Decode a minute.rr blob back to an RR list (ms). */ -export function decodeRr(blob: ArrayBuffer | Uint8Array | null | undefined): number[] { - if (!blob) return [] - const u8 = blob instanceof Uint8Array ? blob : new Uint8Array(blob) - const view = new DataView(u8.buffer, u8.byteOffset, u8.byteLength) - const out: number[] = [] - for (let i = 0; i + 2 <= u8.byteLength; i += 2) out.push(view.getInt16(i, true)) - return out -} From efdf738a585037d66406d86ba0cbd1cc154c1a55 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sun, 21 Jun 2026 17:31:57 +0530 Subject: [PATCH 24/30] perf: drop per-POST raw R2 archive; derive everything from D1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Decoders are validated, so raw re-decodability no longer earns its cost. Stop writing the per-POST raw/…txt object (the dominant R2 Class A cost); RAW_BUCKET is kept ONLY for the hot/seal tier (sealOldDays). Every surfaced signal now derives at ingest into the D1 minute blob: - HR/RR/HRV, steps, SpO2/temp already did. - PPG: ingest decodes the R21 green channel to a per-second mean (RIIV proxy) stored as minute green[]; resp is computed from D1 at the wake-close (respFromMinuteGreen), preferred over RSA-from-RR when confident. estimateResp core unchanged. Removed the R2 re-decode subsystem: biometrics.ts (runBiometrics) deleted, resp.ts R2 I/O removed, queue biometrics/resp jobs + maybeTriggerBiometrics gone, admin run-biometrics/run-resp removed, wipe-raw scoped to raw/ prefix. R2 Class A puts: ~1,440/day/user -> ~0 (seal only). Validated on isolated v3. --- src/biometrics.ts | 292 --------------------------------------- src/biometrics_minute.ts | 12 +- src/index.ts | 58 ++------ src/ingest.ts | 28 ++-- src/ingest_signals.ts | 38 ++++- src/minute_store.ts | 10 +- src/queue.ts | 55 +------- src/resp.ts | 141 ++++--------------- 8 files changed, 110 insertions(+), 524 deletions(-) delete mode 100644 src/biometrics.ts diff --git a/src/biometrics.ts b/src/biometrics.ts deleted file mode 100644 index 3d4f6fe..0000000 --- a/src/biometrics.ts +++ /dev/null @@ -1,292 +0,0 @@ -// biometrics.ts — HRV + relative skin-temp + relative SpO₂, re-decoded from the -// raw V24 records already in R2. These fields ride in every 1 Hz historical record -// (rr intervals @18/19, raw red ADC @64, raw temp ADC @68) — validated on 127,971 -// of our own records (RR 99.7% physiological; |g|≈1.012). No new capture needed. -// -// HRV is the real one: RMSSD over artifact-filtered beat-to-beat intervals during -// the resting/sleep window — the same substrate WHOOP builds recovery on. SpO₂ and -// skin temp are RAW ADCs (the band never sends a finished %/°C), so we only ever -// express them as a deviation from the user's own baseline — relative, never absolute. - -import { parse_r24 } from 'openstrap-protocol/ts/records' -import { - timeDomainHrv, freqDomainHrv, baevskyStressIndex, - calcRecovery, calcStress, calcIllness, - calcHrvStability, calcIrregular, calcReadinessIndex, -} from 'openstrap-analytics' - -const DAY = 86400 - -interface BioEnv { DB: D1Database; RAW_BUCKET: R2Bucket } - -const hexToBytes = (hex: string): Uint8Array => { - const clean = hex.trim() - const m = clean.match(/.{1,2}/g) - if (!m) return new Uint8Array(0) - return new Uint8Array(m.map((b) => parseInt(b, 16))) -} - -const median = (a: number[]): number => { - if (!a.length) return 0 - const s = [...a].sort((x, y) => x - y) - const m = Math.floor(s.length / 2) - return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2 -} - -// One decoded V24 record's biometrics, or null if it isn't a usable V24. -interface BioSample { ts: number; hr: number; rr: number[]; temp: number; spo2: number } -function decodeV24(hex: string): BioSample | null { - const b = hexToBytes(hex) - if (b.length < 89 || b[0] !== 0x2f || b[1] !== 24) return null - const r = parse_r24(b) - if (!r || !(r.ts_epoch > 0)) return null - return { - ts: r.ts_epoch, - hr: r.hr, - rr: r.rr_intervals_ms.filter((x) => x >= 300 && x <= 2000), // physiological - temp: r.skin_temp_raw, - spo2: r.spo2_red_raw, - } -} - -/** - * RMSSD (ms) over a time-ordered RR stream, with successive-difference artifact - * rejection (drop jumps > 200 ms — ectopics / missed beats). The standard - * short-term HRV index; what nocturnal recovery is built on. - */ -export function rmssd(samples: BioSample[]): { rmssd: number | null; nBeats: number } { - const seq: number[] = [] - for (const s of [...samples].sort((a, b) => a.ts - b.ts)) for (const x of s.rr) seq.push(x) - if (seq.length < 30) return { rmssd: null, nBeats: seq.length } - let sumSq = 0, n = 0 - for (let i = 1; i < seq.length; i++) { - const d = seq[i] - seq[i - 1] - if (Math.abs(d) <= 200) { sumSq += d * d; n++ } - } - if (n < 20) return { rmssd: null, nBeats: seq.length } - return { rmssd: Math.round(Math.sqrt(sumSq / n) * 10) / 10, nBeats: seq.length } -} - -async function rawKeysInWindow(bucket: R2Bucket, userId: string, from: number, to: number): Promise { - const out: string[] = [] - let cursor: string | undefined - do { - const listing = await bucket.list({ prefix: `raw/${userId}/`, cursor, limit: 1000 }) - for (const o of listing.objects) { - const m = o.key.match(/-(\d+)-(\d+)\.txt$/) - if (!m) { out.push(o.key); continue } - const minTs = parseInt(m[1]), maxTs = parseInt(m[2]) - if (maxTs >= from && minTs <= to) out.push(o.key) - } - cursor = listing.truncated ? listing.cursor : undefined - } while (cursor) - return out -} - -// Read + decode resting V24 records in [from,to]. "Resting" = HR in a calm band -// so HRV reflects parasympathetic tone, not exercise. Bounded to `maxObjects` -// most-recent objects — a Worker request has a tight CPU/subrequest budget, and -// each object holds ~50–90 records, so ~20 objects ≈ a few hundred beats, which -// is more than enough for a stable RMSSD. -async function restingSamples(env: BioEnv, userId: string, from: number, to: number, maxObjects: number): Promise { - const keys = await rawKeysInWindow(env.RAW_BUCKET, userId, from, to) - // Newest first by the maxTs embedded in the key (…--.txt). - const keyTs = (k: string) => { const m = k.match(/-(\d+)\.txt$/); return m ? parseInt(m[1]) : 0 } - keys.sort((a, b) => keyTs(b) - keyTs(a)) - const out: BioSample[] = [] - for (const key of keys.slice(0, maxObjects)) { - const obj = await env.RAW_BUCKET.get(key) - if (!obj) continue - const text = await obj.text() - for (const line of text.split('\n')) { - const s = decodeV24(line) - if (s && s.ts >= from && s.ts <= to && s.hr > 0 && s.hr <= 70) out.push(s) - } - } - return out -} - -interface NightBio { - date: string - rmssd: number | null; sdnn: number | null; lfhf: number | null; si: number | null - resp: number | null; respConf: number; nBeats: number - temp: number | null; spo2: number | null - rr: number[] // time-ordered RR stream (ms) for stress scoring -} - -/** Collect the time-ordered RR stream over a resting window + full HRV indices. */ -async function computeDay(env: BioEnv, userId: string, date: string, from: number, to: number, maxObjects: number): Promise { - const samples = (await restingSamples(env, userId, from, to, maxObjects)).sort((a, b) => a.ts - b.ts) - const rr: number[] = [] - for (const s of samples) for (const x of s.rr) rr.push(x) - const td = timeDomainHrv(rr) - const fd = freqDomainHrv(rr) - const si = baevskyStressIndex(rr) - const temps = samples.map((s) => s.temp).filter((t) => t > 0) - const spo2s = samples.map((s) => s.spo2).filter((s) => s > 0) - return { - date, - rmssd: td.rmssd, sdnn: td.sdnn, lfhf: fd.lf_hf, si: si.si, - resp: fd.resp_rate, respConf: fd.resp_conf, nBeats: td.n_beats, - temp: temps.length ? median(temps) : null, - spo2: spo2s.length ? median(spo2s) : null, - rr, - } -} - -interface DailyHistRow { date: string; resting_hr: number | null; hrv_rmssd: number | null; hrv_si: number | null; skin_temp_idx: number | null } - -/** - * runBiometrics — for the last `days` days, re-decode resting RR from R2 and - * compute the FULL HRV suite (RMSSD/SDNN/LF-HF/Baevsky-SI/resp), then derive the - * HRV-based metrics that need beat-to-beat data: RECOVERY (Plews lnRMSSD z-score), - * STRESS (Baevsky SI personal-relative), and the multivariate ILLNESS signal - * (Mahalanobis). Writes those + their drivers to the daily row. Heavy (R2 reads) → - * nightly cron / admin only, never inline ingest. - */ -export async function runBiometrics(env: BioEnv, userId: string, days = 3, onlyDate?: string): Promise<{ days: number; computed: number }> { - const now = Math.floor(Date.now() / 1000) - - // Per-(user,day) fan-out: when onlyDate is set, process exactly that one UTC day - // (a single bounded unit). Otherwise the trailing `days`. - const since = onlyDate ?? new Date((now - days * DAY) * 1000).toISOString().slice(0, 10) - const { results: sleeps } = await env.DB.prepare( - 'SELECT date, onset_ts, wake_ts, duration_min FROM sleep WHERE user_id = ? AND date >= ? AND onset_ts IS NOT NULL AND wake_ts IS NOT NULL', - ).bind(userId, since).all<{ date: string; onset_ts: number; wake_ts: number; duration_min: number | null }>() - const sleepByDate = new Map((sleeps ?? []).map((s) => [s.date, s])) - - // Sleep-need baseline (for the composite Readiness sleep component). - const baseRow = await env.DB.prepare('SELECT sleep_need_min FROM baselines WHERE user_id = ?') - .bind(userId).first<{ sleep_need_min: number | null }>() - const sleepNeedMin = baseRow?.sleep_need_min ?? null - - // Prior history (for Plews baseline, personal SI baseline, illness covariance). - const { results: hist } = await env.DB.prepare( - 'SELECT date, resting_hr, hrv_rmssd, hrv_si, skin_temp_idx FROM daily WHERE user_id = ? ORDER BY date DESC LIMIT 60', - ).bind(userId).all() - const histRows = (hist ?? []) - const rhrByDate = new Map(histRows.map((h) => [h.date, h.resting_hr])) - const rmssdHist = histRows.map((h) => h.hrv_rmssd).filter((x): x is number => x != null) - const siHist = histRows.map((h) => h.hrv_si).filter((x): x is number => x != null) - const rhrHist = histRows.map((h) => h.resting_hr).filter((x): x is number => x != null) - const tempHist = histRows.map((h) => h.skin_temp_idx).filter((x): x is number => x != null) - - // Per-request work budget (Worker CPU/subrequest limit). Newest day first. - const PER_DAY = 12 - const TOTAL_OBJECTS = 24 - let spent = 0 - const out: NightBio[] = [] - // Build the explicit list of (dayStart, date) to process. - const dayList: { dayStart: number; date: string }[] = [] - if (onlyDate) { - dayList.push({ dayStart: Math.floor(Date.parse(`${onlyDate}T00:00:00Z`) / 1000), date: onlyDate }) - } else { - for (let d = 0; d < days; d++) { - const ds = Math.floor((now - d * DAY) / DAY) * DAY - dayList.push({ dayStart: ds, date: new Date(ds * 1000).toISOString().slice(0, 10) }) - } - } - for (const { dayStart, date } of dayList) { - const budget = Math.min(PER_DAY, TOTAL_OBJECTS - spent) - // Budget exhausted for this run: do NOT emit a null row. A null here would flow - // into the write loop and overwrite this day's already-computed HRV with null - // (the multi-day clobber bug). Skip it entirely — the day keeps its stored value - // and the next run (or a per-day onlyDate job, which always has full budget) - // recomputes it. computeWindowSteps/restingSamples are bounded per day, so - // multi-day runs starve the tail; skipping is the only non-destructive choice. - if (budget <= 0) continue - const sl = sleepByDate.get(date) - const from = sl ? sl.onset_ts : dayStart - const to = sl ? sl.wake_ts + 60 : dayStart + DAY - out.push(await computeDay(env, userId, date, from, to, budget)) - spent += budget - } - - // Baselines: rolling medians over what we just computed + whatever's stored. - const allRmssd = [...out.map((o) => o.rmssd).filter((x): x is number => x != null), ...rmssdHist] - const allSi = [...out.map((o) => o.si).filter((x): x is number => x != null), ...siHist] - const temps = out.map((o) => o.temp).filter((x): x is number => x != null) - const spo2s = out.map((o) => o.spo2).filter((x): x is number => x != null) - const blHrv = allRmssd.length ? median(allRmssd) : null - const blSi = allSi.length ? median(allSi) : null - const blTemp = temps.length ? median(temps) : null - const blSpo2 = spo2s.length ? median(spo2s) : null - if (blHrv != null || blTemp != null || blSpo2 != null || blSi != null) { - await env.DB.prepare( - 'UPDATE baselines SET hrv_rmssd = COALESCE(?, hrv_rmssd), hrv_si = COALESCE(?, hrv_si), skin_temp_raw = COALESCE(?, skin_temp_raw), spo2_raw = COALESCE(?, spo2_raw) WHERE user_id = ?', - ).bind(blHrv, blSi, blTemp, blSpo2, userId).run() - } - - let computed = 0 - for (const o of out) { - // No usable RMSSD this night → there is nothing to write that wouldn't risk - // erasing a previously-computed value with null. Leave the stored row untouched; - // a thin/empty pass must never regress a good night. (processUser owns the - // non-HRV daily fields, so skipping here loses nothing.) - if (o.rmssd == null) continue - const conf = Math.round(Math.min(1, o.nBeats / 500) * 1000) / 1000 - const tempIdx = (o.temp != null && blTemp != null) ? Math.round((o.temp - blTemp) * 10) / 10 : null - const spo2Idx = (o.spo2 != null && blSpo2 != null) ? Math.round((o.spo2 - blSpo2) * 10) / 10 : null - - // HRV-based metrics (all published algorithms; null when no usable RR). - const recovery = calcRecovery(o.rmssd, rmssdHist, { date: o.date }) - const stress = calcStress(o.rr, siHist, { date: o.date }) - const illness = calcIllness( - { resting_hr: rhrByDate.get(o.date) ?? null, rmssd: o.rmssd, skin_temp: tempIdx }, - { resting_hr: rhrHist, rmssd: rmssdHist, skin_temp: tempHist }, - ) - - // HRV stability (CV of recent nightly RMSSD) + irregular-rhythm screen (RR). - const hrvStab = calcHrvStability([o.rmssd, ...rmssdHist].filter((x): x is number => x != null).slice(0, 14)) - const irregular = calcIrregular(o.rr) - - // Composite Readiness — recovery (HRV) ∩ sleep vs need ∩ nocturnal dip ∩ arousal. - // Read this day's dip + sleep-stress (written by processUser) alongside drivers. - await env.DB.prepare('INSERT OR IGNORE INTO daily(user_id, date) VALUES(?, ?)').bind(userId, o.date).run() - const existing = await env.DB.prepare('SELECT sleep_stress, nocturnal_dip_pct FROM daily WHERE user_id = ? AND date = ?') - .bind(userId, o.date).first<{ sleep_stress: string | null; nocturnal_dip_pct: number | null }>() - // Only the driver keys biometrics OWNS. We merge these into daily.drivers via - // SQL json_patch at write time (below) rather than read-modify-write, so a - // concurrent processUser sweep writing the MAIN drivers can't be clobbered by a - // stale snapshot (the previous read→merge→overwrite had a TOCTOU window: a - // 'sweep' and a 'biometrics' for the same user are distinct queue jobs and run - // concurrently). json_patch is additive + order-independent for disjoint keys. - const bioDrivers: Record = {} - if (recovery.drivers) bioDrivers.recovery = recovery.drivers - if (stress.drivers) bioDrivers.stress = stress.drivers - if (illness.drivers) bioDrivers.illness = illness.drivers - - const sleepStressScore = (() => { try { return existing?.sleep_stress ? JSON.parse(existing.sleep_stress)?.score ?? null : null } catch { return null } })() - const readiness = calcReadinessIndex({ - recovery: recovery.score, - sleepDurationMin: sleepByDate.get(o.date)?.duration_min ?? null, - sleepNeedMin, - dipPct: existing?.nocturnal_dip_pct ?? null, - sleepStress: sleepStressScore, - }) - if (readiness.drivers) bioDrivers.readiness = readiness.drivers - - // Measured HRV indices use COALESCE(?, col): a value computed this run wins, but - // a null (e.g. enough beats for RMSSD but not for LF/HF) never erases a richer - // prior night. Mirrors the resp_rate guard. Derived scores (recovery/readiness/ - // stress/illness/irregular) are recomputed fresh whenever we have an RMSSD (we - // continue'd out of the loop otherwise), so they write directly. - await env.DB.prepare( - 'UPDATE daily SET recovery = ?, hrv_rmssd = COALESCE(?, hrv_rmssd), hrv_conf = ?, ' + - 'hrv_sdnn = COALESCE(?, hrv_sdnn), hrv_lfhf = COALESCE(?, hrv_lfhf), hrv_si = COALESCE(?, hrv_si), ' + - 'hrv_cv = COALESCE(?, hrv_cv), irregular = ?, readiness = ?, ' + - 'resp_rate = COALESCE(?, resp_rate), resp_conf = COALESCE(?, resp_conf), ' + - 'skin_temp_idx = COALESCE(?, skin_temp_idx), spo2_idx = COALESCE(?, spo2_idx), stress = ?, illness = ?, ' + - 'drivers = json_patch(COALESCE(drivers, \'{}\'), ?), updated_at = ? ' + - 'WHERE user_id = ? AND date = ?', - ).bind( - recovery.score, o.rmssd, conf, o.sdnn, o.lfhf, o.si, - hrvStab.cv, JSON.stringify(irregular), readiness.score, - o.respConf >= 0.3 ? o.resp : null, o.respConf >= 0.3 ? o.respConf : null, - tempIdx, spo2Idx, JSON.stringify(stress), JSON.stringify(illness), JSON.stringify(bioDrivers), now, - userId, o.date, - ).run() - computed++ // reached only when o.rmssd != null (we continue'd above otherwise) - } - return { days, computed } -} diff --git a/src/biometrics_minute.ts b/src/biometrics_minute.ts index 9223366..72d0be4 100644 --- a/src/biometrics_minute.ts +++ b/src/biometrics_minute.ts @@ -17,6 +17,7 @@ import { calcSpo2Index, median, } from 'openstrap-analytics' import { readMinutes } from './minute_store' +import { respFromMinuteGreen } from './resp' // [v3] RAW_BUCKET + MINUTE_SOURCE: RR comes via loadDayRr (D1 minute.rr by default, // R2-decoded series when MINUTE_SOURCE='r2'). @@ -55,6 +56,15 @@ export async function runBiometricsMinute( const fd = freqDomainHrv(rr) const si = baevskyStressIndex(rr) + // PPG-resp from the stored R21 green RIIV proxy (D1, replaces the R2 re-decode). + // Present only when the night had a live optical session; else null → RSA-from-RR + // (fd.resp_rate) covers the night. Prefer PPG when it's confident. + const ppgResp = respFromMinuteGreen(recs) + const respRate = ppgResp.confidence >= 0.5 ? ppgResp.resp_rate + : (fd.resp_conf >= 0.3 ? fd.resp_rate : null) + const respConf = ppgResp.confidence >= 0.5 ? ppgResp.confidence + : (fd.resp_conf >= 0.3 ? fd.resp_conf : null) + // 2. Prior history for Plews baseline / personal SI / illness covariance + sleep-need. const { results: hist } = await env.DB.prepare( 'SELECT date, resting_hr, hrv_rmssd, hrv_si, skin_temp_idx FROM daily WHERE user_id = ? ORDER BY date DESC LIMIT 60', @@ -137,7 +147,7 @@ export async function runBiometricsMinute( ).bind( recovery.score, td.rmssd, conf, td.sdnn, fd.lf_hf, si.si, hrvStab.cv, JSON.stringify(irregular), readiness.score, - fd.resp_conf >= 0.3 ? fd.resp_rate : null, fd.resp_conf >= 0.3 ? fd.resp_conf : null, + respRate, respConf, // SpO₂ index only when confidence is meaningful (noisy nights → keep prior, don't regress). spo2.index != null && spo2.confidence >= 0.3 ? spo2.index : null, tempIdx, JSON.stringify(stress), JSON.stringify(illness), JSON.stringify(bioDrivers), now, diff --git a/src/index.ts b/src/index.ts index 79e561a..aab5323 100644 --- a/src/index.ts +++ b/src/index.ts @@ -17,8 +17,6 @@ import { getTrend } from './trend' import { workoutStart, workoutEnd, listWorkouts, getWorkout, deleteWorkout, autoCloseStaleWorkouts } from './workouts' import { getRecords } from './records' import { getNotifications, markNotificationsRead } from './notifications' -import { runRespRate } from './resp' -import { runBiometrics } from './biometrics' import { getAppStatus, adminGetConfig, adminSetConfig } from './appconfig' import { seedInit, seedMinutes, seedAnalytics } from './seed' @@ -272,59 +270,20 @@ app.post('/notifications/read', markNotificationsRead) app.get('/admin/config', adminGetConfig) app.post('/admin/config', adminSetConfig) -// Raw R2 objects auto-expire after this many days (bucket lifecycle rule -// `expire-raw-14d`). The R2-re-decode jobs (HRV/resp/IMU steps) can therefore -// only run over data this recent — older raw is gone, so we cap their `days` -// here. Keep this in sync with the R2 lifecycle rule. -const RAW_RETENTION_DAYS = 14 - +// [drop-r2] re-derive analytics from D1 (no R2). HRV/resp/optical land at the wake-close +// (close_day → runBiometricsMinute); to recompute them for a user, enqueue close_day via +// /admin/enqueue. This endpoint just re-runs the D1 daily/sleep/strain derivation. app.post('/admin/run-analytics', async (c) => { - const body = await c.req.json<{ user_id?: string; days?: number; bio?: boolean }>().catch(() => ({} as any)) + const body = await c.req.json<{ user_id?: string; days?: number }>().catch(() => ({} as any)) const days = body.days ?? 3 - // Biometrics re-decodes from R2 → can only go back as far as raw is retained. - const bioDays = Math.min(days, RAW_RETENTION_DAYS) if (body.user_id) { - // Full re-derive sequence so HRV recovery feeds the coach: - // 1. processUser — minute metrics (strain/RHR/sleep/sessions...) + daily rows - // (reads the `minute` table, retained 90d — not R2 — so `days` is unbounded here) - // 2. runBiometrics — HRV recovery/stress/illness from RR (R2; capped to retention) - // 3. processUser — coach picks up the recovery written in step 2 const r1 = await processUser(c.env.DB, body.user_id, { historyDays: days }) - let bio: any = null - if (body.bio !== false) { - bio = await runBiometrics(c.env, body.user_id, bioDays) - await processUser(c.env.DB, body.user_id, { historyDays: days }) - } - return c.json({ ok: true, ...r1, bio, bio_days: bioDays, bio_capped: bioDays < days }) + return c.json({ ok: true, ...r1 }) } const res = await runAnalytics(c.env.DB, { historyDays: days }) return c.json({ ok: true, ...res }) }) -// Respiratory rate from PPG (R21 re-decoded from R2). Heavy (R2 reads) → admin / -// cron only. Stores resp_rate/resp_conf on daily; gated (conf ≥ 0.5) before surfaced. -// `days` is capped to the R2 retention window (older raw no longer exists). -app.post('/admin/run-resp', async (c) => { - const body = await c.req.json<{ user_id: string; days?: number }>().catch(() => ({} as any)) - if (!body.user_id) return c.json({ error: 'user_id required' }, 400) - const reqDays = body.days ?? 3 - const days = Math.min(reqDays, RAW_RETENTION_DAYS) - const res = await runRespRate(c.env, body.user_id, days) - return c.json({ ok: true, ...res, capped: days < reqDays, max_days: RAW_RETENTION_DAYS }) -}) - -// HRV (RMSSD) + relative skin-temp / SpO₂ from the V24 RR/ADC bytes, re-decoded -// from R2. Heavy (R2 reads) → admin / cron only. Writes daily.hrv_rmssd etc. -// `days` is capped to the R2 retention window (older raw no longer exists). -app.post('/admin/run-biometrics', async (c) => { - const body = await c.req.json<{ user_id: string; days?: number }>().catch(() => ({} as any)) - if (!body.user_id) return c.json({ error: 'user_id required' }, 400) - const reqDays = body.days ?? 3 - const days = Math.min(reqDays, RAW_RETENTION_DAYS) - const res = await runBiometrics(c.env, body.user_id, days) - return c.json({ ok: true, ...res, capped: days < reqDays, max_days: RAW_RETENTION_DAYS }) -}) - // (Steps are AN-2554-only now: counted at ingest into minute.steps, summed into // daily.steps by processUser, served live on-read. No R2 step job / admin endpoint.) @@ -397,11 +356,14 @@ app.post('/admin/enqueue', async (c) => { return c.json({ ok: true, enqueued: msg }) }) +// Clean up the legacy raw/ namespace ONLY. Per-POST raw archival was removed; existing +// raw/ objects also self-expire via the bucket lifecycle rule. This is prefix-scoped so +// it can NEVER touch the sealed minute/ blobs (the hot/seal cold tier we still use). app.post('/admin/wipe-raw', async (c) => { let deleted = 0 let cursor: string | undefined do { - const listing = await c.env.RAW_BUCKET.list({ cursor, limit: 1000 }) + const listing = await c.env.RAW_BUCKET.list({ prefix: 'raw/', cursor, limit: 1000 }) const keys = listing.objects.map((o) => o.key) if (keys.length > 0) { await c.env.RAW_BUCKET.delete(keys) @@ -412,7 +374,7 @@ app.post('/admin/wipe-raw', async (c) => { return c.json({ deleted }) }) -// Prune minute rows older than 90 days (raw stays in R2). +// Prune minute rows older than retention (sealed minute blobs stay in R2). app.post('/admin/prune', async (c) => { const now = Math.floor(Date.now() / 1000) await pruneMinuteDays(c.env, now, MINUTE_RETENTION_DAYS) // day-packed minute_day rows diff --git a/src/ingest.ts b/src/ingest.ts index a789342..8feb01c 100644 --- a/src/ingest.ts +++ b/src/ingest.ts @@ -72,20 +72,11 @@ export async function ingestBatch(c: Context<{ Bindings: IngestEnv; Variables: { // 2. Decode. const samples = decodeBatch(records) - // 3. R2 put the whole batch raw (re-decodable system of record). - let rawKey: string | null = null - if (records.length > 0) { - const tss = samples.map((s) => s.ts).filter((t) => t > 0) - const minTs = tss.length ? Math.min(...tss) : 0 - const maxTs = tss.length ? Math.max(...tss) : 0 - rawKey = `raw/${userId}/${device_id}/${Date.now()}-${minTs}-${maxTs}.txt` - try { - await c.env.RAW_BUCKET.put(rawKey, records.join('\n')) - } catch (e) { - console.error('R2 put failed', e) - rawKey = null - } - } + // 3. (Raw R2 archive removed.) Decoders are validated, and every surfaced signal — + // HR/RR/HRV, steps, SpO₂/temp, and PPG-resp (via the R21 green RIIV proxy) — is now + // derived at ingest and stored in the D1 minute blob, so there is nothing to + // re-decode from raw later. RAW_BUCKET is retained ONLY for the hot/seal tier + // (sealOldDays moves aged minute_day blobs to R2); no per-POST raw object is written. // 4. Rollup → day-packed minute store (ONE row per day, read-merge-write). This is // the cost lever: ~1 row written/day instead of ~1,440. The merge inside writeBatch @@ -109,16 +100,15 @@ export async function ingestBatch(c: Context<{ Bindings: IngestEnv; Variables: { // 6. Enqueue analytics (or mark dirty). Never run inline. if (minutesWritten > 0) await markUserDirty(c.env, userId) - // 7. Respond. `received` = records persisted raw to R2 (the re-decodable system of - // record); `decoded` = records that yielded a surfaceable sample; `minutes_written` - // = per-minute rollups touched. The client shows `received` so the count is honest - // (a 2xx means we stored them all), not 0. + // 7. Respond. `received` = records accepted (a 2xx means we decoded + rolled them into + // the D1 minute store); `decoded` = records that yielded a surfaceable sample; + // `minutes_written` = per-minute rollups touched. The client shows `received` so the + // count is honest, not 0. return c.json({ ok: true, received: records.length, decoded: samples.length, minutes_written: minutesWritten, - raw_key: rawKey, }) } diff --git a/src/ingest_signals.ts b/src/ingest_signals.ts index 5d58ebd..453d0cb 100644 --- a/src/ingest_signals.ts +++ b/src/ingest_signals.ts @@ -26,22 +26,52 @@ export interface MinuteSignal { red_sum?: number ir_sum?: number temp_sum?: number + // PPG RIIV proxy: per-second mean of the R21 green channel, time-ordered. The ONLY + // value estimateResp consumes — so storing it lets resp compute from D1 at the close + // with no raw R2. Present only during live optical sessions (R21 is live-only). + green?: number[] } interface AccelFrame { idx: number; ts: number; mags: number[] } interface OpticalAcc { n: number; red: number; ir: number; temp: number } +// One R21 record's green PPG channel → per-second mean (RIIV proxy). Mirrors +// resp.ts decodeR21Green: rec_type 21, ts@7 (u32 LE), channel A @20 (100×u16 LE). +function r21GreenMean(b: Uint8Array): { ts: number; mean: number } | null { + if (b.length < 620 || b[1] !== 21) return null + const view = new DataView(b.buffer, b.byteOffset, b.byteLength) + const ts = view.getUint32(7, true) + if (!(ts > 0)) return null + let sum = 0, n = 0 + for (let i = 0; i < 100; i++) { + const o = 20 + 2 * i + if (o + 2 <= b.length) { sum += view.getUint16(o, true); n++ } + } + return n >= 50 ? { ts, mean: sum / n } : null +} + /** Build per-minute {steps, rr, optical} from a batch of hex records. Pure; no I/O. */ export function perMinuteSignals(records: string[]): Map { const accelByMin = new Map>() const rrByMin = new Map() const optByMin = new Map() + const greenByMin = new Map() for (const hex of records) { let b: Uint8Array try { b = hexToBytes(hex) } catch { continue } if (b.length < 2) continue + // R21 PPG (rec_type 21): per-second mean green → RIIV proxy for resp (D1, no R2). + const g = r21GreenMean(b) + if (g) { + const m = Math.floor(g.ts / 60) * 60 + const arr = greenByMin.get(m) ?? [] + arr.push({ ts: g.ts, v: g.mean }) + greenByMin.set(m, arr) + continue + } + // R24 (recType @ b[1] === 24): RR intervals via the protocol decoder. if (b[1] === 24 && b.length >= 89) { const r = parse_r24(b) @@ -83,7 +113,7 @@ export function perMinuteSignals(records: string[]): Map { } const out = new Map() - const minutes = new Set([...accelByMin.keys(), ...rrByMin.keys(), ...optByMin.keys()]) + const minutes = new Set([...accelByMin.keys(), ...rrByMin.keys(), ...optByMin.keys(), ...greenByMin.keys()]) for (const m of minutes) { let steps = 0 const frames = accelByMin.get(m) @@ -93,12 +123,18 @@ export function perMinuteSignals(records: string[]): Map { for (const fr of ordered) for (const v of fr.mags) sig.push(v) steps = calcSteps([sig]) } + // PPG green: time-ordered per-second means for the minute (RIIV series → resp at close). + const gArr = greenByMin.get(m) + const green = gArr && gArr.length + ? gArr.sort((a, b) => a.ts - b.ts).map((x) => x.v) + : undefined // Single library gate (300–2000 ms + ectopic |Δ|>200ms drop) — logic lives in // analytics, not duplicated here. const o = optByMin.get(m) out.set(m, { steps, rr: cleanRr(rrByMin.get(m) ?? []), ...(o ? { opt_n: o.n, red_sum: o.red, ir_sum: o.ir, temp_sum: o.temp } : {}), + ...(green ? { green } : {}), }) } return out diff --git a/src/minute_store.ts b/src/minute_store.ts index 0df912e..51debb3 100644 --- a/src/minute_store.ts +++ b/src/minute_store.ts @@ -37,6 +37,11 @@ export interface MinuteRec { // exact and re-uploads can't double-count (edge dedupes by hex). RELATIVE raw ADCs; // SpO₂/°C are derived in the close path, never on-band. Optional (older blobs lack them). opt_n?: number; red_sum?: number; ir_sum?: number; temp_sum?: number + // PPG RIIV proxy: per-second mean of the R21 green channel (the only value + // estimateResp consumes). Present only during live optical sessions (R21 is + // live-stream-only), so usually empty. Replaces the R2 raw store for resp — + // resp is computed from this series at the wake-close. Optional (older blobs lack it). + green?: number[] } export interface StoreEnv { DB: D1Database; RAW_BUCKET?: R2Bucket } @@ -122,7 +127,7 @@ export async function latestMinute(env: StoreEnv, userId: string, sinceTs: numbe export async function writeBatch( env: StoreEnv, userId: string, buckets: MinuteBucket[], - signals: Map, + signals: Map, now: number, ): Promise { if (buckets.length === 0) return 0 @@ -149,6 +154,9 @@ export async function writeBatch( rec.steps += sig?.steps ?? 0 const newRr = sig?.rr ?? [] if (newRr.length >= rec.rr.length) rec.rr = newRr + // PPG green RIIV proxy: same "keep the fuller array" idempotency as rr. + const newGreen = sig?.green ?? [] + if (newGreen.length >= (rec.green?.length ?? 0)) rec.green = newGreen // Optical: additive sums + count (same idempotency basis as steps — edge hex-dedup). if (sig?.opt_n) { rec.opt_n = (rec.opt_n ?? 0) + sig.opt_n diff --git a/src/queue.ts b/src/queue.ts index 01ffc2d..d4c2276 100644 --- a/src/queue.ts +++ b/src/queue.ts @@ -5,14 +5,13 @@ // just enqueues; the consumer fans them out, one bounded unit per invocation. import { processUser } from './analytics' -import { runBiometrics } from './biometrics' import { runBiometricsMinute } from './biometrics_minute' -import { runRespRate } from './resp' import { invalidateDay } from './cache' -// Steps are AN-2554 only, counted at ingest into minute.steps and summed into -// daily.steps by processUser — no R2 step recompute job (steps_imu removed). -export type AnalyticsJob = 'sweep' | 'biometrics' | 'resp' | 'close_day' +// [drop-r2] All HRV/resp/optical now derive from the D1 minute store at the wake-close +// (runBiometricsMinute). The old R2 re-decode jobs ('biometrics'/'resp') and their +// fallback trigger are gone. Steps are AN-2554, counted at ingest into the minute blob. +export type AnalyticsJob = 'sweep' | 'close_day' export interface AnalyticsMessage { user_id: string @@ -29,38 +28,6 @@ interface QueueEnv { ANALYTICS_Q?: Queue } -// After a sweep, if a fresh night just finished (sleep with wake_ts, settled, for a -// date we haven't scored yet), enqueue biometrics + resp for it. This makes -// recovery/HRV appear shortly after each user WAKES — no timezone needed — instead -// of waiting for the fixed nightly run. Fires once per night (bio_last_date guard). -async function maybeTriggerBiometrics(env: QueueEnv, userId: string): Promise { - if (!env.ANALYTICS_Q) return - const sleep = await env.DB.prepare( - 'SELECT date, wake_ts FROM sleep WHERE user_id = ? AND onset_ts IS NOT NULL AND wake_ts IS NOT NULL ORDER BY date DESC LIMIT 1', - ).bind(userId).first<{ date: string; wake_ts: number }>() - if (!sleep) return - const now = Math.floor(Date.now() / 1000) - if (sleep.wake_ts > now - 600) return // woke <10 min ago → let the night settle; next sweep gets it - // Already have HRV for this night (nightly backstop or an earlier fire computed - // it)? Nothing to do — never re-decode R2 for a night we've already scored. - const have = await env.DB.prepare( - 'SELECT 1 FROM daily WHERE user_id = ? AND date = ? AND hrv_rmssd IS NOT NULL', - ).bind(userId, sleep.date).first() - if (have) return - const cur = await env.DB.prepare('SELECT bio_last_date FROM analytics_cursor WHERE user_id = ?') - .bind(userId).first<{ bio_last_date: string | null }>() - // Cost cap: fire the heavy R2 decode at most ONCE per night from the event path. - // If that fire yields null (partial/late R2), the nightly cron retries missing - // nights — so we never spam the decode every 30-min sweep. - if (cur?.bio_last_date && cur.bio_last_date >= sleep.date) return // already fired for this night - await env.ANALYTICS_Q.send({ user_id: userId, job: 'biometrics', day: sleep.date }) - await env.ANALYTICS_Q.send({ user_id: userId, job: 'resp', day: sleep.date }) - await env.DB.prepare( - 'INSERT INTO analytics_cursor (user_id, bio_last_date) VALUES (?,?) ' + - 'ON CONFLICT(user_id) DO UPDATE SET bio_last_date = excluded.bio_last_date', - ).bind(userId, sleep.date).run() -} - // Run one bounded unit of work. Each branch is sized to fit in a single // invocation's budget (one user, one job, a few days of R2 at most). async function runJob(env: QueueEnv, userId: string, job: AnalyticsJob, day?: string, onset_ts?: number, wake_ts?: number): Promise { @@ -81,22 +48,12 @@ async function runJob(env: QueueEnv, userId: string, job: AnalyticsJob, day?: st } await env.DB.prepare('UPDATE analytics_cursor SET dirty = 0 WHERE user_id = ?').bind(userId).run() break - case 'biometrics': - // HRV/temp/SpO₂ from RR (R2) for ONE day (day set) or the trailing window. - await runBiometrics(env, userId, 3, day) - // Legacy multi-day mode re-runs analytics so the coach picks up recovery; in - // per-day mode the next 'sweep' does that (recovery is written either way). - if (!day) await processUser(env.DB, userId, { historyDays: 2 }) - break - case 'resp': - await runRespRate(env, userId, 2, day) - break case 'sweep': default: // The frequent path: derive daily/sleep/strain (D1); steps folded in by processUser. + // HRV/resp/optical are derived at the wake-close (close_day → runBiometricsMinute), + // not here — no R2 re-decode jobs anymore. await processUser(env.DB, userId, { historyDays: 3 }) - // Event-driven: if this sweep just finished a night, fire biometrics for it. - await maybeTriggerBiometrics(env, userId) break } } diff --git a/src/resp.ts b/src/resp.ts index d27678e..bbe7dcf 100644 --- a/src/resp.ts +++ b/src/resp.ts @@ -1,51 +1,24 @@ -// resp.ts — respiratory rate from the optical PPG (R21), re-decoded from the raw -// frames in R2. THE ONLY honest source of breaths/min on this hardware: the green -// PPG channel's slow amplitude/baseline modulation (RIIV) tracks breathing. +// resp.ts — respiratory rate from the optical PPG (R21) green channel's slow +// amplitude/baseline modulation (RIIV), which tracks breathing. THE only honest +// optical source of breaths/min on this hardware. // -// IMPORTANT REALITY: R21 (6-channel optical PPG) is only emitted during the LIVE -// realtime stream (SEND_R10_R11_REALTIME). Overnight, the band flashes R24 (1 Hz -// HR, NO PPG arrays), so on normal nights there is NO R21 in R2 and resp stays -// null. This pipeline computes a number ONLY when there is enough contiguous R21 -// data AND the respiratory periodicity is strong — otherwise confidence 0 and it -// is never surfaced (the /today + /day/sleep gate requires resp_conf ≥ 0.5). -// This mirrors the HRV stance: never fabricate. BETA until validated on real PPG. - -const DAY = 86400 - -interface RespEnv { DB: D1Database; RAW_BUCKET: R2Bucket } - -const hexToBytes = (hex: string): Uint8Array => { - const clean = hex.trim() - const m = clean.match(/.{1,2}/g) - if (!m) return new Uint8Array(0) - return new Uint8Array(m.map((b) => parseInt(b, 16))) -} - -// Decode one R21 record's green PPG channel (channel A @20, 100×u16 LE). Returns -// the record's epoch ts + the green samples, or null if it isn't a usable R21. -// Layout from the reference client — ts@7 (u32), channel A @20 (100 u16). -function decodeR21Green(hex: string): { ts: number; green: number[] } | null { - const b = hexToBytes(hex) - if (b.length < 620) return null - if (b[1] !== 21) return null // rec_type 21 - const view = new DataView(b.buffer, b.byteOffset, b.byteLength) - const ts = view.getUint32(7, true) - if (!(ts > 0)) return null - const green: number[] = [] - for (let i = 0; i < 100; i++) { - const o = 20 + 2 * i - if (o + 2 <= b.length) green.push(view.getUint16(o, true)) - } - return green.length >= 50 ? { ts, green } : null -} +// [drop-r2] The PPG waveform is no longer archived to R2. Instead, ingest stores the +// per-second mean-green RIIV proxy in the D1 minute blob (MinuteRec.green) — the ONLY +// value this algorithm consumes — and resp is computed from D1 at the wake-close via +// respFromMinuteGreen(). estimateResp (the validated autocorrelation core) is unchanged. +// +// REALITY: R21 is emitted ONLY during the live realtime stream; overnight the band +// flashes R24 (1 Hz HR, no PPG), so on normal nights there is no green series and resp +// stays null (RSA-from-RR resp, computed in biometrics_minute, covers those nights). +// Computes a number ONLY with enough contiguous signal AND strong periodicity; else +// confidence 0 and it's never surfaced (display gate requires resp_conf ≥ 0.5). const mean = (a: number[]) => a.reduce((s, v) => s + v, 0) / (a.length || 1) /** - * Estimate respiratory rate (breaths/min) from a window of R21 records via - * autocorrelation of the per-record green level (RIIV proxy). Returns - * {resp_rate, confidence}; confidence 0 when the data is insufficient or the - * periodicity is weak (→ never surfaced). Conservative by design. + * Estimate respiratory rate (breaths/min) from a window of per-record green levels + * via autocorrelation of the RIIV proxy. Returns {resp_rate, confidence}; confidence 0 + * when data is insufficient or periodicity is weak (→ never surfaced). Conservative. */ export function estimateResp(records: { ts: number; green: number[] }[]): { resp_rate: number | null; confidence: number } { if (records.length < 120) return { resp_rate: null, confidence: 0 } @@ -95,77 +68,19 @@ export function estimateResp(records: { ts: number; green: number[] }[]): { resp return { resp_rate: resp, confidence } } -// List R2 raw objects whose key time-range overlaps [from,to]. Keys look like -// raw///--.txt (see ingest.ts). -async function rawKeysInWindow(bucket: R2Bucket, userId: string, from: number, to: number): Promise { - const out: string[] = [] - let cursor: string | undefined - do { - const listing = await bucket.list({ prefix: `raw/${userId}/`, cursor, limit: 1000 }) - for (const o of listing.objects) { - const m = o.key.match(/-(\d+)-(\d+)\.txt$/) - if (!m) { out.push(o.key); continue } // unparseable → include (cheap safety) - const minTs = parseInt(m[1]), maxTs = parseInt(m[2]) - if (maxTs >= from && minTs <= to) out.push(o.key) - } - cursor = listing.truncated ? listing.cursor : undefined - } while (cursor) - return out -} - -/** Compute + store resp rate for one night. Idempotent. */ -async function computeNight(env: RespEnv, userId: string, date: string, from: number, to: number): Promise<{ resp_rate: number | null; confidence: number; r21: number }> { - const keys = await rawKeysInWindow(env.RAW_BUCKET, userId, from, to) - const records: { ts: number; green: number[] }[] = [] - for (const key of keys) { - const obj = await env.RAW_BUCKET.get(key) - if (!obj) continue - const text = await obj.text() - for (const line of text.split('\n')) { - const r = decodeR21Green(line) - if (r && r.ts >= from && r.ts <= to) records.push(r) - } - } - const { resp_rate, confidence } = estimateResp(records) - // Only persist a real reading. A null pass (absent/sparse R21 — the band only - // emits R21 during live streaming, so most nights have none) must NOT erase a - // prior good night: the night is over, its resp doesn't change after the fact. - // A never-written night stays null by default, so the display gate - // (resp_conf >= 0.5) remains authoritative either way. COALESCE can't be used - // here because confidence is 0 (not null) when absent, which would still clobber. - if (resp_rate != null) { - await env.DB.prepare( - 'UPDATE daily SET resp_rate = ?, resp_conf = ? WHERE user_id = ? AND date = ?', - ).bind(resp_rate, confidence, userId, date).run() - } - return { resp_rate, confidence, r21: records.length } -} - /** - * runRespRate — for each of the last `days` nights with a sleep row, re-decode - * R21 from R2 and store resp_rate/resp_conf on daily. Also refreshes the resp - * baseline (median of nights with conf ≥ 0.5). Heavy (R2 reads) → nightly cron / - * admin only, NEVER inline ingest. + * PPG-resp from the D1 minute store. Flattens each minute's stored per-second green + * RIIV proxy into a time-ordered series and runs estimateResp over it. Synthesizes a + * 1 Hz timestamp per sample (ts_min + index) — estimateResp resamples by ts span, so + * the exact sub-minute placement is immaterial. Returns {null, 0} when no usable PPG + * (the normal-night case → resp stays whatever RSA-from-RR produced). */ -export async function runRespRate(env: RespEnv, userId: string, days = 3, onlyDate?: string): Promise<{ nights: number; computed: number }> { - // Per-(user,day) fan-out: onlyDate → just that night; else the trailing `days`. - const since = onlyDate ?? new Date(Date.now() - days * DAY * 1000).toISOString().slice(0, 10) - const dateClause = onlyDate ? 'date = ?' : 'date >= ?' - const { results: nights } = await env.DB.prepare( - `SELECT date, onset_ts, wake_ts FROM sleep WHERE user_id = ? AND ${dateClause} AND onset_ts IS NOT NULL AND wake_ts IS NOT NULL ORDER BY date DESC`, - ).bind(userId, since).all<{ date: string; onset_ts: number; wake_ts: number }>() - - let computed = 0 - const valid: number[] = [] - for (const n of nights ?? []) { - const r = await computeNight(env, userId, n.date, n.onset_ts, n.wake_ts + 60) - if (r.resp_rate != null && r.confidence >= 0.5) { computed++; valid.push(r.resp_rate) } - } - if (valid.length >= 3) { - valid.sort((a, b) => a - b) - const med = valid[Math.floor(valid.length / 2)] - await env.DB.prepare('UPDATE baselines SET resp_rate = ? WHERE user_id = ?') - .bind(Math.round(med * 10) / 10, userId).run() +export function respFromMinuteGreen(recs: { ts_min: number; green?: number[] }[]): { resp_rate: number | null; confidence: number } { + const records: { ts: number; green: number[] }[] = [] + for (const m of recs) { + if (!m.green || !m.green.length) continue + for (let i = 0; i < m.green.length; i++) records.push({ ts: m.ts_min + i, green: [m.green[i]] }) } - return { nights: (nights ?? []).length, computed } + if (!records.length) return { resp_rate: null, confidence: 0 } + return estimateResp(records) } From df5281fe26487c01ffd48d2d1a4ddf2c4c08d329 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sun, 21 Jun 2026 17:51:30 +0530 Subject: [PATCH 25/30] perf(ingest): replace D1 token-bucket with native edge rate limiter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The D1-backed token bucket cost 1 D1 read + 1 D1 WRITE on every ingest POST to maintain a bucket that, at 60s batching, never throttled legitimate traffic (it fully refilled between posts). Swap to the Workers Rate-Limiting binding (env.RATE_LIMITER.limit) — in-memory, per-colo, no D1 I/O, no extra billing. Limit kept equivalent to the old 30/min (limit 30 / 60s). Guarded so the worker runs unthrottled if the binding isn't configured on an environment. Removes the last removable per-POST D1 write (→ 1 write/POST with the dirty-fix). Binding lives in wrangler.v3.toml (validation); add to v2/v1 tomls at cutover. --- src/index.ts | 1 + src/ingest.ts | 49 +++++++++++++++---------------------------------- 2 files changed, 16 insertions(+), 34 deletions(-) diff --git a/src/index.ts b/src/index.ts index aab5323..c5c8277 100644 --- a/src/index.ts +++ b/src/index.ts @@ -23,6 +23,7 @@ import { seedInit, seedMinutes, seedAnalytics } from './seed' type Bindings = { DB: D1Database RAW_BUCKET: R2Bucket + RATE_LIMITER?: { limit(opts: { key: string }): Promise<{ success: boolean }> } ANALYTICS_Q?: Queue JWT_SECRET: string ADMIN_TOKEN?: string diff --git a/src/ingest.ts b/src/ingest.ts index 8feb01c..2aef5f9 100644 --- a/src/ingest.ts +++ b/src/ingest.ts @@ -7,42 +7,22 @@ import { rollupMinutes } from './rollup' import { perMinuteSignals } from './ingest_signals' import { writeBatch } from './minute_store' -// ── Rate limiting: per-user token bucket in a D1 row (RESILIENCE §7). ── -// Refill RATE tokens/sec, cap BURST. One ingest = one token. Over budget → 429. -const RL_BURST = 30 // allow short bursts (bulk reconnect upload) -const RL_REFILL = 0.5 // 0.5 tokens/sec ≈ 30 batches/min sustained - -export async function checkRateLimit(db: D1Database, userId: string): Promise { - const now = Math.floor(Date.now() / 1000) - const row = await db.prepare( - 'SELECT tokens, updated_at FROM rate_limit WHERE user_id = ?', - ).bind(userId).first<{ tokens: number; updated_at: number }>() - - let tokens = RL_BURST - if (row) { - const elapsed = Math.max(0, now - row.updated_at) - tokens = Math.min(RL_BURST, row.tokens + elapsed * RL_REFILL) - } - if (tokens < 1) { - // Persist the (refilled-but-still-empty) state so the clock keeps ticking. - await db.prepare( - 'INSERT INTO rate_limit (user_id, tokens, updated_at) VALUES (?,?,?) ' + - 'ON CONFLICT(user_id) DO UPDATE SET tokens=excluded.tokens, updated_at=excluded.updated_at', - ).bind(userId, tokens, now).run() - return false - } - tokens -= 1 - await db.prepare( - 'INSERT INTO rate_limit (user_id, tokens, updated_at) VALUES (?,?,?) ' + - 'ON CONFLICT(user_id) DO UPDATE SET tokens=excluded.tokens, updated_at=excluded.updated_at', - ).bind(userId, tokens, now).run() - return true -} +// ── Rate limiting: native Workers Rate-Limiting binding (in-memory at the edge, +// per-colo). Replaces the old D1 token-bucket, which cost 1 D1 read + 1 D1 WRITE on +// EVERY ingest POST to maintain a bucket that — at 60s batching — never actually +// throttled legitimate traffic (it fully refilled between posts). The binding has no +// D1 I/O and no separate billing. Limit kept equivalent to the old 30/min (limit 30 / +// 60s, configured as `simple` in wrangler). Per-colo + approximate, which is exactly +// right for abuse/burst protection. Guarded so the worker still runs if the binding +// isn't configured on a given environment (then ingest is unthrottled — the edge +// already self-throttles to ~1 POST/min). +interface RateLimiter { limit(opts: { key: string }): Promise<{ success: boolean }> } interface IngestEnv { DB: D1Database RAW_BUCKET: R2Bucket ANALYTICS_Q?: Queue + RATE_LIMITER?: RateLimiter } // Mark the user dirty so the NEXT 30-min cron sweep enqueues exactly ONE analytics @@ -64,9 +44,10 @@ export async function ingestBatch(c: Context<{ Bindings: IngestEnv; Variables: { const { device_id, records } = await c.req.json<{ device_id: string; records: string[] }>() if (!device_id || !Array.isArray(records)) return c.json({ error: 'Invalid payload' }, 400) - // 1. Rate limit. - if (!(await checkRateLimit(c.env.DB, userId))) { - return c.json({ error: 'Rate limit exceeded' }, 429) + // 1. Rate limit (edge binding; no D1). Skipped if the binding isn't configured. + if (c.env.RATE_LIMITER) { + const { success } = await c.env.RATE_LIMITER.limit({ key: userId }) + if (!success) return c.json({ error: 'Rate limit exceeded' }, 429) } // 2. Decode. From 09c9fd63d9413cc06cda0446a275a076be12c359 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sun, 21 Jun 2026 15:40:46 +0530 Subject: [PATCH 26/30] perf(ingest): only write dirty flag on the 0->1 transition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit markUserDirty did an unconditional upsert per batch (~1,440 D1 row writes/ user/day) to set a flag that stays 1 from first ingest until the wake-close clears it. Add a conditional 'DO UPDATE SET dirty=1 WHERE dirty=0' so a row is written only on the real transition; steady-state batches read the PK and write 0 rows. ~1 dirty-write per wake-cycle instead of per-POST — cuts ingest D1 writes ~1/3 with identical observable state. --- src/ingest.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/ingest.ts b/src/ingest.ts index 2aef5f9..e6894f4 100644 --- a/src/ingest.ts +++ b/src/ingest.ts @@ -32,9 +32,15 @@ interface IngestEnv { // sweep per user per 30 min. Freshness is bounded by the sweep interval (fine — and // recovery still fires promptly on wake via the sleep-detection trigger). async function markUserDirty(env: IngestEnv, userId: string) { + // Cost: `dirty` stays 1 from the first ingest until the wake-close clears it, so an + // unconditional upsert bills a D1 ROW WRITE on every batch (~1,440/user/day) to set a + // flag that's already 1. The conditional DO UPDATE writes a row ONLY on the real 0→1 + // transition (first ingest of a new cycle / after a close) — steady-state batches hit + // the WHERE=false path and write 0 rows (just a cheap PK read). ~1 dirty-write per + // wake-cycle instead of per-POST, cutting ingest D1 writes ~1/3. Same observable state. await env.DB.prepare( 'INSERT INTO analytics_cursor (user_id, last_min_ts, dirty) VALUES (?,0,1) ' + - 'ON CONFLICT(user_id) DO UPDATE SET dirty = 1', + 'ON CONFLICT(user_id) DO UPDATE SET dirty = 1 WHERE analytics_cursor.dirty = 0', ).bind(userId).run() } From 9368888fe65be20128e030f35637be960d7e34ca Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sun, 21 Jun 2026 23:21:02 +0530 Subject: [PATCH 27/30] feat(workouts): ingest HAR classification + motion-typed auto-detection, incremental cron, calibration ledger; wire restlessness/daytime-HRV/desaturation/cycle-gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ingest_signals: classify live minutes (Mannini features → classifyActivityWindow) → store one act_class label/minute (no raw samples) - minute_store: MinuteRec.act_class carried through writeBatch + readMinutes - detectSessions persistence (detectStoreDay + processUser): segments/detected_type/type_confidence/type_source; reconciliation skips manual/auto_live overlaps, never clobbers a confirmed/corrected type on re-derive - migrate_v17: sessions adds segments/detected_type/type_confidence/type_source - #D incremental cron (sweepWorkoutDetection): re-derive dirty users every */10 → auto-workouts surface app-closed, backdated to last data minute - POST /workout/:id/type (confirm/correct) + classifier accuracy in /workouts - GET /day/hrv (daytime ultradian HRV) - processUser: cycle-phase gating on calcAnomaly (track_cycle); calcRestlessness folded into sleep_stress JSON - biometrics_minute: resp as illness feature + cycle-phase gating; calcDesaturation into drivers - tsc clean --- src/analytics.ts | 39 +++++++++++--- src/biometrics_minute.ts | 33 ++++++++++-- src/daydetail.ts | 26 ++++++++- src/db/migrate_v17_workout_typing.sql | 14 +++++ src/db/schema.sql | 6 ++- src/index.ts | 10 +++- src/ingest_signals.ts | 30 ++++++++++- src/minute_store.ts | 10 +++- src/workouts.ts | 77 +++++++++++++++++++++++++-- 9 files changed, 221 insertions(+), 24 deletions(-) create mode 100644 src/db/migrate_v17_workout_typing.sql diff --git a/src/analytics.ts b/src/analytics.ts index d18002a..56d0600 100644 --- a/src/analytics.ts +++ b/src/analytics.ts @@ -11,7 +11,7 @@ import { stageHypnogram, calcSleepRegularity, detectSessions, calcLoad, calcFitnessTrend, calcVo2Max, calcFitnessModel, calcMonotony, - calcAnomaly, calcBaselines, buildCoach, + calcAnomaly, calcBaselines, buildCoach, calcCycle, calcRestlessness, calcSleepStress, calcNocturnalHeart, buildNotifications, type Minute, type Profile, type Baseline, type DayHistory, type DailyStrain, type NightSummary, type SleepValue, type Metric, type Driver, @@ -33,6 +33,7 @@ interface MinuteRow { activity: number | null steps: number | null wrist_on: number | null + act_class?: string | null } const toMinute = (r: MinuteRow): Minute => ({ @@ -44,6 +45,7 @@ const toMinute = (r: MinuteRow): Minute => ({ activity: r.activity ?? 0, steps: r.steps ?? 0, wrist_on: !!r.wrist_on, + ...(r.act_class ? { act_class: r.act_class as Minute['act_class'] } : {}), }) // Build a per-metric flag entry from any Metric. @@ -321,6 +323,18 @@ export async function processUser( } } + // Cycle-phase context (gated on track_cycle) so a luteal RHR/temp rise doesn't + // false-fire the anomaly signal. Loaded once; phase computed per-day below. + let cycleStarts: string[] = [] + const trackRow = await db.prepare('SELECT track_cycle FROM users WHERE id = ?') + .bind(userId).first<{ track_cycle: number | null }>() + if (trackRow?.track_cycle) { + const { results } = await db.prepare( + "SELECT date FROM cycle_log WHERE user_id = ? AND kind = 'start' ORDER BY date", + ).bind(userId).all<{ date: string }>() + cycleStarts = (results ?? []).map((r) => r.date) + } + for (let dayStart = firstDayStart; dayStart <= lastDayStart; dayStart += DAY) { const date = dayKey(dayStart) const dayMin = dayMinutes(dayStart) @@ -383,12 +397,17 @@ export async function processUser( for (const s of sessions) { const sid = `${userId}:${s.start_ts}` statements.push(db.prepare( - 'INSERT INTO sessions (user_id, id, start_ts, end_ts, type, avg_hr, max_hr, strain, calories, hrr60, zones, confidence, status, source) ' + - "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,'done','auto') ON CONFLICT(user_id, id) DO UPDATE SET " + - 'end_ts=excluded.end_ts, type=excluded.type, avg_hr=excluded.avg_hr, max_hr=excluded.max_hr, ' + - 'strain=excluded.strain, calories=excluded.calories, hrr60=excluded.hrr60, zones=excluded.zones, confidence=excluded.confidence', + 'INSERT INTO sessions (user_id, id, start_ts, end_ts, type, avg_hr, max_hr, strain, calories, hrr60, zones, confidence, status, source, segments, detected_type, type_confidence, type_source) ' + + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,'done','auto',?,?,?,'model') ON CONFLICT(user_id, id) DO UPDATE SET " + + 'end_ts=excluded.end_ts, avg_hr=excluded.avg_hr, max_hr=excluded.max_hr, ' + + 'strain=excluded.strain, calories=excluded.calories, hrr60=excluded.hrr60, zones=excluded.zones, confidence=excluded.confidence, ' + + 'segments=excluded.segments, detected_type=excluded.detected_type, ' + + // Preserve a user-confirmed/corrected type across re-derivation; only the model fields refresh. + 'type_confidence=CASE WHEN sessions.type_source IN (\'confirmed\',\'corrected\') THEN sessions.type_confidence ELSE excluded.type_confidence END, ' + + 'type=CASE WHEN sessions.type_source IN (\'confirmed\',\'corrected\') THEN sessions.type ELSE excluded.type END', ).bind(userId, sid, s.start_ts, s.end_ts, s.type, Math.round(s.avg_hr), Math.round(s.max_hr), - s.strain, s.kcal, s.hrr60 == null ? null : Math.round(s.hrr60), JSON.stringify(s.zones), s.confidence)) + s.strain, s.kcal, s.hrr60 == null ? null : Math.round(s.hrr60), JSON.stringify(s.zones), s.confidence, + s.segments ? JSON.stringify(s.segments) : null, s.detected_type ?? s.type, s.type_confidence)) } // -- Wear time (worn minutes). -- @@ -409,6 +428,9 @@ export async function processUser( // -- Sleep stress / nocturnal arousal (HR surges + motion in sleep; NOT HRV; // the HRV-based daytime stress is computed in biometrics.ts from RR). -- const sleepStress = calcSleepStress(sleepWorn, baseline) + // -- Restlessness (movement fragmentation; distinct from arousal). Folded into the + // sleep_stress JSON so no new column / upsert change is needed. -- + const restlessness = calcRestlessness(sleepWorn) // -- Main-metric drivers (the cross-metric "what affected this" graph). -- const mainDrivers: Record = { @@ -471,7 +493,7 @@ export async function processUser( // idx points into the parallel arrays, which now START with `seedLen` // seeded history days — so trailing slices in Pass 3 reach into real history. date, dayStart, idx: seedLen + dayBuffer.length, rhr, strain, zones, calories, sleep, wearMin, daySteps, - sleepStress: JSON.stringify(sleepStress), + sleepStress: JSON.stringify({ ...sleepStress, restlessness }), nocturnal: JSON.stringify(nocturnal), sleepingHr: nocturnal.sleeping_hr_avg, nocturnalElevated: nocturnal.elevated, @@ -523,12 +545,13 @@ export async function processUser( // Readiness + HRV CV/irregular are owned by biometrics.ts (HRV is fresh there). const nocDip = (() => { try { return JSON.parse(nocturnal)?.dip_pct ?? null } catch { return null } })() + const cyclePhase = cycleStarts.length ? calcCycle(cycleStarts, date).phase : null const anomaly = calcAnomaly({ recent_rhr: recentRhr, skin_temp: baseline.skin_temp ?? null, sleep_efficiency: sleep.efficiency != null ? sleep.efficiency : null, // keep a real 0% (not coerced to "no data") baseline_sleep_efficiency: null, - }, baseline) + }, baseline, { cyclePhase }) const flags = JSON.stringify({ strain: flag(strain, 'Strain'), diff --git a/src/biometrics_minute.ts b/src/biometrics_minute.ts index 72d0be4..e2b01d8 100644 --- a/src/biometrics_minute.ts +++ b/src/biometrics_minute.ts @@ -14,8 +14,9 @@ import { timeDomainHrv, freqDomainHrv, baevskyStressIndex, calcRecovery, calcStress, calcIllness, calcHrvStability, calcIrregular, calcReadinessIndex, - calcSpo2Index, median, + calcSpo2Index, calcDesaturation, calcCycle, median, } from 'openstrap-analytics' +import type { CyclePhase } from 'openstrap-analytics' import { readMinutes } from './minute_store' import { respFromMinuteGreen } from './resp' @@ -23,7 +24,7 @@ import { respFromMinuteGreen } from './resp' // R2-decoded series when MINUTE_SOURCE='r2'). interface BioEnv { DB: D1Database; RAW_BUCKET?: R2Bucket; MINUTE_SOURCE?: string } -interface DailyHistRow { date: string; resting_hr: number | null; hrv_rmssd: number | null; hrv_si: number | null; skin_temp_idx: number | null } +interface DailyHistRow { date: string; resting_hr: number | null; hrv_rmssd: number | null; hrv_si: number | null; skin_temp_idx: number | null; resp_rate: number | null } /** * Compute + persist HRV/recovery for ONE physiological night, sourced from D1 @@ -67,15 +68,29 @@ export async function runBiometricsMinute( // 2. Prior history for Plews baseline / personal SI / illness covariance + sleep-need. const { results: hist } = await env.DB.prepare( - 'SELECT date, resting_hr, hrv_rmssd, hrv_si, skin_temp_idx FROM daily WHERE user_id = ? ORDER BY date DESC LIMIT 60', + 'SELECT date, resting_hr, hrv_rmssd, hrv_si, skin_temp_idx, resp_rate FROM daily WHERE user_id = ? ORDER BY date DESC LIMIT 60', ).bind(userId).all() const histRows = hist ?? [] const rmssdHist = histRows.map((h) => h.hrv_rmssd).filter((x): x is number => x != null) const siHist = histRows.map((h) => h.hrv_si).filter((x): x is number => x != null) const rhrHist = histRows.map((h) => h.resting_hr).filter((x): x is number => x != null) const tempHist = histRows.map((h) => h.skin_temp_idx).filter((x): x is number => x != null) + const respHist = histRows.map((h) => h.resp_rate).filter((x): x is number => x != null) const rhrByDate = new Map(histRows.map((h) => [h.date, h.resting_hr])) + // Cycle-phase context (only for users tracking their cycle) so a luteal-phase + // temp/RHR rise isn't mistaken for illness. One cheap gated read; null otherwise. + let cyclePhase: CyclePhase | null = null + const trackRow = await env.DB.prepare('SELECT track_cycle FROM users WHERE id = ?') + .bind(userId).first<{ track_cycle: number | null }>() + if (trackRow?.track_cycle) { + const { results: starts } = await env.DB.prepare( + "SELECT date FROM cycle_log WHERE user_id = ? AND kind = 'start' ORDER BY date", + ).bind(userId).all<{ date: string }>() + const startDates = (starts ?? []).map((s) => s.date) + if (startDates.length) cyclePhase = calcCycle(startDates, date).phase + } + const baseRow = await env.DB.prepare( // spo2_raw holds the rolling baseline red/IR RATIO (v2 uses the ratio, not raw red); // skin_temp_raw holds the rolling baseline temp ADC. @@ -106,8 +121,9 @@ export async function runBiometricsMinute( const recovery = calcRecovery(td.rmssd, rmssdHist, { date }) const stress = calcStress(rr, siHist, { date }) const illness = calcIllness( - { resting_hr: rhrByDate.get(date) ?? null, rmssd: td.rmssd, skin_temp: tempIdx }, - { resting_hr: rhrHist, rmssd: rmssdHist, skin_temp: tempHist }, + { resting_hr: rhrByDate.get(date) ?? null, rmssd: td.rmssd, skin_temp: tempIdx, resp_rate: respRate }, + { resting_hr: rhrHist, rmssd: rmssdHist, skin_temp: tempHist, resp_rate: respHist }, + { cyclePhase }, ) const hrvStab = calcHrvStability([td.rmssd, ...rmssdHist].filter((x): x is number => x != null).slice(0, 14)) const irregular = calcIrregular(rr) @@ -134,6 +150,13 @@ export async function runBiometricsMinute( if (spo2.drivers) bioDrivers.spo2 = spo2.drivers + // Overnight SpO₂ desaturation screen (ODI-style, RELATIVE) from the same red/IR ratios. + // Folded into drivers so no new daily column / UPDATE change is needed. + const desat = calcDesaturation(spo2Ratios, baseRatio) + if (desat.confidence > 0 && desat.events > 0) { + bioDrivers.desaturation = { events: desat.events, odi: desat.odi, deepest_pct: desat.deepest_pct, note: desat.note } + } + // 5. Persist (same null-safe COALESCE + json_patch contract as runBiometrics). SpO₂/temp // now sourced from minute optical aggregates → spo2_idx / skin_temp_idx (RELATIVE). await env.DB.prepare( diff --git a/src/daydetail.ts b/src/daydetail.ts index 792f561..0a2300e 100644 --- a/src/daydetail.ts +++ b/src/daydetail.ts @@ -11,7 +11,7 @@ import type { Context } from 'hono' import { cached, ttlForDate } from './cache' import { readMinutes } from './minute_store' import { ensureTodayWorkouts } from './workouts' -import { stageHypnogram, detectSleepCycles } from 'openstrap-analytics' +import { stageHypnogram, detectSleepCycles, calcDaytimeHrv } from 'openstrap-analytics' type Ctx = Context<{ Bindings: { DB: D1Database; RAW_BUCKET?: R2Bucket }; Variables: { userId: string } }> @@ -375,6 +375,30 @@ export async function getDayStress(c: Ctx) { return c.json(payload) } +// ── /day/hrv ─────────────────────────────────────────────────────────────────── +// Daytime (waking) HRV timeline — the ultradian RMSSD rhythm from minute.rr OUTSIDE +// the main sleep window. Complements nocturnal recovery: a daytime-stress curve. +export async function getDayHrv(c: Ctx) { + const date = (c.req.query('date') || '').trim() + if (!/^\d{4}-\d{2}-\d{2}$/.test(date)) return c.json({ error: 'date=YYYY-MM-DD required' }, 400) + const userId = c.get('userId') as string + const payload = await cached(c.env.DB, userId, `dayhrv:${date}`, ttlForDate(date), async () => { + const start = dayStartOf(date) + const mins = await loadMinutes(c, start, start + DAY) + // Exclude this date's main sleep window so it's genuinely DAYTIME HRV. + const sleep = await c.env.DB.prepare( + 'SELECT onset_ts, wake_ts FROM sleep WHERE user_id = ? AND date = ?', + ).bind(userId, date).first<{ onset_ts: number | null; wake_ts: number | null }>() + const inSleep = (t: number) => sleep?.onset_ts != null && sleep?.wake_ts != null && t >= sleep.onset_ts && t <= sleep.wake_ts + const byMinute = mins + .filter((m) => m.rr && m.rr.length && !inSleep(m.ts_min)) + .map((m) => ({ ts: m.ts_min, rr: m.rr as number[] })) + const hrv = calcDaytimeHrv(byMinute) + return { date, daytime_hrv: hrv } // {rmssd_median, series[{ts,rmssd}], lowest_ts, n_windows, confidence, tier} + }) + return c.json(payload) +} + // ── /day/timeline ──────────────────────────────────────────────────────────── export async function getDayTimeline(c: Ctx) { const date = (c.req.query('date') || '').trim() diff --git a/src/db/migrate_v17_workout_typing.sql b/src/db/migrate_v17_workout_typing.sql new file mode 100644 index 0000000..36df789 --- /dev/null +++ b/src/db/migrate_v17_workout_typing.sql @@ -0,0 +1,14 @@ +-- migrate_v17 — workout typing + calibration ledger. +-- +-- Auto-detected workouts now carry a motion-based TYPE (Mannini HAR classifier, run at +-- ingest on the live high-rate accel; the per-minute class rides minute_day.act_class) +-- plus a graceful PHASE breakdown for multi-activity sessions. We also record what the +-- model said (detected_type) vs what the user confirmed/corrected (type_source) so the +-- app can show how often the classifier was right and we know when it needs retraining. +-- +-- Additive; safe to re-run (column-exists errors are ignored by the migrate runner). + +ALTER TABLE sessions ADD COLUMN segments TEXT; -- JSON [{start_ts,end_ts,type,confidence}] phases +ALTER TABLE sessions ADD COLUMN detected_type TEXT; -- the classifier's call at detection (immutable record) +ALTER TABLE sessions ADD COLUMN type_confidence REAL; -- ESTIMATE confidence in the type +ALTER TABLE sessions ADD COLUMN type_source TEXT; -- 'model' | 'confirmed' | 'corrected' diff --git a/src/db/schema.sql b/src/db/schema.sql index 387f125..6f9fb03 100644 --- a/src/db/schema.sql +++ b/src/db/schema.sql @@ -168,8 +168,12 @@ CREATE TABLE IF NOT EXISTS sessions( avg_hr INTEGER, max_hr INTEGER, strain REAL, calories REAL, hrr60 INTEGER, zones TEXT, confidence REAL, status TEXT, -- 'live' | 'done' - source TEXT, -- 'manual' (user started) | 'auto' (detected) + source TEXT, -- 'manual' (user started) | 'auto' (minute backstop) | 'auto_live' (live-stream detected) title TEXT, -- optional user label + segments TEXT, -- JSON [{start_ts,end_ts,type,confidence}] phases (multi-activity workouts) + detected_type TEXT, -- the HAR classifier's call at detection (calibration ledger) + type_confidence REAL, -- ESTIMATE confidence in the workout type + type_source TEXT, -- 'model' | 'confirmed' | 'corrected' PRIMARY KEY(user_id, id) ); CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions(user_id, start_ts); diff --git a/src/index.ts b/src/index.ts index c5c8277..6ff0d47 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,9 +12,9 @@ import { getHistory } from './history' import { postJournal, getJournal, getJournalInsights } from './journal' import { postCycleLog, deleteCycleLog, getCycle } from './cycle' import { postSpotCheck } from './spotcheck' -import { getDayStrain, getDaySleep, getDaySleepV2, getDayTimeline, getDayStress, getDayHeart, getDayLungs, getDayWear } from './daydetail' +import { getDayStrain, getDaySleep, getDaySleepV2, getDayTimeline, getDayStress, getDayHeart, getDayLungs, getDayWear, getDayHrv } from './daydetail' import { getTrend } from './trend' -import { workoutStart, workoutEnd, listWorkouts, getWorkout, deleteWorkout, autoCloseStaleWorkouts } from './workouts' +import { workoutStart, workoutEnd, listWorkouts, getWorkout, deleteWorkout, autoCloseStaleWorkouts, setWorkoutType, sweepWorkoutDetection } from './workouts' import { getRecords } from './records' import { getNotifications, markNotificationsRead } from './notifications' import { getAppStatus, adminGetConfig, adminSetConfig } from './appconfig' @@ -254,12 +254,14 @@ app.get('/day/timeline', getDayTimeline) app.get('/day/stress', getDayStress) app.get('/day/heart', getDayHeart) app.get('/day/lungs', getDayLungs) +app.get('/day/hrv', getDayHrv) app.get('/day/wear', getDayWear) app.get('/trend/:metric', getTrend) app.post('/workout/start', workoutStart) app.post('/workout/end', workoutEnd) app.get('/workouts', listWorkouts) app.get('/workout/:id', getWorkout) +app.post('/workout/:id/type', setWorkoutType) app.delete('/workout/:id', deleteWorkout) app.get('/records', getRecords) app.get('/notifications', getNotifications) @@ -404,6 +406,10 @@ export default { // the nightly tick keeps autoCloseStaleWorkouts as a safety net for users who // never open the app. try { await runWakeLadder(env) } catch (e) { console.error('wake ladder failed', e) } + // #D incremental workout detection: re-derive TODAY's auto-workouts for users who + // ingested since their last close, so workouts surface ~one tick after they end + // even with the app closed. Throttled per-user + bounded; no hot-path cost. + try { await sweepWorkoutDetection(env) } catch (e) { console.error('wkt sweep failed', e) } // Nightly maintenance ONLY (separate from detection): seal + retention + retry-net. if (event.cron === '30 3 * * *') { try { await autoCloseStaleWorkouts(env.DB) } catch (e) { console.error('autoclose failed', e) } diff --git a/src/ingest_signals.ts b/src/ingest_signals.ts index 453d0cb..44ef3aa 100644 --- a/src/ingest_signals.ts +++ b/src/ingest_signals.ts @@ -13,10 +13,30 @@ // MAX, rr = longer blob), so a re-uploaded batch can't double-count and a fuller // batch wins. -import { calcSteps, cleanRr } from 'openstrap-analytics' +import { calcSteps, cleanRr, extractHarFeaturesFromSmv, classifyActivityWindow } from 'openstrap-analytics' import { frameAccel, hexToBytes, realtimeRr } from 'openstrap-protocol/ts/live' import { parse_r24 } from 'openstrap-protocol/ts/records' +const HAR_FS = 100 // live IMU ≈ 100 Hz (R10 100 samples/s, 0x33 stream) +const HAR_WIN = 400 // 4 s Mannini window +const HAR_MIN = 128 // need ≥~1.3 s for a defensible spectrum + +/** Classify a minute's magnitude (SMV) signal into a dominant HAR class via the + * Mannini feature extractor + threshold classifier. Returns undefined when there's + * too little high-rate accel (e.g. flash 1 Hz minutes carry none). */ +function classifyMinute(sig: number[]): string | undefined { + if (sig.length < HAR_MIN) return undefined + const counts: Record = {} + for (let i = 0; i + HAR_MIN <= sig.length; i += HAR_WIN) { + const w = sig.slice(i, Math.min(i + HAR_WIN, sig.length)) + const { cls } = classifyActivityWindow(extractHarFeaturesFromSmv(w, HAR_FS)) + counts[cls] = (counts[cls] ?? 0) + 1 + } + let best: string | undefined, bestN = -1 + for (const k of Object.keys(counts)) if (counts[k] > bestN) { bestN = counts[k]; best = k } + return best +} + export interface MinuteSignal { steps: number rr: number[] @@ -30,6 +50,9 @@ export interface MinuteSignal { // value estimateResp consumes — so storing it lets resp compute from D1 at the close // with no raw R2. Present only during live optical sessions (R21 is live-only). green?: number[] + // Dominant HAR activity class for the minute, from the live high-rate accel (Mannini + // classifier). One tiny label — drives workout typing. Live minutes only. + act_class?: string } interface AccelFrame { idx: number; ts: number; mags: number[] } @@ -116,12 +139,16 @@ export function perMinuteSignals(records: string[]): Map { const minutes = new Set([...accelByMin.keys(), ...rrByMin.keys(), ...optByMin.keys(), ...greenByMin.keys()]) for (const m of minutes) { let steps = 0 + let act_class: string | undefined const frames = accelByMin.get(m) if (frames && frames.size > 0) { const ordered = [...frames.values()].sort((a, b) => a.ts - b.ts || a.idx - b.idx) const sig: number[] = [] for (const fr of ordered) for (const v of fr.mags) sig.push(v) steps = calcSteps([sig]) + // Classify the minute's motion at ingest (the ONLY place the high-rate accel + // exists) → store just the dominant class label, never the raw samples. + act_class = classifyMinute(sig) } // PPG green: time-ordered per-second means for the minute (RIIV series → resp at close). const gArr = greenByMin.get(m) @@ -135,6 +162,7 @@ export function perMinuteSignals(records: string[]): Map { steps, rr: cleanRr(rrByMin.get(m) ?? []), ...(o ? { opt_n: o.n, red_sum: o.red, ir_sum: o.ir, temp_sum: o.temp } : {}), ...(green ? { green } : {}), + ...(act_class ? { act_class } : {}), }) } return out diff --git a/src/minute_store.ts b/src/minute_store.ts index 51debb3..2402c41 100644 --- a/src/minute_store.ts +++ b/src/minute_store.ts @@ -42,6 +42,11 @@ export interface MinuteRec { // live-stream-only), so usually empty. Replaces the R2 raw store for resp — // resp is computed from this series at the wake-close. Optional (older blobs lack it). green?: number[] + // Dominant HAR activity class for this minute, classified at ingest from the live + // high-rate accel (Mannini, see openstrap-analytics/har). One tiny enum string — NOT + // the raw samples. Present only for live-streamed minutes (flash R24 is 1 Hz → none). + // Feeds workout typing + segmentation in detectSessions. Optional (older blobs lack it). + act_class?: string } export interface StoreEnv { DB: D1Database; RAW_BUCKET?: R2Bucket } @@ -127,7 +132,7 @@ export async function latestMinute(env: StoreEnv, userId: string, sinceTs: numbe export async function writeBatch( env: StoreEnv, userId: string, buckets: MinuteBucket[], - signals: Map, + signals: Map, now: number, ): Promise { if (buckets.length === 0) return 0 @@ -157,6 +162,9 @@ export async function writeBatch( // PPG green RIIV proxy: same "keep the fuller array" idempotency as rr. const newGreen = sig?.green ?? [] if (newGreen.length >= (rec.green?.length ?? 0)) rec.green = newGreen + // HAR activity class (live minutes only): keep when present. Recompute on re-upload + // is deterministic, so a re-drained batch yields the same label. + if (sig?.act_class) rec.act_class = sig.act_class // Optical: additive sums + count (same idempotency basis as steps — edge hex-dedup). if (sig?.opt_n) { rec.opt_n = (rec.opt_n ?? 0) + sig.opt_n diff --git a/src/workouts.ts b/src/workouts.ts index 4123b06..b821797 100644 --- a/src/workouts.ts +++ b/src/workouts.ts @@ -126,13 +126,17 @@ export async function listWorkouts(c: Ctx) { const days = RANGE_DAYS[range] ?? 35 const from = nowSec() - days * DAY const { results } = await c.env.DB.prepare( - 'SELECT id, start_ts, end_ts, type, avg_hr, max_hr, strain, calories, hrr60, zones, status, source, title ' + + 'SELECT id, start_ts, end_ts, type, avg_hr, max_hr, strain, calories, hrr60, zones, status, source, title, ' + + 'segments, detected_type, type_confidence, type_source ' + "FROM sessions WHERE user_id = ? AND start_ts >= ? AND status != 'deleted' ORDER BY start_ts DESC", ).bind(userId, from).all() const rows = (results ?? []).map((r: any) => ({ ...r, duration_min: r.end_ts && r.start_ts ? Math.max(0, Math.round((r.end_ts - r.start_ts) / 60)) : 0, zones: r.zones ? safeParse(r.zones) : null, + segments: r.segments ? safeParse(r.segments) : null, + // distinguish in UI: detected (auto/auto_live) vs selected (manual) + detected: r.source === 'auto' || r.source === 'auto_live', })) // Training-volume summary (HONEST: time/type/zones/calories — no GPS distance, // no rep/weight; we don't fabricate what the band can't measure). @@ -152,6 +156,13 @@ export async function listWorkouts(c: Ctx) { } } const hardest = done.reduce((best, r) => (best == null || (r.strain ?? 0) > (best.strain ?? 0)) ? r : best, null) + // Calibration ledger: of the workouts the user confirmed/corrected, how often did the + // classifier already have it right? Surfaced so we know when the model needs retraining. + const acc = await c.env.DB.prepare( + "SELECT COUNT(*) AS total, SUM(CASE WHEN type = detected_type THEN 1 ELSE 0 END) AS correct " + + "FROM sessions WHERE user_id = ? AND detected_type IS NOT NULL AND type_source IN ('confirmed','corrected')", + ).bind(userId).first<{ total: number; correct: number }>() + const accTotal = acc?.total ?? 0 return c.json({ range, workouts: rows, @@ -162,10 +173,49 @@ export async function listWorkouts(c: Ctx) { by_type: byType, zone_min: zoneTotals, hardest: hardest ? { id: hardest.id, type: hardest.type, strain: hardest.strain } : null, + // model accuracy from user feedback (null until the user has confirmed/corrected any) + classifier: { reviewed: accTotal, correct: acc?.correct ?? 0, accuracy: accTotal > 0 ? Math.round(((acc!.correct) / accTotal) * 100) / 100 : null }, }, }) } +// POST /workout/:id/type { type } — user confirms or corrects an auto-detected type. +// Feeds the calibration ledger (was the model right?) and pins the type so re-derivation +// won't overwrite it. The accumulated (detected_type → corrected type) pairs are the +// labelled dataset that will train the real classifier. +export async function setWorkoutType(c: Ctx) { + const userId = c.get('userId') + const id = c.req.param('id') + const body = await c.req.json<{ type?: string }>().catch(() => ({} as any)) + if (!body.type || !VALID_TYPES.includes(body.type)) return c.json({ error: 'invalid type' }, 400) + const w = await c.env.DB.prepare('SELECT detected_type FROM sessions WHERE user_id = ? AND id = ?') + .bind(userId, id).first<{ detected_type: string | null }>() + if (!w) return c.json({ error: 'not found' }, 404) + const source = w.detected_type && w.detected_type === body.type ? 'confirmed' : 'corrected' + await c.env.DB.prepare( + 'UPDATE sessions SET type = ?, type_source = ?, type_confidence = 1.0 WHERE user_id = ? AND id = ?', + ).bind(body.type, source, userId, id).run() + return c.json({ ok: true, type: body.type, type_source: source }) +} + +/** + * sweepWorkoutDetection(env) — the #D incremental cron. Re-derive TODAY's auto-workouts + * for every user who's ingested since their last close (dirty=1), so a workout surfaces + * ~one tick (≤10 min) after it ends WITHOUT the app being opened. Reuses the throttled + * ensureTodayWorkouts (≤1 detect / 120 s / user) and is bounded per tick. The bout closes + * at its last data minute naturally (re-derivation over available minutes). + */ +export async function sweepWorkoutDetection(env: { DB: D1Database }, limit = 200): Promise { + const { results } = await env.DB.prepare( + 'SELECT user_id FROM analytics_cursor WHERE dirty = 1 LIMIT ?', + ).bind(limit).all<{ user_id: string }>() + let n = 0 + for (const r of results ?? []) { + try { await ensureTodayWorkouts(env.DB, r.user_id); n++ } catch (e) { console.error('wkt detect failed', r.user_id, e) } + } + return n +} + // GET /workout/:id → full breakdown + HR/activity timeline + derived effort // analytics (zone bands w/ bpm ranges + %, min HR, HR drift, time-to-peak, the // HR-recovery curve, steps/cadence, wrist coverage). Derived fields are computed @@ -332,21 +382,38 @@ async function detectStoreDay(db: D1Database, userId: string, dayStart: number, const dayMin: Minute[] = recs.map((r) => ({ ts: r.ts_min, hr_avg: r.hr_avg, hr_min: r.hr_min, hr_max: r.hr_max, hr_n: r.hr_n, activity: r.activity, steps: r.steps, wrist_on: !!r.wrist_on, + act_class: r.act_class as Minute['act_class'], })) const sessions = detectSessions(dayMin, baseline, profile) + // Reconcile: don't clobber the user's manual workouts, deleted tombstones, OR any + // already-typed live-detected sessions (source='auto_live'); only the minute-detector's + // own 'auto' rows are re-derived. Then skip auto bouts that overlap a manual/auto_live + // session so a live-streamed workout isn't double-counted by the backstop. + const { results: keep } = await db.prepare( + "SELECT start_ts, end_ts FROM sessions WHERE user_id = ? AND start_ts < ? AND end_ts > ? AND status != 'deleted' AND source IN ('manual','auto_live')", + ).bind(userId, dayStart + DAY, dayStart).all<{ start_ts: number; end_ts: number }>() + const covered = (keep ?? []) as { start_ts: number; end_ts: number }[] + const overlaps = (s: { start_ts: number; end_ts: number }) => + covered.some((k) => s.start_ts < k.end_ts && s.end_ts > k.start_ts) const stmts: D1PreparedStatement[] = [ db.prepare("DELETE FROM sessions WHERE user_id = ? AND start_ts >= ? AND start_ts < ? AND (source IS NULL OR source = 'auto') AND status != 'deleted'") .bind(userId, dayStart, dayStart + DAY), ] for (const s of sessions) { + if (overlaps(s)) continue // a manual/live-detected session already owns this window const sid = `${userId}:${s.start_ts}` stmts.push(db.prepare( - 'INSERT INTO sessions (user_id, id, start_ts, end_ts, type, avg_hr, max_hr, strain, calories, hrr60, zones, confidence, status, source) ' + - "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,'done','auto') ON CONFLICT(user_id, id) DO UPDATE SET " + + 'INSERT INTO sessions (user_id, id, start_ts, end_ts, type, avg_hr, max_hr, strain, calories, hrr60, zones, confidence, status, source, segments, detected_type, type_confidence, type_source) ' + + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,'done','auto',?,?,?,'model') ON CONFLICT(user_id, id) DO UPDATE SET " + 'end_ts=excluded.end_ts, type=excluded.type, avg_hr=excluded.avg_hr, max_hr=excluded.max_hr, ' + - 'strain=excluded.strain, calories=excluded.calories, hrr60=excluded.hrr60, zones=excluded.zones, confidence=excluded.confidence', + 'strain=excluded.strain, calories=excluded.calories, hrr60=excluded.hrr60, zones=excluded.zones, confidence=excluded.confidence, ' + + // Preserve a user-confirmed/corrected type across re-derivation; only refresh the model fields. + 'segments=excluded.segments, detected_type=excluded.detected_type, ' + + 'type_confidence=CASE WHEN sessions.type_source IN (\'confirmed\',\'corrected\') THEN sessions.type_confidence ELSE excluded.type_confidence END, ' + + 'type=CASE WHEN sessions.type_source IN (\'confirmed\',\'corrected\') THEN sessions.type ELSE excluded.type END', ).bind(userId, sid, s.start_ts, s.end_ts, s.type, Math.round(s.avg_hr), Math.round(s.max_hr), - s.strain, s.kcal, s.hrr60 == null ? null : Math.round(s.hrr60), JSON.stringify(s.zones), s.confidence)) + s.strain, s.kcal, s.hrr60 == null ? null : Math.round(s.hrr60), JSON.stringify(s.zones), s.confidence, + s.segments ? JSON.stringify(s.segments) : null, s.detected_type ?? s.type, s.type_confidence)) } await db.batch(stmts) } From 884540e3e847f1e413f67fb4a2fad6f4e027448a Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sun, 21 Jun 2026 23:24:16 +0530 Subject: [PATCH 28/30] fix(ingest): guard HAR classification so a classifier fault can never break ingest (raw-first) --- src/ingest_signals.ts | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/ingest_signals.ts b/src/ingest_signals.ts index 44ef3aa..93ae5d2 100644 --- a/src/ingest_signals.ts +++ b/src/ingest_signals.ts @@ -26,15 +26,21 @@ const HAR_MIN = 128 // need ≥~1.3 s for a defensible spectrum * too little high-rate accel (e.g. flash 1 Hz minutes carry none). */ function classifyMinute(sig: number[]): string | undefined { if (sig.length < HAR_MIN) return undefined - const counts: Record = {} - for (let i = 0; i + HAR_MIN <= sig.length; i += HAR_WIN) { - const w = sig.slice(i, Math.min(i + HAR_WIN, sig.length)) - const { cls } = classifyActivityWindow(extractHarFeaturesFromSmv(w, HAR_FS)) - counts[cls] = (counts[cls] ?? 0) + 1 + // Defensive: a HAR bug must NEVER break the ingest hot path (raw-first / never lose + // data). Any failure → no class label for the minute, ingest proceeds unaffected. + try { + const counts: Record = {} + for (let i = 0; i + HAR_MIN <= sig.length; i += HAR_WIN) { + const w = sig.slice(i, Math.min(i + HAR_WIN, sig.length)) + const { cls } = classifyActivityWindow(extractHarFeaturesFromSmv(w, HAR_FS)) + counts[cls] = (counts[cls] ?? 0) + 1 + } + let best: string | undefined, bestN = -1 + for (const k of Object.keys(counts)) if (counts[k] > bestN) { bestN = counts[k]; best = k } + return best + } catch { + return undefined } - let best: string | undefined, bestN = -1 - for (const k of Object.keys(counts)) if (counts[k] > bestN) { bestN = counts[k]; best = k } - return best } export interface MinuteSignal { From 051f1e3091906732297d8cdc6efd6a074fbdceff Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Mon, 22 Jun 2026 14:13:15 +0530 Subject: [PATCH 29/30] fix(workouts): stop re-derivation wiping user-confirmed/corrected types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit detectStoreDay and processUser both DELETE auto sessions before re-inserting, and setWorkoutType leaves source='auto' — so a confirmed/corrected workout was deleted on every re-derive (the */10 sweep, every close_day) and reset to the model's guess, erasing the calibration ledger within ~10 min. The INSERT's ON CONFLICT preserve-corrected clause never fired because the row was already gone in the same batch transaction. Exclude type_source IN ('confirmed','corrected') from both DELETEs so the labelled row survives (ON CONFLICT then preserves type/confidence while refreshing the model metrics), and add it to the overlap-skip set so a baseline-drift start_ts shift can't insert a duplicate model row over the user's label. --- src/analytics.ts | 13 ++++++++++++- src/workouts.ts | 15 +++++++++------ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/analytics.ts b/src/analytics.ts index 56d0600..5562f1c 100644 --- a/src/analytics.ts +++ b/src/analytics.ts @@ -390,11 +390,22 @@ export async function processUser( // user-owned and must survive a re-derive. Deleted tombstones are KEPT so a // user-deleted auto session isn't resurrected (its row's status stays // 'deleted'; the re-insert below only updates non-status fields via ON CONFLICT). + // ALSO preserve any auto session the user CONFIRMED/CORRECTED (the calibration + // ledger): exclude it from the DELETE and skip an overlapping re-detection so a + // baseline-drift start_ts shift can't insert a duplicate that re-overwrites the + // user's label. (Without this, every re-derive wiped user corrections.) + const { results: keepRev } = await db.prepare( + "SELECT start_ts, end_ts FROM sessions WHERE user_id = ? AND start_ts < ? AND end_ts > ? AND status != 'deleted' AND type_source IN ('confirmed','corrected')", + ).bind(userId, dayStart + DAY, dayStart).all<{ start_ts: number; end_ts: number }>() + const reviewed = (keepRev ?? []) as { start_ts: number; end_ts: number }[] + const overlapsReviewed = (s: { start_ts: number; end_ts: number }) => + reviewed.some((k) => s.start_ts < k.end_ts && s.end_ts > k.start_ts) statements.push( - db.prepare("DELETE FROM sessions WHERE user_id = ? AND start_ts >= ? AND start_ts < ? AND (source IS NULL OR source = 'auto') AND status != 'deleted'") + db.prepare("DELETE FROM sessions WHERE user_id = ? AND start_ts >= ? AND start_ts < ? AND (source IS NULL OR source = 'auto') AND COALESCE(type_source,'model') NOT IN ('confirmed','corrected') AND status != 'deleted'") .bind(userId, dayStart, dayStart + DAY), ) for (const s of sessions) { + if (overlapsReviewed(s)) continue const sid = `${userId}:${s.start_ts}` statements.push(db.prepare( 'INSERT INTO sessions (user_id, id, start_ts, end_ts, type, avg_hr, max_hr, strain, calories, hrr60, zones, confidence, status, source, segments, detected_type, type_confidence, type_source) ' + diff --git a/src/workouts.ts b/src/workouts.ts index b821797..a619e5d 100644 --- a/src/workouts.ts +++ b/src/workouts.ts @@ -385,18 +385,21 @@ async function detectStoreDay(db: D1Database, userId: string, dayStart: number, act_class: r.act_class as Minute['act_class'], })) const sessions = detectSessions(dayMin, baseline, profile) - // Reconcile: don't clobber the user's manual workouts, deleted tombstones, OR any - // already-typed live-detected sessions (source='auto_live'); only the minute-detector's - // own 'auto' rows are re-derived. Then skip auto bouts that overlap a manual/auto_live - // session so a live-streamed workout isn't double-counted by the backstop. + // Reconcile: don't clobber the user's manual workouts, deleted tombstones, any + // already-typed live-detected sessions (source='auto_live'), OR any auto session the + // user CONFIRMED/CORRECTED (type_source in confirmed/corrected — the calibration + // ledger). Only the minute-detector's own un-reviewed 'auto' rows are re-derived. Then + // skip auto bouts that overlap any preserved session so a live-streamed or + // user-labelled workout isn't double-counted (and a baseline-drift start_ts shift can't + // spawn a duplicate that re-overwrites the user's label). const { results: keep } = await db.prepare( - "SELECT start_ts, end_ts FROM sessions WHERE user_id = ? AND start_ts < ? AND end_ts > ? AND status != 'deleted' AND source IN ('manual','auto_live')", + "SELECT start_ts, end_ts FROM sessions WHERE user_id = ? AND start_ts < ? AND end_ts > ? AND status != 'deleted' AND (source IN ('manual','auto_live') OR type_source IN ('confirmed','corrected'))", ).bind(userId, dayStart + DAY, dayStart).all<{ start_ts: number; end_ts: number }>() const covered = (keep ?? []) as { start_ts: number; end_ts: number }[] const overlaps = (s: { start_ts: number; end_ts: number }) => covered.some((k) => s.start_ts < k.end_ts && s.end_ts > k.start_ts) const stmts: D1PreparedStatement[] = [ - db.prepare("DELETE FROM sessions WHERE user_id = ? AND start_ts >= ? AND start_ts < ? AND (source IS NULL OR source = 'auto') AND status != 'deleted'") + db.prepare("DELETE FROM sessions WHERE user_id = ? AND start_ts >= ? AND start_ts < ? AND (source IS NULL OR source = 'auto') AND COALESCE(type_source,'model') NOT IN ('confirmed','corrected') AND status != 'deleted'") .bind(userId, dayStart, dayStart + DAY), ] for (const s of sessions) { From 903cad850bc258e9fb32a0d984c65cf9ce8d9f88 Mon Sep 17 00:00:00 2001 From: shoshiiiin Date: Wed, 24 Jun 2026 14:24:21 +0200 Subject: [PATCH 30/30] feat: free-tier analytics fallback, baseline seeding, self-hosted AI Coach, Strava sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four self-contained improvements (details in PR): - Free-tier: derive each day INLINE when no ANALYTICS_Q queue is bound, so self-hosters on the Workers Free plan (queues are paid-only) actually get metrics instead of days that are marked closed but never computed. - Seed profile-prior baselines on a user's first run (uses seedBaselines from the analytics package). - Self-hosted AI Coach: OpenAI-compatible /coach/v1 backed by the Workers AI binding — no external provider/key, runs in the free AI allocation. - Strava bidirectional sync: OAuth + pull (activities incl. bike-computer rides) + push (workouts, deduped so rides are never duplicated), with a daily cron. --- src/analytics.ts | 16 ++- src/coach.ts | 144 ++++++++++++++++++++++ src/db/schema.sql | 35 ++++++ src/index.ts | 24 ++++ src/queue.ts | 36 +++--- src/strava.ts | 296 ++++++++++++++++++++++++++++++++++++++++++++++ src/wake_cron.ts | 13 +- wrangler.toml | 39 +++--- 8 files changed, 572 insertions(+), 31 deletions(-) create mode 100644 src/coach.ts create mode 100644 src/strava.ts diff --git a/src/analytics.ts b/src/analytics.ts index 5562f1c..b1f0caa 100644 --- a/src/analytics.ts +++ b/src/analytics.ts @@ -11,7 +11,7 @@ import { stageHypnogram, calcSleepRegularity, detectSessions, calcLoad, calcFitnessTrend, calcVo2Max, calcFitnessModel, calcMonotony, - calcAnomaly, calcBaselines, buildCoach, calcCycle, calcRestlessness, + calcAnomaly, calcBaselines, seedBaselines, buildCoach, calcCycle, calcRestlessness, calcSleepStress, calcNocturnalHeart, buildNotifications, type Minute, type Profile, type Baseline, type DayHistory, type DailyStrain, type NightSummary, type SleepValue, type Metric, type Driver, @@ -181,6 +181,20 @@ export async function processUser( const lastDayStart = oldestDayStart + (dTo - 1) * DAY const profile = await loadProfile(db, userId) + + // Seed population/profile-prior baselines on the first-ever run (INSERT OR IGNORE + // → only when no row exists). Gives strain/HR-zones a personal resting/max HR from + // day 1 and warm-starts the EWMA autonomic baselines (RMSSD/sleeping-HR/SI) from + // physiological norms instead of cold. Real measured data overwrites these: the + // calcBaselines medians below, the nightly EWMA rolls, and the ≥5-night gated + // scores (recovery/stress) — which still need real nights and are never faked. + const seed = seedBaselines(profile) + await db.prepare( + 'INSERT OR IGNORE INTO baselines (user_id, resting_hr, max_hr, sleep_need_min, hrv_rmssd, sleeping_hr, resp_rate, hrv_si, updated_at) ' + + 'VALUES (?,?,?,?,?,?,?,?,?)', + ).bind(userId, seed.resting_hr, seed.max_hr, seed.sleep_need_min, seed.hrv_rmssd, + seed.sleeping_hr, seed.resp_rate, seed.hrv_si, now).run() + let baseline = await loadBaseline(db, userId) // HRV recovery (computed in biometrics.ts from RR) is read back here to drive diff --git a/src/coach.ts b/src/coach.ts new file mode 100644 index 0000000..6ac9873 --- /dev/null +++ b/src/coach.ts @@ -0,0 +1,144 @@ +// coach.ts — self-hosted, OpenAI-compatible LLM endpoint for the in-app AI Coach, +// backed by Cloudflare Workers AI (the `AI` binding). No external provider and no +// OpenAI key: it runs on your own Worker within Cloudflare's free AI allocation. +// +// The app's AI Coach is a BYOK, agentic tool-caller that POSTs OpenAI-shaped +// `/chat/completions` (model, messages, tools, tool_choice) and reads back +// `choices[0].message` (content + tool_calls). Point its base URL at +// `/coach/v1` and set its API key to COACH_KEY. We translate that contract +// to `env.AI.run(...)` and map Workers AI's `{ response, tool_calls }` back to the +// OpenAI shape the coach expects. + +export interface AiBinding { + run(model: string, input: Record): Promise +} + +export interface CoachEnv { + AI: AiBinding + COACH_KEY?: string +} + +interface OpenAIToolCall { + id: string + type: 'function' + function: { name: string; arguments: string } +} + +interface OpenAIMessage { + role: 'assistant' + content: string | null + tool_calls?: OpenAIToolCall[] +} + +// Llama 3.1 8B (fast) is the default: ~0.4s/call and it reliably CONVERGES the +// agentic tool loop (answers in prose after a tool result). The 70B fp8 model is +// available but tends to re-call tools instead of converging here, so it's slower +// in practice for the multi-step coach. The client may pin any of these. +const DEFAULT_MODEL = '@cf/meta/llama-3.1-8b-instruct-fast' + +const COACH_MODELS = [ + '@cf/meta/llama-3.1-8b-instruct-fast', + '@cf/meta/llama-3.3-70b-instruct-fp8-fast', + '@hf/nousresearch/hermes-2-pro-mistral-7b', +] + +// Constant-time-ish bearer check against the configured COACH_KEY. Without a +// configured key the endpoint is closed (never open by default). +function authorized(c: any): boolean { + const want = c.env.COACH_KEY + if (!want) return false + const auth = c.req.header('Authorization') ?? '' + return auth.startsWith('Bearer ') && auth.slice(7) === want +} + +// Pass tools through in OpenAI shape (`{ type:'function', function:{...} }`). +// Workers AI's OpenAI-compatible models validate this exact shape (the unwrapped +// Cloudflare form `{name,description,parameters}` is rejected by some models, e.g. +// llama-3.1-8b-instruct-fast). The coach already sends the OpenAI shape, so forward +// it unchanged. +function normalizeTools(tools: unknown): unknown[] | undefined { + if (!Array.isArray(tools) || tools.length === 0) return undefined + return tools +} + +function extractText(out: any): string { + if (typeof out === 'string') return out + if (typeof out?.response === 'string') return out.response + if (typeof out?.result?.response === 'string') return out.result.response + return '' +} + +function mapToolCalls(out: any): OpenAIToolCall[] | undefined { + const raw = out?.tool_calls ?? out?.result?.tool_calls + if (!Array.isArray(raw) || raw.length === 0) return undefined + return raw.map((tc: any, i: number) => { + const name = tc?.name ?? tc?.function?.name ?? '' + const argsRaw = tc?.arguments ?? tc?.function?.arguments ?? {} + const args = typeof argsRaw === 'string' ? argsRaw : JSON.stringify(argsRaw ?? {}) + return { + id: tc?.id ?? `call_${Date.now()}_${i}`, + type: 'function' as const, + function: { name, arguments: args }, + } + }) +} + +// GET /coach/v1/models — the coach lists models from here (OpenAI /models shape). +export async function coachModels(c: any) { + if (!authorized(c)) return c.json({ error: { message: 'Unauthorized' } }, 401) + return c.json({ + object: 'list', + data: COACH_MODELS.map((id) => ({ id, object: 'model', owned_by: 'cloudflare' })), + }) +} + +// POST /coach/v1/chat/completions — OpenAI-compatible, backed by Workers AI. +export async function coachChatCompletions(c: any) { + if (!authorized(c)) return c.json({ error: { message: 'Unauthorized' } }, 401) + + let body: any + try { + body = await c.req.json() + } catch { + return c.json({ error: { message: 'Invalid JSON body' } }, 400) + } + + const model = typeof body?.model === 'string' && body.model ? body.model : DEFAULT_MODEL + const messages = Array.isArray(body?.messages) ? body.messages : [] + if (messages.length === 0) { + return c.json({ error: { message: 'messages[] is required' } }, 400) + } + + const input: Record = { + messages, + temperature: typeof body?.temperature === 'number' ? body.temperature : 0.3, + max_tokens: typeof body?.max_tokens === 'number' ? body.max_tokens : 1024, + } + const tools = normalizeTools(body?.tools) + if (tools) input.tools = tools + + let out: unknown + try { + out = await c.env.AI.run(model, input) + } catch (e: unknown) { + const msg = e instanceof Error ? e.message : 'Workers AI request failed' + return c.json({ error: { message: `Workers AI error: ${msg}` } }, 502) + } + + const toolCalls = mapToolCalls(out) + const message: OpenAIMessage = toolCalls + ? { role: 'assistant', content: extractText(out) || null, tool_calls: toolCalls } + : { role: 'assistant', content: extractText(out) } + + return c.json({ + id: `chatcmpl-${Date.now()}`, + object: 'chat.completion', + created: Math.floor(Date.now() / 1000), + model, + choices: [{ + index: 0, + message, + finish_reason: toolCalls ? 'tool_calls' : 'stop', + }], + }) +} diff --git a/src/db/schema.sql b/src/db/schema.sql index 6f9fb03..8830d29 100644 --- a/src/db/schema.sql +++ b/src/db/schema.sql @@ -242,3 +242,38 @@ CREATE TABLE IF NOT EXISTS app_config( updated_at INTEGER ); INSERT OR IGNORE INTO app_config (id) VALUES (1); + +-- ── STRAVA (bidirectional sync) ─────────────────────────────────────────────── +-- OAuth tokens per user. access_token is short-lived; refresh via refresh_token. +CREATE TABLE IF NOT EXISTS strava_tokens( + user_id TEXT PRIMARY KEY, + athlete_id INTEGER, + access_token TEXT NOT NULL, + refresh_token TEXT NOT NULL, + expires_at INTEGER NOT NULL, -- unix seconds; refresh when now >= this + scope TEXT, + connected_at INTEGER +); +-- Activities pulled from Strava (Strava → Whoop), kept for the in-app overlay. +CREATE TABLE IF NOT EXISTS strava_activities( + user_id TEXT NOT NULL, + activity_id INTEGER NOT NULL, -- Strava's activity id + start_ts INTEGER, -- unix seconds (UTC) + elapsed_sec INTEGER, + type TEXT, + name TEXT, + distance_m REAL, + avg_hr REAL, max_hr REAL, + raw TEXT, -- the JSON summary, for later enrichment + pulled_at INTEGER, + PRIMARY KEY(user_id, activity_id) +); +CREATE INDEX IF NOT EXISTS idx_strava_act_user ON strava_activities(user_id, start_ts); +-- Which OpenStrap sessions we've pushed to Strava (Whoop → Strava dedupe). +CREATE TABLE IF NOT EXISTS strava_pushed( + user_id TEXT NOT NULL, + session_key TEXT NOT NULL, + activity_id INTEGER, + pushed_at INTEGER, + PRIMARY KEY(user_id, session_key) +); diff --git a/src/index.ts b/src/index.ts index 6ff0d47..35a5e25 100644 --- a/src/index.ts +++ b/src/index.ts @@ -19,6 +19,8 @@ import { getRecords } from './records' import { getNotifications, markNotificationsRead } from './notifications' import { getAppStatus, adminGetConfig, adminSetConfig } from './appconfig' import { seedInit, seedMinutes, seedAnalytics } from './seed' +import { coachChatCompletions, coachModels, type AiBinding } from './coach' +import { stravaConnect, stravaCallback, stravaStatus, stravaDisconnect, stravaSync, stravaActivities, syncAllStrava } from './strava' type Bindings = { DB: D1Database @@ -27,6 +29,10 @@ type Bindings = { ANALYTICS_Q?: Queue JWT_SECRET: string ADMIN_TOKEN?: string + AI: AiBinding // Cloudflare Workers AI (self-hosted AI Coach) + COACH_KEY?: string // bearer the in-app AI Coach uses for /coach/v1/* + STRAVA_CLIENT_ID?: string // Strava API app credentials (bidirectional sync) + STRAVA_CLIENT_SECRET?: string BREVO_API_KEY?: string RESEND_API_KEY?: string EMAIL_FROM?: string @@ -67,6 +73,22 @@ const requireAdmin = async (c: any, next: any) => { await next() } +// ---------- self-hosted AI Coach (OpenAI-compatible, Workers AI) ---------- +// Auth is the COACH_KEY bearer, checked inside the handlers (not requireJwt) so the +// app's BYOK AI-Coach client can point its base URL at `/coach/v1`. +app.post('/coach/v1/chat/completions', coachChatCompletions) +app.get('/coach/v1/models', coachModels) + +// ---------- Strava (bidirectional sync) ---------- +// /connect, /status, /disconnect need the user JWT; /callback is Strava's public +// redirect target (auth is the signed `state` it round-trips). +app.get('/strava/connect', requireJwt, stravaConnect) +app.get('/strava/callback', stravaCallback) +app.get('/strava/status', requireJwt, stravaStatus) +app.post('/strava/disconnect', requireJwt, stravaDisconnect) +app.get('/strava/sync', requireJwt, stravaSync) +app.get('/strava/activities', requireJwt, stravaActivities) + app.use('/ingest/*', requireJwt) app.use('/profile', requireJwt) app.use('/today', requireJwt) @@ -421,6 +443,8 @@ export default { await pruneMinuteDays(env, nowS, MINUTE_RETENTION_DAYS) await env.DB.prepare('DELETE FROM events WHERE ts < ?').bind(nowS - MINUTE_RETENTION_DAYS * DAY).run() try { await retryStaleCloses(env) } catch (e) { console.error('retry-net failed', e) } + // Strava: pull new activities (incl. Wahoo rides) + push non-duplicate workouts. + try { await syncAllStrava(env) } catch (e) { console.error('strava sync failed', e) } } })()) }, diff --git a/src/queue.ts b/src/queue.ts index d4c2276..8812562 100644 --- a/src/queue.ts +++ b/src/queue.ts @@ -28,25 +28,33 @@ interface QueueEnv { ANALYTICS_Q?: Queue } +// [free-tier] Close one physiological day for one user. Exported so the cron can run +// this INLINE when no queue is bound (Cloudflare Queues require Workers Paid). All work +// is D1 + (optional) R2 reads — well under the free plan's 1000 Cloudflare-subrequest +// cap for a single user. Mirrors the 'close_day' queue branch exactly. +export async function closeDay(env: QueueEnv, userId: string, day?: string, onset_ts?: number, wake_ts?: number): Promise { + // [wake-trigger] fired ONCE per physiological day when the user wakes. Derives + // the day (sleep/naps/strain/sessions/baselines/coach via processUser) and folds + // in HRV/recovery from D1 minute.rr — zero R2. Then invalidates the day's Tier-2 + // cache and clears the dirty flag (daytime ingests re-set it; the cron skips + // awake-and-closed users by last_close_date, so no churn). + // processUser derives the day AND folds daily.steps = SUM(minute.steps) (AN-2554 + // counted at ingest) — no separate step job. + await processUser(env.DB, userId, { historyDays: 3 }) + if (day && wake_ts) { + const from = onset_ts ?? (wake_ts - 8 * 3600) + try { await runBiometricsMinute({ DB: env.DB, RAW_BUCKET: env.RAW_BUCKET }, userId, day, from, wake_ts + 60) } catch (e) { console.error('biometrics_minute failed', userId, day, e) } + await invalidateDay(env.DB, userId, day) + } + await env.DB.prepare('UPDATE analytics_cursor SET dirty = 0 WHERE user_id = ?').bind(userId).run() +} + // Run one bounded unit of work. Each branch is sized to fit in a single // invocation's budget (one user, one job, a few days of R2 at most). async function runJob(env: QueueEnv, userId: string, job: AnalyticsJob, day?: string, onset_ts?: number, wake_ts?: number): Promise { switch (job) { case 'close_day': - // [wake-trigger] fired ONCE per physiological day when the user wakes. Derives - // the day (sleep/naps/strain/sessions/baselines/coach via processUser) and folds - // in HRV/recovery from D1 minute.rr — zero R2. Then invalidates the day's Tier-2 - // cache and clears the dirty flag (daytime ingests re-set it; the cron skips - // awake-and-closed users by last_close_date, so no churn). - // processUser derives the day AND folds daily.steps = SUM(minute.steps) (AN-2554 - // counted at ingest) — no separate step job. - await processUser(env.DB, userId, { historyDays: 3 }) - if (day && wake_ts) { - const from = onset_ts ?? (wake_ts - 8 * 3600) - try { await runBiometricsMinute({ DB: env.DB, RAW_BUCKET: env.RAW_BUCKET }, userId, day, from, wake_ts + 60) } catch (e) { console.error('biometrics_minute failed', userId, day, e) } - await invalidateDay(env.DB, userId, day) - } - await env.DB.prepare('UPDATE analytics_cursor SET dirty = 0 WHERE user_id = ?').bind(userId).run() + await closeDay(env, userId, day, onset_ts, wake_ts) break case 'sweep': default: diff --git a/src/strava.ts b/src/strava.ts new file mode 100644 index 0000000..e72c4ff --- /dev/null +++ b/src/strava.ts @@ -0,0 +1,296 @@ +// strava.ts — Strava OAuth + token management (foundation for bidirectional sync). +// +// Connect flow (no client secret ever touches the app): +// app → GET /strava/connect (JWT) → { url } authorize link with a short-lived +// signed `state` carrying the user id +// user authorizes in a browser → Strava redirects to +// GET /strava/callback?code&state → we exchange the code for tokens, store them +// per user, and show a "return to OpenStrap" page. +// +// Access tokens are short-lived (~6h); getStravaAccessToken() transparently +// refreshes via the stored refresh_token and persists the rotation. + +import { signJwt, verifyJwt } from './auth' + +const STRAVA_AUTH = 'https://www.strava.com/oauth/authorize' +const STRAVA_TOKEN = 'https://www.strava.com/oauth/token' +const STRAVA_API = 'https://www.strava.com/api/v3' +// Read all activities + write new ones (both directions). +const SCOPE = 'activity:read_all,activity:write' +const STATE_TTL = 600 // 10 min to complete the browser hop + +export interface StravaEnv { + DB: D1Database + JWT_SECRET: string + STRAVA_CLIENT_ID?: string + STRAVA_CLIENT_SECRET?: string +} + +function configured(env: StravaEnv): boolean { + return !!(env.STRAVA_CLIENT_ID && env.STRAVA_CLIENT_SECRET) +} + +function originOf(c: any): string { + const url = new URL(c.req.url) + return `${url.protocol}//${url.host}` +} + +function htmlPage(message: string, ok: boolean): string { + const accent = ok ? '#34d399' : '#f87171' + return `` + + `` + + `OpenStrap × Strava` + + `` + + `
${ok ? '✅' : '⚠️'}
` + + `

${message}

` + + `

You can close this tab and return to OpenStrap.

` + + `

OpenStrap × Strava

` +} + +// GET /strava/connect (requireJwt) → { url } for the app to open in a browser. +export async function stravaConnect(c: any) { + if (!configured(c.env)) return c.json({ error: 'Strava integration not configured on this backend.' }, 503) + const userId: string = c.get('userId') + const state = await signJwt({ sub: userId, typ: 'strava_state' }, c.env.JWT_SECRET, STATE_TTL) + const redirectUri = `${originOf(c)}/strava/callback` + const url = `${STRAVA_AUTH}?client_id=${encodeURIComponent(c.env.STRAVA_CLIENT_ID)}` + + `&response_type=code&redirect_uri=${encodeURIComponent(redirectUri)}` + + `&approval_prompt=auto&scope=${encodeURIComponent(SCOPE)}&state=${encodeURIComponent(state)}` + return c.json({ url }) +} + +// GET /strava/callback?code&state (public; Strava redirect target) → HTML page. +export async function stravaCallback(c: any) { + if (!configured(c.env)) return c.html(htmlPage('Strava is not configured.', false), 503) + const error = c.req.query('error') + if (error) return c.html(htmlPage('Strava authorization was declined.', false), 400) + const code = c.req.query('code') + const state = c.req.query('state') + if (!code || !state) return c.html(htmlPage('Missing authorization code.', false), 400) + + const payload = await verifyJwt(state, c.env.JWT_SECRET) + if (!payload || (payload as any).typ !== 'strava_state') { + return c.html(htmlPage('This link expired. Start the connection again from the app.', false), 400) + } + const userId = payload.sub + + const form = new URLSearchParams({ + client_id: String(c.env.STRAVA_CLIENT_ID), + client_secret: String(c.env.STRAVA_CLIENT_SECRET), + code, + grant_type: 'authorization_code', + }) + const resp = await fetch(STRAVA_TOKEN, { method: 'POST', body: form }) + if (!resp.ok) return c.html(htmlPage('Strava token exchange failed.', false), 502) + const tok = await resp.json() as { + access_token: string; refresh_token: string; expires_at: number + athlete?: { id?: number } + } + + await c.env.DB.prepare( + 'INSERT INTO strava_tokens (user_id, athlete_id, access_token, refresh_token, expires_at, scope, connected_at) ' + + 'VALUES (?,?,?,?,?,?,?) ON CONFLICT(user_id) DO UPDATE SET ' + + 'athlete_id=excluded.athlete_id, access_token=excluded.access_token, refresh_token=excluded.refresh_token, ' + + 'expires_at=excluded.expires_at, scope=excluded.scope', + ).bind( + userId, tok.athlete?.id ?? null, tok.access_token, tok.refresh_token, tok.expires_at, SCOPE, + Math.floor(Date.now() / 1000), + ).run() + + return c.html(htmlPage('Strava connected!', true)) +} + +// GET /strava/status (requireJwt) → { connected, athlete_id, scope } +export async function stravaStatus(c: any) { + const userId: string = c.get('userId') + const row = (await c.env.DB.prepare( + 'SELECT athlete_id, scope, connected_at FROM strava_tokens WHERE user_id = ?', + ).bind(userId).first()) as { athlete_id: number | null; scope: string | null; connected_at: number | null } | null + return c.json({ + connected: !!row, + athlete_id: row?.athlete_id ?? null, + scope: row?.scope ?? null, + connected_at: row?.connected_at ?? null, + }) +} + +// POST /strava/disconnect (requireJwt) → { ok } +export async function stravaDisconnect(c: any) { + const userId: string = c.get('userId') + await c.env.DB.prepare('DELETE FROM strava_tokens WHERE user_id = ?').bind(userId).run() + return c.json({ ok: true }) +} + +// ── Strava → Whoop : pull recent activities into strava_activities ──────────── +export async function stravaPull( + env: StravaEnv, userId: string, perPage = 30, +): Promise<{ pulled: number }> { + const token = await getStravaAccessToken(env, userId) + if (!token) return { pulled: 0 } + const r = await fetch(`${STRAVA_API}/athlete/activities?per_page=${perPage}`, { + headers: { Authorization: `Bearer ${token}` }, + }) + if (!r.ok) return { pulled: 0 } + const acts = (await r.json()) as Array> + const now = Math.floor(Date.now() / 1000) + let pulled = 0 + for (const a of acts) { + const startTs = typeof a.start_date === 'string' + ? Math.floor(new Date(a.start_date).getTime() / 1000) : null + await env.DB.prepare( + 'INSERT INTO strava_activities (user_id, activity_id, start_ts, elapsed_sec, type, name, distance_m, avg_hr, max_hr, raw, pulled_at) ' + + 'VALUES (?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(user_id, activity_id) DO UPDATE SET ' + + 'start_ts=excluded.start_ts, elapsed_sec=excluded.elapsed_sec, type=excluded.type, name=excluded.name, ' + + 'distance_m=excluded.distance_m, avg_hr=excluded.avg_hr, max_hr=excluded.max_hr, raw=excluded.raw, pulled_at=excluded.pulled_at', + ).bind( + userId, a.id, startTs, a.elapsed_time ?? null, (a.sport_type ?? a.type) ?? null, a.name ?? null, + a.distance ?? null, a.average_heartrate ?? null, a.max_heartrate ?? null, + JSON.stringify({ + id: a.id, name: a.name, type: a.sport_type ?? a.type, start_date: a.start_date, + moving_time: a.moving_time, elapsed_time: a.elapsed_time, distance: a.distance, + total_elevation_gain: a.total_elevation_gain, average_speed: a.average_speed, + average_heartrate: a.average_heartrate, max_heartrate: a.max_heartrate, + average_watts: a.average_watts, kilojoules: a.kilojoules, calories: a.calories, + }), now, + ).run() + pulled++ + } + return { pulled } +} + +// ── Whoop → Strava : push completed OpenStrap workouts as manual activities ──── +// Dedupe against pulled activities (a ride already on Strava via the bike computer +// must NOT be duplicated) and against strava_pushed (idempotent). HR/strain land in +// the description (manual activities don't accept HR streams without a file upload). +const STRAVA_SPORT: Record = { + run: 'Run', running: 'Run', walk: 'Walk', walking: 'Walk', hike: 'Hike', + ride: 'Ride', cycling: 'Ride', bike: 'Ride', + strength: 'WeightTraining', weights: 'WeightTraining', gym: 'Workout', + swim: 'Swim', yoga: 'Yoga', rowing: 'Rowing', elliptical: 'Elliptical', +} + +export async function stravaPush( + env: StravaEnv, userId: string, +): Promise<{ pushed: number; skipped: number }> { + const token = await getStravaAccessToken(env, userId) + if (!token) return { pushed: 0, skipped: 0 } + + const { results: sessions } = await env.DB.prepare( + "SELECT id, start_ts, end_ts, type, avg_hr, max_hr, strain, calories FROM sessions " + + "WHERE user_id = ? AND status = 'done' AND start_ts IS NOT NULL AND end_ts > start_ts " + + "ORDER BY start_ts DESC LIMIT 20", + ).bind(userId).all>() + + let pushed = 0, skipped = 0 + for (const s of sessions ?? []) { + const sid = String(s.id) + const startTs = Number(s.start_ts), endTs = Number(s.end_ts) + + const already = await env.DB.prepare( + 'SELECT 1 FROM strava_pushed WHERE user_id = ? AND session_key = ?', + ).bind(userId, sid).first() + if (already) { skipped++; continue } + + // Overlap with an existing Strava activity (e.g. a Wahoo-synced ride) → skip. + const overlap = await env.DB.prepare( + 'SELECT 1 FROM strava_activities WHERE user_id = ? AND start_ts BETWEEN ? AND ? LIMIT 1', + ).bind(userId, startTs - 1800, endTs + 1800).first() + if (overlap) { + await env.DB.prepare( + 'INSERT OR IGNORE INTO strava_pushed (user_id, session_key, activity_id, pushed_at) VALUES (?,?,?,?)', + ).bind(userId, sid, null, Math.floor(Date.now() / 1000)).run() + skipped++; continue + } + + const typeKey = String(s.type ?? '').toLowerCase() + const sportType = STRAVA_SPORT[typeKey] ?? 'Workout' + const descParts = [ + s.strain != null ? `Strain ${Number(s.strain).toFixed(1)}` : null, + s.avg_hr != null ? `Avg HR ${s.avg_hr}` : null, + s.max_hr != null ? `Max HR ${s.max_hr}` : null, + s.calories != null ? `${Math.round(Number(s.calories))} kcal` : null, + 'via OpenStrap', + ].filter(Boolean) + + const form = new URLSearchParams({ + name: `OpenStrap ${sportType}`, + sport_type: sportType, + start_date_local: new Date(startTs * 1000).toISOString(), + elapsed_time: String(Math.max(60, endTs - startTs)), + description: descParts.join(' · '), + }) + const r = await fetch(`${STRAVA_API}/activities`, { + method: 'POST', + headers: { Authorization: `Bearer ${token}` }, + body: form, + }) + if (!r.ok) { skipped++; continue } + const created = (await r.json()) as { id?: number } + await env.DB.prepare( + 'INSERT OR IGNORE INTO strava_pushed (user_id, session_key, activity_id, pushed_at) VALUES (?,?,?,?)', + ).bind(userId, sid, created.id ?? null, Math.floor(Date.now() / 1000)).run() + pushed++ + } + return { pushed, skipped } +} + +// Cron backstop: pull + push for every connected user (called from the nightly tick). +export async function syncAllStrava(env: StravaEnv): Promise { + const { results } = await env.DB.prepare( + 'SELECT user_id FROM strava_tokens LIMIT 1000', + ).all<{ user_id: string }>() + for (const u of results ?? []) { + try { + await stravaPull(env, u.user_id) + await stravaPush(env, u.user_id) + } catch (e) { + console.error('strava sync failed', u.user_id, e) + } + } +} + +// GET /strava/sync (requireJwt) → pull (Strava→Whoop) then push (Whoop→Strava). +export async function stravaSync(c: any) { + const userId: string = c.get('userId') + const pull = await stravaPull(c.env, userId) + const push = await stravaPush(c.env, userId) + return c.json({ ok: true, ...pull, ...push }) +} + +// GET /strava/activities (requireJwt) → the pulled activities (for the app overlay). +export async function stravaActivities(c: any) { + const userId: string = c.get('userId') + const { results } = await c.env.DB.prepare( + 'SELECT activity_id, start_ts, elapsed_sec, type, name, distance_m, avg_hr, max_hr ' + + 'FROM strava_activities WHERE user_id = ? ORDER BY start_ts DESC LIMIT 50', + ).bind(userId).all() + return c.json({ activities: results ?? [] }) +} + +// A valid access token for `userId`, refreshing + persisting if expired. Returns +// null if the user has not connected Strava or the refresh fails. +export async function getStravaAccessToken(env: StravaEnv, userId: string): Promise { + if (!configured(env)) return null + const row = await env.DB.prepare( + 'SELECT access_token, refresh_token, expires_at FROM strava_tokens WHERE user_id = ?', + ).bind(userId).first<{ access_token: string; refresh_token: string; expires_at: number }>() + if (!row) return null + + const now = Math.floor(Date.now() / 1000) + if (row.expires_at - 60 > now) return row.access_token // still valid (60s skew) + + const form = new URLSearchParams({ + client_id: String(env.STRAVA_CLIENT_ID), + client_secret: String(env.STRAVA_CLIENT_SECRET), + grant_type: 'refresh_token', + refresh_token: row.refresh_token, + }) + const resp = await fetch(STRAVA_TOKEN, { method: 'POST', body: form }) + if (!resp.ok) return null + const tok = await resp.json() as { access_token: string; refresh_token: string; expires_at: number } + await env.DB.prepare( + 'UPDATE strava_tokens SET access_token=?, refresh_token=?, expires_at=? WHERE user_id=?', + ).bind(tok.access_token, tok.refresh_token, tok.expires_at, userId).run() + return tok.access_token +} diff --git a/src/wake_cron.ts b/src/wake_cron.ts index 3fa9ab4..3a97bfa 100644 --- a/src/wake_cron.ts +++ b/src/wake_cron.ts @@ -12,7 +12,7 @@ import { detectWakeState, peekRecentState } from 'openstrap-analytics' import { loadMinutes, loadBaseline } from './analytics' import { loadDayRr } from './dayseries' -import type { AnalyticsMessage } from './queue' +import { closeDay, type AnalyticsMessage } from './queue' interface WakeEnv { DB: D1Database; RAW_BUCKET?: R2Bucket; ANALYTICS_Q?: Queue } @@ -71,6 +71,12 @@ export async function runWakeLadder( user_id: u.user_id, job: 'close_day', day: dayLabel, onset_ts: ws.onset_ts ?? undefined, wake_ts: ws.wake_ts, }) + } else { + // [free-tier] No queue bound (Workers Free): derive the day INLINE. At most one + // user is closed per tick, so this stays well under the 1000 Cloudflare-subrequest + // cap. Without this the day would be marked closed but never actually computed. + try { await closeDay({ DB: env.DB, RAW_BUCKET: env.RAW_BUCKET as R2Bucket }, u.user_id, dayLabel, ws.onset_ts ?? undefined, ws.wake_ts) } + catch (e) { console.error('inline close_day failed', u.user_id, dayLabel, e) } } closed++ } @@ -87,6 +93,11 @@ export async function retryStaleCloses(env: WakeEnv, now = Math.floor(Date.now() let n = 0 for (const u of results ?? []) { if (env.ANALYTICS_Q) { await env.ANALYTICS_Q.send({ user_id: u.user_id, job: 'sweep' }); n++ } + else { + // [free-tier] No queue: re-derive inline as the nightly retry-net backstop. + try { await closeDay({ DB: env.DB, RAW_BUCKET: env.RAW_BUCKET as R2Bucket }, u.user_id) } catch (e) { console.error('inline retry close failed', u.user_id, e) } + n++ + } } return n } diff --git a/wrangler.toml b/wrangler.toml index 1adf2f9..96fdede 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -12,22 +12,31 @@ database_id = "REPLACE_WITH_YOUR_D1_DATABASE_ID" binding = "RAW_BUCKET" bucket_name = "openstrap-raw" -# Analytics queue (producer + consumer). ENABLED on Workers Paid. The cron just -# ENQUEUES one (user, job) message per unit of work; the consumer processes one -# bounded unit per invocation, so heavy R2 jobs (biometrics/resp/steps) never hit -# the 1000-subrequest per-invocation cap. ingest.ts also enqueues a 'sweep' on -# upload (falls back to analytics_cursor.dirty=1 if the binding is absent). -# max_batch_size = 1 → exactly one (user, job) per invocation (guaranteed bounded). -[[queues.producers]] -binding = "ANALYTICS_Q" -queue = "openstrap-analytics" +# Workers AI — powers the self-hosted, OpenAI-compatible AI Coach (src/coach.ts). +# No external key; runs within Cloudflare's free daily AI allocation. +[ai] +binding = "AI" -[[queues.consumers]] -queue = "openstrap-analytics" -max_batch_size = 1 -max_batch_timeout = 10 -max_retries = 3 -dead_letter_queue = "openstrap-analytics-dlq" +# Analytics queue (producer + consumer). Cloudflare Queues require the Workers PAID +# plan ($5/mo). DISABLED here so this deploys on the FREE plan. With no ANALYTICS_Q +# binding, the cron derives each day INLINE instead of enqueuing (see wake_cron.ts / +# queue.ts closeDay) — fine for a personal single-user backend (one close per tick, +# well under the free plan's 1000 Cloudflare-subrequest cap). +# +# To re-enable on Workers Paid: uncomment the four blocks below and +# npx wrangler queues create openstrap-analytics +# npx wrangler queues create openstrap-analytics-dlq +# +# [[queues.producers]] +# binding = "ANALYTICS_Q" +# queue = "openstrap-analytics" +# +# [[queues.consumers]] +# queue = "openstrap-analytics" +# max_batch_size = 1 +# max_batch_timeout = 10 +# max_retries = 3 +# dead_letter_queue = "openstrap-analytics-dlq" # Crons: 30-min sweep (re-derive dirty users → keeps "today" fresh; pure-read # endpoints serve this snapshot) + nightly full re-derive/prune. Any non-nightly