From 0c9a9acaa74ff53db8ef7d6526648fb1f047396f Mon Sep 17 00:00:00 2001 From: erick-gege Date: Tue, 19 May 2026 11:57:58 -0500 Subject: [PATCH 1/7] =?UTF-8?q?feat:=20frontend=20UX=20=E2=80=94=20product?= =?UTF-8?q?=20tour,=20deploy=20stages,=20job=20details?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- estela-api/api/serializers/deploy.py | 15 + estela-api/api/serializers/job.py | 35 ++ estela-api/config/celery.py | 4 + estela-api/core/tasks.py | 74 +++- estela-api/core/views.py | 7 + estela-web/src/pages/DeployListPage/index.tsx | 55 ++- .../src/pages/DeployListPage/styles.scss | 20 + estela-web/src/pages/JobCreateModal/index.tsx | 57 ++- estela-web/src/pages/JobDataPage/index.tsx | 4 +- estela-web/src/pages/JobDetailPage/index.tsx | 398 ++++++++++-------- .../pages/ProjectCronJobListPage/index.tsx | 3 + .../src/pages/ProjectJobListPage/index.tsx | 7 + .../src/pages/SpiderDetailPage/index.tsx | 2 + estela-web/src/pages/SpiderListPage/index.tsx | 2 + .../api/generated-api/models/Deploy.ts | 4 +- .../api/generated-api/models/SpiderJob.ts | 7 + .../src/shared/layouts/ProjectLayout.tsx | 2 + .../src/shared/projectSideNav/index.tsx | 10 +- estela-web/src/tour/TourOverlay.scss | 166 ++++++++ estela-web/src/tour/TourOverlay.tsx | 310 ++++++++++++++ estela-web/src/tour/index.ts | 2 + estela-web/src/tour/steps.ts | 121 ++++++ estela-web/src/tour/store.ts | 186 ++++++++ estela-web/src/tour/types.ts | 55 +++ 24 files changed, 1336 insertions(+), 210 deletions(-) create mode 100644 estela-web/src/tour/TourOverlay.scss create mode 100644 estela-web/src/tour/TourOverlay.tsx create mode 100644 estela-web/src/tour/index.ts create mode 100644 estela-web/src/tour/steps.ts create mode 100644 estela-web/src/tour/store.ts create mode 100644 estela-web/src/tour/types.ts diff --git a/estela-api/api/serializers/deploy.py b/estela-api/api/serializers/deploy.py index 955a9eff..7ce00371 100644 --- a/estela-api/api/serializers/deploy.py +++ b/estela-api/api/serializers/deploy.py @@ -14,6 +14,21 @@ class DeploySerializer(serializers.ModelSerializer): def get_spiders_count(self, obj): return obj.spiders.count() + def to_representation(self, instance): + data = super().to_representation(instance) + if instance.status == Deploy.BUILDING_STATUS: + try: + import redis as redis_lib + from django.conf import settings + + redis_conn = redis_lib.from_url(settings.REDIS_URL) + stage = redis_conn.get(f"deploy_stage:{instance.did}") + if stage: + data["status"] = stage.decode() + except Exception: + pass + return data + class Meta: model = Deploy fields = ["did", "project", "user", "status", "spiders_count", "created"] diff --git a/estela-api/api/serializers/job.py b/estela-api/api/serializers/job.py index 12fe4913..d71abdcd 100644 --- a/estela-api/api/serializers/job.py +++ b/estela-api/api/serializers/job.py @@ -8,6 +8,8 @@ SpiderJobEnvVarSerializer, SpiderJobTagSerializer, ) +import redis as redis_lib +from django.conf import settings from api.utils import ( delete_stats_from_redis, update_stats_from_redis, @@ -38,6 +40,7 @@ class SpiderJobSerializer(serializers.ModelSerializer): spider = serializers.SerializerMethodField("get_spider") storage_size = serializers.SerializerMethodField("get_storage_size") database_insertion_progress = serializers.SerializerMethodField("get_database_insertion_progress") + peak_memory = serializers.SerializerMethodField("get_peak_memory") class Meta: model = SpiderJob @@ -60,6 +63,7 @@ class Meta: "database_insertion_progress", "storage_size", "resource_tier", + "peak_memory", ) def get_spider(self, instance): @@ -85,6 +89,37 @@ def get_database_insertion_progress(self, instance): # Return the actual database insertion progress value from the model return instance.database_insertion_progress + def get_peak_memory(self, instance): + if instance.status == SpiderJob.RUNNING_STATUS: + try: + redis_conn = redis_lib.from_url(settings.REDIS_URL) + raw_stats = redis_conn.hgetall(f"scrapy_stats_{instance.key}") + if raw_stats: + job_stats = {key.decode(): value.decode() for key, value in raw_stats.items()} + mem = job_stats.get("resources/peak_memory_bytes") or job_stats.get("memusage/max") + if mem: + mem_bytes = int(float(mem)) + if mem_bytes > 0: + return mem_bytes + except Exception: + pass + else: + try: + if spiderdata_db_client.get_connection(): + pid = str(instance.spider.project.pid) + job_collection_name = get_collection_name(instance, "stats") + job_stats = spiderdata_db_client.get_job_stats(pid, job_collection_name) + if job_stats: + for stat in job_stats: + mem = stat.get("resources/peak_memory_bytes") + if mem is not None: + mem_bytes = int(float(mem)) + if mem_bytes > 0: + return mem_bytes + except Exception: + pass + return None + class SpiderJobCreateEnvVarSerializer(serializers.Serializer): evid = serializers.IntegerField(required=False, help_text="Env var id.") diff --git a/estela-api/config/celery.py b/estela-api/config/celery.py index f4fc9d91..9662e253 100644 --- a/estela-api/config/celery.py +++ b/estela-api/config/celery.py @@ -27,6 +27,10 @@ "task": "core.tasks.update_mongodb_insertion_progress", "schedule": 60, }, + "update-deploy-stages": { + "task": "core.tasks.update_deploy_stages", + "schedule": 15, + }, } diff --git a/estela-api/core/tasks.py b/estela-api/core/tasks.py index 654bd053..4e94be05 100644 --- a/estela-api/core/tasks.py +++ b/estela-api/core/tasks.py @@ -20,6 +20,7 @@ from config.job_manager import job_manager, spiderdata_db_client from core.models import ( DataStatus, + Deploy, Permission, Project, ProxyProvider, @@ -587,5 +588,76 @@ def update_mongodb_insertion_progress(): logging.info(f"Job {job.jid} excluded after {stall_count} cycles with no progress") except Exception as e: logging.error(f"Error updating progress for job {job.jid}: {str(e)}") - + logging.info(f"Completed MongoDB insertion progress updates") + + +@celery_app.task(name="core.tasks.update_deploy_stages") +def update_deploy_stages(): + deploys = Deploy.objects.filter(status=Deploy.BUILDING_STATUS)[:50] + + if not deploys: + return + + try: + config.load_incluster_config() + core_api = client.CoreV1Api() + batch_api = client.BatchV1Api() + namespace = getattr(settings, "K8S_NAMESPACE", "default") + except Exception: + return + + DOWNLOADING = "DOWNLOADING" + BUILDING = "BUILDING" + + try: + redis_conn = redis.from_url(settings.REDIS_URL) + except Exception: + redis_conn = None + + active_ids = set() + for deploy in deploys: + active_ids.add(deploy.did) + job_name = f"deploy-project-{deploy.did}" + try: + batch_api.read_namespaced_job(job_name, namespace) + pods = core_api.list_namespaced_pod( + namespace, label_selector=f"job-name={job_name}" + ) + if not pods.items: + if redis_conn: + redis_conn.delete(f"deploy_stage:{deploy.did}") + continue + + pod = pods.items[0] + pod_status = pod.status + init_statuses = pod_status.init_container_statuses or [] + new_stage = None + + if init_statuses: + for i, ics in enumerate(init_statuses): + if ics.state and (ics.state.running or ics.state.waiting): + new_stage = DOWNLOADING if i == 0 else BUILDING + break + + if redis_conn: + if new_stage: + redis_conn.set(f"deploy_stage:{deploy.did}", new_stage, ex=300) + else: + redis_conn.delete(f"deploy_stage:{deploy.did}") + except Exception: + pass + + # Clean up Redis keys for deploys that are no longer BUILDING + if redis_conn: + try: + existing_keys = redis_conn.keys("deploy_stage:*") + for key in existing_keys: + try: + did = int(key.decode().split(":")[1]) + if did not in active_ids: + redis_conn.delete(key) + except (ValueError, IndexError): + pass + except Exception: + pass diff --git a/estela-api/core/views.py b/estela-api/core/views.py index 1c679ad2..ee41b4bb 100644 --- a/estela-api/core/views.py +++ b/estela-api/core/views.py @@ -8,6 +8,7 @@ from django.utils.encoding import force_bytes from django.utils.http import urlsafe_base64_encode from rest_framework.authtoken.models import Token +import redis as redis_lib def launch_deploy_job(pid, did, container_image): @@ -42,6 +43,12 @@ def launch_deploy_job(pid, did, container_image): isbuild=True, # Triggers Kaniko 3-container pipeline ) + try: + redis_conn = redis_lib.from_url(settings.REDIS_URL) + redis_conn.set(f"deploy_stage:{did}", "DOWNLOADING", ex=300) + except Exception: + pass + def send_verification_email(user, request): mail_subject = "Activate your estela account." diff --git a/estela-web/src/pages/DeployListPage/index.tsx b/estela-web/src/pages/DeployListPage/index.tsx index 1a4aa75e..6052e172 100644 --- a/estela-web/src/pages/DeployListPage/index.tsx +++ b/estela-web/src/pages/DeployListPage/index.tsx @@ -10,9 +10,10 @@ import WelcomeDeploy from "../../assets/images/welcomeDeploy.svg"; import "./styles.scss"; import { API_BASE_URL } from "../../constants"; import { ApiService, AuthService } from "../../services"; -import { ApiProjectsDeploysListRequest, Deploy, UserDetail } from "../../services/api"; +import { ApiProjectsDeploysListRequest, Deploy, DeployStatusEnum, UserDetail } from "../../services/api"; import { resourceNotAllowedNotification, Spin, PaginationItem } from "../../shared"; import { convertDateToString } from "../../utils"; +import { TourStore } from "../../tour"; const { Content } = Layout; const { Text, Paragraph } = Typography; @@ -29,6 +30,52 @@ interface RouteParams { projectId: string; } +const STAGE_LABELS: Record = { + DOWNLOADING: "Downloading project", + BUILDING: "Building image", +}; + +const STAGE_STEP: Record = { + DOWNLOADING: 1, + BUILDING: 2, +}; + +const DeployStageProgress = ({ stage }: { stage: string }) => { + const label = STAGE_LABELS[stage] || stage; + const stepIndex = STAGE_STEP[stage] || 1; + + return ( +
+
+ + + {label} · step {stepIndex} of 2 + +
+
+ {[1, 2].map((step) => ( +
+ ))} +
+
+ ); +}; + +const ACTIVE_STAGES = [DeployStatusEnum.Downloading, DeployStatusEnum.Building, DeployStatusEnum.Deploying]; + export class DeployListPage extends Component, DeployListPageState> { PAGE_SIZE = 10; state: DeployListPageState = { @@ -83,8 +130,8 @@ export class DeployListPage extends Component, dataIndex: "status", render: (state: string): ReactElement => ( - {state === "BUILDING" ? ( - Waiting + {ACTIVE_STAGES.includes(state as DeployStatusEnum) ? ( + ) : state === "SUCCESS" ? ( Completed ) : ( @@ -98,6 +145,7 @@ export class DeployListPage extends Component, ]; async componentDidMount(): Promise { + TourStore.setRoute("deploys"); await this.getProjectDeploys(1); } @@ -132,6 +180,7 @@ export class DeployListPage extends Component, loaded: true, modalIsOpen: results.count === 0, }); + TourStore.setDeploys(deploys); }, (error: unknown) => { error; diff --git a/estela-web/src/pages/DeployListPage/styles.scss b/estela-web/src/pages/DeployListPage/styles.scss index c72a36ac..a4ed3d7e 100644 --- a/estela-web/src/pages/DeployListPage/styles.scss +++ b/estela-web/src/pages/DeployListPage/styles.scss @@ -4,4 +4,24 @@ .ant-table-thead .ant-table-cell { background-color: white; border:none; +} + +.deploy-stage-dot { + width: 8px; + height: 8px; + border-radius: 50%; + background-color: #4D47C3; + animation: pulseDot 2s infinite; + flex-shrink: 0; +} + +@keyframes pulseDot { + 0% { box-shadow: 0 0 0 0 rgba(77,71,195,0.55); } + 70% { box-shadow: 0 0 0 6px rgba(77,71,195,0); } + 100% { box-shadow: 0 0 0 0 rgba(77,71,195,0); } +} + +@keyframes shimmer { + 0% { background-position: -120px 0; } + 100% { background-position: 120px 0; } } \ No newline at end of file diff --git a/estela-web/src/pages/JobCreateModal/index.tsx b/estela-web/src/pages/JobCreateModal/index.tsx index a3f33b7e..563d5aa5 100644 --- a/estela-web/src/pages/JobCreateModal/index.tsx +++ b/estela-web/src/pages/JobCreateModal/index.tsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect } from "react"; +import React, { useState, useEffect, useRef } from "react"; import { Modal, Button, message, Row, Select, Space, Input, Tag, Checkbox, Tooltip } from "antd"; import { EyeInvisibleOutlined } from "@ant-design/icons"; import type { CheckboxChangeEvent } from "antd/es/checkbox"; @@ -20,6 +20,7 @@ import { ProxySettings } from "../../components/ProxySettingsPage"; import { resourceNotAllowedNotification, invalidDataNotification, incorrectDataNotification } from "../../shared"; import { DEFAULT_RESOURCE_TIER, PREDEFINED_TIERS } from "../../constants"; import { checkExternalError } from "../../defaultComponents"; +import { TourStore } from "../../tour"; import Run from "../../assets/icons/play.svg"; import Add from "../../assets/icons/add.svg"; @@ -153,6 +154,11 @@ export default function JobCreateModal({ pid: projectId, sid: "", }); + const runBtnRef = useRef(null); + + useEffect(() => { + TourStore.setRunButtonEl(runBtnRef.current); + }, []); // MaskedTag component was replaced with inline implementation using EyeInvisibleOutlined @@ -460,6 +466,8 @@ export default function JobCreateModal({ }; apiService.apiProjectsSpidersJobsCreate(requests).then( (response: SpiderJobCreate) => { + TourStore.markStepSeen("step-3"); + sessionStorage.setItem("tour_just_created", "true"); setLoading(false); // Close the modal first if an onClose callback is provided if (onClose) { @@ -552,26 +560,37 @@ export default function JobCreateModal({ } }, [projectEnvVars, spiderEnvVars, jobData]); + useEffect(() => { + TourStore.setNewJobModalOpen(open); + return () => { + TourStore.setNewJobModalOpen(false); + }; + }, [open]); + return ( <> {!hideRunButton && ( - +
+ +
)} {externalComponent} NEW JOB

} footer={null} > - +

Spider

 -

Data persistence

+ setJobData({ ...jobData, resourceTier: value })} className="w-full" @@ -671,7 +683,11 @@ export default function JobCreateModal({
-

Arguments

+
{jobData.args.length > 0 ? (
@@ -731,7 +747,11 @@ export default function JobCreateModal({ {/* Project ENV vars section */} {getFilteredEnvVars(projectEnvVars).length > 0 && (
-

Project Variables

+
{getFilteredEnvVars(projectEnvVars).map( @@ -770,7 +790,11 @@ export default function JobCreateModal({ {/* Spider ENV vars section */} {getFilteredEnvVars(spiderEnvVars).length > 0 && (
-

Spider Variables

+
{getFilteredEnvVars(spiderEnvVars).map( @@ -808,7 +832,11 @@ export default function JobCreateModal({ {/* Job ENV vars section */}
-

Job Variables

+
{jobData.envVars.length > 0 ? (
@@ -889,7 +917,7 @@ export default function JobCreateModal({ -

Proxy

+ {noProxy ? (