From 81948a0864191af85b5edcd78a9158bdfccdd860 Mon Sep 17 00:00:00 2001 From: erick-gege Date: Fri, 19 Jun 2026 20:34:26 -0500 Subject: [PATCH 1/7] feat: improve project list view, breadcrumbs and CLI banner placement --- estela-api/api/serializers/project.py | 16 + estela-api/api/views/project.py | 6 +- estela-api/core/models.py | 20 + estela-api/docs/api.yaml | 68 +++ estela-web/src/pages/DeployListPage/index.tsx | 81 ++-- .../src/pages/ProjectListPage/index.tsx | 420 ++++++++++++++++-- .../services/api/generated-api/apis/ApiApi.ts | 90 ++++ .../api/generated-api/models/DeployUpdate.ts | 8 + .../api/generated-api/models/Project.ts | 14 + .../generated-api/models/SpiderJobUpdate.ts | 8 + estela-web/src/shared/header/index.tsx | 10 +- .../src/shared/layouts/ProjectLayout.tsx | 110 ++++- 12 files changed, 753 insertions(+), 98 deletions(-) diff --git a/estela-api/api/serializers/project.py b/estela-api/api/serializers/project.py index 2b742e45..4583d1b7 100644 --- a/estela-api/api/serializers/project.py +++ b/estela-api/api/serializers/project.py @@ -51,6 +51,20 @@ class ProjectSerializer(serializers.ModelSerializer): container_image = serializers.CharField( read_only=True, help_text="Path of the project's container image." ) + created = serializers.SerializerMethodField() + last_modified = serializers.SerializerMethodField() + + def get_created(self, obj): + value = obj.created_at + if value is None: + return None + return value.isoformat() + + def get_last_modified(self, obj): + value = obj.last_modified + if value is None: + return None + return value.isoformat() class Meta: model = Project @@ -64,6 +78,8 @@ class Meta: "env_vars", "data_status", "data_expiry_days", + "created", + "last_modified", ) diff --git a/estela-api/api/views/project.py b/estela-api/api/views/project.py index 8ae3e3ac..ac3ec883 100644 --- a/estela-api/api/views/project.py +++ b/estela-api/api/views/project.py @@ -56,11 +56,15 @@ def get_parameters(self, request): return page, page_size def get_queryset(self): - return ( + queryset = ( Project.objects.filter(deleted=False) if self.request.user.is_superuser or self.request.user.is_staff else self.request.user.project_set.filter(deleted=False) ) + search = self.request.query_params.get("search", "") + if search: + queryset = queryset.filter(name__icontains=search) + return queryset def perform_create(self, serializer): instance = serializer.save() diff --git a/estela-api/core/models.py b/estela-api/core/models.py index 69cab0b9..2c2bbe3b 100644 --- a/estela-api/core/models.py +++ b/estela-api/core/models.py @@ -84,10 +84,30 @@ class Project(models.Model): deleted = models.BooleanField( default=False, help_text="Whether the project was deleted." ) + created = models.DateTimeField( + auto_now_add=True, help_text="Project creation date.", null=True + ) class Meta: ordering = ["name"] + @property + def created_at(self): + first_deploy = self.deploy_set.order_by("created").values_list("created", flat=True).first() + if first_deploy and (self.created is None or first_deploy < self.created): + return first_deploy + return self.created + + @property + def last_modified(self): + from django.db.models import Max + result = self.spiders.aggregate( + latest_deploy=Max('deploy__created'), + latest_job=Max('jobs__created'), + ) + valid_dates = [d for d in result.values() if d is not None] + return max(valid_dates) if valid_dates else self.created + @property def container_image(self): """Get the production container image for this project.""" diff --git a/estela-api/docs/api.yaml b/estela-api/docs/api.yaml index cce201a9..ed636fc8 100644 --- a/estela-api/docs/api.yaml +++ b/estela-api/docs/api.yaml @@ -698,6 +698,28 @@ paths: in: path required: true type: string + /api/projects/{pid}/deploys/{did}/logs: + get: + operationId: api_projects_deploys_logs + description: '' + parameters: [] + responses: + '200': + description: '' + schema: + $ref: '#/definitions/Deploy' + tags: + - api + parameters: + - name: did + in: path + description: A unique integer value identifying this deploy. + required: true + type: integer + - name: pid + in: path + required: true + type: string /api/projects/{pid}/jobs: get: operationId: api_projects_jobs @@ -1316,6 +1338,32 @@ paths: in: path required: true type: string + /api/projects/{pid}/spiders/{sid}/jobs/{jid}/error_logs: + get: + operationId: api_projects_spiders_jobs_error_logs + description: '' + parameters: [] + responses: + '200': + description: '' + schema: + $ref: '#/definitions/SpiderJob' + tags: + - api + parameters: + - name: jid + in: path + description: A unique integer value identifying this job. + required: true + type: integer + - name: pid + in: path + required: true + type: string + - name: sid + in: path + required: true + type: string /api/projects/{pid}/usage: get: operationId: api_projects_usage @@ -1968,6 +2016,14 @@ definitions: type: integer maximum: 65535 minimum: 0 + created: + title: Created + type: string + readOnly: true + last_modified: + title: Last modified + type: string + readOnly: true ProjectUpdate: type: object properties: @@ -2334,6 +2390,12 @@ definitions: type: string maxLength: 1000 minLength: 1 + error_reason: + title: Error reason + description: Error logs to persist in deploy_logs (Mongo) on failure. + type: string + maxLength: 200000 + x-nullable: true SpiderJob: description: Project jobs. type: object @@ -2797,6 +2859,12 @@ definitions: title: Proxy usage data description: Proxy Usage data. type: string + error_reason: + title: Error reason + description: Error logs to persist in job_logs (Mongo) on failure. + type: string + maxLength: 200000 + x-nullable: true UsageRecord: required: - processing_time diff --git a/estela-web/src/pages/DeployListPage/index.tsx b/estela-web/src/pages/DeployListPage/index.tsx index bc7ab8c3..6e654819 100644 --- a/estela-web/src/pages/DeployListPage/index.tsx +++ b/estela-web/src/pages/DeployListPage/index.tsx @@ -3,9 +3,9 @@ import { Layout, Pagination, Row, Table, Button, Tag, Col, Typography, Modal, To import { ExclamationCircleOutlined } from "@ant-design/icons"; import { RouteComponentProps } from "react-router-dom"; import Copy from "../../assets/icons/copy.svg"; -import Info from "../../assets/icons/info.svg"; import Help from "../../assets/icons/help.svg"; import SpiderIcon from "../../assets/icons/spider.svg"; +import Info from "../../assets/icons/info.svg"; import WelcomeDeploy from "../../assets/images/welcomeDeploy.svg"; import "./styles.scss"; @@ -317,53 +317,40 @@ export class DeployListPage extends Component, -

SPIDER OVERVIEW

- + +

SPIDER OVERVIEW

+ + + - - - - -

- Want to know more about estela? Access our - - - documentation - - - . -

-

- Install the - - - estela CLI - - - to streamline spider deployment, unlock{" "} - advanced developer features, and access all{" "} - estela tools. -

- - - + + + + Install the{" "} + + estela CLI + {" "} + to deploy your spiders and access all estela tools.{" "} + + Read the docs → + + { @@ -52,14 +89,164 @@ export class ProjectListPage extends Component { modalWelcome: false, newProjectName: "", newProjectCategory: ProjectCategoryEnum.NotSpecified, + searchText: "", + sortField: null, + sortOrder: null, + deleteModal: false, + deleteTargetPid: undefined, + deleteTargetName: "", + deleteConfirmText: "", + copiedPid: undefined, }; apiService = ApiService(); static contextType = UserContext; + runSearch = async (search: string): Promise => { + const data = await this.getProjects(1, search); + const projectData = data.data.map((project: Project, id: number) => ({ + name: project.name, + pid: project.pid, + framework: project.framework, + category: project.category, + created: project.created, + lastModified: project.lastModified, + role: + project.users?.find((user) => user.user?.username === AuthService.getUserUsername())?.permission || + "ADMIN", + key: id, + })); + this.setState({ projects: projectData, count: data.count, current: 1 }); + }; + + formatDate = (date?: string): string => { + if (!date) return "—"; + return new Date(date).toLocaleDateString("en-US", { year: "numeric", month: "short", day: "numeric" }); + }; + + handleSort = (field: SortField): void => { + const { sortField, sortOrder } = this.state; + if (sortField !== field) { + this.setState({ sortField: field, sortOrder: "desc" }); + } else if (sortOrder === "desc") { + this.setState({ sortOrder: "asc" }); + } else { + this.setState({ sortField: null, sortOrder: null }); + } + }; + + getSortIcon = (field: SortField): string => { + const { sortField, sortOrder } = this.state; + if (sortField !== field) return ""; + return sortOrder === "asc" ? " ↑" : " ↓"; + }; + + getSortedAndFilteredProjects = (): ProjectList[] => { + const { projects, sortField, sortOrder } = this.state; + + if (!sortField || !sortOrder) return projects; + + return [...projects].sort((a, b) => { + let valA: string | number = ""; + let valB: string | number = ""; + + if (sortField === "name") { + valA = a.name; + valB = b.name; + } else if (sortField === "framework") { + valA = a.framework ?? ""; + valB = b.framework ?? ""; + } else if (sortField === "category") { + valA = a.category ?? ""; + valB = b.category ?? ""; + } else if (sortField === "role") { + valA = a.role; + valB = b.role; + } else if (sortField === "created") { + valA = a.created ? new Date(a.created).getTime() : 0; + valB = b.created ? new Date(b.created).getTime() : 0; + } else if (sortField === "lastModified") { + valA = a.lastModified + ? new Date(a.lastModified).getTime() + : a.created + ? new Date(a.created).getTime() + : 0; + valB = b.lastModified + ? new Date(b.lastModified).getTime() + : b.created + ? new Date(b.created).getTime() + : 0; + } + + if (valA < valB) return sortOrder === "asc" ? -1 : 1; + if (valA > valB) return sortOrder === "asc" ? 1 : -1; + return 0; + }); + }; + + confirmDelete = (pid: string | undefined, name: string): void => { + this.setState({ deleteModal: true, deleteTargetPid: pid, deleteTargetName: name, deleteConfirmText: "" }); + }; + + deleteProject = (): void => { + const { deleteTargetPid } = this.state; + if (!deleteTargetPid) return; + const request: ApiProjectsDeleteRequest = { pid: deleteTargetPid }; + this.apiService.apiProjectsDelete(request).then( + () => { + this.setState({ + deleteModal: false, + deleteTargetPid: undefined, + deleteTargetName: "", + deleteConfirmText: "", + }); + this.onPageChange(1); + }, + () => incorrectDataNotification(), + ); + }; + + getActionMenu = (project: ProjectList): ReactElement => ( + + } + onClick={() => history.push(`/projects/${project.pid}/settings`)} + > + Edit project + + } + onClick={() => history.push(`/projects/${project.pid}/settings`)} + > + Manage tags + + } + onClick={() => history.push(`/projects/${project.pid}/members`)} + > + Team permissions + + + } + danger + onClick={() => this.confirmDelete(project.pid, project.name)} + > + Delete project + + + ); - columns = [ + getColumns = () => [ { - title: "NAME", + title: ( + this.handleSort("name")}> + NAME{this.getSortIcon("name")} + + ), dataIndex: "name", key: "name", render: (name: string, project: ProjectList): ReactElement => ( @@ -79,10 +266,35 @@ export class ProjectListPage extends Component { title: "PID", dataIndex: "pid", key: "pid", - render: (pid: string): ReactElement =>

{pid}

, + render: (pid: string): ReactElement => { + const copied = this.state.copiedPid === pid; + return ( + + {pid ? `${pid.slice(0, 8)}...` : ""} + , + , + ]} + > +

+ This action cannot be undone. Type{" "} + {deleteTargetName} to confirm. +

+ this.setState({ deleteConfirmText: e.target.value })} + onPressEnter={() => { + if (deleteConfirmText === deleteTargetName) this.deleteProject(); + }} + /> + @@ -267,8 +601,22 @@ export class ProjectListPage extends Component { - {projects.map((project: ProjectList, index) => { - return index < 3 ? ( + {[...this.state.projects] + .sort((a, b) => { + const ta = a.lastModified + ? new Date(a.lastModified).getTime() + : a.created + ? new Date(a.created).getTime() + : 0; + const tb = b.lastModified + ? new Date(b.lastModified).getTime() + : b.created + ? new Date(b.created).getTime() + : 0; + return tb - ta; + }) + .slice(0, 5) + .map((project: ProjectList) => ( - ) : ( - - ); - })} + ))} @@ -432,12 +774,22 @@ export class ProjectListPage extends Component { + + this.setState({ searchText: e.target.value })} + onSearch={this.runSearch} + className="border-estela-blue-low rounded-md" + style={{ maxWidth: 400 }} + /> + > { + if (requestParameters.did === null || requestParameters.did === undefined) { + throw new runtime.RequiredError('did','Required parameter requestParameters.did was null or undefined when calling apiProjectsDeploysLogs.'); + } + + if (requestParameters.pid === null || requestParameters.pid === undefined) { + throw new runtime.RequiredError('pid','Required parameter requestParameters.pid was null or undefined when calling apiProjectsDeploysLogs.'); + } + + const queryParameters: any = {}; + + const headerParameters: runtime.HTTPHeaders = {}; + + if (this.configuration && (this.configuration.username !== undefined || this.configuration.password !== undefined)) { + headerParameters["Authorization"] = "Basic " + btoa(this.configuration.username + ":" + this.configuration.password); + } + const response = await this.request({ + path: `/api/projects/{pid}/deploys/{did}/logs`.replace(`{${"did"}}`, encodeURIComponent(String(requestParameters.did))).replace(`{${"pid"}}`, encodeURIComponent(String(requestParameters.pid))), + method: 'GET', + headers: headerParameters, + query: queryParameters, + }); + + return new runtime.JSONApiResponse(response, (jsonValue) => DeployFromJSON(jsonValue)); + } + + /** + */ + async apiProjectsDeploysLogs(requestParameters: ApiProjectsDeploysLogsRequest): Promise { + const response = await this.apiProjectsDeploysLogsRaw(requestParameters); + return await response.value(); + } + /** */ async apiProjectsDeploysPartialUpdateRaw(requestParameters: ApiProjectsDeploysPartialUpdateRequest): Promise> { @@ -1642,6 +1689,10 @@ export class ApiApi extends runtime.BaseAPI { queryParameters['page_size'] = requestParameters.pageSize; } + if (requestParameters.search !== undefined) { + queryParameters['search'] = requestParameters.search; + } + const headerParameters: runtime.HTTPHeaders = {}; if (this.configuration && (this.configuration.username !== undefined || this.configuration.password !== undefined)) { @@ -2263,6 +2314,45 @@ export class ApiApi extends runtime.BaseAPI { return await response.value(); } + /** + */ + async apiProjectsSpidersJobsErrorLogsRaw(requestParameters: ApiProjectsSpidersJobsErrorLogsRequest): Promise> { + if (requestParameters.jid === null || requestParameters.jid === undefined) { + throw new runtime.RequiredError('jid','Required parameter requestParameters.jid was null or undefined when calling apiProjectsSpidersJobsErrorLogs.'); + } + + if (requestParameters.pid === null || requestParameters.pid === undefined) { + throw new runtime.RequiredError('pid','Required parameter requestParameters.pid was null or undefined when calling apiProjectsSpidersJobsErrorLogs.'); + } + + if (requestParameters.sid === null || requestParameters.sid === undefined) { + throw new runtime.RequiredError('sid','Required parameter requestParameters.sid was null or undefined when calling apiProjectsSpidersJobsErrorLogs.'); + } + + const queryParameters: any = {}; + + const headerParameters: runtime.HTTPHeaders = {}; + + if (this.configuration && (this.configuration.username !== undefined || this.configuration.password !== undefined)) { + headerParameters["Authorization"] = "Basic " + btoa(this.configuration.username + ":" + this.configuration.password); + } + const response = await this.request({ + path: `/api/projects/{pid}/spiders/{sid}/jobs/{jid}/error_logs`.replace(`{${"jid"}}`, encodeURIComponent(String(requestParameters.jid))).replace(`{${"pid"}}`, encodeURIComponent(String(requestParameters.pid))).replace(`{${"sid"}}`, encodeURIComponent(String(requestParameters.sid))), + method: 'GET', + headers: headerParameters, + query: queryParameters, + }); + + return new runtime.JSONApiResponse(response, (jsonValue) => SpiderJobFromJSON(jsonValue)); + } + + /** + */ + async apiProjectsSpidersJobsErrorLogs(requestParameters: ApiProjectsSpidersJobsErrorLogsRequest): Promise { + const response = await this.apiProjectsSpidersJobsErrorLogsRaw(requestParameters); + return await response.value(); + } + /** */ async apiProjectsSpidersJobsListRaw(requestParameters: ApiProjectsSpidersJobsListRequest): Promise> { diff --git a/estela-web/src/services/api/generated-api/models/DeployUpdate.ts b/estela-web/src/services/api/generated-api/models/DeployUpdate.ts index b634e339..6f659c5b 100644 --- a/estela-web/src/services/api/generated-api/models/DeployUpdate.ts +++ b/estela-web/src/services/api/generated-api/models/DeployUpdate.ts @@ -37,6 +37,12 @@ export interface DeployUpdate { * @memberof DeployUpdate */ spidersNames?: Array; + /** + * Error logs to persist in deploy_logs (Mongo) on failure. + * @type {string} + * @memberof DeployUpdate + */ + errorReason?: string | null; } /** @@ -64,6 +70,7 @@ export function DeployUpdateFromJSONTyped(json: any, ignoreDiscriminator: boolea 'did': !exists(json, 'did') ? undefined : json['did'], 'status': !exists(json, 'status') ? undefined : json['status'], 'spidersNames': !exists(json, 'spiders_names') ? undefined : json['spiders_names'], + 'errorReason': !exists(json, 'error_reason') ? undefined : json['error_reason'], }; } @@ -78,6 +85,7 @@ export function DeployUpdateToJSON(value?: DeployUpdate | null): any { 'status': value.status, 'spiders_names': value.spidersNames, + 'error_reason': value.errorReason, }; } diff --git a/estela-web/src/services/api/generated-api/models/Project.ts b/estela-web/src/services/api/generated-api/models/Project.ts index 15b6a9dd..57a24d45 100644 --- a/estela-web/src/services/api/generated-api/models/Project.ts +++ b/estela-web/src/services/api/generated-api/models/Project.ts @@ -84,6 +84,18 @@ export interface Project { * @memberof Project */ dataExpiryDays?: number; + /** + * + * @type {string} + * @memberof Project + */ + readonly created?: string; + /** + * + * @type {string} + * @memberof Project + */ + readonly lastModified?: string; } /** @@ -133,6 +145,8 @@ export function ProjectFromJSONTyped(json: any, ignoreDiscriminator: boolean): P 'envVars': !exists(json, 'env_vars') ? undefined : ((json['env_vars'] as Array).map(SpiderJobEnvVarFromJSON)), 'dataStatus': !exists(json, 'data_status') ? undefined : json['data_status'], 'dataExpiryDays': !exists(json, 'data_expiry_days') ? undefined : json['data_expiry_days'], + 'created': !exists(json, 'created') ? undefined : json['created'], + 'lastModified': !exists(json, 'last_modified') ? undefined : json['last_modified'], }; } diff --git a/estela-web/src/services/api/generated-api/models/SpiderJobUpdate.ts b/estela-web/src/services/api/generated-api/models/SpiderJobUpdate.ts index d7fa1032..c11aeed0 100644 --- a/estela-web/src/services/api/generated-api/models/SpiderJobUpdate.ts +++ b/estela-web/src/services/api/generated-api/models/SpiderJobUpdate.ts @@ -73,6 +73,12 @@ export interface SpiderJobUpdate { * @memberof SpiderJobUpdate */ proxyUsageData?: string; + /** + * Error logs to persist in job_logs (Mongo) on failure. + * @type {string} + * @memberof SpiderJobUpdate + */ + errorReason?: string | null; } /** @@ -115,6 +121,7 @@ export function SpiderJobUpdateFromJSONTyped(json: any, ignoreDiscriminator: boo 'dataStatus': !exists(json, 'data_status') ? undefined : json['data_status'], 'dataExpiryDays': !exists(json, 'data_expiry_days') ? undefined : json['data_expiry_days'], 'proxyUsageData': !exists(json, 'proxy_usage_data') ? undefined : json['proxy_usage_data'], + 'errorReason': !exists(json, 'error_reason') ? undefined : json['error_reason'], }; } @@ -135,6 +142,7 @@ export function SpiderJobUpdateToJSON(value?: SpiderJobUpdate | null): any { 'data_status': value.dataStatus, 'data_expiry_days': value.dataExpiryDays, 'proxy_usage_data': value.proxyUsageData, + 'error_reason': value.errorReason, }; } diff --git a/estela-web/src/shared/header/index.tsx b/estela-web/src/shared/header/index.tsx index 05ea164a..2da85537 100644 --- a/estela-web/src/shared/header/index.tsx +++ b/estela-web/src/shared/header/index.tsx @@ -20,6 +20,10 @@ import userDropdownSideNavItems from "ExternalComponents/DropdownComponent"; const { Header, Content } = Layout; type MenuItem = Required["items"][number]; +interface HeaderProps { + breadcrumb?: React.ReactNode; +} + interface HeaderState { notifications: Notification[]; loaded: boolean; @@ -27,7 +31,7 @@ interface HeaderState { news: boolean; } -export class CustomHeader extends Component { +export class CustomHeader extends Component { state: HeaderState = { notifications: [], loaded: false, @@ -283,6 +287,7 @@ export class CustomHeader extends Component { render(): JSX.Element { const { path, loaded, notifications, news } = this.state; + const { breadcrumb } = this.props; return ( <> {loaded ? ( @@ -292,7 +297,8 @@ export class CustomHeader extends Component { estela - {this.getFramework()} + {breadcrumb ? / : null} + {breadcrumb ?? null} = ({ children }) => { const location = useLocation(); - const pathSegments = location.pathname.split("/"); - let lastSegment = pathSegments[pathSegments.length - 1]; - - if (lastSegment === "") { - pathSegments.pop(); - lastSegment = pathSegments[pathSegments.length - 1]; - } + const pathSegments = location.pathname.split("/").filter(Boolean); + const section = pathSegments[2] ?? "dashboard"; - if (!isNaN(lastSegment)) { - pathSegments.pop(); - lastSegment = pathSegments[pathSegments.length - 1]; + let lastSegment = pathSegments[pathSegments.length - 1]; + if (!isNaN(lastSegment as unknown as number)) { + lastSegment = pathSegments[pathSegments.length - 2]; } + const { projectId, spiderId, jobId, cronjobId } = useParams(); const [path, setPath] = useState(lastSegment); - const { projectId } = useParams(); + const [projectName, setProjectName] = useState( + () => sessionStorage.getItem(`project-name-${projectId}`) ?? "", + ); + const [spiderName, setSpiderName] = useState(() => + spiderId ? sessionStorage.getItem(`spider-name-${spiderId}`) ?? "" : "", + ); + const apiService = ApiService(); + + useEffect(() => { + apiService.apiProjectsRead({ pid: projectId }).then((project) => { + setProjectName(project.name); + sessionStorage.setItem(`project-name-${projectId}`, project.name); + }); + }, [projectId]); + + useEffect(() => { + const sid = Number(spiderId); + if (spiderId && !isNaN(sid)) { + apiService.apiProjectsSpidersRead({ pid: projectId, sid }).then((spider) => { + setSpiderName(spider.name); + sessionStorage.setItem(`spider-name-${spiderId}`, spider.name); + }); + } else { + setSpiderName(""); + } + }, [projectId, spiderId]); const updatePathHandler = (newPath: string) => { setPath(newPath); }; + const getCrumbs = (): Crumb[] => { + const project: Crumb = { label: projectName || projectId, path: `/projects/${projectId}/dashboard` }; + + if (section === "dashboard") return [project, { label: "Dashboard" }]; + if (section === "jobs") return [project, { label: "Jobs" }]; + if (section === "cronjobs") return [project, { label: "Schedule" }]; + if (section === "activity") return [project, { label: "Activity" }]; + if (section === "members") return [project, { label: "Members" }]; + if (section === "settings") return [project, { label: "Settings" }]; + if (section === "deploys") return [project, { label: "Deploys" }]; + + if (section === "spiders") { + if (!spiderId) return [project, { label: "Spiders" }]; + if (jobId) { + return [project, { label: "Jobs", path: `/projects/${projectId}/jobs` }, { label: `Job-${jobId}` }]; + } + if (cronjobId) { + return [ + project, + { label: "Schedule", path: `/projects/${projectId}/cronjobs` }, + { label: `Schedule-job-${cronjobId}` }, + ]; + } + return [ + project, + { label: "Spiders", path: `/projects/${projectId}/spiders` }, + { label: spiderName || spiderId }, + ]; + } + + return [project]; + }; + + const crumbs = getCrumbs(); + + const breadcrumb = ( + + {crumbs.map((crumb, index) => ( + + {index > 0 && /} + {crumb.path ? ( + + {crumb.label} + + ) : ( + {crumb.label} + )} + + ))} + + ); + return ( -
+
{children} From 7583f4669b14b6cc875b063bd8936460e1bb48ec Mon Sep 17 00:00:00 2001 From: erick-gege Date: Fri, 19 Jun 2026 20:37:00 -0500 Subject: [PATCH 2/7] add migration --- .../migrations/0040_project_created_field.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 estela-api/core/migrations/0040_project_created_field.py diff --git a/estela-api/core/migrations/0040_project_created_field.py b/estela-api/core/migrations/0040_project_created_field.py new file mode 100644 index 00000000..95adf49a --- /dev/null +++ b/estela-api/core/migrations/0040_project_created_field.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.14 on 2026-06-20 01:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0039_userprofile'), + ] + + operations = [ + migrations.AddField( + model_name='project', + name='created', + field=models.DateTimeField(auto_now_add=True, help_text='Project creation date.', null=True), + ), + migrations.AlterField( + model_name='deploy', + name='status', + field=models.CharField(choices=[('SUCCESS', 'Success'), ('BUILDING', 'Building'), ('DOWNLOADING', 'Downloading'), ('FAILURE', 'Failure'), ('CANCELED', 'Canceled')], default='BUILDING', help_text='Deploy status.', max_length=12), + ), + ] From 72900f9c422cc5fe61e58e04f9f5bbd686a90c80 Mon Sep 17 00:00:00 2001 From: erick-gege Date: Mon, 22 Jun 2026 10:38:52 -0500 Subject: [PATCH 3/7] sorting and filtering for project list --- estela-api/api/views/project.py | 61 +++++ estela-api/docs/api.yaml | 12 +- .../src/pages/ProjectListPage/index.tsx | 240 +++++++----------- .../services/api/generated-api/apis/ApiApi.ts | 5 + 4 files changed, 168 insertions(+), 150 deletions(-) diff --git a/estela-api/api/views/project.py b/estela-api/api/views/project.py index ac3ec883..a7a24791 100644 --- a/estela-api/api/views/project.py +++ b/estela-api/api/views/project.py @@ -2,6 +2,8 @@ from django.conf import settings from django.core.paginator import Paginator +from django.db.models import Case, Max, Min, OuterRef, Subquery, When +from django.db.models.functions import Coalesce, Greatest from django.shortcuts import get_object_or_404 from drf_yasg import openapi from drf_yasg.utils import swagger_auto_schema @@ -55,7 +57,17 @@ def get_parameters(self, request): ) return page, page_size + ALLOWED_ORDERING_FIELDS = { + "name", "-name", + "category", "-category", + "framework", "-framework", + "created", "-created", + "last_modified", "-last_modified", + "role", "-role", + } + def get_queryset(self): + from core.models import Permission as PermissionModel, Spider, SpiderJob, Deploy queryset = ( Project.objects.filter(deleted=False) if self.request.user.is_superuser or self.request.user.is_staff @@ -64,8 +76,57 @@ def get_queryset(self): search = self.request.query_params.get("search", "") if search: queryset = queryset.filter(name__icontains=search) + ordering = self.request.query_params.get("ordering", "") + if ordering in self.ALLOWED_ORDERING_FIELDS: + field = ordering.lstrip("-") + prefix = "-" if ordering.startswith("-") else "" + if field == "created": + first_deploy = Subquery( + Project.objects.filter(pk=OuterRef("pk")).annotate( + fd=Min("deploy__created") + ).values("fd")[:1] + ) + queryset = queryset.annotate( + first_deploy_date=first_deploy + ).annotate( + created_annotation=Case( + When(first_deploy_date__isnull=False, then="first_deploy_date"), + default="created", + ) + ).order_by(f"{prefix}created_annotation") + elif field == "last_modified": + queryset = queryset.annotate( + last_modified_annotation=Coalesce( + Greatest(Max("spiders__deploy__created"), Max("spiders__jobs__created")), + "created", + ) + ).order_by(f"{prefix}last_modified_annotation") + elif field == "role": + role_subquery = Subquery( + PermissionModel.objects.filter( + project=OuterRef("pk"), + user=self.request.user, + ).values("permission")[:1] + ) + queryset = queryset.annotate( + user_role=role_subquery + ).order_by(f"{prefix}user_role") + else: + queryset = queryset.order_by(ordering) return queryset + @swagger_auto_schema( + manual_parameters=[ + openapi.Parameter("page", openapi.IN_QUERY, type=openapi.TYPE_INTEGER, required=False), + openapi.Parameter("page_size", openapi.IN_QUERY, type=openapi.TYPE_INTEGER, required=False), + openapi.Parameter("search", openapi.IN_QUERY, type=openapi.TYPE_STRING, required=False), + openapi.Parameter("ordering", openapi.IN_QUERY, type=openapi.TYPE_STRING, required=False, + description="Order by: name, category, framework, created, last_modified, role (prefix with - for desc)"), + ], + ) + def list(self, request, *args, **kwargs): + return super().list(request, *args, **kwargs) + def perform_create(self, serializer): instance = serializer.save() instance.users.add( diff --git a/estela-api/docs/api.yaml b/estela-api/docs/api.yaml index ed636fc8..ea248060 100644 --- a/estela-api/docs/api.yaml +++ b/estela-api/docs/api.yaml @@ -387,14 +387,22 @@ paths: parameters: - name: page in: query - description: A page number within the paginated result set. required: false type: integer - name: page_size in: query - description: Number of results to return per page. required: false type: integer + - name: search + in: query + required: false + type: string + - name: ordering + in: query + description: 'Order by: name, category, framework, created, last_modified, + role (prefix with - for desc)' + required: false + type: string responses: '200': description: '' diff --git a/estela-web/src/pages/ProjectListPage/index.tsx b/estela-web/src/pages/ProjectListPage/index.tsx index 458b2ed7..371bd10d 100644 --- a/estela-web/src/pages/ProjectListPage/index.tsx +++ b/estela-web/src/pages/ProjectListPage/index.tsx @@ -57,8 +57,10 @@ interface ProjectList { interface ProjectsPageState { projects: ProjectList[]; + recentProjects: ProjectList[]; username: string; loaded: boolean; + tableLoading: boolean; count: number; current: number; modalNewProject: boolean; @@ -81,8 +83,10 @@ export class ProjectListPage extends Component { state: ProjectsPageState = { projects: [], + recentProjects: [], username: "", loaded: false, + tableLoading: false, count: 0, current: 0, modalNewProject: false, @@ -102,20 +106,9 @@ export class ProjectListPage extends Component { apiService = ApiService(); static contextType = UserContext; runSearch = async (search: string): Promise => { - const data = await this.getProjects(1, search); - const projectData = data.data.map((project: Project, id: number) => ({ - name: project.name, - pid: project.pid, - framework: project.framework, - category: project.category, - created: project.created, - lastModified: project.lastModified, - role: - project.users?.find((user) => user.user?.username === AuthService.getUserUsername())?.permission || - "ADMIN", - key: id, - })); - this.setState({ projects: projectData, count: data.count, current: 1 }); + const { sortField, sortOrder } = this.state; + this.setState({ searchText: search }); + await this.loadProjects(1, search, sortField, sortOrder); }; formatDate = (date?: string): string => { @@ -123,15 +116,13 @@ export class ProjectListPage extends Component { return new Date(date).toLocaleDateString("en-US", { year: "numeric", month: "short", day: "numeric" }); }; + SERVER_SORT_FIELDS = new Set(["name", "category", "framework", "created", "lastModified", "role"]); + handleSort = (field: SortField): void => { - const { sortField, sortOrder } = this.state; - if (sortField !== field) { - this.setState({ sortField: field, sortOrder: "desc" }); - } else if (sortOrder === "desc") { - this.setState({ sortOrder: "asc" }); - } else { - this.setState({ sortField: null, sortOrder: null }); - } + const { sortField, sortOrder, searchText } = this.state; + const newOrder: "asc" | "desc" = sortField === field && sortOrder === "desc" ? "asc" : "desc"; + this.setState({ sortField: field, sortOrder: newOrder }); + this.loadProjects(1, searchText, field, newOrder); }; getSortIcon = (field: SortField): string => { @@ -140,49 +131,6 @@ export class ProjectListPage extends Component { return sortOrder === "asc" ? " ↑" : " ↓"; }; - getSortedAndFilteredProjects = (): ProjectList[] => { - const { projects, sortField, sortOrder } = this.state; - - if (!sortField || !sortOrder) return projects; - - return [...projects].sort((a, b) => { - let valA: string | number = ""; - let valB: string | number = ""; - - if (sortField === "name") { - valA = a.name; - valB = b.name; - } else if (sortField === "framework") { - valA = a.framework ?? ""; - valB = b.framework ?? ""; - } else if (sortField === "category") { - valA = a.category ?? ""; - valB = b.category ?? ""; - } else if (sortField === "role") { - valA = a.role; - valB = b.role; - } else if (sortField === "created") { - valA = a.created ? new Date(a.created).getTime() : 0; - valB = b.created ? new Date(b.created).getTime() : 0; - } else if (sortField === "lastModified") { - valA = a.lastModified - ? new Date(a.lastModified).getTime() - : a.created - ? new Date(a.created).getTime() - : 0; - valB = b.lastModified - ? new Date(b.lastModified).getTime() - : b.created - ? new Date(b.created).getTime() - : 0; - } - - if (valA < valB) return sortOrder === "asc" ? -1 : 1; - if (valA > valB) return sortOrder === "asc" ? 1 : -1; - return 0; - }); - }; - confirmDelete = (pid: string | undefined, name: string): void => { this.setState({ deleteModal: true, deleteTargetPid: pid, deleteTargetName: name, deleteConfirmText: "" }); }; @@ -392,23 +340,13 @@ export class ProjectListPage extends Component { const { updateRole } = this.context as UserContextProps; updateRole && updateRole(""); AuthService.removeFramework(); - const data = await this.getProjects(1); - const projectData: ProjectList[] = data.data.map((project: Project, id: number) => { - return { - name: project.name, - category: project.category, - framework: project.framework, - pid: project.pid, - created: project.created, - lastModified: project.lastModified, - role: - project.users?.find((user) => user.user?.username === AuthService.getUserUsername())?.permission || - "ADMIN", - key: id, - }; - }); + const [data, recentData] = await Promise.all([ + this.fetchProjects(1), + this.apiService.apiProjectsList({ page: 1, pageSize: 5, ordering: "-last_modified" }), + ]); this.setState({ - projects: [...projectData], + projects: this.toProjectList(data.data), + recentProjects: this.toProjectList(recentData.results), count: data.count, current: data.current, loaded: true, @@ -457,44 +395,64 @@ export class ProjectListPage extends Component { return String(AuthService.getUserUsername()); }; - async getProjects(page: number, search?: string): Promise<{ data: Project[]; count: number; current: number }> { - const requestParams: ApiProjectsListRequest = { page, pageSize: this.PAGE_SIZE, search }; + async fetchProjects( + page: number, + search?: string, + sortField?: SortField, + sortOrder?: "asc" | "desc" | null, + ): Promise<{ data: Project[]; count: number; current: number }> { + let ordering: string | undefined; + if (sortField && sortOrder && this.SERVER_SORT_FIELDS.has(sortField)) { + const backendField = sortField === "lastModified" ? "last_modified" : sortField; + ordering = sortOrder === "desc" ? `-${backendField}` : backendField; + } + const requestParams: ApiProjectsListRequest = { page, pageSize: this.PAGE_SIZE, search, ordering }; const data = await this.apiService.apiProjectsList(requestParams); this.totalProjects = data.count; return { data: data.results, count: data.count, current: page }; } - onPageChange = async (page: number): Promise => { - this.setState({ loaded: false }); - const data = await this.getProjects(page, this.state.searchText); - const projectData: ProjectList[] = data.data.map((project: Project, id: number) => { - return { - name: project.name, - pid: project.pid, - framework: project.framework, - category: project.category, - created: project.created, - lastModified: project.lastModified, - role: - project.users?.find((user) => user.user?.username === AuthService.getUserUsername())?.permission || - "ADMIN", - key: id, - }; - }); + toProjectList = (projects: Project[]): ProjectList[] => + projects.map((project, id) => ({ + name: project.name, + pid: project.pid, + framework: project.framework, + category: project.category, + created: project.created, + lastModified: project.lastModified, + role: + project.users?.find((user) => user.user?.username === AuthService.getUserUsername())?.permission || + "ADMIN", + key: id, + })); + + loadProjects = async ( + page: number, + search?: string, + sortField?: SortField, + sortOrder?: "asc" | "desc" | null, + ): Promise => { + this.setState({ tableLoading: true }); + const data = await this.fetchProjects(page, search, sortField, sortOrder); this.setState({ - projects: [...projectData], + projects: this.toProjectList(data.data), count: data.count, current: data.current, - loaded: true, - modalWelcome: data.count === 0, + tableLoading: false, }); }; + onPageChange = async (page: number): Promise => { + const { sortField, sortOrder, searchText } = this.state; + await this.loadProjects(page, searchText, sortField, sortOrder); + }; + render(): JSX.Element { const { count, current, loaded, + tableLoading, modalNewProject, modalWelcome, newProjectName, @@ -505,7 +463,7 @@ export class ProjectListPage extends Component { deleteConfirmText, } = this.state; - const displayProjects = this.getSortedAndFilteredProjects(); + const { projects: displayProjects, recentProjects } = this.state; return ( <> @@ -601,51 +559,36 @@ export class ProjectListPage extends Component { - {[...this.state.projects] - .sort((a, b) => { - const ta = a.lastModified - ? new Date(a.lastModified).getTime() - : a.created - ? new Date(a.created).getTime() - : 0; - const tb = b.lastModified - ? new Date(b.lastModified).getTime() - : b.created - ? new Date(b.created).getTime() - : 0; - return tb - ta; - }) - .slice(0, 5) - .map((project: ProjectList) => ( - - ))} + )} + + + ))} @@ -792,6 +735,7 @@ export class ProjectListPage extends Component { dataSource={displayProjects} pagination={false} size="middle" + loading={tableLoading} locale={{ emptyText: this.emptyText }} /> Date: Mon, 22 Jun 2026 20:08:43 -0500 Subject: [PATCH 4/7] feat: add project last_modified field --- estela-api/api/views/project.py | 10 +---- ...ed_field.py => 0040_auto_20260623_0025.py} | 7 +--- .../migrations/0041_auto_20260623_0041.py | 39 +++++++++++++++++++ estela-api/core/models.py | 13 ++----- estela-api/core/signals.py | 15 ++++++- .../src/shared/layouts/ProjectLayout.tsx | 8 +--- 6 files changed, 61 insertions(+), 31 deletions(-) rename estela-api/core/migrations/{0040_project_created_field.py => 0040_auto_20260623_0025.py} (66%) create mode 100644 estela-api/core/migrations/0041_auto_20260623_0041.py diff --git a/estela-api/api/views/project.py b/estela-api/api/views/project.py index a7a24791..825bb863 100644 --- a/estela-api/api/views/project.py +++ b/estela-api/api/views/project.py @@ -2,8 +2,7 @@ from django.conf import settings from django.core.paginator import Paginator -from django.db.models import Case, Max, Min, OuterRef, Subquery, When -from django.db.models.functions import Coalesce, Greatest +from django.db.models import Case, Min, OuterRef, Subquery, When from django.shortcuts import get_object_or_404 from drf_yasg import openapi from drf_yasg.utils import swagger_auto_schema @@ -95,12 +94,7 @@ def get_queryset(self): ) ).order_by(f"{prefix}created_annotation") elif field == "last_modified": - queryset = queryset.annotate( - last_modified_annotation=Coalesce( - Greatest(Max("spiders__deploy__created"), Max("spiders__jobs__created")), - "created", - ) - ).order_by(f"{prefix}last_modified_annotation") + queryset = queryset.order_by(f"{prefix}last_modified") elif field == "role": role_subquery = Subquery( PermissionModel.objects.filter( diff --git a/estela-api/core/migrations/0040_project_created_field.py b/estela-api/core/migrations/0040_auto_20260623_0025.py similarity index 66% rename from estela-api/core/migrations/0040_project_created_field.py rename to estela-api/core/migrations/0040_auto_20260623_0025.py index 95adf49a..06b616c5 100644 --- a/estela-api/core/migrations/0040_project_created_field.py +++ b/estela-api/core/migrations/0040_auto_20260623_0025.py @@ -1,4 +1,4 @@ -# Generated by Django 3.1.14 on 2026-06-20 01:36 +# Generated by Django 3.1.14 on 2026-06-23 00:25 from django.db import migrations, models @@ -10,11 +10,6 @@ class Migration(migrations.Migration): ] operations = [ - migrations.AddField( - model_name='project', - name='created', - field=models.DateTimeField(auto_now_add=True, help_text='Project creation date.', null=True), - ), migrations.AlterField( model_name='deploy', name='status', diff --git a/estela-api/core/migrations/0041_auto_20260623_0041.py b/estela-api/core/migrations/0041_auto_20260623_0041.py new file mode 100644 index 00000000..bb405398 --- /dev/null +++ b/estela-api/core/migrations/0041_auto_20260623_0041.py @@ -0,0 +1,39 @@ +# Generated by Django 3.1.14 on 2026-06-23 00:41 + +from django.db import migrations, models +from django.db.models import Max + + +def populate_last_modified(apps, schema_editor): + Project = apps.get_model("core", "Project") + Spider = apps.get_model("core", "Spider") + Deploy = apps.get_model("core", "Deploy") + SpiderJob = apps.get_model("core", "SpiderJob") + for project in Project.objects.filter(last_modified__isnull=True): + spider_ids = Spider.objects.filter(project=project).values_list("sid", flat=True) + last_deploy = Deploy.objects.filter(spiders__in=spider_ids).aggregate(d=Max("created"))["d"] + last_job = SpiderJob.objects.filter(spider__in=spider_ids).aggregate(j=Max("created"))["j"] + dates = [d for d in [last_deploy, last_job] if d is not None] + project.last_modified = max(dates) if dates else project.created + project.save(update_fields=["last_modified"]) + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0040_auto_20260623_0025'), + ] + + operations = [ + migrations.AddField( + model_name='project', + name='created', + field=models.DateTimeField(auto_now_add=True, help_text='Project creation date.', null=True), + ), + migrations.AddField( + model_name='project', + name='last_modified', + field=models.DateTimeField(help_text='Date of last activity (deploy or job).', null=True), + ), + migrations.RunPython(populate_last_modified, migrations.RunPython.noop), + ] diff --git a/estela-api/core/models.py b/estela-api/core/models.py index 2c2bbe3b..b3b44a53 100644 --- a/estela-api/core/models.py +++ b/estela-api/core/models.py @@ -87,6 +87,9 @@ class Project(models.Model): created = models.DateTimeField( auto_now_add=True, help_text="Project creation date.", null=True ) + last_modified = models.DateTimeField( + null=True, help_text="Date of last activity (deploy or job)." + ) class Meta: ordering = ["name"] @@ -98,16 +101,6 @@ def created_at(self): return first_deploy return self.created - @property - def last_modified(self): - from django.db.models import Max - result = self.spiders.aggregate( - latest_deploy=Max('deploy__created'), - latest_job=Max('jobs__created'), - ) - valid_dates = [d for d in result.values() if d is not None] - return max(valid_dates) if valid_dates else self.created - @property def container_image(self): """Get the production container image for this project.""" diff --git a/estela-api/core/signals.py b/estela-api/core/signals.py index 2db71830..56053a68 100644 --- a/estela-api/core/signals.py +++ b/estela-api/core/signals.py @@ -5,13 +5,26 @@ from django.db.models.signals import post_save from django.dispatch import receiver +from django.utils import timezone + from core.cronjob import disable_cronjob -from core.models import Spider, SpiderCronJob, SpiderJob, UserProfile +from core.models import Deploy, Project, Spider, SpiderCronJob, SpiderJob, UserProfile from core.tasks import get_chain_to_process_usage_data, record_job_coverage_event logger = logging.getLogger(__name__) +@receiver(post_save, sender=SpiderJob, dispatch_uid="update_project_last_modified_on_job") +def update_project_last_modified_on_job(sender, instance, **kwargs): + Project.objects.filter(pid=instance.spider.project_id).update(last_modified=timezone.now()) + + +@receiver(post_save, sender=Deploy, dispatch_uid="update_project_last_modified_on_deploy") +def update_project_last_modified_on_deploy(sender, instance, **kwargs): + project_ids = instance.spiders.values_list("project_id", flat=True).distinct() + Project.objects.filter(pid__in=project_ids).update(last_modified=timezone.now()) + + @receiver(post_save, sender=Spider, dispatch_uid="disable_cronjobs_on_spider_delete") def disable_cronjobs_on_spider_delete(sender, instance, **kwargs): diff --git a/estela-web/src/shared/layouts/ProjectLayout.tsx b/estela-web/src/shared/layouts/ProjectLayout.tsx index f76dad7a..e31e4920 100644 --- a/estela-web/src/shared/layouts/ProjectLayout.tsx +++ b/estela-web/src/shared/layouts/ProjectLayout.tsx @@ -38,7 +38,7 @@ export const ProjectLayout: React.FC = ({ children }) => { () => sessionStorage.getItem(`project-name-${projectId}`) ?? "", ); const [spiderName, setSpiderName] = useState(() => - spiderId ? sessionStorage.getItem(`spider-name-${spiderId}`) ?? "" : "", + spiderId ? sessionStorage.getItem(`spider-name-${spiderId}`) ?? "..." : "", ); const apiService = ApiService(); @@ -88,11 +88,7 @@ export const ProjectLayout: React.FC = ({ children }) => { { label: `Schedule-job-${cronjobId}` }, ]; } - return [ - project, - { label: "Spiders", path: `/projects/${projectId}/spiders` }, - { label: spiderName || spiderId }, - ]; + return [project, { label: "Spiders", path: `/projects/${projectId}/spiders` }, { label: spiderName }]; } return [project]; From bc73a9f41f0401265b8e44b21e3efaa106ac81b8 Mon Sep 17 00:00:00 2001 From: erick-gege Date: Mon, 22 Jun 2026 20:21:36 -0500 Subject: [PATCH 5/7] fix: show loading placeholder in breadcrumbs --- estela-web/src/shared/layouts/ProjectLayout.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/estela-web/src/shared/layouts/ProjectLayout.tsx b/estela-web/src/shared/layouts/ProjectLayout.tsx index e31e4920..dd2ed160 100644 --- a/estela-web/src/shared/layouts/ProjectLayout.tsx +++ b/estela-web/src/shared/layouts/ProjectLayout.tsx @@ -35,7 +35,7 @@ export const ProjectLayout: React.FC = ({ children }) => { const { projectId, spiderId, jobId, cronjobId } = useParams(); const [path, setPath] = useState(lastSegment); const [projectName, setProjectName] = useState( - () => sessionStorage.getItem(`project-name-${projectId}`) ?? "", + () => sessionStorage.getItem(`project-name-${projectId}`) ?? "...", ); const [spiderName, setSpiderName] = useState(() => spiderId ? sessionStorage.getItem(`spider-name-${spiderId}`) ?? "..." : "", @@ -66,7 +66,7 @@ export const ProjectLayout: React.FC = ({ children }) => { }; const getCrumbs = (): Crumb[] => { - const project: Crumb = { label: projectName || projectId, path: `/projects/${projectId}/dashboard` }; + const project: Crumb = { label: projectName, path: `/projects/${projectId}/dashboard` }; if (section === "dashboard") return [project, { label: "Dashboard" }]; if (section === "jobs") return [project, { label: "Jobs" }]; From 476e5c193449f7ef58b105363b9f347afab0d7b1 Mon Sep 17 00:00:00 2001 From: erick-gege Date: Mon, 22 Jun 2026 22:08:05 -0500 Subject: [PATCH 6/7] refactor: simplify project created --- estela-api/api/serializers/project.py | 2 +- estela-api/api/views/project.py | 20 ++++--------------- .../migrations/0041_auto_20260623_0041.py | 17 +++++++++------- estela-api/core/models.py | 7 ------- 4 files changed, 15 insertions(+), 31 deletions(-) diff --git a/estela-api/api/serializers/project.py b/estela-api/api/serializers/project.py index 4583d1b7..4e6e5ee3 100644 --- a/estela-api/api/serializers/project.py +++ b/estela-api/api/serializers/project.py @@ -55,7 +55,7 @@ class ProjectSerializer(serializers.ModelSerializer): last_modified = serializers.SerializerMethodField() def get_created(self, obj): - value = obj.created_at + value = obj.created if value is None: return None return value.isoformat() diff --git a/estela-api/api/views/project.py b/estela-api/api/views/project.py index 825bb863..748af35f 100644 --- a/estela-api/api/views/project.py +++ b/estela-api/api/views/project.py @@ -2,7 +2,7 @@ from django.conf import settings from django.core.paginator import Paginator -from django.db.models import Case, Min, OuterRef, Subquery, When +from django.db.models import OuterRef, Subquery from django.shortcuts import get_object_or_404 from drf_yasg import openapi from drf_yasg.utils import swagger_auto_schema @@ -66,7 +66,7 @@ def get_parameters(self, request): } def get_queryset(self): - from core.models import Permission as PermissionModel, Spider, SpiderJob, Deploy + queryset = ( Project.objects.filter(deleted=False) if self.request.user.is_superuser or self.request.user.is_staff @@ -80,24 +80,12 @@ def get_queryset(self): field = ordering.lstrip("-") prefix = "-" if ordering.startswith("-") else "" if field == "created": - first_deploy = Subquery( - Project.objects.filter(pk=OuterRef("pk")).annotate( - fd=Min("deploy__created") - ).values("fd")[:1] - ) - queryset = queryset.annotate( - first_deploy_date=first_deploy - ).annotate( - created_annotation=Case( - When(first_deploy_date__isnull=False, then="first_deploy_date"), - default="created", - ) - ).order_by(f"{prefix}created_annotation") + queryset = queryset.order_by(f"{prefix}created") elif field == "last_modified": queryset = queryset.order_by(f"{prefix}last_modified") elif field == "role": role_subquery = Subquery( - PermissionModel.objects.filter( + Permission.objects.filter( project=OuterRef("pk"), user=self.request.user, ).values("permission")[:1] diff --git a/estela-api/core/migrations/0041_auto_20260623_0041.py b/estela-api/core/migrations/0041_auto_20260623_0041.py index bb405398..efe5cbaf 100644 --- a/estela-api/core/migrations/0041_auto_20260623_0041.py +++ b/estela-api/core/migrations/0041_auto_20260623_0041.py @@ -1,21 +1,24 @@ # Generated by Django 3.1.14 on 2026-06-23 00:41 from django.db import migrations, models -from django.db.models import Max +from django.db.models import Min, Max -def populate_last_modified(apps, schema_editor): +def populate_dates(apps, schema_editor): Project = apps.get_model("core", "Project") Spider = apps.get_model("core", "Spider") Deploy = apps.get_model("core", "Deploy") SpiderJob = apps.get_model("core", "SpiderJob") - for project in Project.objects.filter(last_modified__isnull=True): + for project in Project.objects.all(): spider_ids = Spider.objects.filter(project=project).values_list("sid", flat=True) + first_deploy = Deploy.objects.filter(spiders__in=spider_ids).aggregate(d=Min("created"))["d"] last_deploy = Deploy.objects.filter(spiders__in=spider_ids).aggregate(d=Max("created"))["d"] last_job = SpiderJob.objects.filter(spider__in=spider_ids).aggregate(j=Max("created"))["j"] - dates = [d for d in [last_deploy, last_job] if d is not None] - project.last_modified = max(dates) if dates else project.created - project.save(update_fields=["last_modified"]) + if project.created is None or (first_deploy and first_deploy < project.created): + project.created = first_deploy + last_dates = [d for d in [last_deploy, last_job] if d is not None] + project.last_modified = max(last_dates) if last_dates else project.created + project.save(update_fields=["created", "last_modified"]) class Migration(migrations.Migration): @@ -35,5 +38,5 @@ class Migration(migrations.Migration): name='last_modified', field=models.DateTimeField(help_text='Date of last activity (deploy or job).', null=True), ), - migrations.RunPython(populate_last_modified, migrations.RunPython.noop), + migrations.RunPython(populate_dates, migrations.RunPython.noop), ] diff --git a/estela-api/core/models.py b/estela-api/core/models.py index b3b44a53..91969c49 100644 --- a/estela-api/core/models.py +++ b/estela-api/core/models.py @@ -94,13 +94,6 @@ class Project(models.Model): class Meta: ordering = ["name"] - @property - def created_at(self): - first_deploy = self.deploy_set.order_by("created").values_list("created", flat=True).first() - if first_deploy and (self.created is None or first_deploy < self.created): - return first_deploy - return self.created - @property def container_image(self): """Get the production container image for this project.""" From 4702dab4de42ceaa238cb700d60da17290f8205d Mon Sep 17 00:00:00 2001 From: erick-gege Date: Tue, 23 Jun 2026 11:15:49 -0500 Subject: [PATCH 7/7] add scraping demo project --- estela-web/src/pages/DeployListPage/index.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/estela-web/src/pages/DeployListPage/index.tsx b/estela-web/src/pages/DeployListPage/index.tsx index 6e654819..65d66d98 100644 --- a/estela-web/src/pages/DeployListPage/index.tsx +++ b/estela-web/src/pages/DeployListPage/index.tsx @@ -383,9 +383,9 @@ export class DeployListPage extends Component,

- $ git clone https://github.com/scrapy/quotesbot.git + $ git clone https://github.com/bitmakerla/scraping-demo-project

-

$ cd quotesbot

+

$ cd scraping-demo-project

$ estela login

Host [http://localhost]: {API_BASE_URL}

Username: {AuthService.getUserUsername()}