diff --git a/estela-api/api/serializers/project.py b/estela-api/api/serializers/project.py index 2b742e45..4e6e5ee3 100644 --- a/estela-api/api/serializers/project.py +++ b/estela-api/api/serializers/project.py @@ -51,6 +51,20 @@ class ProjectSerializer(serializers.ModelSerializer): container_image = serializers.CharField( read_only=True, help_text="Path of the project's container image." ) + created = serializers.SerializerMethodField() + last_modified = serializers.SerializerMethodField() + + def get_created(self, obj): + value = obj.created + if value is None: + return None + return value.isoformat() + + def get_last_modified(self, obj): + value = obj.last_modified + if value is None: + return None + return value.isoformat() class Meta: model = Project @@ -64,6 +78,8 @@ class Meta: "env_vars", "data_status", "data_expiry_days", + "created", + "last_modified", ) diff --git a/estela-api/api/views/project.py b/estela-api/api/views/project.py index 8ae3e3ac..748af35f 100644 --- a/estela-api/api/views/project.py +++ b/estela-api/api/views/project.py @@ -2,6 +2,7 @@ from django.conf import settings from django.core.paginator import Paginator +from django.db.models import OuterRef, Subquery from django.shortcuts import get_object_or_404 from drf_yasg import openapi from drf_yasg.utils import swagger_auto_schema @@ -55,12 +56,58 @@ def get_parameters(self, request): ) return page, page_size + ALLOWED_ORDERING_FIELDS = { + "name", "-name", + "category", "-category", + "framework", "-framework", + "created", "-created", + "last_modified", "-last_modified", + "role", "-role", + } + def get_queryset(self): - return ( + + queryset = ( Project.objects.filter(deleted=False) if self.request.user.is_superuser or self.request.user.is_staff else self.request.user.project_set.filter(deleted=False) ) + search = self.request.query_params.get("search", "") + if search: + queryset = queryset.filter(name__icontains=search) + ordering = self.request.query_params.get("ordering", "") + if ordering in self.ALLOWED_ORDERING_FIELDS: + field = ordering.lstrip("-") + prefix = "-" if ordering.startswith("-") else "" + if field == "created": + queryset = queryset.order_by(f"{prefix}created") + elif field == "last_modified": + queryset = queryset.order_by(f"{prefix}last_modified") + elif field == "role": + role_subquery = Subquery( + Permission.objects.filter( + project=OuterRef("pk"), + user=self.request.user, + ).values("permission")[:1] + ) + queryset = queryset.annotate( + user_role=role_subquery + ).order_by(f"{prefix}user_role") + else: + queryset = queryset.order_by(ordering) + return queryset + + @swagger_auto_schema( + manual_parameters=[ + openapi.Parameter("page", openapi.IN_QUERY, type=openapi.TYPE_INTEGER, required=False), + openapi.Parameter("page_size", openapi.IN_QUERY, type=openapi.TYPE_INTEGER, required=False), + openapi.Parameter("search", openapi.IN_QUERY, type=openapi.TYPE_STRING, required=False), + openapi.Parameter("ordering", openapi.IN_QUERY, type=openapi.TYPE_STRING, required=False, + description="Order by: name, category, framework, created, last_modified, role (prefix with - for desc)"), + ], + ) + def list(self, request, *args, **kwargs): + return super().list(request, *args, **kwargs) def perform_create(self, serializer): instance = serializer.save() diff --git a/estela-api/core/migrations/0040_auto_20260623_0025.py b/estela-api/core/migrations/0040_auto_20260623_0025.py new file mode 100644 index 00000000..06b616c5 --- /dev/null +++ b/estela-api/core/migrations/0040_auto_20260623_0025.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.14 on 2026-06-23 00:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0039_userprofile'), + ] + + operations = [ + migrations.AlterField( + model_name='deploy', + name='status', + field=models.CharField(choices=[('SUCCESS', 'Success'), ('BUILDING', 'Building'), ('DOWNLOADING', 'Downloading'), ('FAILURE', 'Failure'), ('CANCELED', 'Canceled')], default='BUILDING', help_text='Deploy status.', max_length=12), + ), + ] diff --git a/estela-api/core/migrations/0041_auto_20260623_0041.py b/estela-api/core/migrations/0041_auto_20260623_0041.py new file mode 100644 index 00000000..efe5cbaf --- /dev/null +++ b/estela-api/core/migrations/0041_auto_20260623_0041.py @@ -0,0 +1,42 @@ +# Generated by Django 3.1.14 on 2026-06-23 00:41 + +from django.db import migrations, models +from django.db.models import Min, Max + + +def populate_dates(apps, schema_editor): + Project = apps.get_model("core", "Project") + Spider = apps.get_model("core", "Spider") + Deploy = apps.get_model("core", "Deploy") + SpiderJob = apps.get_model("core", "SpiderJob") + for project in Project.objects.all(): + spider_ids = Spider.objects.filter(project=project).values_list("sid", flat=True) + first_deploy = Deploy.objects.filter(spiders__in=spider_ids).aggregate(d=Min("created"))["d"] + last_deploy = Deploy.objects.filter(spiders__in=spider_ids).aggregate(d=Max("created"))["d"] + last_job = SpiderJob.objects.filter(spider__in=spider_ids).aggregate(j=Max("created"))["j"] + if project.created is None or (first_deploy and first_deploy < project.created): + project.created = first_deploy + last_dates = [d for d in [last_deploy, last_job] if d is not None] + project.last_modified = max(last_dates) if last_dates else project.created + project.save(update_fields=["created", "last_modified"]) + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0040_auto_20260623_0025'), + ] + + operations = [ + migrations.AddField( + model_name='project', + name='created', + field=models.DateTimeField(auto_now_add=True, help_text='Project creation date.', null=True), + ), + migrations.AddField( + model_name='project', + name='last_modified', + field=models.DateTimeField(help_text='Date of last activity (deploy or job).', null=True), + ), + migrations.RunPython(populate_dates, migrations.RunPython.noop), + ] diff --git a/estela-api/core/models.py b/estela-api/core/models.py index 69cab0b9..91969c49 100644 --- a/estela-api/core/models.py +++ b/estela-api/core/models.py @@ -84,6 +84,12 @@ class Project(models.Model): deleted = models.BooleanField( default=False, help_text="Whether the project was deleted." ) + created = models.DateTimeField( + auto_now_add=True, help_text="Project creation date.", null=True + ) + last_modified = models.DateTimeField( + null=True, help_text="Date of last activity (deploy or job)." + ) class Meta: ordering = ["name"] diff --git a/estela-api/core/signals.py b/estela-api/core/signals.py index 2db71830..56053a68 100644 --- a/estela-api/core/signals.py +++ b/estela-api/core/signals.py @@ -5,13 +5,26 @@ from django.db.models.signals import post_save from django.dispatch import receiver +from django.utils import timezone + from core.cronjob import disable_cronjob -from core.models import Spider, SpiderCronJob, SpiderJob, UserProfile +from core.models import Deploy, Project, Spider, SpiderCronJob, SpiderJob, UserProfile from core.tasks import get_chain_to_process_usage_data, record_job_coverage_event logger = logging.getLogger(__name__) +@receiver(post_save, sender=SpiderJob, dispatch_uid="update_project_last_modified_on_job") +def update_project_last_modified_on_job(sender, instance, **kwargs): + Project.objects.filter(pid=instance.spider.project_id).update(last_modified=timezone.now()) + + +@receiver(post_save, sender=Deploy, dispatch_uid="update_project_last_modified_on_deploy") +def update_project_last_modified_on_deploy(sender, instance, **kwargs): + project_ids = instance.spiders.values_list("project_id", flat=True).distinct() + Project.objects.filter(pid__in=project_ids).update(last_modified=timezone.now()) + + @receiver(post_save, sender=Spider, dispatch_uid="disable_cronjobs_on_spider_delete") def disable_cronjobs_on_spider_delete(sender, instance, **kwargs): diff --git a/estela-api/docs/api.yaml b/estela-api/docs/api.yaml index cce201a9..ea248060 100644 --- a/estela-api/docs/api.yaml +++ b/estela-api/docs/api.yaml @@ -387,14 +387,22 @@ paths: parameters: - name: page in: query - description: A page number within the paginated result set. required: false type: integer - name: page_size in: query - description: Number of results to return per page. required: false type: integer + - name: search + in: query + required: false + type: string + - name: ordering + in: query + description: 'Order by: name, category, framework, created, last_modified, + role (prefix with - for desc)' + required: false + type: string responses: '200': description: '' @@ -698,6 +706,28 @@ paths: in: path required: true type: string + /api/projects/{pid}/deploys/{did}/logs: + get: + operationId: api_projects_deploys_logs + description: '' + parameters: [] + responses: + '200': + description: '' + schema: + $ref: '#/definitions/Deploy' + tags: + - api + parameters: + - name: did + in: path + description: A unique integer value identifying this deploy. + required: true + type: integer + - name: pid + in: path + required: true + type: string /api/projects/{pid}/jobs: get: operationId: api_projects_jobs @@ -1316,6 +1346,32 @@ paths: in: path required: true type: string + /api/projects/{pid}/spiders/{sid}/jobs/{jid}/error_logs: + get: + operationId: api_projects_spiders_jobs_error_logs + description: '' + parameters: [] + responses: + '200': + description: '' + schema: + $ref: '#/definitions/SpiderJob' + tags: + - api + parameters: + - name: jid + in: path + description: A unique integer value identifying this job. + required: true + type: integer + - name: pid + in: path + required: true + type: string + - name: sid + in: path + required: true + type: string /api/projects/{pid}/usage: get: operationId: api_projects_usage @@ -1968,6 +2024,14 @@ definitions: type: integer maximum: 65535 minimum: 0 + created: + title: Created + type: string + readOnly: true + last_modified: + title: Last modified + type: string + readOnly: true ProjectUpdate: type: object properties: @@ -2334,6 +2398,12 @@ definitions: type: string maxLength: 1000 minLength: 1 + error_reason: + title: Error reason + description: Error logs to persist in deploy_logs (Mongo) on failure. + type: string + maxLength: 200000 + x-nullable: true SpiderJob: description: Project jobs. type: object @@ -2797,6 +2867,12 @@ definitions: title: Proxy usage data description: Proxy Usage data. type: string + error_reason: + title: Error reason + description: Error logs to persist in job_logs (Mongo) on failure. + type: string + maxLength: 200000 + x-nullable: true UsageRecord: required: - processing_time diff --git a/estela-web/src/pages/DeployListPage/index.tsx b/estela-web/src/pages/DeployListPage/index.tsx index bc7ab8c3..65d66d98 100644 --- a/estela-web/src/pages/DeployListPage/index.tsx +++ b/estela-web/src/pages/DeployListPage/index.tsx @@ -3,9 +3,9 @@ import { Layout, Pagination, Row, Table, Button, Tag, Col, Typography, Modal, To import { ExclamationCircleOutlined } from "@ant-design/icons"; import { RouteComponentProps } from "react-router-dom"; import Copy from "../../assets/icons/copy.svg"; -import Info from "../../assets/icons/info.svg"; import Help from "../../assets/icons/help.svg"; import SpiderIcon from "../../assets/icons/spider.svg"; +import Info from "../../assets/icons/info.svg"; import WelcomeDeploy from "../../assets/images/welcomeDeploy.svg"; import "./styles.scss"; @@ -317,53 +317,40 @@ export class DeployListPage extends Component, -

SPIDER OVERVIEW

- + +

SPIDER OVERVIEW

+ + + - - - - -

- Want to know more about estela? Access our - - - documentation - - - . -

-

- Install the - - - estela CLI - - - to streamline spider deployment, unlock{" "} - advanced developer features, and access all{" "} - estela tools. -

- - - + + + + Install the{" "} + + estela CLI + {" "} + to deploy your spiders and access all estela tools.{" "} + + Read the docs → + + ,

- $ git clone https://github.com/scrapy/quotesbot.git + $ git clone https://github.com/bitmakerla/scraping-demo-project

-

$ cd quotesbot

+

$ cd scraping-demo-project

$ estela login

Host [http://localhost]: {API_BASE_URL}

Username: {AuthService.getUserUsername()}

diff --git a/estela-web/src/pages/ProjectListPage/index.tsx b/estela-web/src/pages/ProjectListPage/index.tsx index 95b1becf..371bd10d 100644 --- a/estela-web/src/pages/ProjectListPage/index.tsx +++ b/estela-web/src/pages/ProjectListPage/index.tsx @@ -1,15 +1,39 @@ import React, { Component, Fragment, ReactElement } from "react"; import { Link } from "react-router-dom"; -import { Button, Layout, Pagination, Space, Typography, Row, Col, Tag, Table, Modal, Input, Select } from "antd"; +import { + Button, + Layout, + Pagination, + Space, + Typography, + Row, + Col, + Tag, + Table, + Modal, + Input, + Select, + Dropdown, + Menu, + message, +} from "antd"; +import { SettingOutlined, EditOutlined, TagsOutlined, TeamOutlined, DeleteOutlined } from "@ant-design/icons"; import "./styles.scss"; import { ApiService, AuthService } from "../../services"; import Add from "../../assets/icons/add.svg"; import Bug from "../../assets/icons/bug.svg"; +import Copy from "../../assets/icons/copy.svg"; import FolderDotted from "../../assets/icons/folderDotted.svg"; import WelcomeProjects from "../../assets/images/welcomeProjects.svg"; import history from "../../history"; -import { ApiProjectsListRequest, ApiProjectsCreateRequest, Project, ProjectCategoryEnum } from "../../services/api"; +import { + ApiProjectsListRequest, + ApiProjectsCreateRequest, + ApiProjectsDeleteRequest, + Project, + ProjectCategoryEnum, +} from "../../services/api"; import { incorrectDataNotification, Spin, PaginationItem } from "../../shared"; import { UserContext, UserContextProps } from "../../context/UserContext"; @@ -17,25 +41,40 @@ const { Content } = Layout; const { Option } = Select; const { Text, Paragraph } = Typography; +type SortOrder = "asc" | "desc" | null; +type SortField = "name" | "framework" | "category" | "created" | "lastModified" | "role" | null; + interface ProjectList { name: string; category?: string; pid: string | undefined; role: string; framework: string | undefined; + created?: string; + lastModified?: string; key: number; } interface ProjectsPageState { projects: ProjectList[]; + recentProjects: ProjectList[]; username: string; loaded: boolean; + tableLoading: boolean; count: number; current: number; modalNewProject: boolean; modalWelcome: boolean; newProjectName: string; newProjectCategory: ProjectCategoryEnum; + searchText: string; + sortField: SortField; + sortOrder: SortOrder; + deleteModal: boolean; + deleteTargetPid: string | undefined; + deleteTargetName: string; + deleteConfirmText: string; + copiedPid: string | undefined; } export class ProjectListPage extends Component { @@ -44,22 +83,118 @@ export class ProjectListPage extends Component { state: ProjectsPageState = { projects: [], + recentProjects: [], username: "", loaded: false, + tableLoading: false, count: 0, current: 0, modalNewProject: false, modalWelcome: false, newProjectName: "", newProjectCategory: ProjectCategoryEnum.NotSpecified, + searchText: "", + sortField: null, + sortOrder: null, + deleteModal: false, + deleteTargetPid: undefined, + deleteTargetName: "", + deleteConfirmText: "", + copiedPid: undefined, }; apiService = ApiService(); static contextType = UserContext; + runSearch = async (search: string): Promise => { + const { sortField, sortOrder } = this.state; + this.setState({ searchText: search }); + await this.loadProjects(1, search, sortField, sortOrder); + }; + + formatDate = (date?: string): string => { + if (!date) return "—"; + return new Date(date).toLocaleDateString("en-US", { year: "numeric", month: "short", day: "numeric" }); + }; + + SERVER_SORT_FIELDS = new Set(["name", "category", "framework", "created", "lastModified", "role"]); + + handleSort = (field: SortField): void => { + const { sortField, sortOrder, searchText } = this.state; + const newOrder: "asc" | "desc" = sortField === field && sortOrder === "desc" ? "asc" : "desc"; + this.setState({ sortField: field, sortOrder: newOrder }); + this.loadProjects(1, searchText, field, newOrder); + }; + + getSortIcon = (field: SortField): string => { + const { sortField, sortOrder } = this.state; + if (sortField !== field) return ""; + return sortOrder === "asc" ? " ↑" : " ↓"; + }; + + confirmDelete = (pid: string | undefined, name: string): void => { + this.setState({ deleteModal: true, deleteTargetPid: pid, deleteTargetName: name, deleteConfirmText: "" }); + }; + + deleteProject = (): void => { + const { deleteTargetPid } = this.state; + if (!deleteTargetPid) return; + const request: ApiProjectsDeleteRequest = { pid: deleteTargetPid }; + this.apiService.apiProjectsDelete(request).then( + () => { + this.setState({ + deleteModal: false, + deleteTargetPid: undefined, + deleteTargetName: "", + deleteConfirmText: "", + }); + this.onPageChange(1); + }, + () => incorrectDataNotification(), + ); + }; - columns = [ + getActionMenu = (project: ProjectList): ReactElement => ( + + } + onClick={() => history.push(`/projects/${project.pid}/settings`)} + > + Edit project + + } + onClick={() => history.push(`/projects/${project.pid}/settings`)} + > + Manage tags + + } + onClick={() => history.push(`/projects/${project.pid}/members`)} + > + Team permissions + + + } + danger + onClick={() => this.confirmDelete(project.pid, project.name)} + > + Delete project + + + ); + + getColumns = () => [ { - title: "NAME", + title: ( + this.handleSort("name")}> + NAME{this.getSortIcon("name")} + + ), dataIndex: "name", key: "name", render: (name: string, project: ProjectList): ReactElement => ( @@ -79,10 +214,35 @@ export class ProjectListPage extends Component { title: "PID", dataIndex: "pid", key: "pid", - render: (pid: string): ReactElement =>

{pid}

, + render: (pid: string): ReactElement => { + const copied = this.state.copiedPid === pid; + return ( + + {pid ? `${pid.slice(0, 8)}...` : ""} + , + , + ]} + > +

+ This action cannot be undone. Type{" "} + {deleteTargetName} to confirm. +

+ this.setState({ deleteConfirmText: e.target.value })} + onPressEnter={() => { + if (deleteConfirmText === deleteTargetName) this.deleteProject(); + }} + /> + @@ -267,43 +559,36 @@ export class ProjectListPage extends Component { - {projects.map((project: ProjectList, index) => { - return index < 3 ? ( - - ) : ( - - ); - })} + )} + + + ))} @@ -432,14 +717,25 @@ export class ProjectListPage extends Component { + + this.setState({ searchText: e.target.value })} + onSearch={this.runSearch} + className="border-estela-blue-low rounded-md" + style={{ maxWidth: 400 }} + /> + > { + if (requestParameters.did === null || requestParameters.did === undefined) { + throw new runtime.RequiredError('did','Required parameter requestParameters.did was null or undefined when calling apiProjectsDeploysLogs.'); + } + + if (requestParameters.pid === null || requestParameters.pid === undefined) { + throw new runtime.RequiredError('pid','Required parameter requestParameters.pid was null or undefined when calling apiProjectsDeploysLogs.'); + } + + const queryParameters: any = {}; + + const headerParameters: runtime.HTTPHeaders = {}; + + if (this.configuration && (this.configuration.username !== undefined || this.configuration.password !== undefined)) { + headerParameters["Authorization"] = "Basic " + btoa(this.configuration.username + ":" + this.configuration.password); + } + const response = await this.request({ + path: `/api/projects/{pid}/deploys/{did}/logs`.replace(`{${"did"}}`, encodeURIComponent(String(requestParameters.did))).replace(`{${"pid"}}`, encodeURIComponent(String(requestParameters.pid))), + method: 'GET', + headers: headerParameters, + query: queryParameters, + }); + + return new runtime.JSONApiResponse(response, (jsonValue) => DeployFromJSON(jsonValue)); + } + + /** + */ + async apiProjectsDeploysLogs(requestParameters: ApiProjectsDeploysLogsRequest): Promise { + const response = await this.apiProjectsDeploysLogsRaw(requestParameters); + return await response.value(); + } + /** */ async apiProjectsDeploysPartialUpdateRaw(requestParameters: ApiProjectsDeploysPartialUpdateRequest): Promise> { @@ -1642,6 +1690,14 @@ export class ApiApi extends runtime.BaseAPI { queryParameters['page_size'] = requestParameters.pageSize; } + if (requestParameters.search !== undefined) { + queryParameters['search'] = requestParameters.search; + } + + if (requestParameters.ordering !== undefined) { + queryParameters['ordering'] = requestParameters.ordering; + } + const headerParameters: runtime.HTTPHeaders = {}; if (this.configuration && (this.configuration.username !== undefined || this.configuration.password !== undefined)) { @@ -2263,6 +2319,45 @@ export class ApiApi extends runtime.BaseAPI { return await response.value(); } + /** + */ + async apiProjectsSpidersJobsErrorLogsRaw(requestParameters: ApiProjectsSpidersJobsErrorLogsRequest): Promise> { + if (requestParameters.jid === null || requestParameters.jid === undefined) { + throw new runtime.RequiredError('jid','Required parameter requestParameters.jid was null or undefined when calling apiProjectsSpidersJobsErrorLogs.'); + } + + if (requestParameters.pid === null || requestParameters.pid === undefined) { + throw new runtime.RequiredError('pid','Required parameter requestParameters.pid was null or undefined when calling apiProjectsSpidersJobsErrorLogs.'); + } + + if (requestParameters.sid === null || requestParameters.sid === undefined) { + throw new runtime.RequiredError('sid','Required parameter requestParameters.sid was null or undefined when calling apiProjectsSpidersJobsErrorLogs.'); + } + + const queryParameters: any = {}; + + const headerParameters: runtime.HTTPHeaders = {}; + + if (this.configuration && (this.configuration.username !== undefined || this.configuration.password !== undefined)) { + headerParameters["Authorization"] = "Basic " + btoa(this.configuration.username + ":" + this.configuration.password); + } + const response = await this.request({ + path: `/api/projects/{pid}/spiders/{sid}/jobs/{jid}/error_logs`.replace(`{${"jid"}}`, encodeURIComponent(String(requestParameters.jid))).replace(`{${"pid"}}`, encodeURIComponent(String(requestParameters.pid))).replace(`{${"sid"}}`, encodeURIComponent(String(requestParameters.sid))), + method: 'GET', + headers: headerParameters, + query: queryParameters, + }); + + return new runtime.JSONApiResponse(response, (jsonValue) => SpiderJobFromJSON(jsonValue)); + } + + /** + */ + async apiProjectsSpidersJobsErrorLogs(requestParameters: ApiProjectsSpidersJobsErrorLogsRequest): Promise { + const response = await this.apiProjectsSpidersJobsErrorLogsRaw(requestParameters); + return await response.value(); + } + /** */ async apiProjectsSpidersJobsListRaw(requestParameters: ApiProjectsSpidersJobsListRequest): Promise> { diff --git a/estela-web/src/services/api/generated-api/models/DeployUpdate.ts b/estela-web/src/services/api/generated-api/models/DeployUpdate.ts index b634e339..6f659c5b 100644 --- a/estela-web/src/services/api/generated-api/models/DeployUpdate.ts +++ b/estela-web/src/services/api/generated-api/models/DeployUpdate.ts @@ -37,6 +37,12 @@ export interface DeployUpdate { * @memberof DeployUpdate */ spidersNames?: Array; + /** + * Error logs to persist in deploy_logs (Mongo) on failure. + * @type {string} + * @memberof DeployUpdate + */ + errorReason?: string | null; } /** @@ -64,6 +70,7 @@ export function DeployUpdateFromJSONTyped(json: any, ignoreDiscriminator: boolea 'did': !exists(json, 'did') ? undefined : json['did'], 'status': !exists(json, 'status') ? undefined : json['status'], 'spidersNames': !exists(json, 'spiders_names') ? undefined : json['spiders_names'], + 'errorReason': !exists(json, 'error_reason') ? undefined : json['error_reason'], }; } @@ -78,6 +85,7 @@ export function DeployUpdateToJSON(value?: DeployUpdate | null): any { 'status': value.status, 'spiders_names': value.spidersNames, + 'error_reason': value.errorReason, }; } diff --git a/estela-web/src/services/api/generated-api/models/Project.ts b/estela-web/src/services/api/generated-api/models/Project.ts index 15b6a9dd..57a24d45 100644 --- a/estela-web/src/services/api/generated-api/models/Project.ts +++ b/estela-web/src/services/api/generated-api/models/Project.ts @@ -84,6 +84,18 @@ export interface Project { * @memberof Project */ dataExpiryDays?: number; + /** + * + * @type {string} + * @memberof Project + */ + readonly created?: string; + /** + * + * @type {string} + * @memberof Project + */ + readonly lastModified?: string; } /** @@ -133,6 +145,8 @@ export function ProjectFromJSONTyped(json: any, ignoreDiscriminator: boolean): P 'envVars': !exists(json, 'env_vars') ? undefined : ((json['env_vars'] as Array).map(SpiderJobEnvVarFromJSON)), 'dataStatus': !exists(json, 'data_status') ? undefined : json['data_status'], 'dataExpiryDays': !exists(json, 'data_expiry_days') ? undefined : json['data_expiry_days'], + 'created': !exists(json, 'created') ? undefined : json['created'], + 'lastModified': !exists(json, 'last_modified') ? undefined : json['last_modified'], }; } diff --git a/estela-web/src/services/api/generated-api/models/SpiderJobUpdate.ts b/estela-web/src/services/api/generated-api/models/SpiderJobUpdate.ts index d7fa1032..c11aeed0 100644 --- a/estela-web/src/services/api/generated-api/models/SpiderJobUpdate.ts +++ b/estela-web/src/services/api/generated-api/models/SpiderJobUpdate.ts @@ -73,6 +73,12 @@ export interface SpiderJobUpdate { * @memberof SpiderJobUpdate */ proxyUsageData?: string; + /** + * Error logs to persist in job_logs (Mongo) on failure. + * @type {string} + * @memberof SpiderJobUpdate + */ + errorReason?: string | null; } /** @@ -115,6 +121,7 @@ export function SpiderJobUpdateFromJSONTyped(json: any, ignoreDiscriminator: boo 'dataStatus': !exists(json, 'data_status') ? undefined : json['data_status'], 'dataExpiryDays': !exists(json, 'data_expiry_days') ? undefined : json['data_expiry_days'], 'proxyUsageData': !exists(json, 'proxy_usage_data') ? undefined : json['proxy_usage_data'], + 'errorReason': !exists(json, 'error_reason') ? undefined : json['error_reason'], }; } @@ -135,6 +142,7 @@ export function SpiderJobUpdateToJSON(value?: SpiderJobUpdate | null): any { 'data_status': value.dataStatus, 'data_expiry_days': value.dataExpiryDays, 'proxy_usage_data': value.proxyUsageData, + 'error_reason': value.errorReason, }; } diff --git a/estela-web/src/shared/header/index.tsx b/estela-web/src/shared/header/index.tsx index 05ea164a..2da85537 100644 --- a/estela-web/src/shared/header/index.tsx +++ b/estela-web/src/shared/header/index.tsx @@ -20,6 +20,10 @@ import userDropdownSideNavItems from "ExternalComponents/DropdownComponent"; const { Header, Content } = Layout; type MenuItem = Required["items"][number]; +interface HeaderProps { + breadcrumb?: React.ReactNode; +} + interface HeaderState { notifications: Notification[]; loaded: boolean; @@ -27,7 +31,7 @@ interface HeaderState { news: boolean; } -export class CustomHeader extends Component { +export class CustomHeader extends Component { state: HeaderState = { notifications: [], loaded: false, @@ -283,6 +287,7 @@ export class CustomHeader extends Component { render(): JSX.Element { const { path, loaded, notifications, news } = this.state; + const { breadcrumb } = this.props; return ( <> {loaded ? ( @@ -292,7 +297,8 @@ export class CustomHeader extends Component { estela - {this.getFramework()} + {breadcrumb ? / : null} + {breadcrumb ?? null} = ({ children }) => { const location = useLocation(); - const pathSegments = location.pathname.split("/"); - let lastSegment = pathSegments[pathSegments.length - 1]; - - if (lastSegment === "") { - pathSegments.pop(); - lastSegment = pathSegments[pathSegments.length - 1]; - } + const pathSegments = location.pathname.split("/").filter(Boolean); + const section = pathSegments[2] ?? "dashboard"; - if (!isNaN(lastSegment)) { - pathSegments.pop(); - lastSegment = pathSegments[pathSegments.length - 1]; + let lastSegment = pathSegments[pathSegments.length - 1]; + if (!isNaN(lastSegment as unknown as number)) { + lastSegment = pathSegments[pathSegments.length - 2]; } + const { projectId, spiderId, jobId, cronjobId } = useParams(); const [path, setPath] = useState(lastSegment); - const { projectId } = useParams(); + const [projectName, setProjectName] = useState( + () => sessionStorage.getItem(`project-name-${projectId}`) ?? "...", + ); + const [spiderName, setSpiderName] = useState(() => + spiderId ? sessionStorage.getItem(`spider-name-${spiderId}`) ?? "..." : "", + ); + const apiService = ApiService(); + + useEffect(() => { + apiService.apiProjectsRead({ pid: projectId }).then((project) => { + setProjectName(project.name); + sessionStorage.setItem(`project-name-${projectId}`, project.name); + }); + }, [projectId]); + + useEffect(() => { + const sid = Number(spiderId); + if (spiderId && !isNaN(sid)) { + apiService.apiProjectsSpidersRead({ pid: projectId, sid }).then((spider) => { + setSpiderName(spider.name); + sessionStorage.setItem(`spider-name-${spiderId}`, spider.name); + }); + } else { + setSpiderName(""); + } + }, [projectId, spiderId]); const updatePathHandler = (newPath: string) => { setPath(newPath); }; + const getCrumbs = (): Crumb[] => { + const project: Crumb = { label: projectName, path: `/projects/${projectId}/dashboard` }; + + if (section === "dashboard") return [project, { label: "Dashboard" }]; + if (section === "jobs") return [project, { label: "Jobs" }]; + if (section === "cronjobs") return [project, { label: "Schedule" }]; + if (section === "activity") return [project, { label: "Activity" }]; + if (section === "members") return [project, { label: "Members" }]; + if (section === "settings") return [project, { label: "Settings" }]; + if (section === "deploys") return [project, { label: "Deploys" }]; + + if (section === "spiders") { + if (!spiderId) return [project, { label: "Spiders" }]; + if (jobId) { + return [project, { label: "Jobs", path: `/projects/${projectId}/jobs` }, { label: `Job-${jobId}` }]; + } + if (cronjobId) { + return [ + project, + { label: "Schedule", path: `/projects/${projectId}/cronjobs` }, + { label: `Schedule-job-${cronjobId}` }, + ]; + } + return [project, { label: "Spiders", path: `/projects/${projectId}/spiders` }, { label: spiderName }]; + } + + return [project]; + }; + + const crumbs = getCrumbs(); + + const breadcrumb = ( + + {crumbs.map((crumb, index) => ( + + {index > 0 && /} + {crumb.path ? ( + + {crumb.label} + + ) : ( + {crumb.label} + )} + + ))} + + ); + return ( -
+
{children}