Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions estela-api/api/serializers/cronjob.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import logging

from croniter import croniter
from rest_framework import serializers

Expand All @@ -16,6 +18,8 @@
SpiderJobTag,
)

logger = logging.getLogger(__name__)


class SpiderCronJobSerializer(serializers.ModelSerializer):
cargs = SpiderJobArgSerializer(
Expand Down Expand Up @@ -107,6 +111,22 @@ def create(self, validated_data):
SpiderJobArg.objects.create(cronjob=cronjob, **arg)

for env_var in env_vars_data:
evid = env_var.pop("evid", None)
if evid:
source = SpiderJobEnvVar.objects.filter(evid=evid).first()
if source:
SpiderJobEnvVar.objects.create(
cronjob=cronjob,
name=source.name,
value=source.value,
masked=source.masked,
)
continue
logger.warning(
"evid=%s not found when creating env var '%s' for cronjob %s — skipping.",
evid, env_var.get("name"), cronjob.cjid,
)
continue
SpiderJobEnvVar.objects.create(cronjob=cronjob, **env_var)

for tag_data in tags_data:
Expand Down
6 changes: 6 additions & 0 deletions estela-api/api/serializers/job_specific.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,12 @@ class Meta:


class SpiderJobEnvVarSerializer(serializers.ModelSerializer):
evid = serializers.IntegerField(
required=False,
allow_null=True,
help_text="A unique integer value identifying this job env variable.",
)

class Meta:
model = SpiderJobEnvVar
fields = ("evid", "name", "value", "masked")
Expand Down
10 changes: 7 additions & 3 deletions estela-api/core/cronjob.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,14 @@

from django_celery_beat.models import CrontabSchedule, PeriodicTask

from core.models import SpiderCronJob
from core.tasks import launch_job


def create_cronjob(name, key, args, env_vars, tags, schedule, data_expiry_days=None, resource_tier=None):
minute, hour, day_of_month, month, day_of_week = schedule.split(" ")
cjid, sid, pid = key.split(".")
data = {"cronjob": cjid, "args": args, "env_vars": env_vars, "tags": tags}
data = {"cronjob": cjid, "args": args, "env_vars": [], "tags": tags}
if resource_tier:
data["resource_tier"] = resource_tier
schedule, _ = CrontabSchedule.objects.get_or_create(
Expand All @@ -28,8 +29,11 @@ def create_cronjob(name, key, args, env_vars, tags, schedule, data_expiry_days=N


def run_cronjob_once(data):
env_vars = data.get("cenv_vars") or []
env_vars = [ev for ev in env_vars if ev.get("value") is not None]
cronjob = SpiderCronJob.objects.get(cjid=data.get("cjid"))
env_vars = [
{"name": ev.name, "value": ev.value, "masked": ev.masked}
for ev in cronjob.cenv_vars.all()
]
_data = {
"cronjob": data.get("cjid"),
"args": data.get("cargs"),
Expand Down
13 changes: 13 additions & 0 deletions estela-api/core/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
Project,
ProxyProvider,
Spider,
SpiderCronJob,
SpiderJob,
UsageRecord,
)
Expand Down Expand Up @@ -274,6 +275,18 @@ def launch_job(sid_, data_, data_expiry_days=None, token=None):

resource_tier = data_.pop("resource_tier", None)

cjid = data_.get("cronjob")
if cjid:
cronjob = SpiderCronJob.objects.filter(cjid=cjid).first()
if cronjob:
cronjob_env_vars = {
ev.name: {"name": ev.name, "value": ev.value, "masked": ev.masked}
for ev in cronjob.cenv_vars.all()
}
extra_env_vars = {ev["name"]: ev for ev in data_.get("env_vars", [])}
cronjob_env_vars.update(extra_env_vars)
data_["env_vars"] = list(cronjob_env_vars.values())

serializer = SpiderJobCreateSerializer(data=data_)
serializer.is_valid(raise_exception=True)

Expand Down
2 changes: 1 addition & 1 deletion estela-api/docs/api.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1886,7 +1886,7 @@ definitions:
title: Evid
description: A unique integer value identifying this job env variable.
type: integer
readOnly: true
x-nullable: true
name:
title: Name
description: Env variable name.
Expand Down
7 changes: 7 additions & 0 deletions estela-web/src/pages/CronjobCreateModal/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ interface ArgsData {
}

interface EnvVarsData {
evid?: number;
name: string;
value: string;
key: number;
Expand Down Expand Up @@ -678,14 +679,20 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro

cronjobData.envVars.map((envVar: EnvVarsData) => {
const index = envVarsData.findIndex((element: SpiderJobEnvVar) => element.name === envVar.name);
const resolvedEvid =
envVar.masked && envVar.value === "__MASKED__"
? envVar.evid ?? (index !== -1 ? envVarsData[index].evid : undefined)
: undefined;
if (index != -1) {
envVarsData[index] = {
evid: resolvedEvid,
name: envVar.name,
value: envVar.value,
masked: envVar.masked,
};
} else {
envVarsData.push({
evid: resolvedEvid,
name: envVar.name,
value: envVar.value,
masked: envVar.masked,
Expand Down
8 changes: 8 additions & 0 deletions estela-web/src/pages/JobCreateModal/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ interface ArgsData {
}

interface EnvVarsData {
evid?: number;
name: string;
value: string;
key: number;
Expand Down Expand Up @@ -138,6 +139,7 @@ export default function JobCreateModal({
const [jobData, setJobData] = useState<JobData>({
args: initialArgs.map((arg, index) => ({ ...arg, key: index })),
envVars: initialEnvVars.map((envVar, index) => ({
evid: envVar.evid,
name: envVar.name,
value: envVar.masked ? "__MASKED__" : envVar.value,
masked: envVar.masked || false,
Expand Down Expand Up @@ -448,14 +450,20 @@ export default function JobCreateModal({

jobData.envVars.map((envVar: EnvVarsData) => {
const index = envVarsData.findIndex((element: SpiderJobEnvVar) => element.name === envVar.name);
const resolvedEvid =
envVar.masked && envVar.value === "__MASKED__"
? envVar.evid ?? (index !== -1 ? envVarsData[index].evid : undefined)
: undefined;
if (index != -1) {
envVarsData[index] = {
evid: resolvedEvid,
name: envVar.name,
value: envVar.value,
masked: envVar.masked,
};
} else {
envVarsData.push({
evid: resolvedEvid,
name: envVar.name,
value: envVar.value,
masked: envVar.masked,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ export interface SpiderJobEnvVar {
* @type {number}
* @memberof SpiderJobEnvVar
*/
readonly evid?: number;
evid?: number | null;
/**
* Env variable name.
* @type {string}
Expand Down Expand Up @@ -71,6 +71,7 @@ export function SpiderJobEnvVarToJSON(value?: SpiderJobEnvVar | null): any {
}
return {

'evid': value.evid,
'name': value.name,
'value': value.value,
'masked': value.masked,
Expand Down
Loading