Add CompetencyCategory field to competencies

common/catalogueTypes.ts

@@ -353,7 +353,8 @@ export const catalogueTypes: Record<CatalogueType, CatalogueTypeDefinition> = {
     presencePrompt:
       "Look at the given markdown page and check if there exists a list of skills in a dedicated section. " +
       "We are looking for a list in a dedicated section, do not consider paragraphs or long descriptions. " +
-      "Do not confuse courses with skills. We are looking for skills attained after taking the course described in the page.",
+      "Do not confuse courses with skills. We are looking for skills attained after taking the course described in the page." +
+      "Example of headers listing the items are 'Learning outcomes', 'Course objectives', 'Competencies', etc.",
     desiredOutput:
       "We are looking for a list of skills that are gained after taking the course described in the page. " +
       "If found, take each item exactly as it is in the page and return them. Skip everything else, just the skill list." +
@@ -371,12 +372,20 @@ export const catalogueTypes: Record<CatalogueType, CatalogueTypeDefinition> = {
           "The name of the encompassing or overarching skill or competency or learning program or course that will lead " +
           "to obtaining the skill or competency or learning outcome. " +
           "Usually this value is the same for the entire list but should be set " +
-          "according to the hierarchy structure of the page. This is usually shorter. " +
+          "according to the hierarchy structure of the page. This is usually shorter. No course codes should be included. (ex. ACCT 101)" +
           "Sometimes, this might contain descriptive language about the skill - we should only keep the " +
           'name of the skill and trim phrases such as "competency" or "learning outcome".' +
           "This field should be in title case. If it contains roman numerals, they use use uppercase.",
         required: false,
       },
+      competency_category: {
+        description:
+          "Some pages contain competency under outcomes and course objectives, we need to distinguish between the two. " +
+          "If the competency is under outcomes, set this field to 'outcomes'. " +
+          "Otherwise, set this field to 'course_objectives'. " +
+          "If it's not clear, set this field to 'unknown'.",
+        required: false,
+      },
       language: {
         description:
           "ISO code of the language in which the name property is expressed. Examples - 'en' for English, 'es' for Spanish, 'de' for German, etc.",
@@ -393,10 +402,14 @@ export const catalogueTypes: Record<CatalogueType, CatalogueTypeDefinition> = {
             properties: {
               text: { type: "string" },
               competency_framework: { type: "string" },
+              competency_category: {
+                type: "string",
+                enum: ["outcomes", "course_objectives", "unknown"],
+              },
               language: { type: "string" },
             },
             additionalProperties: false,
-            required: ["competency_framework", "text", "language"],
+            required: ["competency_framework", "text", "language", "competency_category"],
           },
         },
       },

common/types.ts

@@ -162,6 +162,7 @@ export interface CompetencyStructuredData {
   text: string;
   competency_framework: string;
   language?: string;
+  competency_category?: "outcomes" | "course_objectives" | "unknown";
 }
 
 export type TextInclusion<T> = {

server/src/csv.ts

@@ -105,6 +105,17 @@ function getLearningProgramRow(
 
 type Framework = { name: string; id: string };
 
+const COMPETENCY_CATEGORY_MAP: Record<string, string> = {
+  outcomes: "Course Level Student Learning Outcome(s)",
+  course_objectives: "Course Objectives",
+};
+
+function formatCompetencyCategory(
+  value: string | null | undefined
+): string {
+  return COMPETENCY_CATEGORY_MAP[value ?? ""] || "Unknown";
+}
+
 function getCompetencyRow(
   item: Awaited<ReturnType<typeof findDataItems>>["items"][number],
   textVerificationAverage: number,
@@ -143,6 +154,7 @@ function getCompetencyRow(
       text: entityData.text,
       language: entityData.language,
       competency_framework: framework.id,
+      competency_category: entityData.competency_category,
     },
     { textVerificationAverage, textVerificationDetails },
     framework
@@ -177,6 +189,9 @@ function makeCompetencyRow(
     "ceasn:inLanguage": language,
     "ceasn:publisher": "",
     "ceasn:isPartOf": isPartOf,
+    ...(parent && {
+      "Competency Category": formatCompetencyCategory(entity.competency_category),
+    }),
   };
 
   return result;

server/src/openai.ts

@@ -68,11 +68,16 @@ export function estimateCost(
 }
 
 export async function findOpenAiApiKey() {
-  const apiKey = await findSetting<string>("OPENAI_API_KEY", true);
-  if (!apiKey) {
+  const envKey =
+    process.env.OPENAI_API_KEY?.trim()
+  if (envKey) {
+    return envKey;
+  }
+  const dbSetting = await findSetting<string>("OPENAI_API_KEY", true);
+  if (!dbSetting?.value) {
     throw new Error("OpenAI API Key not found");
   }
-  return apiKey.value;
+  return dbSetting.value;
 }
 
 export async function getOpenAi() {

server/src/server.ts

@@ -90,6 +90,7 @@ server.register(async (instance) => {
             path: opts.path,
             type: opts.error.name,
             message: opts.error.message,
+            stack: opts.error.stack,
           },
           "Error in tRPC request"
         );

server/src/workers/index.ts

@@ -11,6 +11,7 @@ import {
 import { default as IORedis } from "ioredis";
 import { closeCluster } from "../extraction/browser";
 import getLogger from "../logging";
+import { inspect } from "node:util";
 
 const logger = getLogger("workers");
 const REDIS_URL = process.env.REDIS_URL || "redis://localhost:6379";
@@ -276,8 +277,11 @@ async function collectJobsForExtraction<T extends { extractionId: number; crawlP
       break;
     }
     for (const job of batch) {
-      if (job.data.extractionId === extractionId) {
+      if (job?.data?.extractionId === extractionId) {
         jobs.push({ data: job.data });
+      } else {
+        logger.info(`Skipping job ${job.id} for extraction ${extractionId} because it's not for the same extraction`);
+        logger.info(`Job data: \n${inspect(job.data)}`);
       }
     }
     start = end;

server/tests/extractions/competencies/coastline.test.ts

@@ -0,0 +1,130 @@
+import { describe, test, expect } from "vitest";
+import { extractCompetencies, EXTRACTION_TIMEOUT } from "../..";
+
+describe("Coastline College", { timeout: EXTRACTION_TIMEOUT }, () => {
+  describe("Coastline College", () => {
+    test("Accounting A101", async () => {
+      const extractions = await extractCompetencies(
+        "https://catalog.cccd.edu/courses/acct-a101/"
+      );
+
+      expect(extractions).toEqual([
+        {
+          "competency_category": "outcomes",
+          "competency_framework": "Financial Accounting",
+          "language": "en",
+          "text": "Demonstrate knowledge of an accounting cycle by performing appropriate accounting functions.",
+        },
+        {
+          "competency_category": "outcomes",
+          "competency_framework": "Financial Accounting",
+          "language": "en",
+          "text": "Demonstrate ability to prepare financial statements for a corporation.",
+        },
+        {
+          "competency_category": "outcomes",
+          "competency_framework": "Financial Accounting",
+          "language": "en",
+          "text": "Prepare accounting entries required for a service versus merchandising business.",
+        },
+        {
+          "competency_category": "course_objectives",
+          "competency_framework": "Financial Accounting",
+          "language": "en",
+          "text": "Explain the nature and purpose of generally accepted accounting principles (GAAP) and International Financial Reporting Standards (IFRS). Explain and apply the components of the conceptual framework for financial accounting and reporting, including the qualitative characteristics of accounting information, the assumptions underlying accounting, the basic principles of financial accounting, and the constraints and limitations on accounting information.",
+        },
+        {
+          "competency_category": "course_objectives",
+          "competency_framework": "Financial Accounting",
+          "language": "en",
+          "text": "Define and use accounting and business terminology.",
+        },
+        {
+          "competency_category": "course_objectives",
+          "competency_framework": "Financial Accounting",
+          "language": "en",
+          "text": "Explain what a system is and how an accounting system is designed to satisfy the needs of specific businesses and users; summarize the purpose of journals and ledgers.",
+        },
+        {
+          "competency_category": "course_objectives",
+          "competency_framework": "Financial Accounting",
+          "language": "en",
+          "text": "Apply transaction analysis, input transactions into the accounting system, process this input, and prepare and interpret the four basic financial statements.",
+        },
+        {
+          "competency_category": "course_objectives",
+          "competency_framework": "Financial Accounting",
+          "language": "en",
+          "text": "Distinguish between cash basis and accrual basis accounting and their impact on the financial statements, including the revenue recognition and matching principles.",
+        },
+        {
+          "competency_category": "course_objectives",
+          "competency_framework": "Financial Accounting",
+          "language": "en",
+          "text": "Identify and illustrate how the principles of internal control are used to manage and control the firm?s resources and minimize risk.",
+        },
+        {
+          "competency_category": "course_objectives",
+          "competency_framework": "Financial Accounting",
+          "language": "en",
+          "text": "Explain the content, form, and purpose of the basic financial statements (including footnotes) and the annual report, and how they satisfy the information needs of investors, creditors, and other users.",
+        },
+        {
+          "competency_category": "course_objectives",
+          "competency_framework": "Financial Accounting",
+          "language": "en",
+          "text": "Explain the nature of current assets and related issues, including the measurement and reporting of cash and cash equivalents, receivables and bad debts, and inventory and cost of goods sold.",
+        },
+        {
+          "competency_category": "course_objectives",
+          "competency_framework": "Financial Accounting",
+          "language": "en",
+          "text": "Explain the valuation and reporting of current liabilities, estimated liabilities, and other contingencies.",
+        },
+        {
+          "competency_category": "course_objectives",
+          "competency_framework": "Financial Accounting",
+          "language": "en",
+          "text": "Identify and illustrate issues relating to long-term asset acquisition, use, cost allocation, and disposal.",
+        },
+        {
+          "competency_category": "course_objectives",
+          "competency_framework": "Financial Accounting",
+          "language": "en",
+          "text": "Distinguish between capital and revenue expenditures.",
+        },
+        {
+          "competency_category": "course_objectives",
+          "competency_framework": "Financial Accounting",
+          "language": "en",
+          "text": "Identify and illustrate issues relating to long-term liabilities, including issuance, valuation, and retirement of debt;(including the time value of money).",
+        },
+        {
+          "competency_category": "course_objectives",
+          "competency_framework": "Financial Accounting",
+          "language": "en",
+          "text": "Identify and illustrate issues relating to stockholders? equity, including issuance, repurchase of capital stock, and dividends.",
+        },
+        {
+          "competency_category": "course_objectives",
+          "competency_framework": "Financial Accounting",
+          "language": "en",
+          "text": "Explain the importance of operating, investing and financing activities reported in the Statement of Cash Flows.",
+        },
+        {
+          "competency_category": "course_objectives",
+          "competency_framework": "Financial Accounting",
+          "language": "en",
+          "text": "Interpret company activity, profitability, liquidity and solvency through selection and application of appropriate financial analysis tools.",
+        },
+        {
+          "competency_category": "course_objectives",
+          "competency_framework": "Financial Accounting",
+          "language": "en",
+          "text": "Identify the ethical implications inherent in financial reporting and be able to apply strategies for addressing them.",
+        },
+
+      ]);
+    });
+  });
+});