Skip to content

Fix content indexing issue #869 #417

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mariusmarin-dev wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Fix content indexing issue #869 #417

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 21 additions & 17 deletions src/main/java/uk/ac/cam/cl/dtg/segue/etl/ContentIndexer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,8 @@ private void processJsonFile(final TreeWalk treeWalk, final Repository repositor
+ ERROR_OCCURRED_SUFFIX + e.getMessage(), context.indexProblemCache);
}
} catch (Exception e) {
log.error(CONTENT_LOG_PREFIX + "Unexpected error while processing file {}: {}", treeWalk.getPathString(), e.getMessage(), e);
log.error(CONTENT_LOG_PREFIX
+ "Unexpected error while processing file {}: {}", treeWalk.getPathString(), e.getMessage(), e);
Content dummyContent = new Content();
dummyContent.setCanonicalSourceFile(treeWalk.getPathString());
this.registerContentProblem(dummyContent,
Expand Down Expand Up @@ -323,7 +324,7 @@ private void validateAndCacheContent(final Content flattenedContent, final Conte

if (flattenedContent instanceof IsaacQuiz) {
List<ContentBase> children = flattenedContent.getChildren();
if (children.stream().anyMatch(c -> !(c instanceof IsaacQuizSection))) {
if (children != null && children.stream().anyMatch(c -> !(c instanceof IsaacQuizSection))) {
log.info("IsaacQuiz ({}) contains top-level non-quiz sections. Skipping.", flattenedContent.getId());
this.registerContentProblem(flattenedContent, "Index failure - Invalid "
+ "content type among quiz sections. Quizzes can only contain quiz sections "
Expand Down Expand Up @@ -528,14 +529,6 @@ private void augmentAnswerContent(final Question question, final String canonica
.forEach(c -> this.augmentChildContent(c, canonicalSourceFile, newParentId, parentPublished));
}
}

if (question.getDefaultFeedback() != null) {
Content defaultFeedback = question.getDefaultFeedback();
if (defaultFeedback.getChildren() != null) {
defaultFeedback.getChildren().stream().map(cb -> (Content) cb)
.forEach(c -> this.augmentChildContent(c, canonicalSourceFile, newParentId, parentPublished));
}
}
}

private void augmentFeedbackContent(final Question question,
Expand Down Expand Up @@ -715,13 +708,15 @@ public synchronized void buildElasticSearchIndex(final String sha,
es.bulkIndex(sha, ContentIndextype.UNIT.toString(), serializeUnits(allUnits, objectMapper));
es.bulkIndex(sha, ContentIndextype.PUBLISHED_UNIT.toString(), serializeUnits(publishedUnits, objectMapper));
endTime = System.nanoTime();
log.info(CONTENT_LOG_PREFIX + "Bulk unit indexing took: {}ms", (endTime - startTime) / NANOSECONDS_IN_A_MILLISECOND);
log.info(CONTENT_LOG_PREFIX
+ "Bulk unit indexing took: {}ms", (endTime - startTime) / NANOSECONDS_IN_A_MILLISECOND);

startTime = System.nanoTime();
es.bulkIndex(sha, ContentIndextype.CONTENT_ERROR.toString(),
serializeContentErrors(indexProblemCache, objectMapper));
endTime = System.nanoTime();
log.info(CONTENT_LOG_PREFIX + "Bulk content error indexing took: {}ms", (endTime - startTime) / NANOSECONDS_IN_A_MILLISECOND);
log.info(CONTENT_LOG_PREFIX
+ "Bulk content error indexing took: {}ms", (endTime - startTime) / NANOSECONDS_IN_A_MILLISECOND);
} catch (JsonProcessingException e) {
log.error(CONTENT_LOG_PREFIX + "Unable to serialise sha or tags");
} catch (SegueSearchException e) {
Expand Down Expand Up @@ -927,7 +922,7 @@ private void validateSymbolicQuestionFormula(final Content content, final IsaacS
final Choice choice,
final Map<Content, List<String>> indexProblemCache) {
if (choice instanceof Formula f) {
if (f.getPythonExpression().contains("\\")) {
if (f.getPythonExpression() != null && f.getPythonExpression().contains("\\")) {
registerContentProblemQuestionFormulaContainsBackslash(content, indexProblemCache, question, choice);
} else if (f.getPythonExpression() == null || f.getPythonExpression().isEmpty()) {
registerContentProblemQuestionFormulaIsEmpty(content, indexProblemCache, question, choice);
Expand Down Expand Up @@ -1028,7 +1023,7 @@ private void registerContentProblemEventMissingOrInvalidEndDate(
if (content instanceof IsaacEventPage eventPage) {
if (eventPage.getEndDate() == null) {
this.registerContentProblem(content, "Event has no end date", indexProblemCache);
} else if (eventPage.getEndDate().isBefore(eventPage.getDate())) {
} else if (eventPage.getDate() != null && eventPage.getEndDate().isBefore(eventPage.getDate())) {
this.registerContentProblem(content, "Event has end date before start date", indexProblemCache);
}
}
Expand All @@ -1045,7 +1040,7 @@ private void registerContentProblemEmailTemplateMissingPainTextContentField(

private void registerContentProblemsChoiceQuestionMissingChoicesOrAnswer(
final Content content, final Map<Content, List<String>> indexProblemCache) {
if (content instanceof ChoiceQuestion question && !(content.getType().equals("isaacQuestion"))) {
if (content instanceof ChoiceQuestion question && !"isaacQuestion".equals(content.getType())) {

if (question.getChoices() == null || question.getChoices().isEmpty()) {
registerContentProblemChoiceQuestionMissingChoices(indexProblemCache, question);
Expand Down Expand Up @@ -1159,7 +1154,7 @@ private void registerContentProblemNestedExpandables(

private void registerContentProblemValueWithChildren(
final Content content, final Map<Content, List<String>> indexProblemCache) {
if (content.getValue() != null && !content.getChildren().isEmpty()) {
if (content.getValue() != null && content.getChildren() != null && !content.getChildren().isEmpty()) {
String id = content.getId();
String firstLine = "Content";
if (id != null) {
Expand Down Expand Up @@ -1221,9 +1216,18 @@ private void recordMissingContentProblems(final Set<String> expectedIds, final M

for (String id : missingContent) {
for (Content src : incomingReferences.get(id)) {
// Diagnose: is the ID present in the cache but as an augmented child ID?
List<String> augmentedMatches = contentById.keySet().stream()
.filter(k -> k.endsWith(Constants.ID_SEPARATOR + id))
.toList();

String diagnosis = augmentedMatches.isEmpty() ? ""
: " (Note: Found augmented forms in index: " + augmentedMatches
+ " — the reference may use a bare ID but the content was indexed as a child)";

this.registerContentProblem(src, "The id '" + id + "' was referenced by "
+ src.getCanonicalSourceFile() + " but the content with that "
+ "ID cannot be found.", indexProblemCache);
+ "ID cannot be found." + diagnosis, indexProblemCache);
}
}
if (!missingContent.isEmpty()) {
Expand Down
Loading