From ebc60aace985e4647bf46e1f647c0bf232c1ac26 Mon Sep 17 00:00:00 2001 From: Dave Martin Date: Tue, 23 Jun 2026 14:02:27 +0200 Subject: [PATCH] fix(occurrence): fix for #526 --- occurrence-download/pom.xml | 7 ++ .../file/dwca/akka/DownloadDwcaActor.java | 51 ++++++++---- .../file/dwca/akka/DownloadDwcaActorTest.java | 79 +++++++++++++++++++ 3 files changed, 123 insertions(+), 14 deletions(-) create mode 100644 occurrence-download/src/test/java/org/gbif/occurrence/download/file/dwca/akka/DownloadDwcaActorTest.java diff --git a/occurrence-download/pom.xml b/occurrence-download/pom.xml index ee9e61aa7..5fc0d6ed8 100644 --- a/occurrence-download/pom.xml +++ b/occurrence-download/pom.xml @@ -486,6 +486,13 @@ + + com.typesafe.akka + akka-testkit_2.12 + ${akka-actor.version} + test + + net.sf.supercsv super-csv diff --git a/occurrence-download/src/main/java/org/gbif/occurrence/download/file/dwca/akka/DownloadDwcaActor.java b/occurrence-download/src/main/java/org/gbif/occurrence/download/file/dwca/akka/DownloadDwcaActor.java index bbc130102..7a17be912 100644 --- a/occurrence-download/src/main/java/org/gbif/occurrence/download/file/dwca/akka/DownloadDwcaActor.java +++ b/occurrence-download/src/main/java/org/gbif/occurrence/download/file/dwca/akka/DownloadDwcaActor.java @@ -17,7 +17,7 @@ import static org.gbif.occurrence.common.download.DownloadUtils.DELIMETERS_MATCH_PATTERN; import akka.actor.AbstractActor; -import com.google.common.base.Throwables; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import java.io.IOException; import java.lang.reflect.InvocationTargetException; @@ -40,8 +40,10 @@ import org.gbif.api.model.event.Event; import org.gbif.api.model.occurrence.Occurrence; import org.gbif.api.model.occurrence.VerbatimOccurrence; +import org.gbif.api.util.TermNormalizationUtils; import org.gbif.api.vocabulary.Extension; import org.gbif.api.vocabulary.MediaType; +import org.gbif.dwc.terms.GbifTerm; import org.gbif.dwc.terms.Term; import org.gbif.terms.utils.TermUtils; import org.gbif.occurrence.common.download.DownloadUtils; @@ -181,18 +183,36 @@ private void writeExtensions(DownloadFileWork work, T record) throws IOException } } - private Map toExtensionRecord(Map row, T record) { + @VisibleForTesting + protected Map toExtensionRecord(Map row, T record) { Map extensionData = new LinkedHashMap<>(); - extensionData.put("gbifid", getRecordKey(record)); - extensionData.put("datasetkey", record.getDatasetKey().toString()); - extensionData.putAll( - row.entrySet().stream() - .collect( - Collectors.toMap( - // the normalization is needed because the avro schemas of the extensions tables - // do it but ES fields aren't normalized so they don't match - e -> normalizeFieldName(HiveColumns.columnFor(e.getKey())), - Map.Entry::getValue))); + extensionData.put(GbifTerm.gbifID.simpleName().toLowerCase(), getRecordKey(record)); + extensionData.put(GbifTerm.datasetKey.simpleName().toLowerCase(), record.getDatasetKey().toString()); + + + Map duplicatesMap = new HashMap<>(); + row.forEach((key, value) -> { + if (duplicatesMap.containsKey(key.simpleName().toLowerCase())) { + duplicatesMap.computeIfPresent(key.simpleName().toLowerCase(), (s, i) -> ++i); + } else { + duplicatesMap.put(key.simpleName().toLowerCase(), 1); + } + }); + + // fix for https://github.com/gbif/occurrence/issues/526 + // handle duplicate keys (which are duplicates when the base url is stripped for the CSV) + // Build the normalized map by iterating so we can log duplicates when they occur. + Map normalized = new LinkedHashMap<>(); + for (Map.Entry e : row.entrySet()) { + Term term = e.getKey(); + String fName = term.simpleName(); + if (duplicatesMap.get(fName) != null && duplicatesMap.get(fName) > 1) { + fName = term.prefixedName(); + } + String normalizeFieldName = TermNormalizationUtils.normalizeFieldName(fName); + normalized.put(normalizeFieldName, e.getValue()); + } + extensionData.putAll(normalized); return extensionData; } @@ -253,9 +273,12 @@ public void doWork(DownloadFileWork work) throws IOException { writeMediaObjects(multimediaCsvWriter, record); writeExtensions(work, record); } + } catch (RuntimeException e) { + getSender().tell(e, getSelf()); // inform our master + throw e; } catch (Exception e) { getSender().tell(e, getSelf()); // inform our master - throw Throwables.propagate(e); + throw (e instanceof RuntimeException) ? (RuntimeException) e : new RuntimeException(e); } }); getSender().tell(new Result(work, datasetUsagesCollector.getDatasetUsages()), getSelf()); @@ -292,7 +315,7 @@ public InnerMediaObject(MediaObject mediaObject, String gbifID) { BeanUtils.copyProperties(this, mediaObject); this.gbifID = gbifID; } catch (IllegalAccessException | InvocationTargetException e) { - throw Throwables.propagate(e); + throw new RuntimeException(e); } } } diff --git a/occurrence-download/src/test/java/org/gbif/occurrence/download/file/dwca/akka/DownloadDwcaActorTest.java b/occurrence-download/src/test/java/org/gbif/occurrence/download/file/dwca/akka/DownloadDwcaActorTest.java new file mode 100644 index 000000000..d4a72181c --- /dev/null +++ b/occurrence-download/src/test/java/org/gbif/occurrence/download/file/dwca/akka/DownloadDwcaActorTest.java @@ -0,0 +1,79 @@ +package org.gbif.occurrence.download.file.dwca.akka; + +import akka.actor.ActorSystem; +import akka.actor.Props; +import akka.testkit.TestActorRef; +import org.gbif.api.model.occurrence.VerbatimOccurrence; +import org.gbif.dwc.terms.*; +import org.junit.jupiter.api.Test; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class DownloadDwcaActorTest { + + @Test + public void testNormalizeExtensionField() { + + ActorSystem system = ActorSystem.create("test"); + TestActorRef actor = TestActorRef.apply( + Props.create(DownloadDwcaActor.class, null, null, null), + system + ); + + UUID datasetKey = UUID.randomUUID(); + VerbatimOccurrence record = new VerbatimOccurrence(); + record.setKey(1L); + record.setDatasetKey(datasetKey); + + Map terms = new LinkedHashMap<>(); + terms.put(AcTerm.subjectCategoryVocabulary, "Specimen/Object"); + terms.put(XmpRightsTerm.UsageTerms, "https://creativecommons.org/publicdomain/zero/1.0/"); + terms.put(DwcTerm.scientificName, "Haloxylon ammodendron"); + terms.put(DcTerm.identifier, "http://n2t.net/ark:/65665/m36c5a41f4-7c60-43f2-ba1d-0860880efacc"); + terms.put(ExifTerm.PixelXDimension, "6729"); + terms.put(XmpRightsTerm.WebStatement, "https://naturalhistory.si.edu/research/nmnh-collections/museum-collections-policies"); + terms.put(DcElement.source, "US National Herbarium, Department of Botany, NMNH, Smithsonian Institution"); + terms.put(DcElement.creator, "Conveyor Belt"); + terms.put(AcTerm.providerLiteral, "Smithsonian Institution, NMNH, Botany"); + terms.put(DcElement.format, "image/jpeg"); + terms.put(AcTerm.licenseLogoURL, "https://www.si.edu/sites/default/files/icons/cc0.svg"); + terms.put(DcElement.rights, "CC0"); + terms.put(DcTerm.description, "Barcode 03538523"); + terms.put(DcTerm.rights, "CC0"); + terms.put(AcTerm.accessURI, "https://collections.nmnh.si.edu/media/?i=14034501&h=2000"); + terms.put(DcElement.type, "image"); + terms.put(ExifTerm.PixelYDimension, "8985"); + terms.put(DcTerm.title, "03538523.tif"); + + Map result = ((DownloadDwcaActor)actor.underlyingActor()).toExtensionRecord(terms, record); + + Map expected = new LinkedHashMap<>(); + expected.put(GbifTerm.gbifID.simpleName().toLowerCase(), "1"); + expected.put(GbifTerm.datasetKey.simpleName().toLowerCase(), datasetKey.toString()); + expected.put(AcTerm.subjectCategoryVocabulary.simpleName().toLowerCase(), "Specimen/Object"); + expected.put(XmpRightsTerm.UsageTerms.simpleName().toLowerCase(), "https://creativecommons.org/publicdomain/zero/1.0/"); + expected.put(DwcTerm.scientificName.simpleName().toLowerCase(), "Haloxylon ammodendron"); + expected.put(DcTerm.identifier.simpleName().toLowerCase(), "http://n2t.net/ark:/65665/m36c5a41f4-7c60-43f2-ba1d-0860880efacc"); + expected.put(ExifTerm.PixelXDimension.simpleName().toLowerCase(), "6729"); + expected.put(XmpRightsTerm.WebStatement.simpleName().toLowerCase(), "https://naturalhistory.si.edu/research/nmnh-collections/museum-collections-policies"); + expected.put(DcElement.source.simpleName().toLowerCase(), "US National Herbarium, Department of Botany, NMNH, Smithsonian Institution"); + expected.put(DcElement.creator.simpleName().toLowerCase(), "Conveyor Belt"); + expected.put(AcTerm.providerLiteral.simpleName().toLowerCase(), "Smithsonian Institution, NMNH, Botany"); + expected.put("format", "image/jpeg"); + expected.put(AcTerm.licenseLogoURL.simpleName().toLowerCase(), "https://www.si.edu/sites/default/files/icons/cc0.svg"); + expected.put("dc_rights", "CC0"); // this one wins + expected.put(DcTerm.description.simpleName().toLowerCase(), "Barcode 03538523"); + expected.put("dcterms_rights", "CC0"); // this one win + expected.put(AcTerm.accessURI.simpleName().toLowerCase(), "https://collections.nmnh.si.edu/media/?i=14034501&h=2000"); + expected.put(DcElement.type.simpleName().toLowerCase(), "image"); + expected.put(ExifTerm.PixelYDimension.simpleName().toLowerCase(), "8985"); + expected.put(DcTerm.title.simpleName().toLowerCase(), "03538523.tif"); + + assertEquals(expected, result); + system.terminate(); + } +}