diff --git a/occurrence-download/pom.xml b/occurrence-download/pom.xml
index ee9e61aa7..5fc0d6ed8 100644
--- a/occurrence-download/pom.xml
+++ b/occurrence-download/pom.xml
@@ -486,6 +486,13 @@
+
+ com.typesafe.akka
+ akka-testkit_2.12
+ ${akka-actor.version}
+ test
+
+
net.sf.supercsv
super-csv
diff --git a/occurrence-download/src/main/java/org/gbif/occurrence/download/file/dwca/akka/DownloadDwcaActor.java b/occurrence-download/src/main/java/org/gbif/occurrence/download/file/dwca/akka/DownloadDwcaActor.java
index bbc130102..7a17be912 100644
--- a/occurrence-download/src/main/java/org/gbif/occurrence/download/file/dwca/akka/DownloadDwcaActor.java
+++ b/occurrence-download/src/main/java/org/gbif/occurrence/download/file/dwca/akka/DownloadDwcaActor.java
@@ -17,7 +17,7 @@
import static org.gbif.occurrence.common.download.DownloadUtils.DELIMETERS_MATCH_PATTERN;
import akka.actor.AbstractActor;
-import com.google.common.base.Throwables;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
@@ -40,8 +40,10 @@
import org.gbif.api.model.event.Event;
import org.gbif.api.model.occurrence.Occurrence;
import org.gbif.api.model.occurrence.VerbatimOccurrence;
+import org.gbif.api.util.TermNormalizationUtils;
import org.gbif.api.vocabulary.Extension;
import org.gbif.api.vocabulary.MediaType;
+import org.gbif.dwc.terms.GbifTerm;
import org.gbif.dwc.terms.Term;
import org.gbif.terms.utils.TermUtils;
import org.gbif.occurrence.common.download.DownloadUtils;
@@ -181,18 +183,36 @@ private void writeExtensions(DownloadFileWork work, T record) throws IOException
}
}
- private Map toExtensionRecord(Map row, T record) {
+ @VisibleForTesting
+ protected Map toExtensionRecord(Map row, T record) {
Map extensionData = new LinkedHashMap<>();
- extensionData.put("gbifid", getRecordKey(record));
- extensionData.put("datasetkey", record.getDatasetKey().toString());
- extensionData.putAll(
- row.entrySet().stream()
- .collect(
- Collectors.toMap(
- // the normalization is needed because the avro schemas of the extensions tables
- // do it but ES fields aren't normalized so they don't match
- e -> normalizeFieldName(HiveColumns.columnFor(e.getKey())),
- Map.Entry::getValue)));
+ extensionData.put(GbifTerm.gbifID.simpleName().toLowerCase(), getRecordKey(record));
+ extensionData.put(GbifTerm.datasetKey.simpleName().toLowerCase(), record.getDatasetKey().toString());
+
+
+ Map duplicatesMap = new HashMap<>();
+ row.forEach((key, value) -> {
+ if (duplicatesMap.containsKey(key.simpleName().toLowerCase())) {
+ duplicatesMap.computeIfPresent(key.simpleName().toLowerCase(), (s, i) -> ++i);
+ } else {
+ duplicatesMap.put(key.simpleName().toLowerCase(), 1);
+ }
+ });
+
+ // fix for https://github.com/gbif/occurrence/issues/526
+ // handle duplicate keys (which are duplicates when the base url is stripped for the CSV)
+ // Build the normalized map by iterating so we can log duplicates when they occur.
+ Map normalized = new LinkedHashMap<>();
+ for (Map.Entry e : row.entrySet()) {
+ Term term = e.getKey();
+ String fName = term.simpleName();
+ if (duplicatesMap.get(fName) != null && duplicatesMap.get(fName) > 1) {
+ fName = term.prefixedName();
+ }
+ String normalizeFieldName = TermNormalizationUtils.normalizeFieldName(fName);
+ normalized.put(normalizeFieldName, e.getValue());
+ }
+ extensionData.putAll(normalized);
return extensionData;
}
@@ -253,9 +273,12 @@ public void doWork(DownloadFileWork work) throws IOException {
writeMediaObjects(multimediaCsvWriter, record);
writeExtensions(work, record);
}
+ } catch (RuntimeException e) {
+ getSender().tell(e, getSelf()); // inform our master
+ throw e;
} catch (Exception e) {
getSender().tell(e, getSelf()); // inform our master
- throw Throwables.propagate(e);
+ throw (e instanceof RuntimeException) ? (RuntimeException) e : new RuntimeException(e);
}
});
getSender().tell(new Result(work, datasetUsagesCollector.getDatasetUsages()), getSelf());
@@ -292,7 +315,7 @@ public InnerMediaObject(MediaObject mediaObject, String gbifID) {
BeanUtils.copyProperties(this, mediaObject);
this.gbifID = gbifID;
} catch (IllegalAccessException | InvocationTargetException e) {
- throw Throwables.propagate(e);
+ throw new RuntimeException(e);
}
}
}
diff --git a/occurrence-download/src/test/java/org/gbif/occurrence/download/file/dwca/akka/DownloadDwcaActorTest.java b/occurrence-download/src/test/java/org/gbif/occurrence/download/file/dwca/akka/DownloadDwcaActorTest.java
new file mode 100644
index 000000000..d4a72181c
--- /dev/null
+++ b/occurrence-download/src/test/java/org/gbif/occurrence/download/file/dwca/akka/DownloadDwcaActorTest.java
@@ -0,0 +1,79 @@
+package org.gbif.occurrence.download.file.dwca.akka;
+
+import akka.actor.ActorSystem;
+import akka.actor.Props;
+import akka.testkit.TestActorRef;
+import org.gbif.api.model.occurrence.VerbatimOccurrence;
+import org.gbif.dwc.terms.*;
+import org.junit.jupiter.api.Test;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class DownloadDwcaActorTest {
+
+ @Test
+ public void testNormalizeExtensionField() {
+
+ ActorSystem system = ActorSystem.create("test");
+ TestActorRef actor = TestActorRef.apply(
+ Props.create(DownloadDwcaActor.class, null, null, null),
+ system
+ );
+
+ UUID datasetKey = UUID.randomUUID();
+ VerbatimOccurrence record = new VerbatimOccurrence();
+ record.setKey(1L);
+ record.setDatasetKey(datasetKey);
+
+ Map terms = new LinkedHashMap<>();
+ terms.put(AcTerm.subjectCategoryVocabulary, "Specimen/Object");
+ terms.put(XmpRightsTerm.UsageTerms, "https://creativecommons.org/publicdomain/zero/1.0/");
+ terms.put(DwcTerm.scientificName, "Haloxylon ammodendron");
+ terms.put(DcTerm.identifier, "http://n2t.net/ark:/65665/m36c5a41f4-7c60-43f2-ba1d-0860880efacc");
+ terms.put(ExifTerm.PixelXDimension, "6729");
+ terms.put(XmpRightsTerm.WebStatement, "https://naturalhistory.si.edu/research/nmnh-collections/museum-collections-policies");
+ terms.put(DcElement.source, "US National Herbarium, Department of Botany, NMNH, Smithsonian Institution");
+ terms.put(DcElement.creator, "Conveyor Belt");
+ terms.put(AcTerm.providerLiteral, "Smithsonian Institution, NMNH, Botany");
+ terms.put(DcElement.format, "image/jpeg");
+ terms.put(AcTerm.licenseLogoURL, "https://www.si.edu/sites/default/files/icons/cc0.svg");
+ terms.put(DcElement.rights, "CC0");
+ terms.put(DcTerm.description, "Barcode 03538523");
+ terms.put(DcTerm.rights, "CC0");
+ terms.put(AcTerm.accessURI, "https://collections.nmnh.si.edu/media/?i=14034501&h=2000");
+ terms.put(DcElement.type, "image");
+ terms.put(ExifTerm.PixelYDimension, "8985");
+ terms.put(DcTerm.title, "03538523.tif");
+
+ Map result = ((DownloadDwcaActor)actor.underlyingActor()).toExtensionRecord(terms, record);
+
+ Map expected = new LinkedHashMap<>();
+ expected.put(GbifTerm.gbifID.simpleName().toLowerCase(), "1");
+ expected.put(GbifTerm.datasetKey.simpleName().toLowerCase(), datasetKey.toString());
+ expected.put(AcTerm.subjectCategoryVocabulary.simpleName().toLowerCase(), "Specimen/Object");
+ expected.put(XmpRightsTerm.UsageTerms.simpleName().toLowerCase(), "https://creativecommons.org/publicdomain/zero/1.0/");
+ expected.put(DwcTerm.scientificName.simpleName().toLowerCase(), "Haloxylon ammodendron");
+ expected.put(DcTerm.identifier.simpleName().toLowerCase(), "http://n2t.net/ark:/65665/m36c5a41f4-7c60-43f2-ba1d-0860880efacc");
+ expected.put(ExifTerm.PixelXDimension.simpleName().toLowerCase(), "6729");
+ expected.put(XmpRightsTerm.WebStatement.simpleName().toLowerCase(), "https://naturalhistory.si.edu/research/nmnh-collections/museum-collections-policies");
+ expected.put(DcElement.source.simpleName().toLowerCase(), "US National Herbarium, Department of Botany, NMNH, Smithsonian Institution");
+ expected.put(DcElement.creator.simpleName().toLowerCase(), "Conveyor Belt");
+ expected.put(AcTerm.providerLiteral.simpleName().toLowerCase(), "Smithsonian Institution, NMNH, Botany");
+ expected.put("format", "image/jpeg");
+ expected.put(AcTerm.licenseLogoURL.simpleName().toLowerCase(), "https://www.si.edu/sites/default/files/icons/cc0.svg");
+ expected.put("dc_rights", "CC0"); // this one wins
+ expected.put(DcTerm.description.simpleName().toLowerCase(), "Barcode 03538523");
+ expected.put("dcterms_rights", "CC0"); // this one win
+ expected.put(AcTerm.accessURI.simpleName().toLowerCase(), "https://collections.nmnh.si.edu/media/?i=14034501&h=2000");
+ expected.put(DcElement.type.simpleName().toLowerCase(), "image");
+ expected.put(ExifTerm.PixelYDimension.simpleName().toLowerCase(), "8985");
+ expected.put(DcTerm.title.simpleName().toLowerCase(), "03538523.tif");
+
+ assertEquals(expected, result);
+ system.terminate();
+ }
+}