fix(occurrence): fix for #526

occurrence-download/pom.xml

@@ -486,6 +486,13 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <dependency>
+      <groupId>com.typesafe.akka</groupId>
+      <artifactId>akka-testkit_2.12</artifactId>
+      <version>${akka-actor.version}</version>
+      <scope>test</scope>
+    </dependency>
+
     <dependency>
       <groupId>net.sf.supercsv</groupId>
       <artifactId>super-csv</artifactId>

occurrence-download/src/main/java/org/gbif/occurrence/download/file/dwca/akka/DownloadDwcaActor.java

@@ -17,7 +17,7 @@
 import static org.gbif.occurrence.common.download.DownloadUtils.DELIMETERS_MATCH_PATTERN;
 
 import akka.actor.AbstractActor;
-import com.google.common.base.Throwables;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Lists;
 import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
@@ -40,8 +40,10 @@
 import org.gbif.api.model.event.Event;
 import org.gbif.api.model.occurrence.Occurrence;
 import org.gbif.api.model.occurrence.VerbatimOccurrence;
+import org.gbif.api.util.TermNormalizationUtils;
 import org.gbif.api.vocabulary.Extension;
 import org.gbif.api.vocabulary.MediaType;
+import org.gbif.dwc.terms.GbifTerm;
 import org.gbif.dwc.terms.Term;
 import org.gbif.terms.utils.TermUtils;
 import org.gbif.occurrence.common.download.DownloadUtils;
@@ -181,18 +183,36 @@ private void writeExtensions(DownloadFileWork work, T record) throws IOException
     }
   }
 
-  private Map<String,String> toExtensionRecord(Map<Term, String> row, T record) {
+  @VisibleForTesting
+  protected Map<String,String> toExtensionRecord(Map<Term, String> row, T record) {
     Map<String,String> extensionData = new LinkedHashMap<>();
-    extensionData.put("gbifid", getRecordKey(record));
-    extensionData.put("datasetkey", record.getDatasetKey().toString());
-    extensionData.putAll(
-        row.entrySet().stream()
-            .collect(
-                Collectors.toMap(
-                    // the normalization is needed because the avro schemas of the extensions tables
-                    // do it but ES fields aren't normalized so they don't match
-                    e -> normalizeFieldName(HiveColumns.columnFor(e.getKey())),
-                    Map.Entry::getValue)));
+    extensionData.put(GbifTerm.gbifID.simpleName().toLowerCase(), getRecordKey(record));
+    extensionData.put(GbifTerm.datasetKey.simpleName().toLowerCase(), record.getDatasetKey().toString());
+
+
+    Map<String, Integer> duplicatesMap = new HashMap<>();
+    row.forEach((key, value) -> {
+      if (duplicatesMap.containsKey(key.simpleName().toLowerCase())) {
+        duplicatesMap.computeIfPresent(key.simpleName().toLowerCase(), (s, i) -> ++i);
+      } else {
+        duplicatesMap.put(key.simpleName().toLowerCase(), 1);
+      }
+    });
+
+    // fix for https://github.com/gbif/occurrence/issues/526
+    // handle duplicate keys (which are duplicates when the base url is stripped for the CSV)
+    // Build the normalized map by iterating so we can log duplicates when they occur.
+    Map<String, String> normalized = new LinkedHashMap<>();
+    for (Map.Entry<Term, String> e : row.entrySet()) {
+      Term term = e.getKey();
+      String fName = term.simpleName();
+      if (duplicatesMap.get(fName) != null && duplicatesMap.get(fName) > 1) {
+        fName = term.prefixedName();
+      }
+      String normalizeFieldName = TermNormalizationUtils.normalizeFieldName(fName);
+      normalized.put(normalizeFieldName, e.getValue());
+    }
+    extensionData.putAll(normalized);
     return extensionData;
   }
 
@@ -253,9 +273,12 @@ public void doWork(DownloadFileWork work) throws IOException {
               writeMediaObjects(multimediaCsvWriter, record);
               writeExtensions(work, record);
             }
+          } catch (RuntimeException e) {
+            getSender().tell(e, getSelf()); // inform our master
+            throw e;
           } catch (Exception e) {
             getSender().tell(e, getSelf()); // inform our master
-            throw Throwables.propagate(e);
+            throw (e instanceof RuntimeException) ? (RuntimeException) e : new RuntimeException(e);
           }
         });
       getSender().tell(new Result(work, datasetUsagesCollector.getDatasetUsages()), getSelf());
@@ -292,7 +315,7 @@ public InnerMediaObject(MediaObject mediaObject, String gbifID) {
         BeanUtils.copyProperties(this, mediaObject);
         this.gbifID = gbifID;
       } catch (IllegalAccessException | InvocationTargetException e) {
-        throw Throwables.propagate(e);
+        throw new RuntimeException(e);
       }
     }
   }

occurrence-download/src/test/java/org/gbif/occurrence/download/file/dwca/akka/DownloadDwcaActorTest.java

@@ -0,0 +1,79 @@
+package org.gbif.occurrence.download.file.dwca.akka;
+
+import akka.actor.ActorSystem;
+import akka.actor.Props;
+import akka.testkit.TestActorRef;
+import org.gbif.api.model.occurrence.VerbatimOccurrence;
+import org.gbif.dwc.terms.*;
+import org.junit.jupiter.api.Test;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class DownloadDwcaActorTest {
+
+  @Test
+  public void testNormalizeExtensionField() {
+
+    ActorSystem system = ActorSystem.create("test");
+    TestActorRef actor =  TestActorRef.apply(
+      Props.create(DownloadDwcaActor.class, null, null, null),
+      system
+    );
+
+    UUID datasetKey = UUID.randomUUID();
+    VerbatimOccurrence record =  new VerbatimOccurrence();
+    record.setKey(1L);
+    record.setDatasetKey(datasetKey);
+
+    Map<Term, String> terms = new LinkedHashMap<>();
+    terms.put(AcTerm.subjectCategoryVocabulary, "Specimen/Object");
+    terms.put(XmpRightsTerm.UsageTerms, "https://creativecommons.org/publicdomain/zero/1.0/");
+    terms.put(DwcTerm.scientificName, "Haloxylon ammodendron");
+    terms.put(DcTerm.identifier, "http://n2t.net/ark:/65665/m36c5a41f4-7c60-43f2-ba1d-0860880efacc");
+    terms.put(ExifTerm.PixelXDimension, "6729");
+    terms.put(XmpRightsTerm.WebStatement, "https://naturalhistory.si.edu/research/nmnh-collections/museum-collections-policies");
+    terms.put(DcElement.source, "US National Herbarium, Department of Botany, NMNH, Smithsonian Institution");
+    terms.put(DcElement.creator, "Conveyor Belt");
+    terms.put(AcTerm.providerLiteral, "Smithsonian Institution, NMNH, Botany");
+    terms.put(DcElement.format, "image/jpeg");
+    terms.put(AcTerm.licenseLogoURL, "https://www.si.edu/sites/default/files/icons/cc0.svg");
+    terms.put(DcElement.rights, "CC0");
+    terms.put(DcTerm.description, "Barcode 03538523");
+    terms.put(DcTerm.rights, "CC0");
+    terms.put(AcTerm.accessURI, "https://collections.nmnh.si.edu/media/?i=14034501&h=2000");
+    terms.put(DcElement.type, "image");
+    terms.put(ExifTerm.PixelYDimension, "8985");
+    terms.put(DcTerm.title, "03538523.tif");
+
+    Map<String,String> result = ((DownloadDwcaActor)actor.underlyingActor()).toExtensionRecord(terms, record);
+
+    Map<String, String> expected = new LinkedHashMap<>();
+    expected.put(GbifTerm.gbifID.simpleName().toLowerCase(), "1");
+    expected.put(GbifTerm.datasetKey.simpleName().toLowerCase(), datasetKey.toString());
+    expected.put(AcTerm.subjectCategoryVocabulary.simpleName().toLowerCase(), "Specimen/Object");
+    expected.put(XmpRightsTerm.UsageTerms.simpleName().toLowerCase(), "https://creativecommons.org/publicdomain/zero/1.0/");
+    expected.put(DwcTerm.scientificName.simpleName().toLowerCase(), "Haloxylon ammodendron");
+    expected.put(DcTerm.identifier.simpleName().toLowerCase(), "http://n2t.net/ark:/65665/m36c5a41f4-7c60-43f2-ba1d-0860880efacc");
+    expected.put(ExifTerm.PixelXDimension.simpleName().toLowerCase(), "6729");
+    expected.put(XmpRightsTerm.WebStatement.simpleName().toLowerCase(), "https://naturalhistory.si.edu/research/nmnh-collections/museum-collections-policies");
+    expected.put(DcElement.source.simpleName().toLowerCase(), "US National Herbarium, Department of Botany, NMNH, Smithsonian Institution");
+    expected.put(DcElement.creator.simpleName().toLowerCase(), "Conveyor Belt");
+    expected.put(AcTerm.providerLiteral.simpleName().toLowerCase(), "Smithsonian Institution, NMNH, Botany");
+    expected.put("format", "image/jpeg");
+    expected.put(AcTerm.licenseLogoURL.simpleName().toLowerCase(), "https://www.si.edu/sites/default/files/icons/cc0.svg");
+    expected.put("dc_rights", "CC0"); // this one wins
+    expected.put(DcTerm.description.simpleName().toLowerCase(), "Barcode 03538523");
+    expected.put("dcterms_rights", "CC0"); // this one win
+    expected.put(AcTerm.accessURI.simpleName().toLowerCase(), "https://collections.nmnh.si.edu/media/?i=14034501&h=2000");
+    expected.put(DcElement.type.simpleName().toLowerCase(), "image");
+    expected.put(ExifTerm.PixelYDimension.simpleName().toLowerCase(), "8985");
+    expected.put(DcTerm.title.simpleName().toLowerCase(), "03538523.tif");
+
+    assertEquals(expected, result);
+    system.terminate();
+  }
+}