Insubric · minettiandrea · Apr 2, 2026 · Apr 2, 2026
diff --git a/client/src/main/scala/ch/wsl/box/client/services/DataAccessObject.scala b/client/src/main/scala/ch/wsl/box/client/services/DataAccessObject.scala
@@ -18,5 +18,5 @@ trait DataAccessObject {
   def delete(kind:String, lang:String, entity:String, id:JSONID)(implicit ec:ExecutionContext):Future[JSONCount]
 
   def tabularMetadata(kind:String, lang:String, entity:String, public:Boolean)(implicit ec:ExecutionContext): Future[JSONMetadata]
-  def list(kind:String, lang:String, entity:String, q:JSONQuery,public:Boolean)(implicit ec:ExecutionContext):Future[Seq[Row]]
+  def list(kind:String, lang:String, entity:String, q:JSONQuery,public:Boolean,metadata:JSONMetadata)(implicit ec:ExecutionContext):Future[Seq[Row]]
 }
diff --git a/client/src/main/scala/ch/wsl/box/client/services/impl/DaoLocalDbImpl.scala b/client/src/main/scala/ch/wsl/box/client/services/impl/DaoLocalDbImpl.scala
@@ -52,11 +52,10 @@ class DaoLocalDbImpl(rest:REST, clientSession: ClientSession) extends DataAccess
     rest.tabularMetadata(kind, lang, entity, public)
   }
 
-  override def list(kind: String, lang: String, entity: String, q: JSONQuery, public: Boolean)(implicit ec: ExecutionContext): Future[Seq[Row]] = {
+  override def list(kind: String, lang: String, entity: String, q: JSONQuery, public: Boolean, metadata:JSONMetadata)(implicit ec: ExecutionContext): Future[Seq[Row]] = {
     for {
-      metadata <- tabularMetadata(kind,lang,entity,public)
-      csv <- rest.csv(kind, clientSession.lang(), entity, q,public).map(_.map(r => RowDb(r, metadata)))
-      local <- DB.localRecord.list(Some(s"kind='$kind' and name='$entity'")).map(_.map(r => RowLocal(r, metadata)))
+      csv <- rest.csv(kind, clientSession.lang(), entity, q,public).map(_.map(r => RowDb(r, metadata,q)))
+      local <- DB.localRecord.list(Some(s"kind='$kind' and name='$entity'")).map(_.map(r => RowLocal(r, metadata,q)))
     } yield {
       val localIds = local.flatMap(_.id)
       local ++ csv.filterNot(r => r.id.exists(localIds.contains))

diff --git a/client/src/main/scala/ch/wsl/box/client/services/impl/DaoPassthroughImpl.scala b/client/src/main/scala/ch/wsl/box/client/services/impl/DaoPassthroughImpl.scala
@@ -45,10 +45,9 @@ class DaoPassthroughImpl(rest:REST, clientSession: ClientSession) extends DataAc
     rest.tabularMetadata(kind, lang, entity, public)
   }
 
-  override def list(kind: String, lang: String, entity: String, q: JSONQuery, public: Boolean)(implicit ec: ExecutionContext): Future[Seq[Row]] = {
+  override def list(kind: String, lang: String, entity: String, q: JSONQuery, public: Boolean,metadata:JSONMetadata)(implicit ec: ExecutionContext): Future[Seq[Row]] = {
     for {
-      metadata <- tabularMetadata(kind,lang,entity,public)
-      csv <- rest.csv(kind, clientSession.lang(), entity, q,public).map(_.map(r => RowDb(r, metadata)))
+      csv <- rest.csv(kind, clientSession.lang(), entity, q,public).map(_.map(r => RowDb(r, metadata,q)))
     } yield csv
   }
 }
diff --git a/client/src/main/scala/ch/wsl/box/client/viewmodel/Row.scala b/client/src/main/scala/ch/wsl/box/client/viewmodel/Row.scala
@@ -1,7 +1,7 @@
 package ch.wsl.box.client.viewmodel
 
 import ch.wsl.box.client.db.LocalRecord
-import ch.wsl.box.model.shared.{JSONID, JSONMetadata}
+import ch.wsl.box.model.shared.{JSONField, JSONID, JSONMetadata, JSONQuery}
 import ch.wsl.box.shared.utils.JSONUtils.EnhancedJson
 import io.circe.Json
 
@@ -11,16 +11,18 @@ sealed trait Row {
   def rowJs:Json
   def id:Option[JSONID]
   def isLocal:Boolean
+
+  protected def fields(q:JSONQuery,metadata: JSONMetadata):Seq[Option[JSONField]] = q.fields.getOrElse(metadata.tabularFields).map(f => metadata.fields.find(_.name == f))
 }
 
-case class RowDb(data: Seq[String],metadata:JSONMetadata) extends Row {
+case class RowDb(data: Seq[String],metadata: JSONMetadata,query:JSONQuery) extends Row {
   def field(name:String) = {
-    metadata.table.zipWithIndex.find{ case (f,_) => f.name == name }.flatMap{ case (f,i) => data.lift(i).filter(_.nonEmpty).map(f.fromString) }
+    fields(query,metadata).zipWithIndex.find{ case (f,_) => f.exists(_.name == name) }.flatMap{ case (f,i) => f.flatMap( f=> data.lift(i).filter(_.nonEmpty).map(f.fromString)) }
   }
 
   def rowJs:Json = {
     Json.fromFields(
-      metadata.tabularFields.map(k => k -> field(k).getOrElse(Json.Null))
+      query.fields.getOrElse(metadata.tabularFields).map(k => k -> field(k).getOrElse(Json.Null))
     )
   }
 
@@ -29,8 +31,8 @@ case class RowDb(data: Seq[String],metadata:JSONMetadata) extends Row {
   override def isLocal: Boolean = false
 }
 
-case class RowLocal(lr:LocalRecord,metadata:JSONMetadata) extends Row {
-  override def data: Seq[String] = metadata.tabularFields.map(lr.data.get)
+case class RowLocal(lr:LocalRecord,metadata:JSONMetadata,query:JSONQuery) extends Row {
+  override def data: Seq[String] = query.fields.getOrElse(metadata.tabularFields).map(lr.data.get)
 
   override def field(name: String): Option[Json] = lr.data.jsOpt(name)
 

diff --git a/client/src/main/scala/ch/wsl/box/client/views/EntityTableView.scala b/client/src/main/scala/ch/wsl/box/client/views/EntityTableView.scala
@@ -415,12 +415,14 @@ case class EntityTablePresenter(model:ModelProperty[EntityTableModel], onSelect:
     logger.info(s"reloading rows page: $page")
     logger.info("filterUpdateHandler "+filterUpdateHandler)
     val qOrig = query(extent)
-    val newQuery = !model.subProp(_.query).get.contains(qOrig)
     model.subProp(_.query).set(Some(qOrig))
-    val q = qOrig.copy(paging = Some(JSONQueryPaging(ClientConf.pageLength, page)))
+    val q = qOrig.copy(
+      paging = Some(JSONQueryPaging(ClientConf.pageLength, page)),
+      fields = Some(model.subProp(_.selectedColumns).get.map(_.name))
+    )
 
     //start request in parallel
-    val csvRequest = services.data.list(model.subProp(_.kind).get, services.clientSession.lang(), model.subProp(_.name).get, q,model.subProp(_.public).get)
+    val csvRequest = services.data.list(model.subProp(_.kind).get, services.clientSession.lang(), model.subProp(_.name).get, q,model.subProp(_.public).get,model.subProp(_.metadata).get.get)
     val idsRequest =  services.rest.ids(model.get.kind, services.clientSession.lang(), model.get.name, q,model.subProp(_.public).get)
     if(hasGeometry() && !defaultClose) {
       loadGeoms(extent)
@@ -441,7 +443,7 @@ case class EntityTablePresenter(model:ModelProperty[EntityTableModel], onSelect:
     val r = for {
       csv <- csvRequest
       ids <- idsRequest
-      lookups <- if(newQuery) lookupReq(csv) else Future.successful( model.subProp(_.lookups).get)
+      lookups <- if(model.subProp(_.lookups).get.isEmpty) lookupReq(csv) else Future.successful( model.subProp(_.lookups).get)
     } yield {
       if(currentCount == reloadCount) {
         model.subProp(_.lookups).set(lookups)
@@ -463,7 +465,9 @@ case class EntityTablePresenter(model:ModelProperty[EntityTableModel], onSelect:
   }
 
 
-
+  model.subProp(_.selectedColumns).listen{ c =>
+    reloadRows(model.subProp(_.ids).get.currentPage)
+  }
 
   def sort(_fieldQuery: ReadableProperty[Option[FieldQuery]]) = (e:Event) => {
     e.preventDefault()

diff --git a/server/src/main/scala/ch/wsl/box/rest/logic/FormActions.scala b/server/src/main/scala/ch/wsl/box/rest/logic/FormActions.scala
@@ -149,12 +149,12 @@ case class FormActions(metadata:JSONMetadata,
   }
 
 
-  private def __list(query:JSONQuery,dropHtml:Boolean = false,fields:JSONMetadata => Seq[String] = _.tabularFields):DBIO[Seq[Seq[(String,Json)]]] = {
+  private def __list(query:JSONQuery,dropHtml:Boolean = false):DBIO[Seq[Seq[(String,Json)]]] = {
 
     _list(query).map{ rows =>
 
         rows.map { row =>
-          val columns = fields(metadata).map { f =>
+          val columns = query.fields.getOrElse(metadata.tabularFields).map { f =>
             (f, listRenderer(row, dropHtml)(f))
           }
 
@@ -168,9 +168,9 @@ case class FormActions(metadata:JSONMetadata,
   }
 
 
-  def csv(query:JSONQuery,fields:JSONMetadata => Seq[String] = _.tabularFields):DBIO[CSVTable] = {
+  def csv(query:JSONQuery):DBIO[CSVTable] = {
 
-    __list(query,fields = fields).map{ rows =>
+    __list(query).map{ rows =>
         CSVTable(title = metadata.label, header = Seq(), rows = rows.map(_.map(_._2.string)), showHeader = false)
     }
   }

diff --git a/server/src/main/scala/ch/wsl/box/rest/logic/Lookup.scala b/server/src/main/scala/ch/wsl/box/rest/logic/Lookup.scala
@@ -8,7 +8,8 @@ import ch.wsl.box.jdbc.PostgresProfile.api._
 import ch.wsl.box.rest.runtime.Registry
 import ch.wsl.box.services.Services
 
-import scala.concurrent.{ExecutionContext, Future}
+import scala.concurrent.duration.DurationInt
+import scala.concurrent.{Await, ExecutionContext, Future}
 
 object Lookup {
 
@@ -18,13 +19,24 @@ object Lookup {
 
   private def remoteLookups(metadata: JSONMetadata):Seq[JSONFieldLookupRemote] =  metadata.fields.flatMap(_.remoteLookup)
 
-  def valuesForEntity(metadata:JSONMetadata)(implicit ec: ExecutionContext, mat:Materializer,services: Services) :DBIO[Map[String,Seq[Json]]] = {
+  def valuesForEntity(metadata:JSONMetadata)(implicit ec: ExecutionContext, mat:Materializer,services: Services,db:FullDatabase) :Future[Map[String,Seq[Json]]] = Future{
 
-    DBIO.sequence{
-        remoteLookups(metadata).map(_.lookupEntity).map{ lookupEntity =>
-          Registry().actions(lookupEntity).findSimple(JSONQuery.empty.limit(10000)).map{ jq => lookupEntity -> jq}
-        }
-      }.map(_.toMap)
+    val z:Seq[(String,Seq[Json])] = Seq[(String,Seq[Json])]()
+
+    remoteLookups(metadata).map(_.lookupEntity).foldRight(z){ case (lookupEntity,acc) =>
+      def le:Future[(String,Seq[Json])] = db.db.run{
+        Registry().actions(lookupEntity).findSimple(JSONQuery.empty.limit(10000)).map{ jq => (lookupEntity,jq) }
+      }
+
+       acc ++ Seq(Await.result(le,10.seconds))
+
+    }.toMap
+
+//    DBIO.sequence{
+//        remoteLookups(metadata).map(_.lookupEntity).map{ lookupEntity =>
+//          Registry().actions(lookupEntity).findSimple(JSONQuery.empty.limit(10000)).map{ jq => lookupEntity -> jq}
+//        }
+//      }.map(_.toMap)
 
   }
 

diff --git a/server/src/main/scala/ch/wsl/box/rest/routes/Exporters.scala b/server/src/main/scala/ch/wsl/box/rest/routes/Exporters.scala
@@ -5,7 +5,7 @@ import akka.http.scaladsl.model.headers.{ContentDispositionTypes, `Content-Dispo
 import akka.http.scaladsl.server.Directives._
 import akka.http.scaladsl.server.{RequestContext, Route}
 import akka.stream.Materializer
-import ch.wsl.box.model.shared.{CSVTable, JSONField, JSONFieldLookupData, JSONFieldLookupExtractor, JSONFieldLookupRemote, JSONMetadata, JSONQuery, XLSTable}
+import ch.wsl.box.model.shared.{CSVTable, JSONField, JSONFieldLookupData, JSONFieldLookupExtractor, JSONFieldLookupRemote, JSONLookups, JSONLookupsRequest, JSONMetadata, JSONQuery, XLSTable}
 import ch.wsl.box.rest.io.xls.XLS
 import ch.wsl.box.rest.logic.{FormActions, Lookup}
 import io.circe.parser.parse
@@ -38,20 +38,24 @@ trait Exporters {
   val name:String
   val metadataFactory: MetadataFactory
   def tabularMetadata(): DBIO[JSONMetadata]
+  def metadata: JSONMetadata
   def actions:FormActions
 
-  def mergeWithForeignKeys(extractFk: Boolean,data:Seq[Json],fk: Map[String,Seq[Json]],metadata:JSONMetadata):Seq[Json] = {
+  def mergeWithForeignKeys(extractFk: Boolean,data:Seq[Json],fk: Seq[JSONLookups],metadata:JSONMetadata, fields:Seq[String]):Seq[Json] = {
     if(extractFk) {
       data.map { row =>
-        val fkData:Json = Json.fromFields(metadata.fields.filter(_.lookup.isDefined).map { f =>
+        val fkData:Json = Json.fromFields(metadata.fields.filter(f => f.lookup.isDefined && fields.contains(f.name)).map { f =>
           f.lookup.get  match {
             case JSONFieldLookupData(data) => f.name -> data.find(_.id == row.js(f.name)).map(_.value).getOrElse(row.get(f.name))
             case JSONFieldLookupExtractor(extractor) => f.name -> extractor.map.get(row.js(extractor.key)).toList.flatten.find(_.id == row.js(f.name)).map(_.value).getOrElse(row.get(f.name))
             case r: JSONFieldLookupRemote => {
-              val local = r.map.localKeysColumn.map(row.js)
-              val remote = fk.get(r.lookupEntity).flatMap(_.find(fkRow => r.map.foreign.keyColumns.map(fkRow.js) == local)).map( remoteRow => r.map.foreign.labelColumns.map(remoteRow.get))
-              val value:String = remote.map(x => x.mkString(" - ")).getOrElse(row.get(f.name))
-              f.name -> value
+
+              val local: Json = row.js(f.name)
+              val res = fk.find(_.fieldName == f.name).flatMap(_.lookups.find(_.id == local).map(_.value)).getOrElse(local.string)
+
+
+
+              f.name -> res
             }
           }
         }.map{ case (k,v) => k -> Json.fromString(v)})
@@ -67,26 +71,44 @@ trait Exporters {
       XLS.importXls(actions.metadata,actions.jsonAction,db.db)
     } ~
     get {
-        parameters('q,'fk.?) { case (q,fk) =>
+        parameters('q,'fk.?,'fields.?) { case (q,fk,fields) =>
           val extractFk = fk.forall(_ == "resolve_fk")
           val query = parse(q).right.get.as[JSONQuery].right.get
-          val io = for {
-            metadata <- DBIO.from(boxDb.adminDb.run(tabularMetadata()))
-            formActions = FormActions(metadata, registry, metadataFactory)
-            fkValues <- Lookup.valuesForEntity(metadata)
-            data <- formActions.list(query, true, metadata.exportFieldsNoGeom.map(_.name))
-            xlsTable = XLSTable(
-              title = name,
-              header = metadata.exportFieldsNoGeom.map(_.title),
-              rows = mergeWithForeignKeys(extractFk,data,fkValues,metadata).map(row => metadata.exportFieldsNoGeom.map(cell => row.get(cell.name)))
-            )
-          } yield {
-            XLS.route(xlsTable)
+          val m = metadata
+          val fut: Future[Route]  = {
+            for {
+              route <- {
+
+                val formActions = FormActions(m, registry, metadataFactory)
+                val rFields = fields.map(_.split(",").toSeq)
+                val requestedFields = rFields.orElse(query.fields).getOrElse(m.tabularFields)
+
+                val f = requestedFields.flatMap(x => m.fields.find(_.name == x))
+
+                val fkFields = m.fields.filter(f => f.lookup.isDefined && requestedFields.contains(f.name))
+
+                val io = for {
+                  fkValues <- actions.lookups(JSONLookupsRequest(fkFields.map(_.name),query))
+                  data <- formActions.list(query, true,requestedFields)
+                  xlsTable = XLSTable(
+                    title = name,
+                    header = f.map(_.title),
+                    rows = mergeWithForeignKeys(extractFk, data, fkValues, m,requestedFields).map(row => f.map(cell => row.get(cell.name)))
+                  )
+                } yield {
+                  XLS.route(xlsTable)
+                }
+                db.db.run(io)
+              }
+            } yield route
           }
-          onSuccess(db.db.run(io))(x => x)
-        }
+
+          rc:RequestContext => fut.flatMap(x => x(rc))
+
+
       }
   }
+    }
 
   def exportCsv(q:String,fk:Option[String],_fields:Option[String])(implicit session:BoxSession, db:FullDatabase, mat:Materializer, ec:ExecutionContext, services:Services): Route = {
 
@@ -101,23 +123,36 @@ trait Exporters {
           }
         }
 
-    val fut: Future[Route] = boxDb.adminDb.run(tabularMetadata()).flatMap { metadata =>
-
-      val formActions = FormActions(metadata, registry, metadataFactory)
-      val fields =  selectedFields(metadata.exportFieldsNoGeom)
-      val io = for {
-        fkValues <- Lookup.valuesForEntity(metadata)
-        data <- formActions.list(query, true, fields.map(_.name))
-        csvTable = CSVTable(
-          title = name,
-          header = fields.map(_.title),
-          rows = mergeWithForeignKeys(extractFk, data, fkValues, metadata).map(row => fields.map(cell => row.get(cell.name)))
-        )
-      } yield {
-        CSV.download(csvTable)
+    val fut: Future[Route] = {
+      for{
+
+       route <- {
+
+
+         val m = metadata
+        val formActions = FormActions(m, registry, metadataFactory)
+
+         val requestedFields = m.fields.filter(f => query.fields.getOrElse(m.tabularFields).contains(f.name))
+         val fkFields = m.fields.filter(f => f.lookup.isDefined && requestedFields.exists(_.name == f.name))
+
+        val fields =  selectedFields(requestedFields)
+
+
+
+        val io = for {
+          fkValues <- actions.lookups(JSONLookupsRequest(fkFields.map(_.name),query))
+          data <- formActions.list(query, true, fields.map(_.name))
+          csvTable = CSVTable(
+            title = name,
+            header = fields.map(_.title),
+            rows = mergeWithForeignKeys(extractFk, data, fkValues, metadata,fields.map(_.name)).map(row => fields.map(cell => row.get(cell.name)))
+          )
+        } yield {
+          CSV.download(csvTable)
+        }
+        db.db.run(io)
       }
-      db.db.run(io)
-    }
+    } yield route }
 
     rc:RequestContext => fut.flatMap(x => x(rc))
   }

diff --git a/server/src/main/scala/ch/wsl/box/rest/routes/Form.scala b/server/src/main/scala/ch/wsl/box/rest/routes/Form.scala
@@ -89,9 +89,9 @@ case class Form(
 
   def csvTable(query:JSONQuery):Future[CSVTable] = {
     for {
-      metadata <- boxDb.adminDb.run(tabularMetadata())
-      formActions = FormActions(metadata, registry, metadataFactory)
-      csv <- db.run(formActions.csv(query))
+//      metadata <- boxDb.adminDb.run(tabularMetadata())
+//      formActions = FormActions(metadata, registry, metadataFactory)
+      csv <- db.run(actions.csv(query))
     } yield csv
   }
 

diff --git a/server/src/main/scala/ch/wsl/box/rest/routes/Table.scala b/server/src/main/scala/ch/wsl/box/rest/routes/Table.scala
@@ -94,7 +94,6 @@ case class Table[T <: ch.wsl.box.jdbc.PostgresProfile.api.Table[M] with UpdateTa
       parameters('q) { q =>
         val query = parse(q).right.get.as[JSONQuery].right.get
         val io = for {
-          fkValues <- Lookup.valuesForEntity(jsonMetadata)(ec,mat,services).map(Some(_))
           data <- jsonActions.findSimple(query)
         } yield {
           val table = XLSTable(