diff --git a/client/src/main/scala/ch/wsl/box/client/services/DataAccessObject.scala b/client/src/main/scala/ch/wsl/box/client/services/DataAccessObject.scala index 8fdba8fc..19584d8e 100644 --- a/client/src/main/scala/ch/wsl/box/client/services/DataAccessObject.scala +++ b/client/src/main/scala/ch/wsl/box/client/services/DataAccessObject.scala @@ -18,5 +18,5 @@ trait DataAccessObject { def delete(kind:String, lang:String, entity:String, id:JSONID)(implicit ec:ExecutionContext):Future[JSONCount] def tabularMetadata(kind:String, lang:String, entity:String, public:Boolean)(implicit ec:ExecutionContext): Future[JSONMetadata] - def list(kind:String, lang:String, entity:String, q:JSONQuery,public:Boolean)(implicit ec:ExecutionContext):Future[Seq[Row]] + def list(kind:String, lang:String, entity:String, q:JSONQuery,public:Boolean,metadata:JSONMetadata)(implicit ec:ExecutionContext):Future[Seq[Row]] } diff --git a/client/src/main/scala/ch/wsl/box/client/services/impl/DaoLocalDbImpl.scala b/client/src/main/scala/ch/wsl/box/client/services/impl/DaoLocalDbImpl.scala index a11388b8..5647b02f 100644 --- a/client/src/main/scala/ch/wsl/box/client/services/impl/DaoLocalDbImpl.scala +++ b/client/src/main/scala/ch/wsl/box/client/services/impl/DaoLocalDbImpl.scala @@ -52,11 +52,10 @@ class DaoLocalDbImpl(rest:REST, clientSession: ClientSession) extends DataAccess rest.tabularMetadata(kind, lang, entity, public) } - override def list(kind: String, lang: String, entity: String, q: JSONQuery, public: Boolean)(implicit ec: ExecutionContext): Future[Seq[Row]] = { + override def list(kind: String, lang: String, entity: String, q: JSONQuery, public: Boolean, metadata:JSONMetadata)(implicit ec: ExecutionContext): Future[Seq[Row]] = { for { - metadata <- tabularMetadata(kind,lang,entity,public) - csv <- rest.csv(kind, clientSession.lang(), entity, q,public).map(_.map(r => RowDb(r, metadata))) - local <- DB.localRecord.list(Some(s"kind='$kind' and name='$entity'")).map(_.map(r => RowLocal(r, metadata))) + csv <- rest.csv(kind, clientSession.lang(), entity, q,public).map(_.map(r => RowDb(r, metadata,q))) + local <- DB.localRecord.list(Some(s"kind='$kind' and name='$entity'")).map(_.map(r => RowLocal(r, metadata,q))) } yield { val localIds = local.flatMap(_.id) local ++ csv.filterNot(r => r.id.exists(localIds.contains)) diff --git a/client/src/main/scala/ch/wsl/box/client/services/impl/DaoPassthroughImpl.scala b/client/src/main/scala/ch/wsl/box/client/services/impl/DaoPassthroughImpl.scala index 5db0bd23..6d4df8f7 100644 --- a/client/src/main/scala/ch/wsl/box/client/services/impl/DaoPassthroughImpl.scala +++ b/client/src/main/scala/ch/wsl/box/client/services/impl/DaoPassthroughImpl.scala @@ -45,10 +45,9 @@ class DaoPassthroughImpl(rest:REST, clientSession: ClientSession) extends DataAc rest.tabularMetadata(kind, lang, entity, public) } - override def list(kind: String, lang: String, entity: String, q: JSONQuery, public: Boolean)(implicit ec: ExecutionContext): Future[Seq[Row]] = { + override def list(kind: String, lang: String, entity: String, q: JSONQuery, public: Boolean,metadata:JSONMetadata)(implicit ec: ExecutionContext): Future[Seq[Row]] = { for { - metadata <- tabularMetadata(kind,lang,entity,public) - csv <- rest.csv(kind, clientSession.lang(), entity, q,public).map(_.map(r => RowDb(r, metadata))) + csv <- rest.csv(kind, clientSession.lang(), entity, q,public).map(_.map(r => RowDb(r, metadata,q))) } yield csv } } diff --git a/client/src/main/scala/ch/wsl/box/client/viewmodel/Row.scala b/client/src/main/scala/ch/wsl/box/client/viewmodel/Row.scala index 0307c80c..768acc56 100644 --- a/client/src/main/scala/ch/wsl/box/client/viewmodel/Row.scala +++ b/client/src/main/scala/ch/wsl/box/client/viewmodel/Row.scala @@ -1,7 +1,7 @@ package ch.wsl.box.client.viewmodel import ch.wsl.box.client.db.LocalRecord -import ch.wsl.box.model.shared.{JSONID, JSONMetadata} +import ch.wsl.box.model.shared.{JSONField, JSONID, JSONMetadata, JSONQuery} import ch.wsl.box.shared.utils.JSONUtils.EnhancedJson import io.circe.Json @@ -11,16 +11,18 @@ sealed trait Row { def rowJs:Json def id:Option[JSONID] def isLocal:Boolean + + protected def fields(q:JSONQuery,metadata: JSONMetadata):Seq[Option[JSONField]] = q.fields.getOrElse(metadata.tabularFields).map(f => metadata.fields.find(_.name == f)) } -case class RowDb(data: Seq[String],metadata:JSONMetadata) extends Row { +case class RowDb(data: Seq[String],metadata: JSONMetadata,query:JSONQuery) extends Row { def field(name:String) = { - metadata.table.zipWithIndex.find{ case (f,_) => f.name == name }.flatMap{ case (f,i) => data.lift(i).filter(_.nonEmpty).map(f.fromString) } + fields(query,metadata).zipWithIndex.find{ case (f,_) => f.exists(_.name == name) }.flatMap{ case (f,i) => f.flatMap( f=> data.lift(i).filter(_.nonEmpty).map(f.fromString)) } } def rowJs:Json = { Json.fromFields( - metadata.tabularFields.map(k => k -> field(k).getOrElse(Json.Null)) + query.fields.getOrElse(metadata.tabularFields).map(k => k -> field(k).getOrElse(Json.Null)) ) } @@ -29,8 +31,8 @@ case class RowDb(data: Seq[String],metadata:JSONMetadata) extends Row { override def isLocal: Boolean = false } -case class RowLocal(lr:LocalRecord,metadata:JSONMetadata) extends Row { - override def data: Seq[String] = metadata.tabularFields.map(lr.data.get) +case class RowLocal(lr:LocalRecord,metadata:JSONMetadata,query:JSONQuery) extends Row { + override def data: Seq[String] = query.fields.getOrElse(metadata.tabularFields).map(lr.data.get) override def field(name: String): Option[Json] = lr.data.jsOpt(name) diff --git a/client/src/main/scala/ch/wsl/box/client/views/EntityTableView.scala b/client/src/main/scala/ch/wsl/box/client/views/EntityTableView.scala index fe84a424..e6761a41 100755 --- a/client/src/main/scala/ch/wsl/box/client/views/EntityTableView.scala +++ b/client/src/main/scala/ch/wsl/box/client/views/EntityTableView.scala @@ -415,12 +415,14 @@ case class EntityTablePresenter(model:ModelProperty[EntityTableModel], onSelect: logger.info(s"reloading rows page: $page") logger.info("filterUpdateHandler "+filterUpdateHandler) val qOrig = query(extent) - val newQuery = !model.subProp(_.query).get.contains(qOrig) model.subProp(_.query).set(Some(qOrig)) - val q = qOrig.copy(paging = Some(JSONQueryPaging(ClientConf.pageLength, page))) + val q = qOrig.copy( + paging = Some(JSONQueryPaging(ClientConf.pageLength, page)), + fields = Some(model.subProp(_.selectedColumns).get.map(_.name)) + ) //start request in parallel - val csvRequest = services.data.list(model.subProp(_.kind).get, services.clientSession.lang(), model.subProp(_.name).get, q,model.subProp(_.public).get) + val csvRequest = services.data.list(model.subProp(_.kind).get, services.clientSession.lang(), model.subProp(_.name).get, q,model.subProp(_.public).get,model.subProp(_.metadata).get.get) val idsRequest = services.rest.ids(model.get.kind, services.clientSession.lang(), model.get.name, q,model.subProp(_.public).get) if(hasGeometry() && !defaultClose) { loadGeoms(extent) @@ -441,7 +443,7 @@ case class EntityTablePresenter(model:ModelProperty[EntityTableModel], onSelect: val r = for { csv <- csvRequest ids <- idsRequest - lookups <- if(newQuery) lookupReq(csv) else Future.successful( model.subProp(_.lookups).get) + lookups <- if(model.subProp(_.lookups).get.isEmpty) lookupReq(csv) else Future.successful( model.subProp(_.lookups).get) } yield { if(currentCount == reloadCount) { model.subProp(_.lookups).set(lookups) @@ -463,7 +465,9 @@ case class EntityTablePresenter(model:ModelProperty[EntityTableModel], onSelect: } - + model.subProp(_.selectedColumns).listen{ c => + reloadRows(model.subProp(_.ids).get.currentPage) + } def sort(_fieldQuery: ReadableProperty[Option[FieldQuery]]) = (e:Event) => { e.preventDefault() diff --git a/server/src/main/scala/ch/wsl/box/rest/logic/FormActions.scala b/server/src/main/scala/ch/wsl/box/rest/logic/FormActions.scala index dd663f69..66d48778 100755 --- a/server/src/main/scala/ch/wsl/box/rest/logic/FormActions.scala +++ b/server/src/main/scala/ch/wsl/box/rest/logic/FormActions.scala @@ -149,12 +149,12 @@ case class FormActions(metadata:JSONMetadata, } - private def __list(query:JSONQuery,dropHtml:Boolean = false,fields:JSONMetadata => Seq[String] = _.tabularFields):DBIO[Seq[Seq[(String,Json)]]] = { + private def __list(query:JSONQuery,dropHtml:Boolean = false):DBIO[Seq[Seq[(String,Json)]]] = { _list(query).map{ rows => rows.map { row => - val columns = fields(metadata).map { f => + val columns = query.fields.getOrElse(metadata.tabularFields).map { f => (f, listRenderer(row, dropHtml)(f)) } @@ -168,9 +168,9 @@ case class FormActions(metadata:JSONMetadata, } - def csv(query:JSONQuery,fields:JSONMetadata => Seq[String] = _.tabularFields):DBIO[CSVTable] = { + def csv(query:JSONQuery):DBIO[CSVTable] = { - __list(query,fields = fields).map{ rows => + __list(query).map{ rows => CSVTable(title = metadata.label, header = Seq(), rows = rows.map(_.map(_._2.string)), showHeader = false) } } diff --git a/server/src/main/scala/ch/wsl/box/rest/logic/Lookup.scala b/server/src/main/scala/ch/wsl/box/rest/logic/Lookup.scala index db90b455..7d6be033 100755 --- a/server/src/main/scala/ch/wsl/box/rest/logic/Lookup.scala +++ b/server/src/main/scala/ch/wsl/box/rest/logic/Lookup.scala @@ -8,7 +8,8 @@ import ch.wsl.box.jdbc.PostgresProfile.api._ import ch.wsl.box.rest.runtime.Registry import ch.wsl.box.services.Services -import scala.concurrent.{ExecutionContext, Future} +import scala.concurrent.duration.DurationInt +import scala.concurrent.{Await, ExecutionContext, Future} object Lookup { @@ -18,13 +19,24 @@ object Lookup { private def remoteLookups(metadata: JSONMetadata):Seq[JSONFieldLookupRemote] = metadata.fields.flatMap(_.remoteLookup) - def valuesForEntity(metadata:JSONMetadata)(implicit ec: ExecutionContext, mat:Materializer,services: Services) :DBIO[Map[String,Seq[Json]]] = { + def valuesForEntity(metadata:JSONMetadata)(implicit ec: ExecutionContext, mat:Materializer,services: Services,db:FullDatabase) :Future[Map[String,Seq[Json]]] = Future{ - DBIO.sequence{ - remoteLookups(metadata).map(_.lookupEntity).map{ lookupEntity => - Registry().actions(lookupEntity).findSimple(JSONQuery.empty.limit(10000)).map{ jq => lookupEntity -> jq} - } - }.map(_.toMap) + val z:Seq[(String,Seq[Json])] = Seq[(String,Seq[Json])]() + + remoteLookups(metadata).map(_.lookupEntity).foldRight(z){ case (lookupEntity,acc) => + def le:Future[(String,Seq[Json])] = db.db.run{ + Registry().actions(lookupEntity).findSimple(JSONQuery.empty.limit(10000)).map{ jq => (lookupEntity,jq) } + } + + acc ++ Seq(Await.result(le,10.seconds)) + + }.toMap + +// DBIO.sequence{ +// remoteLookups(metadata).map(_.lookupEntity).map{ lookupEntity => +// Registry().actions(lookupEntity).findSimple(JSONQuery.empty.limit(10000)).map{ jq => lookupEntity -> jq} +// } +// }.map(_.toMap) } diff --git a/server/src/main/scala/ch/wsl/box/rest/routes/Exporters.scala b/server/src/main/scala/ch/wsl/box/rest/routes/Exporters.scala index e5888564..ea3872b5 100644 --- a/server/src/main/scala/ch/wsl/box/rest/routes/Exporters.scala +++ b/server/src/main/scala/ch/wsl/box/rest/routes/Exporters.scala @@ -5,7 +5,7 @@ import akka.http.scaladsl.model.headers.{ContentDispositionTypes, `Content-Dispo import akka.http.scaladsl.server.Directives._ import akka.http.scaladsl.server.{RequestContext, Route} import akka.stream.Materializer -import ch.wsl.box.model.shared.{CSVTable, JSONField, JSONFieldLookupData, JSONFieldLookupExtractor, JSONFieldLookupRemote, JSONMetadata, JSONQuery, XLSTable} +import ch.wsl.box.model.shared.{CSVTable, JSONField, JSONFieldLookupData, JSONFieldLookupExtractor, JSONFieldLookupRemote, JSONLookups, JSONLookupsRequest, JSONMetadata, JSONQuery, XLSTable} import ch.wsl.box.rest.io.xls.XLS import ch.wsl.box.rest.logic.{FormActions, Lookup} import io.circe.parser.parse @@ -38,20 +38,24 @@ trait Exporters { val name:String val metadataFactory: MetadataFactory def tabularMetadata(): DBIO[JSONMetadata] + def metadata: JSONMetadata def actions:FormActions - def mergeWithForeignKeys(extractFk: Boolean,data:Seq[Json],fk: Map[String,Seq[Json]],metadata:JSONMetadata):Seq[Json] = { + def mergeWithForeignKeys(extractFk: Boolean,data:Seq[Json],fk: Seq[JSONLookups],metadata:JSONMetadata, fields:Seq[String]):Seq[Json] = { if(extractFk) { data.map { row => - val fkData:Json = Json.fromFields(metadata.fields.filter(_.lookup.isDefined).map { f => + val fkData:Json = Json.fromFields(metadata.fields.filter(f => f.lookup.isDefined && fields.contains(f.name)).map { f => f.lookup.get match { case JSONFieldLookupData(data) => f.name -> data.find(_.id == row.js(f.name)).map(_.value).getOrElse(row.get(f.name)) case JSONFieldLookupExtractor(extractor) => f.name -> extractor.map.get(row.js(extractor.key)).toList.flatten.find(_.id == row.js(f.name)).map(_.value).getOrElse(row.get(f.name)) case r: JSONFieldLookupRemote => { - val local = r.map.localKeysColumn.map(row.js) - val remote = fk.get(r.lookupEntity).flatMap(_.find(fkRow => r.map.foreign.keyColumns.map(fkRow.js) == local)).map( remoteRow => r.map.foreign.labelColumns.map(remoteRow.get)) - val value:String = remote.map(x => x.mkString(" - ")).getOrElse(row.get(f.name)) - f.name -> value + + val local: Json = row.js(f.name) + val res = fk.find(_.fieldName == f.name).flatMap(_.lookups.find(_.id == local).map(_.value)).getOrElse(local.string) + + + + f.name -> res } } }.map{ case (k,v) => k -> Json.fromString(v)}) @@ -67,26 +71,44 @@ trait Exporters { XLS.importXls(actions.metadata,actions.jsonAction,db.db) } ~ get { - parameters('q,'fk.?) { case (q,fk) => + parameters('q,'fk.?,'fields.?) { case (q,fk,fields) => val extractFk = fk.forall(_ == "resolve_fk") val query = parse(q).right.get.as[JSONQuery].right.get - val io = for { - metadata <- DBIO.from(boxDb.adminDb.run(tabularMetadata())) - formActions = FormActions(metadata, registry, metadataFactory) - fkValues <- Lookup.valuesForEntity(metadata) - data <- formActions.list(query, true, metadata.exportFieldsNoGeom.map(_.name)) - xlsTable = XLSTable( - title = name, - header = metadata.exportFieldsNoGeom.map(_.title), - rows = mergeWithForeignKeys(extractFk,data,fkValues,metadata).map(row => metadata.exportFieldsNoGeom.map(cell => row.get(cell.name))) - ) - } yield { - XLS.route(xlsTable) + val m = metadata + val fut: Future[Route] = { + for { + route <- { + + val formActions = FormActions(m, registry, metadataFactory) + val rFields = fields.map(_.split(",").toSeq) + val requestedFields = rFields.orElse(query.fields).getOrElse(m.tabularFields) + + val f = requestedFields.flatMap(x => m.fields.find(_.name == x)) + + val fkFields = m.fields.filter(f => f.lookup.isDefined && requestedFields.contains(f.name)) + + val io = for { + fkValues <- actions.lookups(JSONLookupsRequest(fkFields.map(_.name),query)) + data <- formActions.list(query, true,requestedFields) + xlsTable = XLSTable( + title = name, + header = f.map(_.title), + rows = mergeWithForeignKeys(extractFk, data, fkValues, m,requestedFields).map(row => f.map(cell => row.get(cell.name))) + ) + } yield { + XLS.route(xlsTable) + } + db.db.run(io) + } + } yield route } - onSuccess(db.db.run(io))(x => x) - } + + rc:RequestContext => fut.flatMap(x => x(rc)) + + } } + } def exportCsv(q:String,fk:Option[String],_fields:Option[String])(implicit session:BoxSession, db:FullDatabase, mat:Materializer, ec:ExecutionContext, services:Services): Route = { @@ -101,23 +123,36 @@ trait Exporters { } } - val fut: Future[Route] = boxDb.adminDb.run(tabularMetadata()).flatMap { metadata => - - val formActions = FormActions(metadata, registry, metadataFactory) - val fields = selectedFields(metadata.exportFieldsNoGeom) - val io = for { - fkValues <- Lookup.valuesForEntity(metadata) - data <- formActions.list(query, true, fields.map(_.name)) - csvTable = CSVTable( - title = name, - header = fields.map(_.title), - rows = mergeWithForeignKeys(extractFk, data, fkValues, metadata).map(row => fields.map(cell => row.get(cell.name))) - ) - } yield { - CSV.download(csvTable) + val fut: Future[Route] = { + for{ + + route <- { + + + val m = metadata + val formActions = FormActions(m, registry, metadataFactory) + + val requestedFields = m.fields.filter(f => query.fields.getOrElse(m.tabularFields).contains(f.name)) + val fkFields = m.fields.filter(f => f.lookup.isDefined && requestedFields.exists(_.name == f.name)) + + val fields = selectedFields(requestedFields) + + + + val io = for { + fkValues <- actions.lookups(JSONLookupsRequest(fkFields.map(_.name),query)) + data <- formActions.list(query, true, fields.map(_.name)) + csvTable = CSVTable( + title = name, + header = fields.map(_.title), + rows = mergeWithForeignKeys(extractFk, data, fkValues, metadata,fields.map(_.name)).map(row => fields.map(cell => row.get(cell.name))) + ) + } yield { + CSV.download(csvTable) + } + db.db.run(io) } - db.db.run(io) - } + } yield route } rc:RequestContext => fut.flatMap(x => x(rc)) } diff --git a/server/src/main/scala/ch/wsl/box/rest/routes/Form.scala b/server/src/main/scala/ch/wsl/box/rest/routes/Form.scala index 6cb4e2a8..ea02b3f3 100755 --- a/server/src/main/scala/ch/wsl/box/rest/routes/Form.scala +++ b/server/src/main/scala/ch/wsl/box/rest/routes/Form.scala @@ -89,9 +89,9 @@ case class Form( def csvTable(query:JSONQuery):Future[CSVTable] = { for { - metadata <- boxDb.adminDb.run(tabularMetadata()) - formActions = FormActions(metadata, registry, metadataFactory) - csv <- db.run(formActions.csv(query)) +// metadata <- boxDb.adminDb.run(tabularMetadata()) +// formActions = FormActions(metadata, registry, metadataFactory) + csv <- db.run(actions.csv(query)) } yield csv } diff --git a/server/src/main/scala/ch/wsl/box/rest/routes/Table.scala b/server/src/main/scala/ch/wsl/box/rest/routes/Table.scala index 13480c42..e9f70637 100755 --- a/server/src/main/scala/ch/wsl/box/rest/routes/Table.scala +++ b/server/src/main/scala/ch/wsl/box/rest/routes/Table.scala @@ -94,7 +94,6 @@ case class Table[T <: ch.wsl.box.jdbc.PostgresProfile.api.Table[M] with UpdateTa parameters('q) { q => val query = parse(q).right.get.as[JSONQuery].right.get val io = for { - fkValues <- Lookup.valuesForEntity(jsonMetadata)(ec,mat,services).map(Some(_)) data <- jsonActions.findSimple(query) } yield { val table = XLSTable(