From 842828e20e39ee4aab1356a4bbf69a96f5dd07d7 Mon Sep 17 00:00:00 2001 From: Eduardo Pareja-Tobes Date: Sat, 2 Jan 2016 18:48:48 +0100 Subject: [PATCH 1/3] fighting with scalac; I won --- src/main/scala/defaultLocations.scala | 2 +- src/main/scala/package.scala | 1 + src/main/scala/projects.scala | 96 ++++++++++++++-------- src/test/scala/DefaultLocationsTests.scala | 44 +++++----- src/test/scala/exampleProject.scala | 48 +++++++---- src/test/scala/genericTests.scala | 27 ++++++ 6 files changed, 148 insertions(+), 70 deletions(-) create mode 100644 src/test/scala/genericTests.scala diff --git a/src/main/scala/defaultLocations.scala b/src/main/scala/defaultLocations.scala index ae25f39..f562cef 100644 --- a/src/main/scala/defaultLocations.scala +++ b/src/main/scala/defaultLocations.scala @@ -10,7 +10,7 @@ class defaultS3LocationForTask[T <: AnyTask](val task: T) extends DepFn1[ ] { - implicit def default[D <: AnyData]: AnyApp1At[this.type, D] { type Y = D := S3Resource } = + implicit def default[D <: AnyData]: AnyApp1At[defaultS3LocationForTask[T], D] { type Y = D := S3Resource } = App1 { d: D => d := S3Resource(task.s3Output / d.label) } } diff --git a/src/main/scala/package.scala b/src/main/scala/package.scala index f9d1572..84fccbe 100644 --- a/src/main/scala/package.scala +++ b/src/main/scala/package.scala @@ -5,5 +5,6 @@ import ohnosequences.datasets._ package object projects { + type NoData = |[AnyData] val noData: |[AnyData] = |[AnyData] } diff --git a/src/main/scala/projects.scala b/src/main/scala/projects.scala index 3883326..e0cb97b 100644 --- a/src/main/scala/projects.scala +++ b/src/main/scala/projects.scala @@ -43,7 +43,6 @@ trait AnyProject { lazy val s3Input : S3Folder = s3 / "data" / "in" / lazy val s3Output : S3Folder = s3 / "data" / "out" / - // TODO one role per project, created when initialized // lazy val role: Role = projects } @@ -72,44 +71,32 @@ trait AnyTask extends AnyType { Both inputs and outputs are records of `AnyData`. Normally you would define a task as an `object`, with nested `object`s for the input and output. Then you just need to set the corresponding types and values. */ - type Input <: AnyDataSet + type Input <: AnyProductType { type Types <: AnyKList.Of[AnyData] } val input: Input - type Output <: AnyDataSet + type Output <: AnyProductType { type Types <: AnyKList.Of[AnyData] } val output: Output // NOTE in a future better world we could use this lazy val branch = name - /* - This is a depfn which when applied on data: `task.defaultS3Location(d)` yields the default S3 location for `d`. You can use it for building data Loquat data mappings, for example, by mapping over the types of the input/output records. - */ - case object defaultS3Location extends defaultS3LocationForTask(this) - - /* the returned depfn will let you add a qualifier after the standard output path for this task. See the tests for an example of its use */ - def defaultLocationWithQualifier(qual: String): insertAfter = insertAfter(s3Output.key, qual) - - def defaultOutputS3Location[ - O <: Output#Raw - ](implicit - mapper: AnyApp2At[ - mapKList[defaultS3Location.type, AnyDenotation { type Value = S3Resource }], - defaultS3Location.type, - Output#Keys#Types - ] { type Y = O } - ) - : O = - mapper(defaultS3Location, output.keys.types) - val deadline: LocalDate + + def deadlinePassed: Boolean = deadline.isAfter(LocalDate.now) } -// abstract class Task[P <: AnyProject](val project: P)(val deadline: LocalDate) extends AnyTask { -// -// type Project = P -// -// lazy val name: String = toString -// } +case object AnyTask { + + implicit def denotationSyntax[T <: AnyTask, V <: T#Raw](td: T := V): TaskDenotationSyntax[T,V] = + TaskDenotationSyntax(td) +} +case class TaskDenotationSyntax[T <: AnyTask, V <: T#Raw](val td: T := V) { + + def isDeadlineOK: Boolean = td.value.head match { + case Failed | Cancelled | Expired | Completed => true + case Specified | InReview |Started => td.tpe.deadlinePassed + } +} /* This is a helper constructor for doing something like @@ -139,16 +126,37 @@ extends AnyTask { type Project = P - type Input = DataSet[I] - lazy val input: Input = new DataSet(inputData) {} + type Input = I + lazy val input: Input = inputData + + type Output = O + lazy val output: Output = outputData + + lazy val name: String = toString +} + - type Output = DataSet[O] - lazy val output: Output = new DataSet(outputData) {} +abstract class Task2[ + P <: AnyProject +]( + val project: P +)( + val deadline: LocalDate +) +extends AnyTask { + + type Project = P lazy val name: String = toString } +trait AnyTaskRaw { + + type T <: AnyTask + type Inputs = List[Any] +} + sealed trait AnyTaskState case object Specified extends AnyTaskState { @@ -184,3 +192,25 @@ abstract class ProjectTasks[P <: AnyProject, Ks <: AnyProductType { type Types < noDuplicates: noDuplicates isTrueOn Ks#Types ) extends RecordType(tasks) + +case object getState extends DepFn1[AnyDenotation, (AnyTask, AnyTaskState)] { + + implicit def default[ + T <: AnyTask, + V <: T#Raw + ] + : AnyApp1At[this.type, T := V] { type Y = (T,V#Head) } = + this at { tv: T := V => (tv.tpe, tv.value.head) } +} +// +// case object taskDeadlineOK extends DepFn1[(AnyTask, TaskState), Boolean] { +// +// implicit def default[T <: AnyTask, V <: T#Raw] +// : AnyApp1At[taskDeadlineOK.type, T := V] { type Y = Boolean } = +// App1 { +// tv: T := V => tv.value.head match { +// case Failed | Cancelled | Expired | Completed => true +// case Specified | InReview |Started => tv.tpe.deadlinePassed +// } +// } +// } diff --git a/src/test/scala/DefaultLocationsTests.scala b/src/test/scala/DefaultLocationsTests.scala index a9efc39..8b9c971 100644 --- a/src/test/scala/DefaultLocationsTests.scala +++ b/src/test/scala/DefaultLocationsTests.scala @@ -24,32 +24,32 @@ class DefaultLocationsTest extends FunSuite { test("default S3 locations") { - assert { - (doSomething.input.keys.types map doSomething.defaultS3Location) === ( - (x := S3Resource(doSomething.s3Output / x.label)) :: *[AnyDenotation] - ) - } - - assert { - (doSomething.input.keys.types map doSomething.defaultS3Location) === doSomething.defaultOutputS3Location - } + // assert { + // (doSomething.input.types map doSomething.defaultS3Location) === ( + // (x := S3Resource(doSomething.s3Output / x.label)) :: *[AnyDenotation] + // ) + // } + // + // assert { + // (doSomething.input.types map doSomething.defaultS3Location) === doSomething.defaultOutputS3Location + // } } test("can map over default locations") { // NOTE just something which serves as a classifier for different denotations of the same resource - val samples = Set("hola", "scalac", "que tal") - - val s3PerSample = samples map { - s => { - - val addSamplePrefix = doSomething.defaultLocationWithQualifier(s) - import addSamplePrefix._ - - doSomething.defaultOutputS3Location map addSamplePrefix - } - } - - println { s3PerSample } + // val samples = Set("hola", "scalac", "que tal") + // + // val s3PerSample = samples map { + // s => { + // + // val addSamplePrefix = doSomething.defaultLocationWithQualifier(s) + // import addSamplePrefix._ + // + // doSomething.defaultOutputS3Location map addSamplePrefix + // } + // } + // + // println { s3PerSample } } } diff --git a/src/test/scala/exampleProject.scala b/src/test/scala/exampleProject.scala index 8c1405c..0ef70ae 100644 --- a/src/test/scala/exampleProject.scala +++ b/src/test/scala/exampleProject.scala @@ -15,15 +15,35 @@ case object preparePaellaTasks extends ProjectTasks(preparePaella)( case object rice extends Data("La Fallera") case object seafood extends Data("Preparado de Paella Pescanova") -case object buyRice extends Task(preparePaella)(noData)(rice :×: |[AnyData])(LocalDate.of(2016,3,2)) -case object buySeafood extends Task(preparePaella)(noData)(seafood :×: |[AnyData])(LocalDate.of(2016,3,2)) +case object buyRice extends Task2(preparePaella)(LocalDate.of(2016,3,2)) { + + type Input = NoData; val input = noData + type Output = rice.type :×: |[AnyData]; val output = rice :×: |[AnyData] + + val outputLocations = List( rice(buyRice.s3Output / rice.label) :: *[AnyDenotation] ) +} +case object buySeafood extends Task2(preparePaella)(LocalDate.of(2016,3,2)) { + + type Input = NoData; val input = noData + type Output = seafood.type :×: |[AnyData]; val output = seafood :×: |[AnyData] + + val outputLocations = List( seafood(buySeafood.s3Output / seafood.label) :: *[AnyDenotation] ) +} case object projectState { // Now imagine that I already got my rice, but no seafood yet val current = preparePaellaTasks := { - buyRice( Completed :: List(*[AnyDenotation]) :: List(buyRice.defaultOutputS3Location) :: *[Any] ) :: - buySeafood( Specified :: List(*[AnyDenotation]) :: List(buySeafood.defaultOutputS3Location) :: *[Any] ) :: + buyRice( + Completed :: + List(*[AnyDenotation]) :: + buyRice.outputLocations :: *[Any] + ) :: + buySeafood( + Specified :: + List(*[AnyDenotation]) :: + buySeafood.outputLocations :: *[Any] + ) :: *[AnyDenotation] } } @@ -36,15 +56,15 @@ abstract class GenericTasksTests[ PT <: ProjectTasks[P, Ks] ](val pt: PT) extends org.scalatest.FunSuite { - test("dummy test with tasks") { - - // println { pt.keys.types.asList toString } - } + test("dummy test with tasks") {} } -class buh extends GenericTasksTests[ - preparePaella.type, - buyRice.type :×: - buySeafood.type :×: |[AnyTask], - preparePaellaTasks.type -](preparePaellaTasks) + +// class preparePaellaDefaultTests extends GenericProjectStateTest(preparePaella)(preparePaellaTasks.keys)(preparePaellaTasks)(projectState.current) + +object uhoh { + + val uh = projectState.current.value.head.isDeadlineOK + + val aaaa = projectState.current.value map getState +} diff --git a/src/test/scala/genericTests.scala b/src/test/scala/genericTests.scala new file mode 100644 index 0000000..552acd1 --- /dev/null +++ b/src/test/scala/genericTests.scala @@ -0,0 +1,27 @@ +package era7.projects.test + +import era7.projects._ +import ohnosequences.cosas._, types._, klists._, fns._ + +// abstract class GenericProjectStateTest[ +// P <: AnyProject, +// Ks <: AnyProductType { type Types <: AnyKList { type Bound <: AnyTask } }, +// PT <: ProjectTasks[P, Ks], +// PTS <: PT#Raw, +// BL <: AnyKList { type Bound = Boolean } +// ](val p: P)(val pks: Ks)(val pt: PT)(val state: PT := PTS)(implicit +// mapper: AnyApp2At[ +// mapKList[taskDeadlineOK.type, Boolean], +// taskDeadlineOK.type, +// PTS +// ] { +// type Y = BL +// } +// ) extends org.scalatest.FunSuite { +// +// test("check deadlines") +// { +// +// // println { mapper(taskDeadlineOK, state.value) } +// } +// } From 966d160a48529c22928158bfe3830f207b9f432b Mon Sep 17 00:00:00 2001 From: Eduardo Pareja-Tobes Date: Sat, 2 Jan 2016 19:57:52 +0100 Subject: [PATCH 2/3] reestablish generic deadlines test --- src/main/scala/projects.scala | 25 ++++++++-------- src/test/scala/exampleProject.scala | 13 ++++++--- src/test/scala/genericTests.scala | 44 ++++++++++++++--------------- 3 files changed, 44 insertions(+), 38 deletions(-) diff --git a/src/main/scala/projects.scala b/src/main/scala/projects.scala index e0cb97b..d52f292 100644 --- a/src/main/scala/projects.scala +++ b/src/main/scala/projects.scala @@ -202,15 +202,16 @@ case object getState extends DepFn1[AnyDenotation, (AnyTask, AnyTaskState)] { : AnyApp1At[this.type, T := V] { type Y = (T,V#Head) } = this at { tv: T := V => (tv.tpe, tv.value.head) } } -// -// case object taskDeadlineOK extends DepFn1[(AnyTask, TaskState), Boolean] { -// -// implicit def default[T <: AnyTask, V <: T#Raw] -// : AnyApp1At[taskDeadlineOK.type, T := V] { type Y = Boolean } = -// App1 { -// tv: T := V => tv.value.head match { -// case Failed | Cancelled | Expired | Completed => true -// case Specified | InReview |Started => tv.tpe.deadlinePassed -// } -// } -// } + +// TODO return the task if not or somethikn similar +case object taskDeadlineOK extends DepFn1[AnyDenotation, Boolean] { + + implicit def default[T <: AnyTask, V <: T#Raw] + : AnyApp1At[taskDeadlineOK.type, T := V] { type Y = Boolean } = + App1 { + tv: T := V => tv.value.head match { + case Failed | Cancelled | Expired | Completed => true + case Specified | InReview |Started => tv.tpe.deadlinePassed + } + } +} diff --git a/src/test/scala/exampleProject.scala b/src/test/scala/exampleProject.scala index 0ef70ae..03748d9 100644 --- a/src/test/scala/exampleProject.scala +++ b/src/test/scala/exampleProject.scala @@ -59,12 +59,17 @@ abstract class GenericTasksTests[ test("dummy test with tasks") {} } +class ExampleProjectTests extends org.scalatest.FunSuite { -// class preparePaellaDefaultTests extends GenericProjectStateTest(preparePaella)(preparePaellaTasks.keys)(preparePaellaTasks)(projectState.current) + test("deadlines are still in the future") { -object uhoh { + assertResult(true) { - val uh = projectState.current.value.head.isDeadlineOK + val zzz: List[Boolean] = (projectState.current.value map taskDeadlineOK).asList - val aaaa = projectState.current.value map getState + zzz.foldLeft(true)(_ && _) + } + } } + +class preparePaellaDefaultTests extends GenericProjectStateTest(preparePaella)(preparePaellaTasks.keys)(preparePaellaTasks)(projectState.current) diff --git a/src/test/scala/genericTests.scala b/src/test/scala/genericTests.scala index 552acd1..d463253 100644 --- a/src/test/scala/genericTests.scala +++ b/src/test/scala/genericTests.scala @@ -3,25 +3,25 @@ package era7.projects.test import era7.projects._ import ohnosequences.cosas._, types._, klists._, fns._ -// abstract class GenericProjectStateTest[ -// P <: AnyProject, -// Ks <: AnyProductType { type Types <: AnyKList { type Bound <: AnyTask } }, -// PT <: ProjectTasks[P, Ks], -// PTS <: PT#Raw, -// BL <: AnyKList { type Bound = Boolean } -// ](val p: P)(val pks: Ks)(val pt: PT)(val state: PT := PTS)(implicit -// mapper: AnyApp2At[ -// mapKList[taskDeadlineOK.type, Boolean], -// taskDeadlineOK.type, -// PTS -// ] { -// type Y = BL -// } -// ) extends org.scalatest.FunSuite { -// -// test("check deadlines") -// { -// -// // println { mapper(taskDeadlineOK, state.value) } -// } -// } +abstract class GenericProjectStateTest[ + P <: AnyProject, + Ks <: AnyProductType { type Types <: AnyKList { type Bound <: AnyTask } }, + PT <: ProjectTasks[P, Ks], + PTS <: PT#Raw, + BL <: AnyKList { type Bound = Boolean } +](val p: P)(val pks: Ks)(val pt: PT)(val state: PT := PTS)(implicit + mapper: AnyApp2At[ + mapKList[taskDeadlineOK.type, Boolean], + taskDeadlineOK.type, + PTS + ] { + type Y = BL + } +) extends org.scalatest.FunSuite { + + test("check deadlines") + { + + // println { mapper(taskDeadlineOK, state.value) } + } +} From 7970ac7b7d2a951e7e310602eff457b3c9bd7f0b Mon Sep 17 00:00:00 2001 From: Eduardo Pareja-Tobes Date: Mon, 4 Jan 2016 17:44:24 +0100 Subject: [PATCH 3/3] simplify locations, general improvements --- src/main/scala/defaultLocations.scala | 8 ++- src/main/scala/projects.scala | 68 +++++++--------------- src/test/scala/DefaultLocationsTests.scala | 2 +- src/test/scala/exampleProject.scala | 20 +++---- 4 files changed, 36 insertions(+), 62 deletions(-) diff --git a/src/main/scala/defaultLocations.scala b/src/main/scala/defaultLocations.scala index f562cef..3e7eb9d 100644 --- a/src/main/scala/defaultLocations.scala +++ b/src/main/scala/defaultLocations.scala @@ -4,13 +4,15 @@ import ohnosequences.datasets._ import ohnosequences.cosas._, types._, fns._, klists._ import ohnosequences.awstools.s3._ -class defaultS3LocationForTask[T <: AnyTask](val task: T) extends DepFn1[ +case class defaultS3LocationForTask[T <: AnyTask]() extends DepFn1[ AnyData, AnyDenotation { type Value = S3Resource } ] -{ - implicit def default[D <: AnyData]: AnyApp1At[defaultS3LocationForTask[T], D] { type Y = D := S3Resource } = +case object defaultS3LocationForTask { + + implicit def defApp[T <: AnyTask, D <: AnyData](implicit task: T) + : AnyApp1At[defaultS3LocationForTask[T], D] { type Y = D := S3Resource } = App1 { d: D => d := S3Resource(task.s3Output / d.label) } } diff --git a/src/main/scala/projects.scala b/src/main/scala/projects.scala index d52f292..2de1099 100644 --- a/src/main/scala/projects.scala +++ b/src/main/scala/projects.scala @@ -55,11 +55,12 @@ abstract class Project(val name: String) extends AnyProject */ trait AnyTask extends AnyType { - type Raw = AnyTaskState :: List[Input#Raw] :: List[Output#Raw] :: *[Any] - type Project <: AnyProject val project: Project + val deadline: LocalDate + def deadlinePassed: Boolean = deadline.isAfter(LocalDate.now) + val name: String lazy val fullName : String = s"${project.name}.${name}" lazy val label : String = fullName @@ -77,12 +78,18 @@ trait AnyTask extends AnyType { type Output <: AnyProductType { type Types <: AnyKList.Of[AnyData] } val output: Output + type Raw = AnyTaskState :: List[Input#Raw] :: List[Output#Raw] :: *[Any] + // NOTE in a future better world we could use this lazy val branch = name - val deadline: LocalDate - - def deadlinePassed: Boolean = deadline.isAfter(LocalDate.now) + def defaultS3Locations[O <: AnyKList { type Bound = AnyDenotation { type Value = S3Resource } }](implicit + mapper: AnyApp2At[ + mapKList[defaultS3LocationForTask[this.type], AnyDenotation { type Value = S3Resource }], + defaultS3LocationForTask[this.type], Output#Types + ] { type Y = O } + ) + : O = mapper(defaultS3LocationForTask[this.type], output.types) } case object AnyTask { @@ -102,41 +109,17 @@ case class TaskDenotationSyntax[T <: AnyTask, V <: T#Raw](val td: T := V) { This is a helper constructor for doing something like ``` scala - case object doSomething extends Task(project)(input)(output)(date)` - ``` -*/ -class Task[ - P <: AnyProject, - I <: AnyProductType { type Types <: AnyKList.Of[AnyData] }, - O <: AnyProductType { type Types <: AnyKList.Of[AnyData] } -]( - val project: P -)( - val inputData: I -)( - val outputData: O -)( - val deadline: LocalDate -)( - implicit - proof1: noDuplicates isTrueOn I#Types, - proof2: noDuplicates isTrueOn O#Types -) -extends AnyTask { - - type Project = P + case object doSomething extends Task(project)(date) { - type Input = I - lazy val input: Input = inputData - - type Output = O - lazy val output: Output = outputData - - lazy val name: String = toString -} + type Input = NoData + val input = noData - -abstract class Task2[ + type Output = NoData + val output = noData + } + ``` +*/ +abstract class Task[ P <: AnyProject ]( val project: P @@ -150,13 +133,6 @@ extends AnyTask { lazy val name: String = toString } - -trait AnyTaskRaw { - - type T <: AnyTask - type Inputs = List[Any] -} - sealed trait AnyTaskState case object Specified extends AnyTaskState { @@ -200,7 +176,7 @@ case object getState extends DepFn1[AnyDenotation, (AnyTask, AnyTaskState)] { V <: T#Raw ] : AnyApp1At[this.type, T := V] { type Y = (T,V#Head) } = - this at { tv: T := V => (tv.tpe, tv.value.head) } + getState at { tv: T := V => (tv.tpe, tv.value.head) } } // TODO return the task if not or somethikn similar diff --git a/src/test/scala/DefaultLocationsTests.scala b/src/test/scala/DefaultLocationsTests.scala index 8b9c971..f036e18 100644 --- a/src/test/scala/DefaultLocationsTests.scala +++ b/src/test/scala/DefaultLocationsTests.scala @@ -15,7 +15,7 @@ case object example { case object buh extends era7.projects.Project("buh") - case object doSomething extends Task(buh)(x :×: |[AnyData])(x :×: |[AnyData])(LocalDate.of(2016,3,2)) + // case object doSomething extends Task(buh)(x :×: |[AnyData])(x :×: |[AnyData])(LocalDate.of(2016,3,2)) } class DefaultLocationsTest extends FunSuite { diff --git a/src/test/scala/exampleProject.scala b/src/test/scala/exampleProject.scala index 03748d9..1126dca 100644 --- a/src/test/scala/exampleProject.scala +++ b/src/test/scala/exampleProject.scala @@ -15,19 +15,15 @@ case object preparePaellaTasks extends ProjectTasks(preparePaella)( case object rice extends Data("La Fallera") case object seafood extends Data("Preparado de Paella Pescanova") -case object buyRice extends Task2(preparePaella)(LocalDate.of(2016,3,2)) { +case object buyRice extends Task(preparePaella)(LocalDate.of(2016,3,2)) { type Input = NoData; val input = noData type Output = rice.type :×: |[AnyData]; val output = rice :×: |[AnyData] - - val outputLocations = List( rice(buyRice.s3Output / rice.label) :: *[AnyDenotation] ) } -case object buySeafood extends Task2(preparePaella)(LocalDate.of(2016,3,2)) { +case object buySeafood extends Task(preparePaella)(LocalDate.of(2016,3,2)) { type Input = NoData; val input = noData type Output = seafood.type :×: |[AnyData]; val output = seafood :×: |[AnyData] - - val outputLocations = List( seafood(buySeafood.s3Output / seafood.label) :: *[AnyDenotation] ) } case object projectState { @@ -35,14 +31,14 @@ case object projectState { // Now imagine that I already got my rice, but no seafood yet val current = preparePaellaTasks := { buyRice( - Completed :: - List(*[AnyDenotation]) :: - buyRice.outputLocations :: *[Any] + Completed :: + List(*[AnyDenotation]) :: + List(buyRice.defaultS3Locations) :: *[Any] ) :: buySeafood( - Specified :: - List(*[AnyDenotation]) :: - buySeafood.outputLocations :: *[Any] + Specified :: + List(*[AnyDenotation]) :: + List(buySeafood.defaultS3Locations) :: *[Any] ) :: *[AnyDenotation] }