[WIP] Explore BESTWAIT hardware steering for scheduler optimization

kernel/CMakeLists.txt

@@ -16,7 +16,6 @@ set(
   "data/readylist/readylist.ref.c"
   "data/intconfig/intconfig.ref.c"
   "data/stacks/stacks.ref.S"
-  "data/runlist/runlist.ref.c"
   "data/globals/globals.ref.c"
   "data/context/context.v${ARCHV}opt.S"
   "data/vm/vm.ref.c"

kernel/bugs

@@ -37,17 +37,3 @@ Other specing / assertions
 	* For every data element in a struct, spec where it should be updated
 	* Add collection of assertions that we can check on function entry/exit
 	* Add structure verifier that can make sure structures are sane
-
-
-
-runlist valid states
-====================
-
-An empty runlist is apparently not a valid state; should probably
-document this in the spec and maybe make an assertion.
-
-
-
-
-
-

kernel/build/offsets/offsets.ref.c

@@ -119,7 +119,7 @@ int main(int argc, char **argv)
 	PRINT_KG_OFFSET(hthreads_mask);
 	PRINT_KG_OFFSET(hthreads);
 
-#ifdef CLUSTER_SCHED
+#if CLUSTER_SCHED
 	PRINT_KG_OFFSET(cluster_clusters);
 	PRINT_KG_OFFSET(cluster_hthreads);
 	PRINT_KG_OFFSET(cluster_mask);

kernel/data/globals/globals.h

@@ -97,7 +97,7 @@ typedef struct {
 	pa_t gpio_reg;
 #endif
 
-#ifdef CLUSTER_SCHED
+#if CLUSTER_SCHED
 	u32_t cluster_clusters;   // number of clusters
 	u32_t cluster_hthreads;   // hardware threads per cluster
 	u32_t cluster_mask[4];    // bitmask of threads in cluster
@@ -216,7 +216,7 @@ static inline H2K_kg_t PURITY *H2K_gp_llvm()
 #undef PURITY
 #endif
 
-#ifdef CLUSTER_SCHED
+#if CLUSTER_SCHED
 
 static inline u32_t H2K_hthread_cluster(u32_t hthread) {
 	return (hthread / H2K_gp->cluster_hthreads);

kernel/data/globals/globals.ref.c

@@ -69,7 +69,7 @@ void H2K_kg_init(u32_t phys_offset, u32_t multicore_shift, u32_t devpage_offset,
 		have_hvx = (H2K_cfg_table(CFG_TABLE_COPROC_TYPE) & CFG_TABLE_COPROC_TYPE_HVX_MASK) != 0;
 		have_silver = (H2K_cfg_table(CFG_TABLE_COPROC_TYPE) & CFG_TABLE_COPROC_TYPE_SILVER_MASK) != 0;
 		H2K_kg.coproc_contexts = (have_hvx || have_silver ? H2K_cfg_table(CFG_TABLE_COPROC_CONTEXTS) : 0);
-#ifdef CLUSTER_SCHED
+#if CLUSTER_SCHED
 		/* FIXME: need a cfg_table entry for this */
 		H2K_kg.cluster_clusters = (u32_t)(Q6_R_popcount_P(H2K_cfg_table(CFG_TABLE_HTHREADS_MASK)) > 8 ? 4 : 2); // hack
 		H2K_kg.cluster_hthreads = (u32_t)(Q6_R_popcount_P(H2K_cfg_table(CFG_TABLE_HTHREADS_MASK)) / H2K_kg.cluster_clusters);
@@ -144,7 +144,7 @@ void H2K_kg_init(u32_t phys_offset, u32_t multicore_shift, u32_t devpage_offset,
 		H2K_kg.hmx_units = (H2K_cfg_table(CFG_TABLE_HMX_INT8_RATE) != 0);  // exists?
 		H2K_kg.info_boot_flags.boot_have_hmx = (H2K_kg.hmx_units > 0);
 #endif
-#ifdef CLUSTER_SCHED
+#if CLUSTER_SCHED
 		H2K_kg.coproc_max_save = ((H2K_kg.coproc_contexts + H2K_kg.hmx_units) / H2K_kg.cluster_clusters) + (((H2K_kg.coproc_contexts + H2K_kg.hmx_units) % H2K_kg.cluster_clusters) != 0);
 		H2K_kg.coproc_max_save = (H2K_kg.coproc_max < CLUSTER_SCHED_MIN_COPROCS ? CLUSTER_SCHED_MIN_COPROCS : H2K_kg.coproc_max);
 #endif	
@@ -230,7 +230,7 @@ void H2K_kg_init(u32_t phys_offset, u32_t multicore_shift, u32_t devpage_offset,
 
 }
 
-#ifdef CLUSTER_SCHED
+#if CLUSTER_SCHED
 void H2K_cluster_config(void) {
 	u32_t i;
 

kernel/data/readylist/readylist.h

@@ -13,7 +13,6 @@
 #include <globals.h>
 #include <hw.h>
 #include <log.h>
-#include <runlist.h>
 
 /* Get the best ready priority */
 static inline u32_t H2K_ready_best_prio()
@@ -86,7 +85,29 @@ static inline void H2K_ready_remove(H2K_thread_context *thread)
 	if (H2K_gp->ready[prio] == NULL) H2K_ready_clear_prio(prio);
 }
 
-#ifdef CLUSTER_SCHED
+static inline void H2K_ready_append_arm(H2K_thread_context *thread)
+{
+	H2K_ready_append(thread);
+	H2K_set_bestwait(H2K_ready_best_prio());
+}
+
+/* Take the thread and place it in the ready structure,
+ * the first thread to be scheduled at its priority */
+static inline void H2K_ready_insert_arm(H2K_thread_context *thread)
+{
+	H2K_ready_insert(thread);
+	H2K_set_bestwait(H2K_ready_best_prio());
+}
+
+/* Remove a specific thread from the ready list */
+/* The caller guarantees that the thread is actually in the ready list correctly */
+static inline void H2K_ready_remove_arm(H2K_thread_context *thread)
+{
+	H2K_ready_remove(thread);
+	H2K_set_bestwait(H2K_ready_best_prio());
+}
+
+#if CLUSTER_SCHED
 static inline void H2K_update_coprocs(u32_t hthread, u32_t hthread_xe, u32_t hthread_xe2, u32_t hthread_xe3,u32_t head_xe, u32_t head_xe2, u32_t head_xe3) {
 	xex(hthread, head_xe, head_xe2, head_xe3, hthread_xe, hthread_xe2, hthread_xe3);
 	if (hthread_xe) {
@@ -123,7 +144,7 @@ static inline void H2K_update_coprocs(u32_t hthread, u32_t hthread_xe, u32_t hth
 static inline H2K_thread_context *H2K_ready_head(u32_t prio, u32_t hthread) {
 	H2K_thread_context *head = H2K_gp->ready[prio];
 
-#ifdef CLUSTER_SCHED
+#if CLUSTER_SCHED
 	if ((!H2K_gp->cluster_sched) || H2K_gp->coproc_max == -1) {
 		return head;
 	}
@@ -223,7 +244,7 @@ static inline H2K_thread_context *H2K_ready_getbest(u32_t hthread)
 	H2K_log("hthread %d  getbest\n", hthread);
 	prio = H2K_ready_best_prio();
 	if (prio >= MAX_PRIOS) {  // !H2K_ready_any_valid(), go to sleep
-#ifdef CLUSTER_SCHED
+#if CLUSTER_SCHED
 		if (!H2K_gp->cluster_sched) {
 			return NULL;
 		}
@@ -246,7 +267,7 @@ static inline H2K_thread_context *H2K_ready_getbest(u32_t hthread)
 
 	ret = H2K_ready_head(prio, hthread);
 	if (ret != NULL) {
-		H2K_ready_remove(ret);
+		H2K_ready_remove_arm(ret);
 	}
 	return ret;
 }

kernel/data/readylist/readylist.spec

@@ -4,6 +4,26 @@
 
 .. module:: readylist
 
+Overview
+--------
+
+The readylist API is split into two groups:
+
+* **Base functions** (:c:func:`H2K_ready_append`, :c:func:`H2K_ready_insert`,
+  :c:func:`H2K_ready_remove`) — manipulate the ready list without touching the
+  hardware BESTWAIT comparator. Used at mid-reschedule sites where a following
+  :c:func:`H2K_dosched()` call on the same HW thread will re-arm BESTWAIT.
+
+* **Arming variants** (:c:func:`H2K_ready_append_arm`, :c:func:`H2K_ready_insert_arm`,
+  :c:func:`H2K_ready_remove_arm`) — manipulate the ready list AND arm the hardware
+  BESTWAIT comparator with the new best ready priority. Used at wake/sleep-transition
+  sites where the thread is switched to immediately, or at :c:func:`H2K_ready_getbest()`
+  to ensure BESTWAIT is always re-armed after removing the best thread.
+
+The BESTWAIT comparator is a hardware mechanism that raises a reschedule interrupt
+whenever any HW thread's STID.PRIO is strictly worse (higher number) than the armed
+BESTWAIT value. 
+
 H2K_kg.ready and H2K_kg.ready_valids
 ------------------------------------
 
@@ -200,6 +220,81 @@ more elements in the ring, we call :c:func:`H2K_ready_clear_prio()` with the
 thread's priority.
 
 
+H2K_ready_append_arm
+--------------------
+
+.. c:function:: static inline void H2K_ready_append_arm(H2K_thread_context *thread)
+
+	:param thread: the thread to add
+
+Description
+~~~~~~~~~~~
+
+Appends the thread to the ready list and arms the hardware BESTWAIT comparator.
+
+Functionality
+~~~~~~~~~~~~~
+
+Calls :c:func:`H2K_ready_append()` to add the thread to the ready list, then
+calls :c:func:`H2K_set_bestwait()` with the result of :c:func:`H2K_ready_best_prio()`
+to arm the hardware reschedule-interrupt comparator with the best ready priority.
+
+Used at wake/sleep-transition sites where a thread is being placed on the ready 
+list and will be switched to immediately or returned without a following :c:func:`H2K_dosched()`
+call on the same HW thread. The BESTWAIT arm ensures the hardware comparator is active to
+trigger a reschedule interrupt if a running thread's priority becomes worse than
+the newly-ready thread's priority.
+
+
+H2K_ready_insert_arm
+--------------------
+
+.. c:function:: static inline void H2K_ready_insert_arm(H2K_thread_context *thread)
+
+	:param thread: the thread to add
+
+Description
+~~~~~~~~~~~
+
+Inserts the thread to the ready list and arms the hardware BESTWAIT comparator.
+
+Functionality
+~~~~~~~~~~~~~
+
+Calls :c:func:`H2K_ready_insert()` to add the thread to the ready list, then
+calls :c:func:`H2K_set_bestwait()` with the result of :c:func:`H2K_ready_best_prio()`
+to arm the hardware reschedule-interrupt comparator with the best ready priority.
+
+Used at wake/sleep-transition sites where a thread is being inserted at the head
+of the ready list (higher priority within the same priority level) and will be
+switched to immediately without a following :c:func:`H2K_dosched()` call.
+
+
+H2K_ready_remove_arm
+--------------------
+
+.. c:function:: static inline void H2K_ready_remove_arm(H2K_thread_context *thread)
+
+	:param thread: the thread to remove
+
+Description
+~~~~~~~~~~~
+
+Removes the thread from the ready list and arms the hardware BESTWAIT comparator.
+
+Functionality
+~~~~~~~~~~~~~
+
+Calls :c:func:`H2K_ready_remove()` to remove the thread from the ready list, then
+calls :c:func:`H2K_set_bestwait()` with the result of :c:func:`H2K_ready_best_prio()`
+to arm the hardware reschedule-interrupt comparator with the best ready priority.
+
+Used in :c:func:`H2K_ready_getbest()` to ensure BESTWAIT is always re-armed after
+removing the best thread from the ready list. Also used in :c:func:`H2K_resched()`
+to immediately close the disarmed window after a hardware reschedule interrupt
+fires (which resets BESTWAIT to 0x1FF).
+
+
 H2K_ready_getbest
 -----------------
 
@@ -211,7 +306,7 @@ H2K_ready_getbest
 Description
 ~~~~~~~~~~~
 
-Removes the best priority thread from the ready list.
+Removes the best priority thread from the ready list and re-arms BESTWAIT.
 
 Functionality
 ~~~~~~~~~~~~~
@@ -222,7 +317,9 @@ We call :c:func:`H2K_ready_best_prio()` to obtain the priority of the best prior
 
 We then get the thread pointed to by the H2K_kg.ready pointer at the correct priority.
 
-This thread is removed from the ready list by calling :c:func:`H2K_ready_remove()`, and returned.
+This thread is removed from the ready list by calling :c:func:`H2K_ready_remove_arm()`,
+which also re-arms BESTWAIT with the new best priority (or 0x1FF if no threads remain),
+and the thread is returned.
 
 
 
@@ -232,6 +329,9 @@ This thread is removed from the ready list by calling :c:func:`H2K_ready_remove(
 Testing
 -------
 
+Base Functions (H2K_ready_append, H2K_ready_insert, H2K_ready_remove, H2K_ready_getbest)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
 Samples
 ~~~~~~~
 
@@ -264,6 +364,12 @@ Harness
 ~~~~~~~
 
 The readylist module is reasonably self-contained, so the test harness will only
-use the header file and object file.  
+use the header file and object file.
+
 
+BESTWAIT Arming Functions (H2K_ready_append_arm, H2K_ready_insert_arm, H2K_ready_remove_arm)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+The `_arm` variants are tested separately in the H2K_bestwait test suite
+(``kernel/data/readylist/tests/H2K_bestwait/test.c``), which validates the
+hardware BESTWAIT/SCHEDCFG comparator behavior and the arming discipline.

kernel/data/readylist/tests/H2K_bestwait/Makefile

@@ -0,0 +1,7 @@
+STANDALONE=1
+
+OBJS+=test.o
+EXEC=test.elf
+
+include Makefile.inc
+OSLIB=

kernel/sched/lowprio/test/Makefile.inc → kernel/data/readylist/tests/H2K_bestwait/Makefile.inc

@@ -1,7 +1,5 @@
 #  Need to define how to get back to the main H2 dir
-H2DIR=${UPDIR}../../../..
+H2DIR=${UPDIR}../../../../../..
 
 #  Everything else defined here
 include ${H2DIR}/scripts/Makefile.inc.test
-
-