8257588: Make os::_page_sizes a bitmask

src/hotspot/os/aix/os_aix.cpp

@@ -3509,7 +3509,7 @@ void os::init(void) {
 
   // For now UseLargePages is just ignored.
   FLAG_SET_ERGO(bool, UseLargePages, false);
-  _page_sizes[0] = 0;
+  _page_sizes.add(Aix::_page_size);
 
   // debug trace
   trcVerbose("os::vm_page_size %s", describe_pagesize(os::vm_page_size()));

src/hotspot/os/bsd/os_bsd.cpp

@@ -3326,7 +3326,7 @@ void os::init(void) {
   if (Bsd::page_size() == -1) {
     fatal("os_bsd.cpp: os::init: sysconf failed (%s)", os::strerror(errno));
   }
-  init_page_sizes((size_t) Bsd::page_size());
+  _page_sizes.add(Bsd::page_size());
 
   Bsd::initialize_system_info();
 

src/hotspot/os/linux/os_linux.cpp

@@ -4108,9 +4108,7 @@ size_t os::Linux::setup_large_page_size() {
   _large_page_size = Linux::find_large_page_size();
   const size_t default_page_size = (size_t)Linux::page_size();
   if (_large_page_size > default_page_size) {
-    _page_sizes[0] = _large_page_size;
-    _page_sizes[1] = default_page_size;
-    _page_sizes[2] = 0;
+    _page_sizes.add(_large_page_size);
   }
 
   return _large_page_size;
@@ -5538,7 +5536,7 @@ void os::init(void) {
     fatal("os_linux.cpp: os::init: sysconf failed (%s)",
           os::strerror(errno));
   }
-  init_page_sizes((size_t) Linux::page_size());
+  _page_sizes.add(Linux::page_size());
 
   Linux::initialize_system_info();
 

src/hotspot/os/solaris/os_solaris.cpp

@@ -2273,9 +2273,10 @@ size_t os::Solaris::page_size_for_alignment(size_t alignment) {
          SIZE_FORMAT " is not aligned to " SIZE_FORMAT,
          alignment, (size_t) vm_page_size());
 
-  for (int i = 0; _page_sizes[i] != 0; i++) {
-    if (is_aligned(alignment, _page_sizes[i])) {
-      return _page_sizes[i];
+  for (size_t page_size = page_sizes().largest(); page_size != 0;
+       page_size = page_sizes().next_smaller(page_size)) {
+    if (is_aligned(alignment, page_size)) {
+      return page_size;
     }
   }
 
@@ -2787,17 +2788,30 @@ bool os::Solaris::mpss_sanity_check(bool warn, size_t* page_size) {
   }
 
   // Fill the array of page sizes.
-  int n = (*gps_func)(_page_sizes, page_sizes_max);
+  size_t raw_os_page_sizes[os::Solaris::page_sizes_max];
+  int n = (*gps_func)(raw_os_page_sizes, os::Solaris::page_sizes_max);
   assert(n > 0, "Solaris bug?");
 
-  if (n == page_sizes_max) {
+  for (int i = 0; i < n; ++i) {
+    _page_sizes.add(raw_os_page_sizes[i]);
+  }
+
+  if (n == os::Solaris::page_sizes_max) {
     // Add a sentinel value (necessary only if the array was completely filled
     // since it is static (zeroed at initialization)).
-    _page_sizes[--n] = 0;
-    DEBUG_ONLY(warning("increase the size of the os::_page_sizes array.");)
+    raw_os_page_sizes[--n] = 0;
+    DEBUG_ONLY(warning("increase the size of the raw_os_page_sizes array.");)
+  }
+  assert(raw_os_page_sizes[n] == 0, "missing sentinel");
+
+  // Log the initially available page sizes.
+  LogTarget(Info, pagesize) log_available;
+  if (log_available.is_enabled()) {
+    LogStream out(log_available);
+    out.print("available page sizes: ");
+    _page_sizes.print_on(&out);
+    out.cr();
   }
-  assert(_page_sizes[n] == 0, "missing sentinel");
-  trace_page_sizes("available page sizes", _page_sizes, n);
 
   if (n == 1) return false;     // Only one page size available.
 
@@ -2809,25 +2823,38 @@ bool os::Solaris::mpss_sanity_check(bool warn, size_t* page_size) {
   // A better policy could get rid of the 4M limit by taking the sizes of the
   // important VM memory regions (java heap and possibly the code cache) into
   // account.
-  insertion_sort_descending(_page_sizes, n);
+  insertion_sort_descending(raw_os_page_sizes, n);
   const size_t size_limit =
     FLAG_IS_DEFAULT(LargePageSizeInBytes) ? 4 * M : LargePageSizeInBytes;
   int beg;
-  for (beg = 0; beg < n && _page_sizes[beg] > size_limit; ++beg) /* empty */;
+  for (beg = 0; beg < n && raw_os_page_sizes[beg] > size_limit; ++beg) /* empty */;
   const int end = MIN2((int)usable_count, n) - 1;
   for (int cur = 0; cur < end; ++cur, ++beg) {
-    _page_sizes[cur] = _page_sizes[beg];
+    raw_os_page_sizes[cur] = raw_os_page_sizes[beg];
   }
-  _page_sizes[end] = vm_page_size();
-  _page_sizes[end + 1] = 0;
+  raw_os_page_sizes[end] = vm_page_size();
+  raw_os_page_sizes[end + 1] = 0;
 
-  if (_page_sizes[end] > _page_sizes[end - 1]) {
+  if (raw_os_page_sizes[end] > raw_os_page_sizes[end - 1]) {
     // Default page size is not the smallest; sort again.
-    insertion_sort_descending(_page_sizes, end + 1);
+    insertion_sort_descending(raw_os_page_sizes, end + 1);
+  }
+
+  _page_sizes = os::PageSizes();
+  for (int i = 0; i < (end + 1); ++i) {
+    _page_sizes.add(raw_os_page_sizes[i]);
   }
-  *page_size = _page_sizes[0];
 
-  trace_page_sizes("usable page sizes", _page_sizes, end + 1);
+  *page_size = raw_os_page_sizes[0];
+
+  // Log the final usable page sizes.
+  LogTarget(Info, pagesize) log_usable;
+  if (log_usable.is_enabled()) {
+    LogStream out(log_usable);
+    out.print("usable page sizes: ");
+    _page_sizes.print_on(&out);
+    out.cr();
+  }
   return true;
 }
 
@@ -2842,12 +2869,7 @@ void os::large_page_init() {
 }
 
 bool os::Solaris::is_valid_page_size(size_t bytes) {
-  for (int i = 0; _page_sizes[i] != 0; i++) {
-    if (_page_sizes[i] == bytes) {
-      return true;
-    }
-  }
-  return false;
+  return _page_sizes.contains(bytes);
 }
 
 bool os::Solaris::setup_large_pages(caddr_t start, size_t bytes, size_t align) {
@@ -4162,7 +4184,7 @@ void os::init(void) {
   if (page_size == -1) {
     fatal("os_solaris.cpp: os::init: sysconf failed (%s)", os::strerror(errno));
   }
-  init_page_sizes((size_t) page_size);
+  _page_sizes.add(page_size);
 
   Solaris::initialize_system_info();
 

src/hotspot/os/solaris/os_solaris.hpp

@@ -64,6 +64,7 @@ class Solaris {
   static int_fnP_cond_tP_i_vP _cond_init;
   static int_fnP_cond_tP _cond_destroy;
   static int _cond_scope;
+  static const int page_sizes_max = 9;
 
   static bool _synchronization_initialized;
 

src/hotspot/os/windows/os_windows.cpp

@@ -3056,9 +3056,7 @@ void os::large_page_init() {
 
   const size_t default_page_size = (size_t) vm_page_size();
   if (success && _large_page_size > default_page_size) {
-    _page_sizes[0] = _large_page_size;
-    _page_sizes[1] = default_page_size;
-    _page_sizes[2] = 0;
+    _page_sizes.add(_large_page_size);
   }
 
   cleanup_after_large_page_init();
@@ -4192,7 +4190,7 @@ void os::init(void) {
 
   win32::initialize_system_info();
   win32::setmode_streams();
-  init_page_sizes((size_t) win32::vm_page_size());
+  _page_sizes.add(win32::vm_page_size());
 
   // This may be overridden later when argument processing is done.
   FLAG_SET_ERGO(bool, UseLargePagesIndividualAllocation, false);

src/hotspot/share/runtime/os.cpp

@@ -64,8 +64,10 @@
 #include "services/nmtCommon.hpp"
 #include "services/threadService.hpp"
 #include "utilities/align.hpp"
+#include "utilities/count_trailing_zeros.hpp"
 #include "utilities/defaultStream.hpp"
 #include "utilities/events.hpp"
+#include "utilities/globalDefinitions.hpp"
 
 # include <signal.h>
 # include <errno.h>
@@ -77,7 +79,7 @@ uintptr_t         os::_serialize_page_mask = 0;
 volatile unsigned int os::_rand_seed      = 1;
 int               os::_processor_count    = 0;
 int               os::_initial_active_processor_count = 0;
-size_t            os::_page_sizes[os::page_sizes_max];
+os::PageSizes     os::_page_sizes;
 
 #ifndef PRODUCT
 julong os::num_mallocs = 0;         // # of calls to malloc/realloc
@@ -1490,8 +1492,8 @@ size_t os::page_size_for_region(size_t region_size, size_t min_pages, bool must_
   if (UseLargePages) {
     const size_t max_page_size = region_size / min_pages;
 
-    for (size_t i = 0; _page_sizes[i] != 0; ++i) {
-      const size_t page_size = _page_sizes[i];
+    for (size_t page_size = page_sizes().largest(); page_size != 0;
+         page_size = page_sizes().next_smaller(page_size)) {
       if (page_size <= max_page_size) {
         if (!must_be_aligned || is_aligned(region_size, page_size)) {
           return page_size;
@@ -1636,19 +1638,6 @@ const char* os::errno_name(int e) {
   return errno_to_string(e, true);
 }
 
-void os::trace_page_sizes(const char* str, const size_t* page_sizes, int count) {
-  LogTarget(Info, pagesize) log;
-  if (log.is_enabled()) {
-    LogStream out(log);
-
-    out.print("%s: ", str);
-    for (int i = 0; i < count; ++i) {
-      out.print(" " SIZE_FORMAT, page_sizes[i]);
-    }
-    out.cr();
-  }
-}
-
 #define trace_page_size_params(size) byte_size_in_exact_unit(size), exact_unit_for_byte_size(size)
 
 void os::trace_page_sizes(const char* str,
@@ -1915,6 +1904,76 @@ void os::realign_memory(char *addr, size_t bytes, size_t alignment_hint) {
   pd_realign_memory(addr, bytes, alignment_hint);
 }
 
+
+////// Implementation of PageSizes
+
+void os::PageSizes::add(size_t page_size) {
+  assert(is_power_of_2(page_size), "page_size must be a power of 2: " SIZE_FORMAT_HEX, page_size);
+  _v |= page_size;
+}
+
+bool os::PageSizes::contains(size_t page_size) const {
+  assert(is_power_of_2(page_size), "page_size must be a power of 2: " SIZE_FORMAT_HEX, page_size);
+  return (_v & page_size) != 0;
+}
+
+size_t os::PageSizes::next_smaller(size_t page_size) const {
+  assert(is_power_of_2(page_size), "page_size must be a power of 2: " SIZE_FORMAT_HEX, page_size);
+  size_t v2 = _v & (page_size - 1);
+  if (v2 == 0) {
+    return 0;
+  }
+  return round_down_power_of_2_size_t(v2);
+}
+
+size_t os::PageSizes::next_larger(size_t page_size) const {
+  assert(is_power_of_2(page_size), "page_size must be a power of 2: " SIZE_FORMAT_HEX, page_size);
+  if (page_size == max_power_of_2_size_t()) { // Shift by 32/64 would be UB
+    return 0;
+  }
+  // Remove current and smaller page sizes
+  size_t v2 = _v & ~(page_size + (page_size - 1));
+  if (v2 == 0) {
+    return 0;
+  }
+  return (size_t)1 << count_trailing_zeros(v2);
+}
+
+size_t os::PageSizes::largest() const {
+  const size_t max = max_power_of_2_size_t();
+  if (contains(max)) {
+    return max;
+  }
+  return next_smaller(max);
+}
+
+size_t os::PageSizes::smallest() const {
+  // Strictly speaking the set should not contain sizes < os::vm_page_size().
+  // But this is not enforced.
+  return next_larger(1);
+}
+
+void os::PageSizes::print_on(outputStream* st) const {
+  bool first = true;
+  for (size_t sz = smallest(); sz != 0; sz = next_larger(sz)) {
+    if (first) {
+      first = false;
+    } else {
+      st->print_raw(", ");
+    }
+    if (sz < M) {
+      st->print(SIZE_FORMAT "k", sz / K);
+    } else if (sz < G) {
+      st->print(SIZE_FORMAT "M", sz / M);
+    } else {
+      st->print(SIZE_FORMAT "G", sz / G);
+    }
+  }
+  if (first) {
+    st->print("empty");
+  }
+}
+
 #ifndef _WINDOWS
 /* try to switch state from state "from" to state "to"
  * returns the state set after the method is complete

src/hotspot/share/runtime/os.hpp

@@ -100,21 +100,32 @@ class os: AllStatic {
   friend class JVMCIVMStructs;
   friend class MallocTracker;
  public:
-  enum { page_sizes_max = 9 }; // Size of _page_sizes array (8 plus a sentinel)
+
+  // A simple value class holding a set of page sizes (similar to sigset_t)
+  class PageSizes {
+    size_t _v; // actually a bitmap.
+  public:
+    PageSizes() : _v(0) {}
+    void add(size_t pagesize);
+    bool contains(size_t pagesize) const;
+    // Given a page size, return the next smaller page size in this set, or 0.
+    size_t next_smaller(size_t pagesize) const;
+    // Given a page size, return the next larger page size in this set, or 0.
+    size_t next_larger(size_t pagesize) const;
+    // Returns the largest page size in this set, or 0 if set is empty.
+    size_t largest() const;
+    // Returns the smallest page size in this set, or 0 if set is empty.
+    size_t smallest() const;
+    // Prints one line of comma separated, human readable page sizes, "empty" if empty.
+    void print_on(outputStream* st) const;
+  };
 
  private:
   static OSThread*          _starting_thread;
   static address            _polling_page;
   static volatile int32_t * _mem_serialize_page;
   static uintptr_t          _serialize_page_mask;
- public:
-  static size_t             _page_sizes[page_sizes_max];
-
- private:
-  static void init_page_sizes(size_t default_page_size) {
-    _page_sizes[0] = default_page_size;
-    _page_sizes[1] = 0; // sentinel
-  }
+  static PageSizes          _page_sizes;
 
 
   MACOS_ONLY(static char*  pd_reserve_memory(size_t bytes, char* addr = 0,
@@ -297,6 +308,10 @@ class os: AllStatic {
   // Return the default page size.
   static int    vm_page_size();
 
+  // The set of page sizes which the VM is allowed to use (may be a subset of
+  //  the page sizes actually available on the platform).
+  static const PageSizes& page_sizes() { return _page_sizes; }
+
   // Returns the page size to use for a region of memory.
   // region_size / min_pages will always be greater than or equal to the
   // returned value. The returned value will divide region_size.
@@ -308,10 +323,7 @@ class os: AllStatic {
   static size_t page_size_for_region_unaligned(size_t region_size, size_t min_pages);
 
   // Return the largest page size that can be used
-  static size_t max_page_size() {
-    // The _page_sizes array is sorted in descending order.
-    return _page_sizes[0];
-  }
+  static size_t max_page_size() { return page_sizes().largest(); }
 
   // Return a lower bound for page sizes. Also works before os::init completed.
   static size_t min_page_size() { return 4 * K; }