From 37ca9024ef59d99cae0bd7e25b2e6d3c1e085f97 Mon Sep 17 00:00:00 2001 From: Thomas Stuefe Date: Mon, 17 Jul 2023 04:56:10 +0000 Subject: [PATCH 1/2] 8310233: Fix THP detection on Linux Reviewed-by: jsjolen, dholmes --- src/hotspot/os/linux/hugepages.cpp | 228 ++++++++++++++++++ src/hotspot/os/linux/hugepages.hpp | 110 +++++++++ src/hotspot/os/linux/os_linux.cpp | 204 ++++++---------- src/hotspot/os/linux/os_linux.hpp | 6 - .../share/utilities/globalDefinitions.hpp | 3 + .../runtime/os/HugePageConfiguration.java | 221 +++++++++++++++++ .../jtreg/runtime/os/HugePageDetection.java | 75 ++++++ 7 files changed, 716 insertions(+), 131 deletions(-) create mode 100644 src/hotspot/os/linux/hugepages.cpp create mode 100644 src/hotspot/os/linux/hugepages.hpp create mode 100644 test/hotspot/jtreg/runtime/os/HugePageConfiguration.java create mode 100644 test/hotspot/jtreg/runtime/os/HugePageDetection.java diff --git a/src/hotspot/os/linux/hugepages.cpp b/src/hotspot/os/linux/hugepages.cpp new file mode 100644 index 00000000000..299c27a8c1d --- /dev/null +++ b/src/hotspot/os/linux/hugepages.cpp @@ -0,0 +1,228 @@ +/* + * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "hugepages.hpp" + +#include "logging/log.hpp" +#include "logging/logStream.hpp" +#include "runtime/os.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/ostream.hpp" + +#include + +StaticHugePageSupport::StaticHugePageSupport() : + _initialized(false), _pagesizes(), _default_hugepage_size(SIZE_MAX) {} + +os::PageSizes StaticHugePageSupport::pagesizes() const { + assert(_initialized, "Not initialized"); + return _pagesizes; +} + +size_t StaticHugePageSupport::default_hugepage_size() const { + assert(_initialized, "Not initialized"); + return _default_hugepage_size; +} + +// Scan /proc/meminfo and return value of Hugepagesize +static size_t scan_default_hugepagesize() { + size_t pagesize = 0; + + // large_page_size on Linux is used to round up heap size. x86 uses either + // 2M or 4M page, depending on whether PAE (Physical Address Extensions) + // mode is enabled. AMD64/EM64T uses 2M page in 64bit mode. IA64 can use + // page as large as 1G. + // + // Here we try to figure out page size by parsing /proc/meminfo and looking + // for a line with the following format: + // Hugepagesize: 2048 kB + // + // If we can't determine the value (e.g. /proc is not mounted, or the text + // format has been changed), we'll set largest page size to 0 + + FILE *fp = os::fopen("/proc/meminfo", "r"); + if (fp) { + while (!feof(fp)) { + int x = 0; + char buf[16]; + if (fscanf(fp, "Hugepagesize: %d", &x) == 1) { + if (x && fgets(buf, sizeof(buf), fp) && strcmp(buf, " kB\n") == 0) { + pagesize = x * K; + break; + } + } else { + // skip to next line + for (;;) { + int ch = fgetc(fp); + if (ch == EOF || ch == (int)'\n') break; + } + } + } + fclose(fp); + } + + return pagesize; +} + +// Given a file that contains a single (integral) number, return that number in (*out) and true; +// in case of an error, return false. +static bool read_number_file(const char* file, size_t* out) { + FILE* f = ::fopen(file, "r"); + bool rc = false; + if (f != nullptr) { + uint64_t i = 0; + if (::fscanf(f, SIZE_FORMAT, out) == 1) { + rc = true; + } + ::fclose(f); + } + return rc; +} + +static const char* const sys_hugepages = "/sys/kernel/mm/hugepages"; + +// Scan all directories in /sys/kernel/mm/hugepages/hugepages-xxxx +// to discover the available page sizes +static os::PageSizes scan_hugepages() { + + os::PageSizes pagesizes; + + DIR *dir = opendir(sys_hugepages); + + struct dirent *entry; + size_t pagesize; + while ((entry = readdir(dir)) != nullptr) { + if (entry->d_type == DT_DIR && + sscanf(entry->d_name, "hugepages-%zukB", &pagesize) == 1) { + // The kernel is using kB, hotspot uses bytes + // Add each found Large Page Size to page_sizes + pagesize *= K; + pagesizes.add(pagesize); + } + } + closedir(dir); + + return pagesizes; +} + +void StaticHugePageSupport::print_on(outputStream* os) { + if (_initialized) { + os->print_cr("Static hugepage support:"); + for (size_t s = _pagesizes.smallest(); s != 0; s = _pagesizes.next_larger(s)) { + os->print_cr(" hugepage size: " EXACTFMT, EXACTFMTARGS(s)); + } + os->print_cr(" default hugepage size: " EXACTFMT, EXACTFMTARGS(_default_hugepage_size)); + } else { + os->print_cr(" unknown."); + } +} + +void StaticHugePageSupport::scan_os() { + _pagesizes = scan_hugepages(); + _default_hugepage_size = scan_default_hugepagesize(); + assert(_pagesizes.contains(_default_hugepage_size), + "Unexpected configuration: default pagesize (" SIZE_FORMAT ") " + "has no associated directory in /sys/kernel/mm/hugepages..", _default_hugepage_size); + _initialized = true; + LogTarget(Info, pagesize) lt; + if (lt.is_enabled()) { + LogStream ls(lt); + print_on(&ls); + } +} + +THPSupport::THPSupport() : + _initialized(false), _mode(THPMode::never), _pagesize(SIZE_MAX) {} + + +THPMode THPSupport::mode() const { + assert(_initialized, "Not initialized"); + return _mode; +} + +size_t THPSupport::pagesize() const { + assert(_initialized, "Not initialized"); + return _pagesize; +} + +void THPSupport::scan_os() { + // Scan /sys/kernel/mm/transparent_hugepage/enabled + // see mm/huge_memory.c + _mode = THPMode::never; + const char* filename = "/sys/kernel/mm/transparent_hugepage/enabled"; + FILE* f = ::fopen(filename, "r"); + if (f != nullptr) { + char buf[64]; + char* s = fgets(buf, sizeof(buf), f); + assert(s == buf, "Should have worked"); + if (::strstr(buf, "[madvise]") != nullptr) { + _mode = THPMode::madvise; + } else if (::strstr(buf, "[always]") != nullptr) { + _mode = THPMode::always; + } else { + assert(::strstr(buf, "[never]") != nullptr, "Weird content of %s: %s", filename, buf); + } + fclose(f); + } + + // Scan large page size for THP from hpage_pmd_size + _pagesize = 0; + if (read_number_file("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size", &_pagesize)) { + assert(_pagesize > 0, "Expected"); + } + _initialized = true; + + LogTarget(Info, pagesize) lt; + if (lt.is_enabled()) { + LogStream ls(lt); + print_on(&ls); + } +} + +void THPSupport::print_on(outputStream* os) { + if (_initialized) { + os->print_cr("Transparent hugepage (THP) support:"); + os->print_cr(" THP mode: %s", + (_mode == THPMode::always ? "always" : (_mode == THPMode::never ? "never" : "madvise"))); + os->print_cr(" THP pagesize: " EXACTFMT, EXACTFMTARGS(_pagesize)); + } else { + os->print_cr(" unknown."); + } +} + +StaticHugePageSupport HugePages::_static_hugepage_support; +THPSupport HugePages::_thp_support; + +void HugePages::initialize() { + _static_hugepage_support.scan_os(); + _thp_support.scan_os(); +} + +void HugePages::print_on(outputStream* os) { + _static_hugepage_support.print_on(os); + _thp_support.print_on(os); +} diff --git a/src/hotspot/os/linux/hugepages.hpp b/src/hotspot/os/linux/hugepages.hpp new file mode 100644 index 00000000000..2f378fdc224 --- /dev/null +++ b/src/hotspot/os/linux/hugepages.hpp @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef OS_LINUX_HUGEPAGES_HPP +#define OS_LINUX_HUGEPAGES_HPP + +#include "memory/allStatic.hpp" +#include "runtime/os.hpp" // for os::PageSizes +#include "utilities/globalDefinitions.hpp" + +class outputStream; + +// Header contains the interface that reads OS information about +// available hugepage support: +// - class StaticHugePageSupport - about static (non-THP) hugepages +// - class THPSupport - about transparent huge pages +// and: +// - class HugePages - a static umbrella wrapper + +// Information about static (non-thp) hugepages +class StaticHugePageSupport { + bool _initialized; + + // All supported hugepage sizes (sizes for which entries exist + // in /sys/kernel/mm/hugepages/hugepage-xxx) + os::PageSizes _pagesizes; + + // Contains the default hugepage. The "default hugepage size" is the one that + // - is marked in /proc/meminfo as "Hugepagesize" + // - is the size one gets when using mmap(MAP_HUGETLB) when omitting size specifiers like MAP_HUGE_SHIFT) + size_t _default_hugepage_size; + +public: + StaticHugePageSupport(); + + void scan_os(); + + os::PageSizes pagesizes() const; + size_t default_hugepage_size() const; + void print_on(outputStream* os); +}; + +enum class THPMode { always, never, madvise }; + +// 2) for transparent hugepages +class THPSupport { + bool _initialized; + + // See /sys/kernel/mm/transparent_hugepages/enabled + THPMode _mode; + + // Contains the THP page size + size_t _pagesize; + +public: + + THPSupport(); + + // Queries the OS, fills in object + void scan_os(); + + THPMode mode() const; + size_t pagesize() const; + void print_on(outputStream* os); +}; + +// Umbrella static interface +class HugePages : public AllStatic { + + static StaticHugePageSupport _static_hugepage_support; + static THPSupport _thp_support; + +public: + + static const StaticHugePageSupport& static_info() { return _static_hugepage_support; } + static const THPSupport& thp_info() { return _thp_support; } + + static size_t default_static_hugepage_size() { return _static_hugepage_support.default_hugepage_size(); } + static bool supports_static_hugepages() { return default_static_hugepage_size() > 0; } + static THPMode thp_mode() { return _thp_support.mode(); } + static bool supports_thp() { return thp_mode() == THPMode::madvise || thp_mode() == THPMode::always; } + static size_t thp_pagesize() { return _thp_support.pagesize(); } + + static void initialize(); + static void print_on(outputStream* os); +}; + +#endif // OS_LINUX_HUGEPAGES_HPP diff --git a/src/hotspot/os/linux/os_linux.cpp b/src/hotspot/os/linux/os_linux.cpp index 82c6b511b62..1ea989281c2 100644 --- a/src/hotspot/os/linux/os_linux.cpp +++ b/src/hotspot/os/linux/os_linux.cpp @@ -29,6 +29,7 @@ #include "code/vtableStubs.hpp" #include "compiler/compileBroker.hpp" #include "compiler/disassembler.hpp" +#include "hugepages.hpp" #include "interpreter/interpreter.hpp" #include "jvm.h" #include "jvmtifiles/jvmti.h" @@ -170,7 +171,6 @@ pthread_t os::Linux::_main_thread; bool os::Linux::_supports_fast_thread_cpu_time = false; const char * os::Linux::_libc_version = nullptr; const char * os::Linux::_libpthread_version = nullptr; -size_t os::Linux::_default_large_page_size = 0; #ifdef __GLIBC__ // We want to be buildable and runnable on older and newer glibcs, so resolve both @@ -933,7 +933,7 @@ bool os::create_thread(Thread* thread, ThreadType thr_type, // Add an additional page to the stack size to reduce its chances of getting large page aligned // so that the stack does not get backed by a transparent huge page. - size_t default_large_page_size = os::Linux::default_large_page_size(); + size_t default_large_page_size = HugePages::default_static_hugepage_size(); if (default_large_page_size != 0 && stack_size >= default_large_page_size && is_aligned(stack_size, default_large_page_size)) { @@ -3556,7 +3556,7 @@ bool os::Linux::transparent_huge_pages_sanity_check(bool warn, } int os::Linux::hugetlbfs_page_size_flag(size_t page_size) { - if (page_size != default_large_page_size()) { + if (page_size != HugePages::default_static_hugepage_size()) { return (exact_log2(page_size) << MAP_HUGE_SHIFT); } return 0; @@ -3664,79 +3664,6 @@ static void set_coredump_filter(CoredumpFilterBit bit) { static size_t _large_page_size = 0; -static size_t scan_default_large_page_size() { - size_t default_large_page_size = 0; - - // large_page_size on Linux is used to round up heap size. x86 uses either - // 2M or 4M page, depending on whether PAE (Physical Address Extensions) - // mode is enabled. AMD64/EM64T uses 2M page in 64bit mode. IA64 can use - // page as large as 1G. - // - // Here we try to figure out page size by parsing /proc/meminfo and looking - // for a line with the following format: - // Hugepagesize: 2048 kB - // - // If we can't determine the value (e.g. /proc is not mounted, or the text - // format has been changed), we'll set largest page size to 0 - - FILE *fp = os::fopen("/proc/meminfo", "r"); - if (fp) { - while (!feof(fp)) { - int x = 0; - char buf[16]; - if (fscanf(fp, "Hugepagesize: %d", &x) == 1) { - if (x && fgets(buf, sizeof(buf), fp) && strcmp(buf, " kB\n") == 0) { - default_large_page_size = x * K; - break; - } - } else { - // skip to next line - for (;;) { - int ch = fgetc(fp); - if (ch == EOF || ch == (int)'\n') break; - } - } - } - fclose(fp); - } - - return default_large_page_size; -} - -static os::PageSizes scan_multiple_page_support() { - // Scan /sys/kernel/mm/hugepages - // to discover the available page sizes - const char* sys_hugepages = "/sys/kernel/mm/hugepages"; - os::PageSizes page_sizes; - - DIR *dir = opendir(sys_hugepages); - - struct dirent *entry; - size_t page_size; - while ((entry = readdir(dir)) != nullptr) { - if (entry->d_type == DT_DIR && - sscanf(entry->d_name, "hugepages-%zukB", &page_size) == 1) { - // The kernel is using kB, hotspot uses bytes - // Add each found Large Page Size to page_sizes - page_sizes.add(page_size * K); - } - } - closedir(dir); - - LogTarget(Debug, pagesize) lt; - if (lt.is_enabled()) { - LogStream ls(lt); - ls.print("Large Page sizes: "); - page_sizes.print_on(&ls); - } - - return page_sizes; -} - -size_t os::Linux::default_large_page_size() { - return _default_large_page_size; -} - void warn_no_large_pages_configured() { if (!FLAG_IS_DEFAULT(UseLargePages)) { log_warning(pagesize)("UseLargePages disabled, no large pages configured and available on the system."); @@ -3789,10 +3716,29 @@ bool os::Linux::setup_large_page_type(size_t page_size) { return false; } +struct LargePageInitializationLoggerMark { + ~LargePageInitializationLoggerMark() { + LogTarget(Info, pagesize) lt; + if (lt.is_enabled()) { + LogStream ls(lt); + if (UseLargePages) { + ls.print_cr("UseLargePages=1, UseTransparentHugePages=%d, UseHugeTLBFS=%d, UseSHM=%d", + UseTransparentHugePages, UseHugeTLBFS, UseSHM); + ls.print("Large page support enabled. Usable page sizes: "); + os::page_sizes().print_on(&ls); + ls.print_cr(". Default large page size: " EXACTFMT ".", EXACTFMTARGS(os::large_page_size())); + } else { + ls.print("Large page support disabled."); + } + } + } +}; + void os::large_page_init() { - // Always initialize the default large page size even if large pages are not being used. - size_t default_large_page_size = scan_default_large_page_size(); - os::Linux::_default_large_page_size = default_large_page_size; + LargePageInitializationLoggerMark logger; + + // Query OS information first. + HugePages::initialize(); // 1) Handle the case where we do not want to use huge pages if (!UseLargePages && @@ -3813,7 +3759,8 @@ void os::large_page_init() { } // 2) check if large pages are configured - if (default_large_page_size == 0) { + if ( ( UseTransparentHugePages && HugePages::supports_thp() == false) || + (!UseTransparentHugePages && HugePages::supports_static_hugepages() == false) ) { // No large pages configured, return. warn_no_large_pages_configured(); UseLargePages = false; @@ -3822,57 +3769,64 @@ void os::large_page_init() { UseSHM = false; return; } - os::PageSizes all_large_pages = scan_multiple_page_support(); - - // 3) Consistency check and post-processing - - // It is unclear if /sys/kernel/mm/hugepages/ and /proc/meminfo could disagree. Manually - // re-add the default page size to the list of page sizes to be sure. - all_large_pages.add(default_large_page_size); - - // Check LargePageSizeInBytes matches an available page size and if so set _large_page_size - // using LargePageSizeInBytes as the maximum allowed large page size. If LargePageSizeInBytes - // doesn't match an available page size set _large_page_size to default_large_page_size - // and use it as the maximum. - if (FLAG_IS_DEFAULT(LargePageSizeInBytes) || - LargePageSizeInBytes == 0 || - LargePageSizeInBytes == default_large_page_size) { - _large_page_size = default_large_page_size; - log_info(pagesize)("Using the default large page size: " SIZE_FORMAT "%s", - byte_size_in_exact_unit(_large_page_size), - exact_unit_for_byte_size(_large_page_size)); + + if (UseTransparentHugePages) { + // In THP mode: + // - os::large_page_size() is the *THP page size* + // - os::pagesizes() has two members, the THP page size and the system page size + assert(HugePages::supports_thp() && HugePages::thp_pagesize() > 0, "Missing OS info"); + _large_page_size = HugePages::thp_pagesize(); + _page_sizes.add(_large_page_size); + _page_sizes.add(os::vm_page_size()); + } else { - if (all_large_pages.contains(LargePageSizeInBytes)) { - _large_page_size = LargePageSizeInBytes; - log_info(pagesize)("Overriding default large page size (" SIZE_FORMAT "%s) " - "using LargePageSizeInBytes: " SIZE_FORMAT "%s", - byte_size_in_exact_unit(default_large_page_size), - exact_unit_for_byte_size(default_large_page_size), - byte_size_in_exact_unit(_large_page_size), - exact_unit_for_byte_size(_large_page_size)); - } else { + + // In static hugepage mode: + // - os::large_page_size() is the default static hugepage size (/proc/meminfo "Hugepagesize") + // - os::pagesizes() contains all hugepage sizes the kernel supports, regardless whether there + // are pages configured in the pool or not (from /sys/kernel/hugepages/hugepage-xxxx ...) + os::PageSizes all_large_pages = HugePages::static_info().pagesizes(); + const size_t default_large_page_size = HugePages::default_static_hugepage_size(); + + // 3) Consistency check and post-processing + + // Check LargePageSizeInBytes matches an available page size and if so set _large_page_size + // using LargePageSizeInBytes as the maximum allowed large page size. If LargePageSizeInBytes + // doesn't match an available page size set _large_page_size to default_large_page_size + // and use it as the maximum. + if (FLAG_IS_DEFAULT(LargePageSizeInBytes) || + LargePageSizeInBytes == 0 || + LargePageSizeInBytes == default_large_page_size) { _large_page_size = default_large_page_size; - log_info(pagesize)("LargePageSizeInBytes is not a valid large page size (" SIZE_FORMAT "%s) " - "using the default large page size: " SIZE_FORMAT "%s", - byte_size_in_exact_unit(LargePageSizeInBytes), - exact_unit_for_byte_size(LargePageSizeInBytes), + log_info(pagesize)("Using the default large page size: " SIZE_FORMAT "%s", byte_size_in_exact_unit(_large_page_size), exact_unit_for_byte_size(_large_page_size)); + } else { + if (all_large_pages.contains(LargePageSizeInBytes)) { + _large_page_size = LargePageSizeInBytes; + log_info(pagesize)("Overriding default large page size (" SIZE_FORMAT "%s) " + "using LargePageSizeInBytes: " SIZE_FORMAT "%s", + byte_size_in_exact_unit(default_large_page_size), + exact_unit_for_byte_size(default_large_page_size), + byte_size_in_exact_unit(_large_page_size), + exact_unit_for_byte_size(_large_page_size)); + } else { + _large_page_size = default_large_page_size; + log_info(pagesize)("LargePageSizeInBytes is not a valid large page size (" SIZE_FORMAT "%s) " + "using the default large page size: " SIZE_FORMAT "%s", + byte_size_in_exact_unit(LargePageSizeInBytes), + exact_unit_for_byte_size(LargePageSizeInBytes), + byte_size_in_exact_unit(_large_page_size), + exact_unit_for_byte_size(_large_page_size)); + } } - } - // Populate _page_sizes with large page sizes less than or equal to - // _large_page_size. - for (size_t page_size = _large_page_size; page_size != 0; - page_size = all_large_pages.next_smaller(page_size)) { - _page_sizes.add(page_size); - } - - LogTarget(Info, pagesize) lt; - if (lt.is_enabled()) { - LogStream ls(lt); - ls.print("Usable page sizes: "); - _page_sizes.print_on(&ls); + // Populate _page_sizes with large page sizes less than or equal to + // _large_page_size. + for (size_t page_size = _large_page_size; page_size != 0; + page_size = all_large_pages.next_smaller(page_size)) { + _page_sizes.add(page_size); + } } // Now determine the type of large pages to use: diff --git a/src/hotspot/os/linux/os_linux.hpp b/src/hotspot/os/linux/os_linux.hpp index a7cf69f3164..ace7e4ab2dd 100644 --- a/src/hotspot/os/linux/os_linux.hpp +++ b/src/hotspot/os/linux/os_linux.hpp @@ -49,8 +49,6 @@ class os::Linux { static GrowableArray* _cpu_to_node; static GrowableArray* _nindex_to_node; - static size_t _default_large_page_size; - static julong available_memory_in_container(); protected: @@ -77,10 +75,6 @@ class os::Linux { static GrowableArray* cpu_to_node() { return _cpu_to_node; } static GrowableArray* nindex_to_node() { return _nindex_to_node; } - static size_t default_large_page_size(); - static size_t scan_default_large_page_size(); - static os::PageSizes scan_multiple_page_support(); - static bool setup_large_page_type(size_t page_size); static bool transparent_huge_pages_sanity_check(bool warn, size_t pages_size); static bool hugetlbfs_sanity_check(bool warn, size_t page_size); diff --git a/src/hotspot/share/utilities/globalDefinitions.hpp b/src/hotspot/share/utilities/globalDefinitions.hpp index a1547e7ed43..762020741db 100644 --- a/src/hotspot/share/utilities/globalDefinitions.hpp +++ b/src/hotspot/share/utilities/globalDefinitions.hpp @@ -409,6 +409,9 @@ inline size_t byte_size_in_exact_unit(size_t s) { return s; } +#define EXACTFMT SIZE_FORMAT "%s" +#define EXACTFMTARGS(s) byte_size_in_exact_unit(s), exact_unit_for_byte_size(s) + // Memory size transition formatting. #define HEAP_CHANGE_FORMAT "%s: " SIZE_FORMAT "K(" SIZE_FORMAT "K)->" SIZE_FORMAT "K(" SIZE_FORMAT "K)" diff --git a/test/hotspot/jtreg/runtime/os/HugePageConfiguration.java b/test/hotspot/jtreg/runtime/os/HugePageConfiguration.java new file mode 100644 index 00000000000..a0590b7739d --- /dev/null +++ b/test/hotspot/jtreg/runtime/os/HugePageConfiguration.java @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, Red Hat Inc. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +import jdk.test.lib.process.OutputAnalyzer; + +import java.io.*; +import java.util.Set; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +class HugePageConfiguration { + + Set _staticHugePageSizes; + long _staticDefaultHugePageSize; + + enum THPMode {always, never, madvise, unknown} + THPMode _thpMode; + long _thpPageSize; + + public Set getStaticHugePageSizes() { + return _staticHugePageSizes; + } + + public long getStaticDefaultHugePageSize() { + return _staticDefaultHugePageSize; + } + + public THPMode getThpMode() { + return _thpMode; + } + + public long getThpPageSize() { + return _thpPageSize; + } + + public HugePageConfiguration(Set _staticHugePageSizes, long _staticDefaultHugePageSize, THPMode _thpMode, long _thpPageSize) { + this._staticHugePageSizes = _staticHugePageSizes; + this._staticDefaultHugePageSize = _staticDefaultHugePageSize; + this._thpMode = _thpMode; + this._thpPageSize = _thpPageSize; + } + + @Override + public String toString() { + return "Configuration{" + + "_staticHugePageSizes=" + _staticHugePageSizes + + ", _staticDefaultHugePageSize=" + _staticDefaultHugePageSize + + ", _thpMode=" + _thpMode + + ", _thpPageSize=" + _thpPageSize + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + HugePageConfiguration that = (HugePageConfiguration) o; + return _staticDefaultHugePageSize == that._staticDefaultHugePageSize && _thpPageSize == that._thpPageSize && Objects.equals(_staticHugePageSizes, that._staticHugePageSizes) && _thpMode == that._thpMode; + } + + @Override + public int hashCode() { + return Objects.hash(_staticHugePageSizes, _staticDefaultHugePageSize, _thpMode, _thpPageSize); + } + + private static long readDefaultHugePageSizeFromOS() { + Pattern pat = Pattern.compile("Hugepagesize: *(\\d+) +kB"); + long result = 0; + try (Scanner scanner = new Scanner(new File("/proc/meminfo"))) { + while (scanner.hasNextLine()) { + Matcher mat = pat.matcher(scanner.nextLine()); + if (mat.matches()) { + scanner.close(); + return Long.parseLong(mat.group(1)) * 1024; + } + } + } catch (FileNotFoundException e) { + System.out.println("Could not open /proc/meminfo"); + } + return 0; + } + + private static Set readSupportedHugePagesFromOS() { + TreeSet pagesizes = new TreeSet<>(); + Pattern pat = Pattern.compile("hugepages-(\\d+)kB"); + File[] subdirs = new File("/sys/kernel/mm/hugepages").listFiles(); + if (subdirs != null) { + for (File f : subdirs) { + String name = f.getName(); + Matcher mat = pat.matcher(name); + if (mat.matches()) { + long pagesize = Long.parseLong(mat.group(1)) * 1024; + pagesizes.add(pagesize); + } + } + } + return pagesizes; + } + + private static THPMode readTHPModeFromOS() { + THPMode mode = THPMode.unknown; + String file = "/sys/kernel/mm/transparent_hugepage/enabled"; + try (FileReader fr = new FileReader(file); + BufferedReader reader = new BufferedReader(fr)) { + String s = reader.readLine(); + if (s.contains("[never]")) { + mode = THPMode.never; + } else if (s.contains("[always]")) { + mode = THPMode.always; + } else if (s.contains("[madvise]")) { + mode = THPMode.madvise; + } else { + throw new RuntimeException("Unexpected content of " + file + ": " + s); + } + } catch (IOException e) { + System.out.println("Failed to read " + file); + mode = THPMode.unknown; + } + return mode; + } + + private static long readTHPPageSizeFromOS() { + long pagesize = 0; + String file = "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size"; + try (FileReader fr = new FileReader(file); + BufferedReader reader = new BufferedReader(fr)) { + String s = reader.readLine(); + pagesize = Long.parseLong(s); + } catch (IOException | NumberFormatException e) { /* ignored */ } + return pagesize; + } + + // Fill object with info read from proc file system + public static HugePageConfiguration readFromOS() { + return new HugePageConfiguration(readSupportedHugePagesFromOS(), + readDefaultHugePageSizeFromOS(), + readTHPModeFromOS(), + readTHPPageSizeFromOS()); + } + + private static long parseSIUnit(String num, String unit) { + long n = Long.parseLong(num); + return switch (unit) { + case "K" -> n * 1024; + case "M" -> n * 1024 * 1024; + case "G" -> n * 1024 * 1024 * 1024; + default -> throw new RuntimeException("Invalid unit " + unit); + }; + } + + public static HugePageConfiguration readFromJVMLog(OutputAnalyzer output) { + // Expects output from -Xlog:pagesize + // Example: + // [0.001s][info][pagesize] Static hugepage support: + // [0.001s][info][pagesize] hugepage size: 2M + // [0.001s][info][pagesize] hugepage size: 1G + // [0.001s][info][pagesize] default hugepage size: 2M + // [0.001s][info][pagesize] Transparent hugepage (THP) support: + // [0.001s][info][pagesize] THP mode: madvise + // [0.001s][info][pagesize] THP pagesize: 2M + TreeSet hugepages = new TreeSet<>(); + long defaultHugepageSize = 0; + THPMode thpMode = THPMode.never; + long thpPageSize = 0; + Pattern patternHugepageSize = Pattern.compile(".*\\[pagesize] *hugepage size: (\\d+)([KMG])"); + Pattern patternDefaultHugepageSize = Pattern.compile(".*\\[pagesize] *default hugepage size: (\\d+)([KMG]) *"); + Pattern patternTHPPageSize = Pattern.compile(".*\\[pagesize] *THP pagesize: (\\d+)([KMG])"); + Pattern patternTHPMode = Pattern.compile(".*\\[pagesize] *THP mode: (\\S+)"); + List lines = output.asLines(); + for (String s : lines) { + Matcher mat = patternHugepageSize.matcher(s); + if (mat.matches()) { + hugepages.add(parseSIUnit(mat.group(1), mat.group(2))); + continue; + } + if (defaultHugepageSize == 0) { + mat = patternDefaultHugepageSize.matcher(s); + if (mat.matches()) { + defaultHugepageSize = parseSIUnit(mat.group(1), mat.group(2)); + continue; + } + } + if (thpPageSize == 0) { + mat = patternTHPPageSize.matcher(s); + if (mat.matches()) { + thpPageSize = parseSIUnit(mat.group(1), mat.group(2)); + continue; + } + } + mat = patternTHPMode.matcher(s); + if (mat.matches()) { + thpMode = THPMode.valueOf(mat.group(1)); + } + } + + return new HugePageConfiguration(hugepages, defaultHugepageSize, thpMode, thpPageSize); + } + +} diff --git a/test/hotspot/jtreg/runtime/os/HugePageDetection.java b/test/hotspot/jtreg/runtime/os/HugePageDetection.java new file mode 100644 index 00000000000..40d67198c5f --- /dev/null +++ b/test/hotspot/jtreg/runtime/os/HugePageDetection.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, Red Hat Inc. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test + * @summary Test that the JVM detects the OS hugepage/THP settings correctly. + * @library /test/lib + * @requires os.family == "linux" + * @modules java.base/jdk.internal.misc + * java.management + * @run driver HugePageDetection + */ + +import java.util.*; +import jdk.test.lib.process.OutputAnalyzer; +import jdk.test.lib.process.ProcessTools; + +public class HugePageDetection { + + public static void main(String[] args) throws Exception { + + ArrayList finalargs = new ArrayList(); + String[] defaultArgs = { + "-Xlog:pagesize", "-Xmx64M", "-XX:-CreateCoredumpOnCrash" + }; + finalargs.addAll(Arrays.asList(defaultArgs)); + finalargs.add("-version"); + + ProcessBuilder pb = ProcessTools.createJavaProcessBuilder( + new String[] {"-Xlog:pagesize", "-Xmx64M", "-version"}); + + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + output.reportDiagnosticSummary(); + output.shouldHaveExitValue(0); + + // The configuration detected by the JVM should match the OS settings + + HugePageConfiguration configurationFromOS = HugePageConfiguration.readFromOS(); + System.out.println("Configuration read from OS: " + configurationFromOS); + + HugePageConfiguration configurationFromLog = HugePageConfiguration.readFromJVMLog(output); + System.out.println("Configuration read from JVM log: " + configurationFromLog); + + if (configurationFromOS.equals(configurationFromLog)) { + System.out.println("Okay"); + } else { + throw new RuntimeException("Configurations differ"); + } + + // If we want to run + + } + +} From 1c9691b1f762812aa090b88507d60a2e2e5f4933 Mon Sep 17 00:00:00 2001 From: Jan Lahoda Date: Mon, 17 Jul 2023 06:43:16 +0000 Subject: [PATCH 2/2] 8312093: Incorrect javadoc comment text Reviewed-by: jlaskey --- .../sun/tools/javac/parser/JavaTokenizer.java | 5 +++- .../tools/javac/parser/JavacParserTest.java | 30 ++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/jdk.compiler/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java b/src/jdk.compiler/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java index b0b176b36df..6f6c3a192e8 100644 --- a/src/jdk.compiler/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java +++ b/src/jdk.compiler/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java @@ -1546,7 +1546,10 @@ protected void scanDocComment() { return; } - skip('*'); + if (skip('*') != 0 && is('/')) { + return ; + } + skipWhitespace(); if (isEOLN()) { diff --git a/test/langtools/tools/javac/parser/JavacParserTest.java b/test/langtools/tools/javac/parser/JavacParserTest.java index a22e15bc1f2..b6c48070c81 100644 --- a/test/langtools/tools/javac/parser/JavacParserTest.java +++ b/test/langtools/tools/javac/parser/JavacParserTest.java @@ -23,7 +23,7 @@ /* * @test - * @bug 7073631 7159445 7156633 8028235 8065753 8205418 8205913 8228451 8237041 8253584 8246774 8256411 8256149 8259050 8266436 8267221 8271928 8275097 8293897 8295401 8304671 8310326 + * @bug 7073631 7159445 7156633 8028235 8065753 8205418 8205913 8228451 8237041 8253584 8246774 8256411 8256149 8259050 8266436 8267221 8271928 8275097 8293897 8295401 8304671 8310326 8312093 * @summary tests error and diagnostics positions * @author Jan Lahoda * @modules jdk.compiler/com.sun.tools.javac.api @@ -2420,6 +2420,34 @@ public Void visitClass(ClassTree node, Void p) { }.scan(cut, null); } + @Test //JDK-8312093 + void testJavadoc() throws IOException { + String code = """ + public class Test { + /***/ + void main() { + } + } + """; + DiagnosticCollector coll = + new DiagnosticCollector<>(); + JavacTaskImpl ct = (JavacTaskImpl) tool.getTask(null, fm, coll, null, + null, Arrays.asList(new MyFileObject(code))); + Trees trees = Trees.instance(ct); + CompilationUnitTree cut = ct.parse().iterator().next(); + new TreePathScanner() { + @Override + public Void visitMethod(MethodTree node, Void p) { + if (!node.getName().contentEquals("main")) { + return null; + } + String comment = trees.getDocComment(getCurrentPath()); + assertEquals("Expecting empty comment", "", comment); + return null; + } + }.scan(cut, null); + } + void run(String[] args) throws Exception { int passed = 0, failed = 0; final Pattern p = (args != null && args.length > 0)