[SYCL] Move device library install out of lib/ and into share/

clang/include/clang/Driver/SyclInstallationDetector.h

@@ -45,6 +45,7 @@ class SYCLInstallationDetector {
   const Driver &D;
   SmallString<0> SYCLRTLibPath;
   llvm::SmallVector<llvm::SmallString<128>, 4> InstallationCandidates;
+  llvm::Triple HostTriple;
 };
 
 } // namespace driver

clang/lib/Driver/ToolChains/Clang.cpp

@@ -11720,12 +11720,8 @@ void LinkerWrapper::ConstructJob(Compilation &C, const JobAction &JA,
     SYCLInstallationDetector SYCLInstallation(D, getToolChain().getTriple(),
                                               Args);
     SYCLInstallation.getSYCLDeviceLibPath(LibLocCandidates);
-    SmallString<128> LibName("libsycl-crt");
+    SmallString<128> LibName("libsycl-crt.bc");
     bool IsNewOffload = D.getUseNewOffloadingDriver();
-    StringRef LibSuffix = TheTriple.isWindowsMSVCEnvironment()
-                              ? (IsNewOffload ? ".new.obj" : ".obj")
-                              : (IsNewOffload ? ".new.o" : ".o");
-    llvm::sys::path::replace_extension(LibName, LibSuffix);
     for (const auto &LibLoc : LibLocCandidates) {
       SmallString<128> FullLibName(LibLoc);
       llvm::sys::path::append(FullLibName, LibName);

clang/lib/Driver/ToolChains/SYCL.cpp

@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 #include "SYCL.h"
+#include "clang/Basic/Version.h"
 #include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
@@ -25,7 +26,7 @@ using namespace llvm::opt;
 SYCLInstallationDetector::SYCLInstallationDetector(
     const Driver &D, const llvm::Triple &HostTriple,
     const llvm::opt::ArgList &Args)
-    : D(D), InstallationCandidates() {
+    : D(D), InstallationCandidates(), HostTriple(HostTriple) {
   // Detect the presence of the SYCL runtime library (libsycl.so) in the
   // filesystem. This is used to determine whether a usable SYCL installation
   // is available for the current driver invocation.
@@ -98,12 +99,23 @@ void SYCLInstallationDetector::addLibspirvLinkArgs(
 
 void SYCLInstallationDetector::getSYCLDeviceLibPath(
     llvm::SmallVector<llvm::SmallString<128>, 4> &DeviceLibPaths) const {
+#define TOSTR2(X) #X
+#define TOSTR(X) TOSTR2(X)
+  Twine ShareDirSuffix = Twine("/share/dpcpp-") + TOSTR(DPCPP_VERSION_MAJOR);
+#undef TOSTR
+#undef TOSTR2
   for (const auto &IC : InstallationCandidates) {
+    if (!HostTriple.isWindowsMSVCEnvironment()) {
+      llvm::SmallString<128> InstallDataPath(IC.str());
+      InstallDataPath.append(ShareDirSuffix.str());
+      DeviceLibPaths.emplace_back(InstallDataPath);
+    }
     llvm::SmallString<128> InstallLibPath(IC.str());
     InstallLibPath.append("/lib");
     DeviceLibPaths.emplace_back(InstallLibPath);
   }
-
+  if (!HostTriple.isWindowsMSVCEnvironment())
+    DeviceLibPaths.emplace_back(D.SysRoot + ShareDirSuffix.str());
   DeviceLibPaths.emplace_back(D.SysRoot + "/lib");
 }
 

clang/test/Driver/sycl-device-lib-win.cpp

@@ -9,14 +9,14 @@
 /// test behavior of device library default link
 // RUN: %clangxx -fsycl %s --sysroot=%S/Inputs/SYCL -### 2>&1 \
 // RUN:   | FileCheck %s -check-prefix=SYCL_DEVICE_LIB_LINK_DEFAULT
-// SYCL_DEVICE_LIB_LINK_DEFAULT: llvm-link{{.*}} "{{.*}}libsycl-crt.bc"
-// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: "{{.*}}libsycl-cmath.bc"
-// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: "{{.*}}libsycl-imf.bc"
-// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: "{{.*}}libsycl-imf-fp64.bc"
-// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: "{{.*}}libsycl-imf-bf16.bc"
-// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: "{{.*}}libsycl-fallback-imf.bc"
-// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: "{{.*}}libsycl-fallback-imf-fp64.bc"
-// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: "{{.*}}libsycl-fallback-imf-bf16.bc"
+// SYCL_DEVICE_LIB_LINK_DEFAULT: llvm-link{{.*}} "{{.*}}lib\\libsycl-crt.bc"
+// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: "{{.*}}lib\\libsycl-cmath.bc"
+// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: "{{.*}}lib\\libsycl-imf.bc"
+// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: "{{.*}}lib\\libsycl-imf-fp64.bc"
+// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: "{{.*}}lib\\libsycl-imf-bf16.bc"
+// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: "{{.*}}lib\\libsycl-fallback-imf.bc"
+// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: "{{.*}}lib\\libsycl-fallback-imf-fp64.bc"
+// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: "{{.*}}lib\\libsycl-fallback-imf-bf16.bc"
 
 /// ###########################################################################
 

clang/test/Driver/sycl-device-lib.cpp

@@ -9,14 +9,14 @@
 /// test behavior of device library default link
 // RUN: %clangxx -fsycl --offload-new-driver %s --sysroot=%S/Inputs/SYCL -### 2>&1 \
 // RUN:   | FileCheck %s -check-prefix=SYCL_DEVICE_LIB_LINK_DEFAULT
-// SYCL_DEVICE_LIB_LINK_DEFAULT: clang{{.*}} "-mlink-builtin-bitcode" "{{.*}}libsycl-crt.bc"
-// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: {{.*}}libsycl-cmath.bc
-// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: {{.*}}libsycl-imf.bc
-// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: {{.*}}libsycl-imf-fp64.bc
-// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: {{.*}}libsycl-imf-bf16.bc
-// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: {{.*}}libsycl-fallback-imf.bc
-// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: {{.*}}libsycl-fallback-imf-fp64.bc
-// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: {{.*}}libsycl-fallback-imf-bf16.bc
+// SYCL_DEVICE_LIB_LINK_DEFAULT: clang{{.*}} "-mlink-builtin-bitcode" "{{.*}}/share/dpcpp-{{[0-9]+}}/libsycl-crt.bc"
+// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: {{.*}}/share/dpcpp-{{[0-9]+}}/libsycl-cmath.bc
+// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: {{.*}}/share/dpcpp-{{[0-9]+}}/libsycl-imf.bc
+// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: {{.*}}/share/dpcpp-{{[0-9]+}}/libsycl-imf-fp64.bc
+// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: {{.*}}/share/dpcpp-{{[0-9]+}}/libsycl-imf-bf16.bc
+// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: {{.*}}/share/dpcpp-{{[0-9]+}}/libsycl-fallback-imf.bc
+// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: {{.*}}/share/dpcpp-{{[0-9]+}}/libsycl-fallback-imf-fp64.bc
+// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: {{.*}}/share/dpcpp-{{[0-9]+}}/libsycl-fallback-imf-bf16.bc
 // SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: "-mlink-builtin-bitcode-postopt"
 
 /// ###########################################################################

libdevice/cmake/modules/SYCLLibdevice.cmake

@@ -8,9 +8,18 @@ else()
   set(devicelib_host_static_obj libsycl-devicelib-host.a)
 endif()
 set(bc-suffix bc)
-set(bc_binary_dir "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}")
 set(install_dest_obj lib${LLVM_LIBDIR_SUFFIX})
-set(install_dest_bc lib${LLVM_LIBDIR_SUFFIX})
+
+if(WIN32)
+  # On Windows, install to lib/
+  set(install_dest_bc lib${LLVM_LIBDIR_SUFFIX})
+  set(bc_binary_dir "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}")
+else()
+  # On other platforms, install to share/dpcpp-<major_ver>/
+  set(bc_dir "share/dpcpp-${DPCPP_VERSION_MAJOR}")
+  set(install_dest_bc ${bc_dir})
+  set(bc_binary_dir "${CMAKE_BINARY_DIR}/${bc_dir}")
+endif()
 
 string(CONCAT sycl_targets_opt
   "-fsycl-targets="

sycl-jit/jit-compiler/lib/rtc/DeviceCompilation.cpp

@@ -167,6 +167,17 @@ template <> struct std::hash<auto_pch_key> {
 };
 
 namespace {
+std::string getLibPathSuffix() {
+#define TOSTR2(X) #X
+#define TOSTR(X) TOSTR2(X)
+#ifdef _WIN32
+  return "/lib/";
+#else
+  return "/share/dpcpp-" TOSTR(DPCPP_VERSION_MAJOR) "/";
+#endif
+#undef TOSTR
+#undef TOSTR2
+}
 class SYCLToolchain {
   static auto &getToolchainFS() {
     // TODO: For some reason, removing `thread_local` results in data races
@@ -847,10 +858,11 @@ Error jit_compiler::linkDeviceLibraries(llvm::Module &Module,
     std::string TripleName = (Format == BinaryFormat::PTX)
                                  ? "nvptx64-nvidia-cuda"
                                  : "amdgcn-amd-amdhsa";
+
     std::string LibPath =
         (LibName.find("libspirv") != std::string::npos)
             ? (TC.getLibclcDir() + TripleName + "/" + LibName).str()
-            : (TC.getPrefix() + "/lib/" + LibName).str();
+            : (TC.getPrefix() + getLibPathSuffix() + LibName).str();
 
     ModuleUPtr LibModule;
     if (auto Error =
@@ -1111,7 +1123,8 @@ jit_compiler::performPostLink(ModuleUPtr Module,
     auto &Ctx = Modules.front()->getContext();
     auto WrapLibraryInDevImg = [&](const std::string &LibName) -> Error {
       std::string LibPath =
-          (SYCLToolchain::instance().getPrefix() + "/lib/" + LibName).str();
+          (SYCLToolchain::instance().getPrefix() + getLibPathSuffix() + LibName)
+              .str();
       ModuleUPtr LibModule;
       if (auto Error = SYCLToolchain::instance()
                            .loadBitcodeLibrary(LibPath, Ctx)

sycl/doc/extensions/supported/C-CXX-StandardLibrary.rst

@@ -240,24 +240,3 @@ In order to indicate support for a particular set of functions,
 underlying runtime have to support the corresponding OpenCL (PI)
 extension. See ``../../design/DeviceLibExtensions.rst`` for
 a list of supported functions and corresponding extensions.
-
-Fallback implementation
-=======================
-
-If a device compiler does not indicate "native" support for a
-particular function, a fallback library is linked at JIT time by the
-SYCL Runtime. This library is distributed with the SYCL Runtime and
-resides in the same directory as the `libsycl.so` or `sycl.dll`.
-
-A fallback library is implemented as a device-agnostic SPIR-V program,
-and it is supposed to work for any device that supports SPIR-V.
-
-Every set of functions is implemented in a separate fallback
-library. For example, a fallback for `cl_intel_devicelib_cassert`
-extension is provided as `libsycl-fallback-cassert.spv`
-
-For AOT compilation, fallback libraries are provided as object files
-(e.g. `libsycl-fallback-cassert.o`) which contain device code in LLVM
-IR format. Device code in these object files is equivalent to device
-code in the `*.spv` files. Those object files are located in compiler
-package's 'lib/' folder.

sycl/test-e2e/CMakeLists.txt

@@ -81,6 +81,19 @@ if(hip_FOUND)
   set(HIP_INCLUDE "${hip_INCLUDE_DIRS}")
 endif()
 
+execute_process(
+      COMMAND clang++ --version
+      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+      RESULT_VARIABLE CMPLR_RESULT
+      OUTPUT_VARIABLE CMPLR_VER
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+
+if(CMPLR_RESULT EQUAL 0 AND CMPLR_VER)
+  string(REGEX MATCH "DPC\\+\\+ compiler ([0-9]+)" _ ${CMPLR_VER})
+  set(DPCPP_VERSION_MAJOR "${CMAKE_MATCH_1}")
+endif()
+
 if(SYCL_TEST_E2E_STANDALONE)
   configure_file("${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in"
                  "${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py")

sycl/test-e2e/lit.cfg.py

@@ -215,16 +215,13 @@ def __exit__(self, exc_type, exc_value, exc_traceback):
 
 
 config.substitutions.append(("%sycl_libs_dir", config.sycl_libs_dir))
+config.substitutions.append(("%sycl_static_libs_dir", config.sycl_device_libs_dir))
 if platform.system() == "Windows":
-    config.substitutions.append(
-        ("%sycl_static_libs_dir", config.sycl_libs_dir + "/../lib")
-    )
     config.substitutions.append(("%obj_ext", ".obj"))
     config.substitutions.append(
         ("%sycl_include", "-Xclang -isystem -Xclang " + config.sycl_include)
     )
 elif platform.system() == "Linux":
-    config.substitutions.append(("%sycl_static_libs_dir", config.sycl_libs_dir))
     config.substitutions.append(("%obj_ext", ".o"))
     config.substitutions.append(("%sycl_include", "-isystem " + config.sycl_include))
 

sycl/test-e2e/lit.site.cfg.py.in

@@ -26,6 +26,7 @@ config.sycl_tools_dir = config.llvm_tools_dir
 config.sycl_include = os.path.join(config.dpcpp_root_dir, 'include')
 config.sycl_obj_root = "@CMAKE_CURRENT_BINARY_DIR@"
 config.sycl_libs_dir =  os.path.join(config.dpcpp_root_dir, ('bin' if platform.system() == "Windows" else 'lib'))
+config.sycl_device_libs_dir = os.path.join(config.dpcpp_root_dir, ('lib' if platform.system() == "Windows" else "share/dpcpp-@DPCPP_VERSION_MAJOR@"))
 
 config.opencl_libs_dir = (os.path.dirname("@OpenCL_LIBRARY@") if "@OpenCL_LIBRARY@" else "")
 config.level_zero_libs_dir = "@LEVEL_ZERO_LIBS_DIR@"