From e7f8fa9c56dfbc18fc8aa2fcfa11684d1b1fe61c Mon Sep 17 00:00:00 2001 From: Alessandro Di Federico Date: Sun, 29 Jan 2023 10:14:50 +0100 Subject: [PATCH 001/147] CachedReachabilityAA constructor: drop templates --- llvm/lib/Transforms/IPO/AttributorAttributes.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp index 42158e4e05dd..5cb037925efa 100644 --- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp +++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp @@ -3494,7 +3494,7 @@ template struct CachedReachabilityAA : public BaseTy { using RQITy = ReachabilityQueryInfo; - CachedReachabilityAA(const IRPosition &IRP, Attributor &A) + CachedReachabilityAA(const IRPosition &IRP, Attributor &A) : BaseTy(IRP, A) {} /// See AbstractAttribute::isQueryAA. From b5e66fc1bce22217ffd250c418a0f60f817ffc46 Mon Sep 17 00:00:00 2001 From: Alessandro Di Federico Date: Sun, 29 Jan 2023 10:14:20 +0100 Subject: [PATCH 002/147] libc++ tests: link libc++abi too --- libcxx/test/configs/llvm-libc++-shared.cfg.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcxx/test/configs/llvm-libc++-shared.cfg.in b/libcxx/test/configs/llvm-libc++-shared.cfg.in index 7228c7e8d467..709eb17cbd76 100644 --- a/libcxx/test/configs/llvm-libc++-shared.cfg.in +++ b/libcxx/test/configs/llvm-libc++-shared.cfg.in @@ -10,7 +10,7 @@ config.substitutions.append(('%{compile_flags}', '-nostdinc++ -I %{include} -I %{target-include} -I %{libcxx}/test/support' )) config.substitutions.append(('%{link_flags}', - '-nostdlib++ -L %{lib} -Wl,-rpath,%{lib} -lc++ -pthread' + '-nostdlib++ -L %{lib} -Wl,-rpath,%{lib} -lc++ -lc++abi -pthread' )) config.substitutions.append(('%{exec}', '%{executor} --execdir %T -- ' From 2f98e7204cd0b0d9323e11770decddd41e0f59ec Mon Sep 17 00:00:00 2001 From: Alessandro Di Federico Date: Sun, 29 Jan 2023 20:20:38 +0100 Subject: [PATCH 003/147] cstdlib: implement aligned_alloc --- libcxx/include/cstdlib | 5 +++++ .../depr.c.headers/stdlib_h.aligned_alloc.compile.pass.cpp | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/libcxx/include/cstdlib b/libcxx/include/cstdlib index 25c9de5165c2..e69ea70e6042 100644 --- a/libcxx/include/cstdlib +++ b/libcxx/include/cstdlib @@ -85,6 +85,7 @@ void *aligned_alloc(size_t alignment, size_t size); // C11 #include <__config> #include +#include #ifndef _LIBCPP_STDLIB_H # error tried including but didn't find libc++'s header. \ @@ -98,6 +99,10 @@ void *aligned_alloc(size_t alignment, size_t size); // C11 # pragma GCC system_header #endif +extern "C" inline void *aligned_alloc(size_t alignment, size_t size) { + return memalign(alignment, size); +} + _LIBCPP_BEGIN_NAMESPACE_STD using ::size_t _LIBCPP_USING_IF_EXISTS; diff --git a/libcxx/test/std/depr/depr.c.headers/stdlib_h.aligned_alloc.compile.pass.cpp b/libcxx/test/std/depr/depr.c.headers/stdlib_h.aligned_alloc.compile.pass.cpp index a02a5bbbdbcc..8639c43fbe50 100644 --- a/libcxx/test/std/depr/depr.c.headers/stdlib_h.aligned_alloc.compile.pass.cpp +++ b/libcxx/test/std/depr/depr.c.headers/stdlib_h.aligned_alloc.compile.pass.cpp @@ -21,7 +21,7 @@ // ::aligned_alloc is available starting with Android P (API 28) // XFAIL: target={{.+}}-android{{(eabi)?(21|22|23|24|25|26|27)}} -#include +#include #include static_assert(std::is_same::value, ""); From 96c0bae1982e7505fffe759aaa91aa9cec46c93b Mon Sep 17 00:00:00 2001 From: Alessandro Di Federico Date: Tue, 31 Jan 2023 23:25:08 +0100 Subject: [PATCH 004/147] libcxx: expect failures due to old glibc --- libcxx/test/std/depr/depr.c.headers/uchar_h.compile.pass.cpp | 2 ++ .../support.runtime/ctime.timespec.compile.pass.cpp | 2 ++ libcxx/test/std/strings/c.strings/cuchar.compile.pass.cpp | 2 ++ 3 files changed, 6 insertions(+) diff --git a/libcxx/test/std/depr/depr.c.headers/uchar_h.compile.pass.cpp b/libcxx/test/std/depr/depr.c.headers/uchar_h.compile.pass.cpp index 016e51d99c59..f093107f3767 100644 --- a/libcxx/test/std/depr/depr.c.headers/uchar_h.compile.pass.cpp +++ b/libcxx/test/std/depr/depr.c.headers/uchar_h.compile.pass.cpp @@ -14,6 +14,8 @@ // The system-provided seems to be broken on AIX // XFAIL: LIBCXX-AIX-FIXME +// XFAIL: target={{.+}} + // #include diff --git a/libcxx/test/std/language.support/support.runtime/ctime.timespec.compile.pass.cpp b/libcxx/test/std/language.support/support.runtime/ctime.timespec.compile.pass.cpp index 37c7b6723478..03250ae8218a 100644 --- a/libcxx/test/std/language.support/support.runtime/ctime.timespec.compile.pass.cpp +++ b/libcxx/test/std/language.support/support.runtime/ctime.timespec.compile.pass.cpp @@ -20,6 +20,8 @@ // ::timespec_get is available starting with Android Q (API 29) // XFAIL: target={{.+}}-android{{(eabi)?(21|22|23|24|25|26|27|28)}} +// XFAIL: target={{.+}} + #include #include diff --git a/libcxx/test/std/strings/c.strings/cuchar.compile.pass.cpp b/libcxx/test/std/strings/c.strings/cuchar.compile.pass.cpp index 909f2c025bdf..69160aedebe4 100644 --- a/libcxx/test/std/strings/c.strings/cuchar.compile.pass.cpp +++ b/libcxx/test/std/strings/c.strings/cuchar.compile.pass.cpp @@ -14,6 +14,8 @@ // The system-provided seems to be broken on AIX // XFAIL: LIBCXX-AIX-FIXME +// XFAIL: target={{.+}} + // #include From 40999c81da57c2df0e0e808cb5282725e27013e2 Mon Sep 17 00:00:00 2001 From: Alessandro Di Federico Date: Sun, 29 Jan 2023 20:20:57 +0100 Subject: [PATCH 005/147] runtimes: forward CMAKE_C{,XX}_COMPILER --- llvm/cmake/modules/LLVMExternalProjectUtils.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/llvm/cmake/modules/LLVMExternalProjectUtils.cmake b/llvm/cmake/modules/LLVMExternalProjectUtils.cmake index d760eee07958..d92c58f1966f 100644 --- a/llvm/cmake/modules/LLVMExternalProjectUtils.cmake +++ b/llvm/cmake/modules/LLVMExternalProjectUtils.cmake @@ -291,6 +291,9 @@ function(llvm_ExternalProject_Add name source_dir) list(APPEND compiler_args -DCMAKE_ASM_COMPILER_TARGET=${ARG_TARGET_TRIPLE}) endif() + list(APPEND compiler_args -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}) + list(APPEND compiler_args -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}) + ExternalProject_Add(${name} DEPENDS ${ARG_DEPENDS} llvm-config ${name}-clobber From 799f199ad24a87a16b53b74cdf616672db53dbbf Mon Sep 17 00:00:00 2001 From: Alessandro Di Federico Date: Sun, 4 Feb 2024 16:58:29 +0100 Subject: [PATCH 006/147] Ignore mold's $plt and $got symbols --- clang/test/LibClang/symbols.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clang/test/LibClang/symbols.test b/clang/test/LibClang/symbols.test index fd2ff8bc6cd4..5c48f4e62526 100644 --- a/clang/test/LibClang/symbols.test +++ b/clang/test/LibClang/symbols.test @@ -1,6 +1,6 @@ # Check that there are no unversioned clang symbols in libclang.so -RUN: llvm-nm -Dj --defined-only %libclang | grep -v -e '@@LLVM_[0-9]\+$' | not grep '^clang' +RUN: llvm-nm -Dj --defined-only %libclang | grep -v -e '@@LLVM_[0-9]\+$' | grep -vF '$plt' | grep -vF '$got' | not grep '^clang' # Check that here are no local clang_ symbols (ignoring symbols with .cold or # .localalias suffxies.) -RUN: llvm-nm %libclang | not grep '[a-z] clang_[^.]\+$' +RUN: llvm-nm %libclang | grep -vF '$plt' | grep -vF '$got' | not grep '[a-z] clang_[^.]\+$' From 118a82a1e7e8a688a608ac7490e9deb2c6e7249f Mon Sep 17 00:00:00 2001 From: Alessandro Di Federico Date: Fri, 17 Apr 2026 18:40:16 +0200 Subject: [PATCH 007/147] Silence noisy profiling counter warning Remove the "Running out of static counters" warning from InstrProfilingValue.c as it pollutes stderr and breaks lit tests when the compiler is built with profiling. --- compiler-rt/lib/profile/InstrProfilingValue.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/compiler-rt/lib/profile/InstrProfilingValue.c b/compiler-rt/lib/profile/InstrProfilingValue.c index c819a38553f3..174a4a624b34 100644 --- a/compiler-rt/lib/profile/InstrProfilingValue.c +++ b/compiler-rt/lib/profile/InstrProfilingValue.c @@ -116,13 +116,6 @@ static ValueProfNode *allocateOneNode(void) { /* Early check to avoid value wrapping around. */ if (CurrentVNode + 1 > EndVNode) { - if (OutOfNodesWarnings++ < INSTR_PROF_MAX_VP_WARNS) { - PROF_WARN("Unable to track new values: %s. " - " Consider using option -mllvm -vp-counters-per-site= to " - "allocate more" - " value profile counters at compile time. \n", - "Running out of static counters"); - } return 0; } Node = COMPILER_RT_PTR_FETCH_ADD(ValueProfNode, CurrentVNode, 1); From 8feb943dd1c78495c4e02d9163faa9d22c32e11a Mon Sep 17 00:00:00 2001 From: Alessandro Di Federico Date: Wed, 18 Feb 2026 11:26:11 +0100 Subject: [PATCH 008/147] Add -fcase-insensitive-paths flag When enabled, file lookups that fail are retried with a case-insensitive directory scan. This is useful when compiling against SDKs (e.g. the Windows SDK) whose header paths use a different case than the #include directives. The flag adds a CaseInsensitivePaths boolean to FileSystemOptions, marshalled from the new -fcase-insensitive-paths CC1 flag, and implements the fallback in FileManager::getStatValue(). --- clang/include/clang/Basic/FileSystemOptions.h | 5 ++ clang/include/clang/Driver/Options.td | 4 ++ clang/lib/Basic/FileManager.cpp | 59 ++++++++++++++++--- clang/lib/Driver/ToolChains/Clang.cpp | 1 + clang/test/Driver/fcase-insensitive-paths.c | 3 + clang/test/Lexer/fcase-insensitive-paths.c | 17 ++++++ 6 files changed, 81 insertions(+), 8 deletions(-) create mode 100644 clang/test/Driver/fcase-insensitive-paths.c create mode 100644 clang/test/Lexer/fcase-insensitive-paths.c diff --git a/clang/include/clang/Basic/FileSystemOptions.h b/clang/include/clang/Basic/FileSystemOptions.h index 458af0c7b659..63fa3ba41b90 100644 --- a/clang/include/clang/Basic/FileSystemOptions.h +++ b/clang/include/clang/Basic/FileSystemOptions.h @@ -24,6 +24,11 @@ class FileSystemOptions { /// If set, paths are resolved as if the working directory was /// set to the value of WorkingDir. std::string WorkingDir; + + /// If set, file lookups that fail are retried with a case-insensitive + /// directory scan. Useful for compiling against SDKs whose header paths + /// use a different case than the #include directives. + bool CaseInsensitivePaths = false; }; } // end namespace clang diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index 652c15afcce8..66e9511dc798 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -4376,6 +4376,10 @@ def working_directory : Separate<["-"], "working-directory">, Flags<[CC1Option]> MarshallingInfoString>; def working_directory_EQ : Joined<["-"], "working-directory=">, Flags<[CC1Option]>, Alias; +def fcase_insensitive_paths : Flag<["-"], "fcase-insensitive-paths">, + Flags<[CC1Option]>, + HelpText<"Retry failed file lookups with a case-insensitive directory scan">, + MarshallingInfoFlag>; // Double dash options, which are usually an alias for one of the previous // options. diff --git a/clang/lib/Basic/FileManager.cpp b/clang/lib/Basic/FileManager.cpp index e8d0f20019eb..e923c0015d7f 100644 --- a/clang/lib/Basic/FileManager.cpp +++ b/clang/lib/Basic/FileManager.cpp @@ -577,6 +577,43 @@ FileManager::getBufferForFileImpl(StringRef Filename, int64_t FileSize, isVolatile); } +/// Walk a path component-by-component, resolving each level with a +/// case-insensitive directory scan. Returns the real-cased path on success. +static std::optional +findPathCaseInsensitive(StringRef CaseInsensitivePath) { + using llvm::sys::fs::directory_iterator; + namespace path = llvm::sys::path; + + SmallString<256> Resolved = path::root_path(CaseInsensitivePath); + StringRef Relative = path::relative_path(CaseInsensitivePath); + if (Relative.empty()) + return std::nullopt; + + for (StringRef Component : + llvm::make_range(path::begin(Relative), path::end(Relative))) { + std::error_code EC; + std::string Match; + unsigned Hits = 0; + for (directory_iterator It(Resolved, EC), End; It != End && !EC; + It.increment(EC)) { + StringRef EntryName = path::filename(It->path()); + if (EntryName.equals_insensitive(Component)) { + if (Hits == 0) + Match = EntryName.str(); + ++Hits; + } + } + if (EC || Hits == 0) + return std::nullopt; + if (Hits > 1) + llvm::errs() << "warning: ambiguous case-insensitive match for '" + << Component << "' in '" << Resolved + << "'; picking '" << Match << "'\n"; + path::append(Resolved, Match); + } + return Resolved.str().str(); +} + /// getStatValue - Get the 'stat' information for the specified path, /// using the cache to accelerate it if possible. This returns true /// if the path points to a virtual file or does not exist, or returns @@ -587,15 +624,21 @@ FileManager::getStatValue(StringRef Path, llvm::vfs::Status &Status, bool isFile, std::unique_ptr *F) { // FIXME: FileSystemOpts shouldn't be passed in here, all paths should be // absolute! - if (FileSystemOpts.WorkingDir.empty()) - return FileSystemStatCache::get(Path, Status, isFile, F, - StatCache.get(), *FS); - SmallString<128> FilePath(Path); - FixupRelativePath(FilePath); - - return FileSystemStatCache::get(FilePath.c_str(), Status, isFile, F, - StatCache.get(), *FS); + if (!FileSystemOpts.WorkingDir.empty()) + FixupRelativePath(FilePath); + + auto stat = [&](StringRef P) { + return FileSystemStatCache::get(P, Status, isFile, F, StatCache.get(), *FS); + }; + + auto EC = stat(FilePath); + if (!EC) + return EC; + if (FileSystemOpts.CaseInsensitivePaths) + if (auto Resolved = findPathCaseInsensitive(FilePath)) + return stat(*Resolved); + return EC; } std::error_code diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index ec6860113b7e..c5ec17b47089 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -5743,6 +5743,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, CmdArgs.push_back(D.ResourceDir.c_str()); Args.AddLastArg(CmdArgs, options::OPT_working_directory); + Args.AddLastArg(CmdArgs, options::OPT_fcase_insensitive_paths); RenderARCMigrateToolOptions(D, Args, CmdArgs); diff --git a/clang/test/Driver/fcase-insensitive-paths.c b/clang/test/Driver/fcase-insensitive-paths.c new file mode 100644 index 000000000000..ef2f470a61ac --- /dev/null +++ b/clang/test/Driver/fcase-insensitive-paths.c @@ -0,0 +1,3 @@ +// RUN: %clang -### -fcase-insensitive-paths -c %s 2>&1 | FileCheck %s + +// CHECK: "-fcase-insensitive-paths" diff --git a/clang/test/Lexer/fcase-insensitive-paths.c b/clang/test/Lexer/fcase-insensitive-paths.c new file mode 100644 index 000000000000..6d92bcb46aef --- /dev/null +++ b/clang/test/Lexer/fcase-insensitive-paths.c @@ -0,0 +1,17 @@ +// Verify that -fcase-insensitive-paths retries failed header lookups with a +// case-insensitive directory scan. Only meaningful on a case-sensitive FS. + +// UNSUPPORTED: case-insensitive-filesystem + +// RUN: rm -rf %t && mkdir -p %t/inc +// RUN: echo 'int from_header;' > %t/inc/Header.h + +// Without the flag, the case-mismatched include fails. +// RUN: not %clang_cc1 -fsyntax-only -I %t/inc %s 2>&1 | FileCheck --check-prefix=OFF %s +// OFF: 'header.h' file not found + +// With the flag, the lookup succeeds. +// RUN: %clang_cc1 -fsyntax-only -fcase-insensitive-paths -I %t/inc %s + +#include "header.h" +int use_it(void) { return from_header; } From 53d114159304d175492010db3faff42f2584fd07 Mon Sep 17 00:00:00 2001 From: Alessandro Di Federico Date: Wed, 18 Feb 2026 11:28:04 +0100 Subject: [PATCH 009/147] Retain decl-only subprograms in CodeView When -fno-eliminate-unused-debug-types is active, retain declaration-only subprograms via DBuilder.retainType(SP) so they survive into the output even when the function is never defined or called. In CodeViewDebug::emitDebugInfoForRetainedTypes(), handle DISubprogram entries by emitting LF_FUNC_ID records for declaration-only subprograms. Call EmitFunctionDecl from EmitTopLevelDecl for every top-level non-template function declaration so its prototype is materialized as a DISubprogram in the first place. Together this puts function prototypes from headers into the PDB even when the TU defines none of them. --- clang/lib/CodeGen/CGDebugInfo.cpp | 12 +++++++++++- clang/lib/CodeGen/CodeGenModule.cpp | 19 +++++++++++++++++++ ...debug-info-codeview-emit-decl-subprogram.c | 16 ++++++++++++++++ llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp | 5 +++++ 4 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 clang/test/CodeGen/debug-info-codeview-emit-decl-subprogram.c diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp index 3bde43cc1db3..65f8ab067742 100644 --- a/clang/lib/CodeGen/CGDebugInfo.cpp +++ b/clang/lib/CodeGen/CGDebugInfo.cpp @@ -4189,6 +4189,7 @@ void CGDebugInfo::EmitFunctionDecl(GlobalDecl GD, SourceLocation Loc, llvm::DINode::DIFlags Flags = llvm::DINode::FlagZero; llvm::DIFile *Unit = getOrCreateFile(Loc); bool IsDeclForCallSite = Fn ? true : false; + llvm::DIScope *FDContext = IsDeclForCallSite ? Unit : getDeclContextDescriptor(D); llvm::DINodeArray TParamsArray; @@ -4219,10 +4220,17 @@ void CGDebugInfo::EmitFunctionDecl(GlobalDecl GD, SourceLocation Loc, llvm::DINodeArray Annotations = CollectBTFDeclTagAnnotations(D); llvm::DISubroutineType *STy = getOrCreateFunctionType(D, FnType, Unit); + // Only link to a prior declaration when creating a call-site entry. + // For standalone declarations the SP *is* the declaration; passing + // getFunctionDeclaration() here would let DISubprogram::get() unique + // the new node to the existing member SP, producing a self-referencing + // declaration link that causes infinite recursion in the DWARF emitter. + llvm::DISubprogram *Decl = + IsDeclForCallSite ? getFunctionDeclaration(D) : nullptr; llvm::DISubprogram *SP = DBuilder.createFunction(FDContext, Name, LinkageName, Unit, LineNo, STy, ScopeLine, Flags, SPFlags, TParamsArray.get(), - getFunctionDeclaration(D), nullptr, Annotations); + Decl, nullptr, Annotations); // Preserve btf_decl_tag attributes for parameters of extern functions // for BPF target. The parameters created in this loop are attached as @@ -4243,6 +4251,8 @@ void CGDebugInfo::EmitFunctionDecl(GlobalDecl GD, SourceLocation Loc, if (IsDeclForCallSite) Fn->setSubprogram(SP); + else if (CGM.getCodeGenOpts().hasMaybeUnusedDebugInfo()) + DBuilder.retainType(SP); DBuilder.finalizeSubprogram(SP); } diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index 12d602fed693..85d1b33127c0 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -6276,6 +6276,24 @@ void CodeGenModule::EmitDeclContext(const DeclContext *DC) { } } +/// Emit a DISubprogram for a function declaration so its prototype ends +/// up in the debug output (LF_FUNC_ID in CodeView) even when the TU +/// has no definition. +static void emitFunctionDecl(CodeGenModule &CGM, const FunctionDecl *FD) { + // Only emit for declaration-only functions; definitions already get + // a DISubprogram through the normal codegen path. + if (FD->doesThisDeclarationHaveABody()) + return; + // Skip declarations whose type can't be represented in debug info + // (e.g. undeduced auto return types, dependent template types). + const auto *FPT = FD->getType()->getAs(); + if (!FPT || FPT->isDependentType() || FPT->isUndeducedType()) + return; + CGDebugInfo *DI = CGM.getModuleDebugInfo(); + if (DI) + DI->EmitFunctionDecl(GlobalDecl(FD), FD->getLocation(), QualType(FPT, 0)); +} + /// EmitTopLevelDecl - Emit code for a single top level declaration. void CodeGenModule::EmitTopLevelDecl(Decl *D) { // Ignore dependent declarations. @@ -6295,6 +6313,7 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) { // Always provide some coverage mapping // even for the functions that aren't emitted. AddDeferredUnusedCoverageMapping(D); + emitFunctionDecl(*this, cast(D)); break; case Decl::CXXDeductionGuide: diff --git a/clang/test/CodeGen/debug-info-codeview-emit-decl-subprogram.c b/clang/test/CodeGen/debug-info-codeview-emit-decl-subprogram.c new file mode 100644 index 000000000000..c8378f3419a1 --- /dev/null +++ b/clang/test/CodeGen/debug-info-codeview-emit-decl-subprogram.c @@ -0,0 +1,16 @@ +// Verify that a declaration-only function gets its DISubprogram retained +// and lowered to an LF_FUNC_ID record in the CodeView output. + +// REQUIRES: x86-registered-target + +// RUN: %clang_cc1 -triple x86_64-windows-msvc -gcodeview \ +// RUN: -debug-info-kind=unused-types -emit-obj -o %t.o %s +// RUN: llvm-readobj %t.o --codeview | FileCheck %s + +void declared_fn(int); + +// CHECK: FuncId ({{.*}}) { +// CHECK-NEXT: TypeLeafKind: LF_FUNC_ID +// CHECK: FunctionType: void (int) +// CHECK-NEXT: Name: declared_fn +// CHECK-NEXT: } diff --git a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp index 0a67c4b6beb6..c4223dd516d9 100644 --- a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp @@ -3298,6 +3298,11 @@ void CodeViewDebug::emitDebugInfoForRetainedTypes() { if (DIType *RT = dyn_cast(Ty)) { getTypeIndex(RT); // FIXME: Add to global/local DTU list. + } else if (auto *SP = dyn_cast(Ty)) { + // Emit LF_FUNC_ID for declaration-only subprograms so that + // function prototypes from headers appear in the PDB. + if (!SP->isDefinition()) + getFuncIdForSubprogram(SP); } } } From 5ab7de2fa5ac287305b24892443554f7a8ba690a Mon Sep 17 00:00:00 2001 From: Alessandro Di Federico Date: Wed, 18 Feb 2026 11:27:33 +0100 Subject: [PATCH 010/147] Add -fdebug-info-allowed-file flag Adds the -fdebug-info-allowed-file= CC1 flag (repeatable). When set, only declarations from listed files get rich debug info: - types (CXXRecord, Typedef, Record, Enum) are retained - extern function declarations emit their LF_FUNC_ID prototypes This keeps PDB/DWARF output focused on the headers of interest (e.g. SDK headers) rather than emitting debug info for the entire translation unit. --- clang/include/clang/Basic/CodeGenOptions.h | 4 +++ clang/include/clang/Driver/Options.td | 5 ++++ clang/lib/CodeGen/CodeGenModule.cpp | 31 ++++++++++++++++---- clang/lib/Driver/ToolChains/Clang.cpp | 1 + clang/test/CodeGen/debug-info-allowed-file.c | 26 ++++++++++++++++ clang/test/Driver/fdebug-info-allowed-file.c | 3 ++ 6 files changed, 65 insertions(+), 5 deletions(-) create mode 100644 clang/test/CodeGen/debug-info-allowed-file.c create mode 100644 clang/test/Driver/fdebug-info-allowed-file.c diff --git a/clang/include/clang/Basic/CodeGenOptions.h b/clang/include/clang/Basic/CodeGenOptions.h index 4175fe3072ab..87d0a2ca5626 100644 --- a/clang/include/clang/Basic/CodeGenOptions.h +++ b/clang/include/clang/Basic/CodeGenOptions.h @@ -398,6 +398,10 @@ class CodeGenOptions : public CodeGenOptionsBase { /// by sanitizer coverage pass. std::vector SanitizeCoverageAllowlistFiles; + /// Files whose declarations get rich debug info (types retained and + /// declaration-only subprograms emitted). + std::vector DebugInfoAllowedFiles; + /// The guard style used for stack protector to get a initial value, this /// value usually be gotten from TLS or get from __stack_chk_guard, or some /// other styles we may implement in the future. diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index 66e9511dc798..d73ffa11176e 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -1282,6 +1282,11 @@ def fdebug_compilation_dir_EQ : Joined<["-"], "fdebug-compilation-dir=">, Group, Flags<[CC1Option, CC1AsOption, CoreOption]>, HelpText<"The compilation directory to embed in the debug info">, MarshallingInfoString>; +def fdebug_info_allowed_file_EQ : Joined<["-"], "fdebug-info-allowed-file=">, + Group, Flags<[CC1Option, CoreOption]>, + HelpText<"Emit rich debug info only for declarations from the given file " + "(repeatable)">, + MarshallingInfoStringVector>; def fdebug_compilation_dir : Separate<["-"], "fdebug-compilation-dir">, Group, Flags<[CC1Option, CC1AsOption, CoreOption]>, Alias; diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index 85d1b33127c0..9db3ec470738 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -6276,6 +6276,22 @@ void CodeGenModule::EmitDeclContext(const DeclContext *DC) { } } +/// Returns true if D comes from a file selected for rich debug info. +/// If -fdebug-info-allowed-file is not set, all declarations pass. +static bool isDeclInAllowedFiles(const Decl *D, CodeGenModule &CGM) { + const auto &Allowed = CGM.getCodeGenOpts().DebugInfoAllowedFiles; + if (Allowed.empty()) + return true; + auto &SM = CGM.getContext().getSourceManager(); + auto Filename = SM.getFilename(SM.getExpansionLoc(D->getLocation())); + if (Filename.empty()) + return false; + for (const auto &F : Allowed) + if (Filename == F) + return true; + return false; +} + /// Emit a DISubprogram for a function declaration so its prototype ends /// up in the debug output (LF_FUNC_ID in CodeView) even when the TU /// has no definition. @@ -6289,6 +6305,8 @@ static void emitFunctionDecl(CodeGenModule &CGM, const FunctionDecl *FD) { const auto *FPT = FD->getType()->getAs(); if (!FPT || FPT->isDependentType() || FPT->isUndeducedType()) return; + if (!isDeclInAllowedFiles(FD, CGM)) + return; CGDebugInfo *DI = CGM.getModuleDebugInfo(); if (DI) DI->EmitFunctionDecl(GlobalDecl(FD), FD->getLocation(), QualType(FPT, 0)); @@ -6350,7 +6368,7 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) { case Decl::CXXRecord: { CXXRecordDecl *CRD = cast(D); if (CGDebugInfo *DI = getModuleDebugInfo()) { - if (CRD->hasDefinition()) + if (CRD->hasDefinition() && isDeclInAllowedFiles(D, *this)) DI->EmitAndRetainType(getContext().getRecordType(cast(D))); if (auto *ES = D->getASTContext().getExternalSource()) if (ES->hasExternalDefinitions(D) == ExternalASTSource::EK_Never) @@ -6572,19 +6590,22 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) { case Decl::Typedef: case Decl::TypeAlias: // using foo = bar; [C++11] if (CGDebugInfo *DI = getModuleDebugInfo()) - DI->EmitAndRetainType( - getContext().getTypedefType(cast(D))); + if (isDeclInAllowedFiles(D, *this)) + DI->EmitAndRetainType( + getContext().getTypedefType(cast(D))); break; case Decl::Record: if (CGDebugInfo *DI = getModuleDebugInfo()) - if (cast(D)->getDefinition()) + if (cast(D)->getDefinition() && + isDeclInAllowedFiles(D, *this)) DI->EmitAndRetainType(getContext().getRecordType(cast(D))); break; case Decl::Enum: if (CGDebugInfo *DI = getModuleDebugInfo()) - if (cast(D)->getDefinition()) + if (cast(D)->getDefinition() && + isDeclInAllowedFiles(D, *this)) DI->EmitAndRetainType(getContext().getEnumType(cast(D))); break; diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index c5ec17b47089..ac223fe97e3f 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -5744,6 +5744,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, Args.AddLastArg(CmdArgs, options::OPT_working_directory); Args.AddLastArg(CmdArgs, options::OPT_fcase_insensitive_paths); + Args.AddAllArgs(CmdArgs, options::OPT_fdebug_info_allowed_file_EQ); RenderARCMigrateToolOptions(D, Args, CmdArgs); diff --git a/clang/test/CodeGen/debug-info-allowed-file.c b/clang/test/CodeGen/debug-info-allowed-file.c new file mode 100644 index 000000000000..3ec308b10325 --- /dev/null +++ b/clang/test/CodeGen/debug-info-allowed-file.c @@ -0,0 +1,26 @@ +// Verify that -fdebug-info-allowed-file restricts rich debug-info emission to +// declarations from the listed files. + +// RUN: rm -rf %t && mkdir -p %t +// RUN: echo 'struct allowed_struct {}; void allowed_fn(int);' > %t/allowed.h +// RUN: echo 'struct excluded_struct {}; void excluded_fn(int);' > %t/excluded.h + +// With the filter, only decls from allowed.h appear in the debug output. +// RUN: %clang_cc1 -debug-info-kind=unused-types -emit-llvm -o - %s -I %t \ +// RUN: -fdebug-info-allowed-file=%t/allowed.h \ +// RUN: | FileCheck %s --check-prefix=FILTERED + +// Without the filter, both decls are emitted. +// RUN: %clang_cc1 -debug-info-kind=unused-types -emit-llvm -o - %s -I %t \ +// RUN: | FileCheck %s --check-prefix=UNFILTERED + +#include "allowed.h" +#include "excluded.h" + +// FILTERED-DAG: name: "allowed_struct" +// FILTERED-DAG: name: "allowed_fn" +// FILTERED-NOT: name: "excluded_struct" +// FILTERED-NOT: name: "excluded_fn" + +// UNFILTERED-DAG: name: "allowed_struct" +// UNFILTERED-DAG: name: "excluded_struct" diff --git a/clang/test/Driver/fdebug-info-allowed-file.c b/clang/test/Driver/fdebug-info-allowed-file.c new file mode 100644 index 000000000000..e8678d18bef6 --- /dev/null +++ b/clang/test/Driver/fdebug-info-allowed-file.c @@ -0,0 +1,3 @@ +// RUN: %clang -### -fdebug-info-allowed-file=a.h -fdebug-info-allowed-file=b.h -c %s 2>&1 | FileCheck %s + +// CHECK: "-fdebug-info-allowed-file=a.h" "-fdebug-info-allowed-file=b.h" From e966bb52c876de8da25b301e960f886234c78007 Mon Sep 17 00:00:00 2001 From: Alessandro Di Federico Date: Fri, 27 Feb 2026 16:50:30 +0100 Subject: [PATCH 011/147] Add LF_ALIAS support to CodeView type system Promote LF_ALIAS (0x150a) from a bare CV_TYPE enum entry to a full TYPE_RECORD with an AliasRecord class. Emit LF_ALIAS records in lowerTypeAlias instead of returning the underlying type index directly, and remove the typedef look-through in getCompleteTypeIndex since aliases are now first-class type records. Add visitor, serialization, YAML, and dump support for AliasRecord across all CodeView infrastructure. Based on https://github.com/llvm/llvm-project/pull/153936. --- .../llvm/DebugInfo/CodeView/CodeViewTypes.def | 2 +- .../llvm/DebugInfo/CodeView/TypeRecord.h | 13 ++++ llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp | 13 ++-- llvm/lib/DebugInfo/CodeView/RecordName.cpp | 5 ++ .../DebugInfo/CodeView/TypeDumpVisitor.cpp | 6 ++ .../DebugInfo/CodeView/TypeIndexDiscovery.cpp | 3 + .../DebugInfo/CodeView/TypeRecordMapping.cpp | 6 ++ llvm/lib/ObjectYAML/CodeViewYAMLTypes.cpp | 5 ++ .../DebugInfo/COFF/array-odr-violation.ll | 14 +++-- llvm/test/DebugInfo/COFF/global_rust.ll | 2 +- llvm/test/DebugInfo/COFF/int8-char-type.ll | 15 ++++- llvm/test/DebugInfo/COFF/integer-128.ll | 4 +- llvm/test/DebugInfo/COFF/nested-types.ll | 8 ++- llvm/test/DebugInfo/COFF/typedef.ll | 11 +++- .../DebugInfo/COFF/types-array-advanced.ll | 11 +++- llvm/test/DebugInfo/COFF/types-cvarargs.ll | 2 +- llvm/test/DebugInfo/COFF/udts-complete.ll | 22 ++++--- llvm/test/DebugInfo/COFF/udts.ll | 16 ++--- llvm/test/DebugInfo/PDB/Inputs/typedef.ll | 47 ++++++++++++++ llvm/test/DebugInfo/PDB/Inputs/typedef.yaml | 12 ++++ llvm/test/DebugInfo/PDB/typedef.test | 62 +++++++++++++++++++ llvm/tools/llvm-pdbutil/MinimalTypeDumper.cpp | 7 +++ 22 files changed, 242 insertions(+), 44 deletions(-) create mode 100644 llvm/test/DebugInfo/PDB/Inputs/typedef.ll create mode 100644 llvm/test/DebugInfo/PDB/Inputs/typedef.yaml create mode 100644 llvm/test/DebugInfo/PDB/typedef.test diff --git a/llvm/include/llvm/DebugInfo/CodeView/CodeViewTypes.def b/llvm/include/llvm/DebugInfo/CodeView/CodeViewTypes.def index a31111eb80a4..aa3beea75c75 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/CodeViewTypes.def +++ b/llvm/include/llvm/DebugInfo/CodeView/CodeViewTypes.def @@ -53,6 +53,7 @@ TYPE_RECORD(LF_ENUM, 0x1507, Enum) TYPE_RECORD(LF_TYPESERVER2, 0x1515, TypeServer2) TYPE_RECORD(LF_VFTABLE, 0x151d, VFTable) TYPE_RECORD(LF_VTSHAPE, 0x000a, VFTableShape) +TYPE_RECORD(LF_ALIAS, 0x150a, Alias) TYPE_RECORD(LF_BITFIELD, 0x1205, BitField) @@ -181,7 +182,6 @@ CV_TYPE(LF_MANAGED_ST, 0x140f) CV_TYPE(LF_ST_MAX, 0x1500) CV_TYPE(LF_TYPESERVER, 0x1501) CV_TYPE(LF_DIMARRAY, 0x1508) -CV_TYPE(LF_ALIAS, 0x150a) CV_TYPE(LF_DEFARG, 0x150b) CV_TYPE(LF_FRIENDFCN, 0x150c) CV_TYPE(LF_NESTTYPEEX, 0x1512) diff --git a/llvm/include/llvm/DebugInfo/CodeView/TypeRecord.h b/llvm/include/llvm/DebugInfo/CodeView/TypeRecord.h index 24a4accab845..924b9520038b 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/TypeRecord.h +++ b/llvm/include/llvm/DebugInfo/CodeView/TypeRecord.h @@ -953,6 +953,19 @@ class EndPrecompRecord : public TypeRecord { uint32_t Signature = 0; }; +// LF_ALIAS +class AliasRecord : public TypeRecord { +public: + AliasRecord() = default; + explicit AliasRecord(TypeRecordKind Kind) : TypeRecord(Kind) {} + AliasRecord(TypeIndex UnderlyingType, StringRef Name) + : TypeRecord(TypeRecordKind::Alias), UnderlyingType(UnderlyingType), + Name(Name) {} + + TypeIndex UnderlyingType; + StringRef Name; +}; + } // end namespace codeview } // end namespace llvm diff --git a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp index c4223dd516d9..31bf54b3d845 100644 --- a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp @@ -1684,6 +1684,9 @@ TypeIndex CodeViewDebug::lowerTypeAlias(const DIDerivedType *Ty) { addToUDTs(Ty); + AliasRecord AR(UnderlyingTypeIndex, TypeName); + auto AliasIndex = TypeTable.writeLeafType(AR); + if (UnderlyingTypeIndex == TypeIndex(SimpleTypeKind::Int32Long) && TypeName == "HRESULT") return TypeIndex(SimpleTypeKind::HResult); @@ -1691,7 +1694,7 @@ TypeIndex CodeViewDebug::lowerTypeAlias(const DIDerivedType *Ty) { TypeName == "wchar_t") return TypeIndex(SimpleTypeKind::WideCharacter); - return UnderlyingTypeIndex; + return AliasIndex; } TypeIndex CodeViewDebug::lowerTypeArray(const DICompositeType *Ty) { @@ -2703,14 +2706,6 @@ TypeIndex CodeViewDebug::getCompleteTypeIndex(const DIType *Ty) { if (!Ty) return TypeIndex::Void(); - // Look through typedefs when getting the complete type index. Call - // getTypeIndex on the typdef to ensure that any UDTs are accumulated and are - // emitted only once. - if (Ty->getTag() == dwarf::DW_TAG_typedef) - (void)getTypeIndex(Ty); - while (Ty->getTag() == dwarf::DW_TAG_typedef) - Ty = cast(Ty)->getBaseType(); - // If this is a non-record type, the complete type index is the same as the // normal type index. Just call getTypeIndex. switch (Ty->getTag()) { diff --git a/llvm/lib/DebugInfo/CodeView/RecordName.cpp b/llvm/lib/DebugInfo/CodeView/RecordName.cpp index 5fbbc4a5d497..bc5aa4670e01 100644 --- a/llvm/lib/DebugInfo/CodeView/RecordName.cpp +++ b/llvm/lib/DebugInfo/CodeView/RecordName.cpp @@ -251,6 +251,11 @@ Error TypeNameComputer::visitKnownRecord(CVType &CVR, return Error::success(); } +Error TypeNameComputer::visitKnownRecord(CVType &CVR, AliasRecord &Alias) { + Name = Alias.Name; + return Error::success(); +} + std::string llvm::codeview::computeTypeName(TypeCollection &Types, TypeIndex Index) { TypeNameComputer Computer(Types); diff --git a/llvm/lib/DebugInfo/CodeView/TypeDumpVisitor.cpp b/llvm/lib/DebugInfo/CodeView/TypeDumpVisitor.cpp index df7e42df1afc..f2f7bc6ceb44 100644 --- a/llvm/lib/DebugInfo/CodeView/TypeDumpVisitor.cpp +++ b/llvm/lib/DebugInfo/CodeView/TypeDumpVisitor.cpp @@ -567,3 +567,9 @@ Error TypeDumpVisitor::visitKnownRecord(CVType &CVR, W->printHex("Signature", EndPrecomp.getSignature()); return Error::success(); } + +Error TypeDumpVisitor::visitKnownRecord(CVType &CVR, AliasRecord &Alias) { + printTypeIndex("UnderlyingType", Alias.UnderlyingType); + W->printString("Name", Alias.Name); + return Error::success(); +} diff --git a/llvm/lib/DebugInfo/CodeView/TypeIndexDiscovery.cpp b/llvm/lib/DebugInfo/CodeView/TypeIndexDiscovery.cpp index 682747a2b81f..9396f2ac341a 100644 --- a/llvm/lib/DebugInfo/CodeView/TypeIndexDiscovery.cpp +++ b/llvm/lib/DebugInfo/CodeView/TypeIndexDiscovery.cpp @@ -308,6 +308,9 @@ static void discoverTypeIndices(ArrayRef Content, TypeLeafKind Kind, case TypeLeafKind::LF_MODIFIER: Refs.push_back({TiRefKind::TypeRef, 0, 1}); break; + case TypeLeafKind::LF_ALIAS: + Refs.push_back({TiRefKind::TypeRef, 0, 1}); + break; case TypeLeafKind::LF_PROCEDURE: Refs.push_back({TiRefKind::TypeRef, 0, 1}); Refs.push_back({TiRefKind::TypeRef, 8, 1}); diff --git a/llvm/lib/DebugInfo/CodeView/TypeRecordMapping.cpp b/llvm/lib/DebugInfo/CodeView/TypeRecordMapping.cpp index 0bc65f8d0359..eaebd0865f62 100644 --- a/llvm/lib/DebugInfo/CodeView/TypeRecordMapping.cpp +++ b/llvm/lib/DebugInfo/CodeView/TypeRecordMapping.cpp @@ -752,3 +752,9 @@ Error TypeRecordMapping::visitKnownRecord(CVType &CVR, error(IO.mapInteger(EndPrecomp.Signature, "Signature")); return Error::success(); } + +Error TypeRecordMapping::visitKnownRecord(CVType &CVR, AliasRecord &Alias) { + error(IO.mapInteger(Alias.UnderlyingType, "UnderlyingType")); + error(IO.mapStringZ(Alias.Name, "Name")); + return Error::success(); +} diff --git a/llvm/lib/ObjectYAML/CodeViewYAMLTypes.cpp b/llvm/lib/ObjectYAML/CodeViewYAMLTypes.cpp index e4e2b2a6d21a..a5c6fc2a7b9b 100644 --- a/llvm/lib/ObjectYAML/CodeViewYAMLTypes.cpp +++ b/llvm/lib/ObjectYAML/CodeViewYAMLTypes.cpp @@ -613,6 +613,11 @@ template <> void LeafRecordImpl::map(IO &IO) { IO.mapRequired("Signature", Record.Signature); } +template <> void LeafRecordImpl::map(IO &IO) { + IO.mapRequired("UnderlyingType", Record.UnderlyingType); + IO.mapRequired("Name", Record.Name); +} + template <> void MemberRecordImpl::map(IO &IO) { MappingTraits::mapping(IO, Record); } diff --git a/llvm/test/DebugInfo/COFF/array-odr-violation.ll b/llvm/test/DebugInfo/COFF/array-odr-violation.ll index 0283515c269b..3a1f8c852331 100644 --- a/llvm/test/DebugInfo/COFF/array-odr-violation.ll +++ b/llvm/test/DebugInfo/COFF/array-odr-violation.ll @@ -17,21 +17,27 @@ ; FIXME: sizeof(a) in the user program is 1, but we claim it is 4 because ; sometimes the frontend lies to us. See array-types-advanced.ll for an example. ; -; CHECK: # Array (0x1004) +; CHECK: # Alias (0x1004) +; CHECK: .short 0xe # Record length +; CHECK: .short 0x150a # Record kind: LF_ALIAS +; CHECK: .long 0x1003 # UnderlyingType: YYSTYPE +; CHECK: .asciz "YYSTYPE" # Name + +; CHECK: # Array (0x1005) ; CHECK: .short 0xe # Record length ; CHECK: .short 0x1503 # Record kind: LF_ARRAY -; CHECK: .long 0x1003 # ElementType: YYSTYPE +; CHECK: .long 0x1004 # ElementType: YYSTYPE ; CHECK: .long 0x23 # IndexType: unsigned __int64 ; CHECK: .short 0x4 # SizeOf ; CHECK: .byte 0 # Name ; CHECK: .byte 241 -; CHECK: # Union (0x1006) +; CHECK: # Union (0x1007) ; CHECK: .short 0x22 # Record length ; CHECK: .short 0x1506 # Record kind: LF_UNION ; CHECK: .short 0x1 # MemberCount ; CHECK: .short 0x600 # Properties ( HasUniqueName (0x200) | Sealed (0x400) ) -; CHECK: .long 0x1005 # FieldList: +; CHECK: .long 0x1006 # FieldList: ; CHECK: .short 0x4 # SizeOf ; CHECK: .asciz "YYSTYPE" # Name ; CHECK: .asciz ".?ATYYSTYPE@@" # LinkageName diff --git a/llvm/test/DebugInfo/COFF/global_rust.ll b/llvm/test/DebugInfo/COFF/global_rust.ll index 1bbfe0f49623..e107771b58fe 100644 --- a/llvm/test/DebugInfo/COFF/global_rust.ll +++ b/llvm/test/DebugInfo/COFF/global_rust.ll @@ -29,7 +29,7 @@ ; CHECK: DataSym { ; CHECK: Kind: S_LDATA32 (0x110C) ; CHECK: DataOffset: .rdata+0x8 -; CHECK: Type: impl$::vtable_type$ (0x101D) +; CHECK: Type: impl$::vtable_type$ (0x1020) ; CHECK: DisplayName: impl$::vtable$ ; CHECK: LinkageName: .rdata ; CHECK: } diff --git a/llvm/test/DebugInfo/COFF/int8-char-type.ll b/llvm/test/DebugInfo/COFF/int8-char-type.ll index 3dd5aae5421b..6e3afae5cd7a 100644 --- a/llvm/test/DebugInfo/COFF/int8-char-type.ll +++ b/llvm/test/DebugInfo/COFF/int8-char-type.ll @@ -5,10 +5,21 @@ ; DW_ATE_[un]signed encoding for all integer types if they don't have distinct ; integer types for characters types. This was PR30552. +; CHECK: Alias (0x1000) { +; CHECK-NEXT: TypeLeafKind: LF_ALIAS (0x150A) +; CHECK-NEXT: UnderlyingType: signed char (0x10) +; CHECK-NEXT: Name: int8_t +; CHECK-NEXT: } +; CHECK: Alias (0x1001) { +; CHECK-NEXT: TypeLeafKind: LF_ALIAS (0x150A) +; CHECK-NEXT: UnderlyingType: unsigned char (0x20) +; CHECK-NEXT: Name: uint8_t +; CHECK-NEXT: } + ; CHECK-LABEL: GlobalData { ; CHECK-NEXT: Kind: S_GDATA32 (0x110D) ; CHECK-NEXT: DataOffset: -; CHECK-NEXT: Type: signed char (0x10) +; CHECK-NEXT: Type: int8_t (0x1000) ; CHECK-NEXT: DisplayName: x ; CHECK-NEXT: LinkageName: x ; CHECK-NEXT: } @@ -16,7 +27,7 @@ ; CHECK-LABEL: GlobalData { ; CHECK-NEXT: Kind: S_GDATA32 (0x110D) ; CHECK-NEXT: DataOffset: -; CHECK-NEXT: Type: unsigned char (0x20) +; CHECK-NEXT: Type: uint8_t (0x1001) ; CHECK-NEXT: DisplayName: y ; CHECK-NEXT: LinkageName: y ; CHECK-NEXT: } diff --git a/llvm/test/DebugInfo/COFF/integer-128.ll b/llvm/test/DebugInfo/COFF/integer-128.ll index 57ebdcc144af..e29e37388544 100644 --- a/llvm/test/DebugInfo/COFF/integer-128.ll +++ b/llvm/test/DebugInfo/COFF/integer-128.ll @@ -18,7 +18,7 @@ ; ASM-LABEL: .long 241 # Symbol subsection for globals ; ; ASM-LABEL: .short 4359 # Record kind: S_CONSTANT -; ASM-NEXT: .long 4110 # Type +; ASM-NEXT: .long 4111 # Type ; ASM-NEXT: .byte 0x0a, 0x80, 0xff, 0xff # Value ; ASM-NEXT: .byte 0xff, 0xff, 0xff, 0xff ; ASM-NEXT: .byte 0xff, 0xff @@ -26,7 +26,7 @@ ; ASM-NEXT: .p2align 2 ; ; ASM-LABEL: .short 4359 # Record kind: S_CONSTANT -; ASM-NEXT: .long 4111 # Type +; ASM-NEXT: .long 4112 # Type ; ASM-NEXT: .byte 0x09, 0x80, 0x00, 0x00 # Value ; ASM-NEXT: .byte 0x00, 0x00, 0x00, 0x00 ; ASM-NEXT: .byte 0x00, 0x80 diff --git a/llvm/test/DebugInfo/COFF/nested-types.ll b/llvm/test/DebugInfo/COFF/nested-types.ll index c68e4314925a..276054db6545 100644 --- a/llvm/test/DebugInfo/COFF/nested-types.ll +++ b/llvm/test/DebugInfo/COFF/nested-types.ll @@ -72,6 +72,12 @@ target triple = "x86_64-pc-windows-msvc19.0.24215" ; CHECK-NEXT: LinkageName: .?AW4InnerEnum@HasNested@@ ; CHECK-NEXT: } +; CHECK: Alias ([[INNERALIAS:0x.*]]) { +; CHECK-NEXT: TypeLeafKind: LF_ALIAS (0x150A) +; CHECK-NEXT: UnderlyingType: int (0x74) +; CHECK-NEXT: Name: InnerTypedef +; CHECK-NEXT: } + ; CHECK: FieldList ([[UNNAMED_MEMBERS:0x.*]]) { ; CHECK-NEXT: TypeLeafKind: LF_FIELDLIST (0x1203) ; CHECK-NEXT: Enumerator { @@ -120,7 +126,7 @@ target triple = "x86_64-pc-windows-msvc19.0.24215" ; CHECK-NEXT: } ; CHECK-NEXT: NestedType { ; CHECK-NEXT: TypeLeafKind: LF_NESTTYPE (0x1510) -; CHECK-NEXT: Type: int (0x74) +; CHECK-NEXT: Type: InnerTypedef ([[INNERALIAS]]) ; CHECK-NEXT: Name: InnerTypedef ; CHECK-NEXT: } ; CHECK-NEXT: NestedType { diff --git a/llvm/test/DebugInfo/COFF/typedef.ll b/llvm/test/DebugInfo/COFF/typedef.ll index 892bd4dead0a..9b15beea5786 100644 --- a/llvm/test/DebugInfo/COFF/typedef.ll +++ b/llvm/test/DebugInfo/COFF/typedef.ll @@ -1,9 +1,16 @@ ; RUN: llc < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s +; CHECK: CodeViewTypes [ +; CHECK: Alias (0x1003) { +; CHECK: TypeLeafKind: LF_ALIAS (0x150A) +; CHECK: UnderlyingType: wchar_t (0x71) +; CHECK: Name: XYZ +; CHECK: } +; CHECK: ] ; CHECK: CodeViewDebugInfo [ ; CHECK: Subsection [ ; CHECK: LocalSym { -; CHECK: Type: wchar_t (0x71) +; CHECK: Type: XYZ (0x1003) ; CHECK: Flags [ (0x0) ; CHECK: ] ; CHECK: VarName: foo @@ -11,7 +18,7 @@ ; CHECK: Subsection [ ; CHECK: SubSectionType: Symbols (0xF1) ; CHECK: UDTSym { -; CHECK: Type: wchar_t (0x71) +; CHECK: Type: XYZ (0x1003) ; CHECK: UDTName: XYZ ; CHECK: } ; CHECK: ] diff --git a/llvm/test/DebugInfo/COFF/types-array-advanced.ll b/llvm/test/DebugInfo/COFF/types-array-advanced.ll index f44510813372..eba2963d0001 100644 --- a/llvm/test/DebugInfo/COFF/types-array-advanced.ll +++ b/llvm/test/DebugInfo/COFF/types-array-advanced.ll @@ -157,12 +157,17 @@ ; CHECK: Volatile (0x2) ; CHECK: ] ; CHECK: } -; CHECK: Array (0x1010) { +; CHECK: Alias (0x1010) { +; CHECK: TypeLeafKind: LF_ALIAS (0x150A) +; CHECK: UnderlyingType: const volatile int (0x100F) +; CHECK: Name: T_INT +; CHECK: } +; CHECK: Array (0x1011) { ; CHECK: TypeLeafKind: LF_ARRAY (0x1503) -; CHECK: ElementType: const volatile int (0x100F) +; CHECK: ElementType: T_INT (0x1010) ; CHECK: IndexType: unsigned long (0x22) ; CHECK: SizeOf: 16 -; CHECK: Name: +; CHECK: Name: ; CHECK: } ; CHECK: ] diff --git a/llvm/test/DebugInfo/COFF/types-cvarargs.ll b/llvm/test/DebugInfo/COFF/types-cvarargs.ll index 68c7a7740b71..fd4b2048928a 100644 --- a/llvm/test/DebugInfo/COFF/types-cvarargs.ll +++ b/llvm/test/DebugInfo/COFF/types-cvarargs.ll @@ -34,7 +34,7 @@ ; CHECK: } ; CHECK: UDTSym { ; CHECK: Kind: S_UDT (0x1108) -; CHECK: Type: void (int, float, )* (0x100F) +; CHECK: Type: FuncTypedef (0x1010) ; CHECK: UDTName: FuncTypedef ; CHECK: } ; CHECK: ] diff --git a/llvm/test/DebugInfo/COFF/udts-complete.ll b/llvm/test/DebugInfo/COFF/udts-complete.ll index e5ee8521ab48..28cc6d81b1d7 100644 --- a/llvm/test/DebugInfo/COFF/udts-complete.ll +++ b/llvm/test/DebugInfo/COFF/udts-complete.ll @@ -1,8 +1,8 @@ ; RUN: llc -filetype=obj %s -o %t.obj ; RUN: llvm-pdbutil dump -symbols -types %t.obj | FileCheck %s -; Make sure that `gv`, `Bar`, and `Baz` all point to the complete type index of -; Foo (0x1002), and not the forward declaration index (0x1000). +; Make sure that `gv`, `Bar`, and `Baz` reference proper type indices. +; With LF_ALIAS, typedefs get their own type records. ; C++ source: ; struct Foo { @@ -19,26 +19,28 @@ ; CHECK: unique name: `.?AUFoo@@` ; CHECK: vtable: , base list: , field list: ; CHECK: options: forward ref | has unique name, sizeof 0 -; CHECK: 0x1001 | LF_FIELDLIST [size = 16] +; CHECK: 0x1001 | LF_ALIAS [size = 12] alias = Bar, underlying type = 0x1000 +; CHECK: 0x1002 | LF_ALIAS [size = 12] alias = Baz, underlying type = 0x1001 +; CHECK: 0x1003 | LF_FIELDLIST [size = 16] ; CHECK: - LF_MEMBER [name = `x`, Type = 0x0074 (int), offset = 0, attrs = public] -; CHECK: 0x1002 | LF_STRUCTURE [size = 36] `Foo` +; CHECK: 0x1004 | LF_STRUCTURE [size = 36] `Foo` ; CHECK: unique name: `.?AUFoo@@` -; CHECK: vtable: , base list: , field list: 0x1001 +; CHECK: vtable: , base list: , field list: 0x1003 ; CHECK: options: has unique name, sizeof 4 -; CHECK: 0x1004 | LF_UDT_SRC_LINE [size = 16] -; CHECK: udt = 0x1002, file = 4099, line = 1 +; CHECK: 0x1006 | LF_UDT_SRC_LINE [size = 16] +; CHECK: udt = 0x1004, file = 4101, line = 1 ; CHECK: Symbols ; CHECK: ============================================================ ; CHECK: Mod 0000 | `.debug$S`: ; CHECK: 0 | S_GDATA32 [size = 20] `gv` -; CHECK: type = 0x1002 (Foo), addr = 0000:0000 +; CHECK: type = 0x1002 (Baz), addr = 0000:0000 ; CHECK: 0 | S_UDT [size = 12] `Bar` -; CHECK: original type = 0x1002 +; CHECK: original type = 0x1001 ; CHECK: 0 | S_UDT [size = 12] `Baz` ; CHECK: original type = 0x1002 ; CHECK: 0 | S_UDT [size = 12] `Foo` -; CHECK: original type = 0x1002 +; CHECK: original type = 0x1004 ; ModuleID = 't.cpp' source_filename = "t.cpp" diff --git a/llvm/test/DebugInfo/COFF/udts.ll b/llvm/test/DebugInfo/COFF/udts.ll index 86d2c7a6c079..3dc752f5ed83 100644 --- a/llvm/test/DebugInfo/COFF/udts.ll +++ b/llvm/test/DebugInfo/COFF/udts.ll @@ -36,7 +36,7 @@ ; READOBJ: } ; READOBJ: UDTSym { ; READOBJ-NEXT: Kind: S_UDT (0x1108) -; READOBJ-NEXT: Type: int (0x74) +; READOBJ-NEXT: Type: FOO (0x{{[0-9A-F]+}}) ; READOBJ-NEXT: UDTName: f::FOO ; READOBJ-NEXT: } ; READOBJ-NEXT: ProcEnd { @@ -79,13 +79,13 @@ ; PDBUTIL: Symbols ; PDBUTIL-NEXT: ============================================================ ; PDBUTIL-NOT: S_UDT {{.*}} `A::C` -; PDBUTIL: S_UDT [size = 16] `f::FOO` -; PDBUTIL: S_UDT [size = 16] `g::pun` -; PDBUTIL: S_UDT [size = 12] `S` -; PDBUTIL: S_UDT [size = 12] `A` -; PDBUTIL: S_UDT [size = 16] `A::D` -; PDBUTIL: S_UDT [size = 12] `U` -; PDBUTIL: S_UDT [size = 12] `U` +; PDBUTIL: S_UDT [size = {{[0-9]+}}] `f::FOO` +; PDBUTIL: S_UDT [size = {{[0-9]+}}] `g::pun` +; PDBUTIL: S_UDT [size = {{[0-9]+}}] `S` +; PDBUTIL: S_UDT [size = {{[0-9]+}}] `A` +; PDBUTIL: S_UDT [size = {{[0-9]+}}] `A::D` +; PDBUTIL: S_UDT [size = {{[0-9]+}}] `U` +; PDBUTIL: S_UDT [size = {{[0-9]+}}] `U` source_filename = "test/DebugInfo/COFF/udts.ll" target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128" diff --git a/llvm/test/DebugInfo/PDB/Inputs/typedef.ll b/llvm/test/DebugInfo/PDB/Inputs/typedef.ll new file mode 100644 index 000000000000..7c5b7bec7828 --- /dev/null +++ b/llvm/test/DebugInfo/PDB/Inputs/typedef.ll @@ -0,0 +1,47 @@ +; ModuleID = 'typedef.cpp' +source_filename = "typedef.cpp" +target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128" +target triple = "x86_64-pc-windows-msvc19.44.35214" + +; Function Attrs: mustprogress noinline norecurse nounwind optnone uwtable +define dso_local noundef i32 @main() #0 !dbg !14 { +entry: + %retval = alloca i32, align 4 + %val = alloca i8, align 1 + %val2 = alloca i64, align 8 + store i32 0, ptr %retval, align 4 + call void @llvm.dbg.declare(metadata ptr %val, metadata !19, metadata !DIExpression()), !dbg !22 + store i8 15, ptr %val, align 1, !dbg !22 + call void @llvm.dbg.declare(metadata ptr %val2, metadata !23, metadata !DIExpression()), !dbg !26 + store i64 -1, ptr %val2, align 8, !dbg !26 + ret i32 0, !dbg !27 +} + +; Function Attrs: nounwind readnone speculatable willreturn +declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 + +attributes #0 = { mustprogress noinline norecurse nounwind optnone uwtable } +attributes #1 = { nounwind readnone speculatable willreturn } + +!llvm.dbg.cu = !{!2} +!llvm.module.flags = !{!7, !8} +!llvm.ident = !{!13} + +!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !5, producer: "clang version 22.0.0git", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None) +!5 = !DIFile(filename: "typedef.cpp", directory: "llvm-project") +!7 = !{i32 2, !"CodeView", i32 1} +!8 = !{i32 2, !"Debug Info Version", i32 3} +!13 = !{!"clang version 22.0.0git"} +!14 = distinct !DISubprogram(name: "main", scope: !5, file: !5, line: 6, type: !15, scopeLine: 6, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !2) +!15 = !DISubroutineType(types: !16) +!16 = !{!17} +!17 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!19 = !DILocalVariable(name: "val", scope: !14, file: !5, line: 7, type: !20) +!20 = !DIDerivedType(tag: DW_TAG_typedef, name: "u8", file: !5, line: 3, baseType: !21) +!21 = !DIBasicType(name: "unsigned char", size: 8, encoding: DW_ATE_unsigned_char) +!22 = !DILocation(line: 7, scope: !14) +!23 = !DILocalVariable(name: "val2", scope: !14, file: !5, line: 8, type: !24) +!24 = !DIDerivedType(tag: DW_TAG_typedef, name: "i64", file: !5, line: 4, baseType: !25) +!25 = !DIBasicType(name: "long long", size: 64, encoding: DW_ATE_signed) +!26 = !DILocation(line: 8, scope: !14) +!27 = !DILocation(line: 10, scope: !14) diff --git a/llvm/test/DebugInfo/PDB/Inputs/typedef.yaml b/llvm/test/DebugInfo/PDB/Inputs/typedef.yaml new file mode 100644 index 000000000000..c02b11cce613 --- /dev/null +++ b/llvm/test/DebugInfo/PDB/Inputs/typedef.yaml @@ -0,0 +1,12 @@ +--- +TpiStream: + Records: + - Kind: LF_ALIAS + Alias: + UnderlyingType: 32 + Name: u8 + - Kind: LF_ALIAS + Alias: + UnderlyingType: 19 + Name: i64 +... diff --git a/llvm/test/DebugInfo/PDB/typedef.test b/llvm/test/DebugInfo/PDB/typedef.test new file mode 100644 index 000000000000..666036f27b03 --- /dev/null +++ b/llvm/test/DebugInfo/PDB/typedef.test @@ -0,0 +1,62 @@ +Built with clang (22+) because MSVC does not output lf_alias for typedefs + +Build typedef.ll with clang -S -emit-llvm -g: + +``` +void *__purecall = 0; + +typedef unsigned char u8; +using i64 = long long; + +int main() { + u8 val = 15; + i64 val2 = -1; + + return 0; +} +``` + +RUN: llc -O0 %p/Inputs/typedef.ll --filetype=obj -o %t.obj +RUN: llvm-pdbutil dump --types %t.obj | FileCheck --check-prefix=OBJTYPE %s + +OBJTYPE: 0x1003 | LF_ALIAS [size = 12] alias = u8, underlying type = 0x0020 (unsigned char) +OBJTYPE-NEXT: 0x1004 | LF_ALIAS [size = 12] alias = i64, underlying type = 0x0013 (__int64) + +RUN: llvm-pdbutil dump --symbols %t.obj | FileCheck --check-prefix=OBJSYM %s + +OBJSYM: 0 | S_LOCAL [size = 16] `val` +OBJSYM-NEXT: type=0x1003 (u8), flags = none +OBJSYM: 0 | S_LOCAL [size = 16] `val2` +OBJSYM: type=0x1004 (i64), flags = none +OBJSYM: 0 | S_UDT [size = 12] `u8` +OBJSYM-NEXT: original type = 0x1003 +OBJSYM-NEXT: 0 | S_UDT [size = 12] `i64` +OBJSYM-NEXT: original type = 0x1004 + + +typedef.yaml file generated via `clang -fno-rtti -g -O0 typedef.cpp` and `llvm-pdbutil pdb2yaml -tpi-stream`, and then manually +reduced to only the typedef nodes + +RUN: llvm-pdbutil yaml2pdb -pdb=%t.yaml.pdb %p/Inputs/typedef.yaml +RUN: llvm-pdbutil dump --types %t.yaml.pdb \ +RUN: | FileCheck --check-prefix=TYPES %s + +Also check that the YAML output hasn't drifted +RUN: llvm-pdbutil pdb2yaml -tpi-stream %t.yaml.pdb \ +RUN: | FileCheck --check-prefixes=YAML %s + +TYPES: Types (TPI Stream) +TYPES-NEXT:============================================================ +TYPES-NEXT: Showing 2 records +TYPES-NEXT: 0x1000 | LF_ALIAS [size = 12] alias = u8, underlying type = 0x0020 (unsigned char) +TYPES-NEXT: 0x1001 | LF_ALIAS [size = 12] alias = i64, underlying type = 0x0013 (__int64) + + +YAML: - Kind: LF_ALIAS +YAML-NEXT: Alias: +YAML-NEXT: UnderlyingType: 32 +YAML-NEXT: Name: u8 +YAML-NEXT: - Kind: LF_ALIAS +YAML-NEXT: Alias: +YAML-NEXT: UnderlyingType: 19 +YAML-NEXT: Name: i64 diff --git a/llvm/tools/llvm-pdbutil/MinimalTypeDumper.cpp b/llvm/tools/llvm-pdbutil/MinimalTypeDumper.cpp index be7e487673fb..e33628ea8340 100644 --- a/llvm/tools/llvm-pdbutil/MinimalTypeDumper.cpp +++ b/llvm/tools/llvm-pdbutil/MinimalTypeDumper.cpp @@ -521,6 +521,13 @@ Error MinimalTypeDumpVisitor::visitKnownRecord(CVType &CVR, return Error::success(); } +Error MinimalTypeDumpVisitor::visitKnownRecord(CVType &CVR, + AliasRecord &Alias) { + P.format(" alias = {0}, underlying type = {1}", Alias.Name, + Alias.UnderlyingType); + return Error::success(); +} + Error MinimalTypeDumpVisitor::visitKnownMember(CVMemberRecord &CVR, NestedTypeRecord &Nested) { P.format(" [name = `{0}`, parent = {1}]", Nested.Name, Nested.Type); From 5e2892486adcf5f703e20a9642e73b666154a45a Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Thu, 26 Jan 2023 15:09:48 +0100 Subject: [PATCH 012/147] [LVI] Use !range metadata for supported intrinsics Even if the intrinsic is supported by ConstantRange, we should still make use of !range metadata. This doesn't matter much now, but is important if we want to support ctlz style intrinsics, which always have KnownBits-based !range metadata attached, which might be better than what we can compute using ranges. --- llvm/lib/Analysis/LazyValueInfo.cpp | 8 +++++--- .../Transforms/CorrelatedValuePropagation/range.ll | 13 +++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp index 0e9fab667e6e..944b4a2938da 100644 --- a/llvm/lib/Analysis/LazyValueInfo.cpp +++ b/llvm/lib/Analysis/LazyValueInfo.cpp @@ -990,10 +990,11 @@ LazyValueInfoImpl::solveBlockValueOverflowIntrinsic(WithOverflowInst *WO, std::optional LazyValueInfoImpl::solveBlockValueIntrinsic(IntrinsicInst *II, BasicBlock *BB) { + ValueLatticeElement MetadataVal = getFromRangeMetadata(II); if (!ConstantRange::isIntrinsicSupported(II->getIntrinsicID())) { LLVM_DEBUG(dbgs() << " compute BB '" << BB->getName() << "' - unknown intrinsic.\n"); - return getFromRangeMetadata(II); + return MetadataVal; } SmallVector OpRanges; @@ -1004,8 +1005,9 @@ LazyValueInfoImpl::solveBlockValueIntrinsic(IntrinsicInst *II, BasicBlock *BB) { OpRanges.push_back(*Range); } - return ValueLatticeElement::getRange( - ConstantRange::intrinsic(II->getIntrinsicID(), OpRanges)); + return intersect(ValueLatticeElement::getRange(ConstantRange::intrinsic( + II->getIntrinsicID(), OpRanges)), + MetadataVal); } std::optional diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/range.ll b/llvm/test/Transforms/CorrelatedValuePropagation/range.ll index 435a096a6e1e..26b31ce8ee09 100644 --- a/llvm/test/Transforms/CorrelatedValuePropagation/range.ll +++ b/llvm/test/Transforms/CorrelatedValuePropagation/range.ll @@ -946,5 +946,18 @@ define i1 @intrinsic_range(i16 %x) { ret i1 %res } +define i1 @supported_intrinsic_range(i16 %x) { +; CHECK-LABEL: @supported_intrinsic_range( +; CHECK-NEXT: [[ABS:%.*]] = call i16 @llvm.abs.i16(i16 [[X:%.*]], i1 false), !range [[RNG5]] +; CHECK-NEXT: [[TRUNC:%.*]] = trunc i16 [[ABS]] to i8 +; CHECK-NEXT: ret i1 true +; + %abs = call i16 @llvm.abs.i16(i16 %x, i1 false), !range !{i16 0, i16 8} + %trunc = trunc i16 %abs to i8 + %res = icmp ult i8 %trunc, 8 + ret i1 %res +} + declare i16 @llvm.ctlz.i16(i16, i1) +declare i16 @llvm.abs.i16(i16, i1) declare void @llvm.assume(i1) From 4524bab6321eaa2b65e358c8b4a70d6b40311469 Mon Sep 17 00:00:00 2001 From: Antonio Frighetto Date: Fri, 17 Feb 2023 09:50:49 +0100 Subject: [PATCH 013/147] [ConstantRange] Handle `Intrinsic::ctlz` Introduce ConstantRange support for ctlz intrinsic, including exhaustive testing. Among other things, LVI may now be able to propagate information about cltz constant ranges lattice values. Differential Revision: https://reviews.llvm.org/D142234 --- llvm/include/llvm/IR/ConstantRange.h | 4 ++ llvm/lib/IR/ConstantRange.cpp | 46 ++++++++++++++++ .../CorrelatedValuePropagation/range.ll | 52 +++++++++++++++++++ llvm/unittests/IR/ConstantRangeTest.cpp | 15 ++++++ 4 files changed, 117 insertions(+) diff --git a/llvm/include/llvm/IR/ConstantRange.h b/llvm/include/llvm/IR/ConstantRange.h index 0b9ac18d0a92..ca36732e4e2e 100644 --- a/llvm/include/llvm/IR/ConstantRange.h +++ b/llvm/include/llvm/IR/ConstantRange.h @@ -526,6 +526,10 @@ class [[nodiscard]] ConstantRange { /// \p IntMinIsPoison is false. ConstantRange abs(bool IntMinIsPoison = false) const; + /// Calculate ctlz range. If \p ZeroIsPoison is set, the range is computed + /// ignoring a possible zero value contained in the input range. + ConstantRange ctlz(bool ZeroIsPoison = false) const; + /// Represents whether an operation on the given constant range is known to /// always or never overflow. enum class OverflowResult { diff --git a/llvm/lib/IR/ConstantRange.cpp b/llvm/lib/IR/ConstantRange.cpp index 0dbccaa1a66a..ab94053df45a 100644 --- a/llvm/lib/IR/ConstantRange.cpp +++ b/llvm/lib/IR/ConstantRange.cpp @@ -945,6 +945,7 @@ bool ConstantRange::isIntrinsicSupported(Intrinsic::ID IntrinsicID) { case Intrinsic::smin: case Intrinsic::smax: case Intrinsic::abs: + case Intrinsic::ctlz: return true; default: return false; @@ -976,6 +977,12 @@ ConstantRange ConstantRange::intrinsic(Intrinsic::ID IntrinsicID, assert(IntMinIsPoison->getBitWidth() == 1 && "Must be boolean"); return Ops[0].abs(IntMinIsPoison->getBoolValue()); } + case Intrinsic::ctlz: { + const APInt *ZeroIsPoison = Ops[1].getSingleElement(); + assert(ZeroIsPoison && "Must be known (immarg)"); + assert(ZeroIsPoison->getBitWidth() == 1 && "Must be boolean"); + return Ops[0].ctlz(ZeroIsPoison->getBoolValue()); + } default: assert(!isIntrinsicSupported(IntrinsicID) && "Shouldn't be supported"); llvm_unreachable("Unsupported intrinsic"); @@ -1667,6 +1674,45 @@ ConstantRange ConstantRange::abs(bool IntMinIsPoison) const { APIntOps::umax(-SMin, SMax) + 1); } +ConstantRange ConstantRange::ctlz(bool ZeroIsPoison) const { + if (isEmptySet()) + return getEmpty(); + + APInt Zero = APInt::getZero(getBitWidth()); + if (ZeroIsPoison && contains(Zero)) { + // ZeroIsPoison is set, and zero is contained. We discern three cases, in + // which a zero can appear: + // 1) Lower is zero, handling cases of kind [0, 1), [0, 2), etc. + // 2) Upper is zero, wrapped set, handling cases of kind [3, 0], etc. + // 3) Zero contained in a wrapped set, e.g., [3, 2), [3, 1), etc. + + if (getLower().isZero()) { + if ((getUpper() - 1).isZero()) { + // We have in input interval of kind [0, 1). In this case we cannot + // really help but return empty-set. + return getEmpty(); + } + + // Compute the resulting range by excluding zero from Lower. + return ConstantRange( + APInt(getBitWidth(), (getUpper() - 1).countLeadingZeros()), + APInt(getBitWidth(), (getLower() + 1).countLeadingZeros() + 1)); + } else if ((getUpper() - 1).isZero()) { + // Compute the resulting range by excluding zero from Upper. + return ConstantRange( + Zero, APInt(getBitWidth(), getLower().countLeadingZeros() + 1)); + } else { + return ConstantRange(Zero, APInt(getBitWidth(), getBitWidth())); + } + } + + // Zero is either safe or not in the range. The output range is composed by + // the result of countLeadingZero of the two extremes. + return getNonEmpty( + APInt(getBitWidth(), getUnsignedMax().countLeadingZeros()), + APInt(getBitWidth(), getUnsignedMin().countLeadingZeros() + 1)); +} + ConstantRange::OverflowResult ConstantRange::unsignedAddMayOverflow( const ConstantRange &Other) const { if (isEmptySet() || Other.isEmptySet()) diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/range.ll b/llvm/test/Transforms/CorrelatedValuePropagation/range.ll index 26b31ce8ee09..0d3b547f102b 100644 --- a/llvm/test/Transforms/CorrelatedValuePropagation/range.ll +++ b/llvm/test/Transforms/CorrelatedValuePropagation/range.ll @@ -958,6 +958,58 @@ define i1 @supported_intrinsic_range(i16 %x) { ret i1 %res } +define i1 @ctlz_fold(i16 %x) { +; CHECK-LABEL: @ctlz_fold( +; CHECK-NEXT: [[CMP:%.*]] = icmp ult i16 [[X:%.*]], 256 +; CHECK-NEXT: br i1 [[CMP]], label [[IF:%.*]], label [[ELSE:%.*]] +; CHECK: if: +; CHECK-NEXT: [[CTLZ:%.*]] = call i16 @llvm.ctlz.i16(i16 [[X]], i1 false) +; CHECK-NEXT: ret i1 true +; CHECK: else: +; CHECK-NEXT: [[CTLZ2:%.*]] = call i16 @llvm.ctlz.i16(i16 [[X]], i1 false) +; CHECK-NEXT: ret i1 true +; + %cmp = icmp ult i16 %x, 256 + br i1 %cmp, label %if, label %else + +if: + %ctlz = call i16 @llvm.ctlz.i16(i16 %x, i1 false) + %res = icmp uge i16 %ctlz, 8 + ret i1 %res + +else: + %ctlz2 = call i16 @llvm.ctlz.i16(i16 %x, i1 false) + %res2 = icmp ult i16 %ctlz2, 8 + ret i1 %res2 +} + +define i1 @ctlz_nofold(i16 %x) { +; CHECK-LABEL: @ctlz_nofold( +; CHECK-NEXT: [[CMP:%.*]] = icmp ult i16 [[X:%.*]], 256 +; CHECK-NEXT: br i1 [[CMP]], label [[IF:%.*]], label [[ELSE:%.*]] +; CHECK: if: +; CHECK-NEXT: [[CTLZ:%.*]] = call i16 @llvm.ctlz.i16(i16 [[X]], i1 false) +; CHECK-NEXT: [[RES:%.*]] = icmp uge i16 [[CTLZ]], 9 +; CHECK-NEXT: ret i1 [[RES]] +; CHECK: else: +; CHECK-NEXT: [[CTLZ2:%.*]] = call i16 @llvm.ctlz.i16(i16 [[X]], i1 false) +; CHECK-NEXT: [[RES2:%.*]] = icmp ult i16 [[CTLZ2]], 7 +; CHECK-NEXT: ret i1 [[RES2]] +; + %cmp = icmp ult i16 %x, 256 + br i1 %cmp, label %if, label %else + +if: + %ctlz = call i16 @llvm.ctlz.i16(i16 %x, i1 false) + %res = icmp uge i16 %ctlz, 9 + ret i1 %res + +else: + %ctlz2 = call i16 @llvm.ctlz.i16(i16 %x, i1 false) + %res2 = icmp ult i16 %ctlz2, 7 + ret i1 %res2 +} + declare i16 @llvm.ctlz.i16(i16, i1) declare i16 @llvm.abs.i16(i16, i1) declare void @llvm.assume(i1) diff --git a/llvm/unittests/IR/ConstantRangeTest.cpp b/llvm/unittests/IR/ConstantRangeTest.cpp index b54746b00536..5d1931ebf602 100644 --- a/llvm/unittests/IR/ConstantRangeTest.cpp +++ b/llvm/unittests/IR/ConstantRangeTest.cpp @@ -2397,6 +2397,21 @@ TEST_F(ConstantRangeTest, Abs) { }); } +TEST_F(ConstantRangeTest, Ctlz) { + TestUnaryOpExhaustive([](const ConstantRange &CR) { return CR.ctlz(); }, + [](const APInt &N) { + return APInt(N.getBitWidth(), N.countLeadingZeros()); + }); + + TestUnaryOpExhaustive( + [](const ConstantRange &CR) { return CR.ctlz(/*ZeroIsPoison=*/true); }, + [](const APInt &N) -> std::optional { + if (N.isZero()) + return std::nullopt; + return APInt(N.getBitWidth(), N.countLeadingZeros()); + }); +} + TEST_F(ConstantRangeTest, castOps) { ConstantRange A(APInt(16, 66), APInt(16, 128)); ConstantRange FpToI8 = A.castOp(Instruction::FPToSI, 8); From 9ad18d894c37aa299ed88ad0dbb24f348846ebd3 Mon Sep 17 00:00:00 2001 From: Alessandro Di Federico Date: Tue, 21 May 2019 23:13:13 +0200 Subject: [PATCH 014/147] MachO: improve relocation information --- llvm/include/llvm/BinaryFormat/MachO.h | 2 ++ llvm/include/llvm/Object/MachO.h | 6 ++++-- llvm/lib/Object/MachOObjectFile.cpp | 18 ++++++++++++++---- .../MachO/Inputs/macho-rebase-set-type-imm | Bin 8432 -> 0 bytes .../tools/llvm-objdump/MachO/bad-bind.test | 3 --- 5 files changed, 20 insertions(+), 9 deletions(-) delete mode 100755 llvm/test/tools/llvm-objdump/MachO/Inputs/macho-rebase-set-type-imm diff --git a/llvm/include/llvm/BinaryFormat/MachO.h b/llvm/include/llvm/BinaryFormat/MachO.h index d51af31fb14f..f7941e0441db 100644 --- a/llvm/include/llvm/BinaryFormat/MachO.h +++ b/llvm/include/llvm/BinaryFormat/MachO.h @@ -232,6 +232,7 @@ enum DataRegionType { }; enum RebaseType { + REBASE_TYPE_INVALID = 0u, REBASE_TYPE_POINTER = 1u, REBASE_TYPE_TEXT_ABSOLUTE32 = 2u, REBASE_TYPE_TEXT_PCREL32 = 3u @@ -252,6 +253,7 @@ enum RebaseOpcode { }; enum BindType { + BIND_TYPE_INVALID = 0u, BIND_TYPE_POINTER = 1u, BIND_TYPE_TEXT_ABSOLUTE32 = 2u, BIND_TYPE_TEXT_PCREL32 = 3u diff --git a/llvm/include/llvm/Object/MachO.h b/llvm/include/llvm/Object/MachO.h index 56e7c8580b4e..c9e2c3118100 100644 --- a/llvm/include/llvm/Object/MachO.h +++ b/llvm/include/llvm/Object/MachO.h @@ -173,6 +173,7 @@ class MachORebaseEntry { int32_t segmentIndex() const; uint64_t segmentOffset() const; StringRef typeName() const; + MachO::RebaseType type() const; StringRef segmentName() const; StringRef sectionName() const; uint64_t address() const; @@ -196,7 +197,7 @@ class MachORebaseEntry { int32_t SegmentIndex = -1; uint64_t RemainingLoopCount = 0; uint64_t AdvanceAmount = 0; - uint8_t RebaseType = 0; + MachO::RebaseType RebaseType = MachO::REBASE_TYPE_INVALID; uint8_t PointerSize; bool Done = false; }; @@ -219,6 +220,7 @@ class MachOBindEntry { int32_t segmentIndex() const; uint64_t segmentOffset() const; StringRef typeName() const; + MachO::BindType type() const; StringRef symbolName() const; uint32_t flags() const; int64_t addend() const; @@ -253,7 +255,7 @@ class MachOBindEntry { int64_t Addend = 0; uint64_t RemainingLoopCount = 0; uint64_t AdvanceAmount = 0; - uint8_t BindType = 0; + MachO::BindType BindType = MachO::BIND_TYPE_INVALID; uint8_t PointerSize; Kind TableKind; bool Done = false; diff --git a/llvm/lib/Object/MachOObjectFile.cpp b/llvm/lib/Object/MachOObjectFile.cpp index 9c0b85cf7416..35d8a32fb7c5 100644 --- a/llvm/lib/Object/MachOObjectFile.cpp +++ b/llvm/lib/Object/MachOObjectFile.cpp @@ -3518,8 +3518,9 @@ void MachORebaseEntry::moveNext() { DEBUG_WITH_TYPE("mach-o-rebase", dbgs() << "REBASE_OPCODE_DONE\n"); break; case MachO::REBASE_OPCODE_SET_TYPE_IMM: - RebaseType = ImmValue; - if (RebaseType > MachO::REBASE_TYPE_TEXT_PCREL32) { + RebaseType = static_cast(ImmValue); + if (RebaseType <= MachO::REBASE_TYPE_INVALID + || RebaseType < MachO::REBASE_TYPE_POINTER) { *E = malformedError("for REBASE_OPCODE_SET_TYPE_IMM bad bind type: " + Twine((int)RebaseType) + " for opcode at: 0x" + Twine::utohexstr(OpcodeStart - Opcodes.begin())); @@ -3748,8 +3749,12 @@ int32_t MachORebaseEntry::segmentIndex() const { return SegmentIndex; } uint64_t MachORebaseEntry::segmentOffset() const { return SegmentOffset; } +MachO::RebaseType MachORebaseEntry::type() const { return RebaseType; } + StringRef MachORebaseEntry::typeName() const { switch (RebaseType) { + case MachO::REBASE_TYPE_INVALID: + return "invalid"; case MachO::REBASE_TYPE_POINTER: return "pointer"; case MachO::REBASE_TYPE_TEXT_ABSOLUTE32: @@ -3978,8 +3983,9 @@ void MachOBindEntry::moveNext() { } break; case MachO::BIND_OPCODE_SET_TYPE_IMM: - BindType = ImmValue; - if (ImmValue > MachO::BIND_TYPE_TEXT_PCREL32) { + BindType = static_cast(ImmValue); + if (ImmValue <= MachO::BIND_TYPE_INVALID + || ImmValue > MachO::BIND_TYPE_TEXT_PCREL32) { *E = malformedError("for BIND_OPCODE_SET_TYPE_IMM bad bind type: " + Twine((int)ImmValue) + " for opcode at: 0x" + Twine::utohexstr(OpcodeStart - Opcodes.begin())); @@ -4292,8 +4298,12 @@ int32_t MachOBindEntry::segmentIndex() const { return SegmentIndex; } uint64_t MachOBindEntry::segmentOffset() const { return SegmentOffset; } +MachO::BindType MachOBindEntry::type() const { return BindType; } + StringRef MachOBindEntry::typeName() const { switch (BindType) { + case MachO::BIND_TYPE_INVALID: + return "invalid"; case MachO::BIND_TYPE_POINTER: return "pointer"; case MachO::BIND_TYPE_TEXT_ABSOLUTE32: diff --git a/llvm/test/tools/llvm-objdump/MachO/Inputs/macho-rebase-set-type-imm b/llvm/test/tools/llvm-objdump/MachO/Inputs/macho-rebase-set-type-imm deleted file mode 100755 index 947db0ee915f435659ff6b833bef998f0f6b3ee9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8432 zcmeHM&ui2`6rQx!E^W0bsQ8;$TB#n|Dm{4YmM*5WXn$-$1Q~a`sk@l{K{iWUL8ufD zD-`NYFCO&bp-7M36{;TeU+|z(L80J9L|osO%yg5~?b*XTcsu##z4_+l%M4^QAAWuR z)5@6E%-C`hW2}>$wJ>&{8MtD{$W3z2C*zkB*AvrM(Hk}`4F8G1(?=6(Bo6a^6nsa;AYfW+4GrsLi=6Y(crqSH zqxBG4Cvv?6)(LoJt(+HZUS7(li#9L0PD4IPgY_1y3(wP=Pn5v9o!W~}r}4;-iG1K7 zAJ%aY*Z0-L*p>0b=$PCSM&v#T!?yi~fhSm3H|^U�-D#7`0MsubVKH!Um~D9`r#x zOBnm&CAHWq?c|f>K|`+eV1MxXxe?i4B6yHm3b79IflApKuyg5w%#xiU3_C%NI!4x? z-R^mD@%@t@>*i4D@l+4>pSuH1n_;fD zjFTFhiQmwqdiSSUt!$gs*)6lWxxu#ZckII3YT{G%jk)sLxbdpi`xJ?6V)fKRYT`4s z-h0%pWzqTdORZKjE!!>{3q{Ay(5tUT-6s(v3$}Q-#k=niweXyH55_hAUv$t&-5X7S zXbyP|#qLZqZBf_sj@H;3$Q#%)@$C_AFiLeR1(X6x0i}RaKq;UUPzopolmbctrGQdE zDeyllaJXBaLVaDsIGpni!+Dy{hw1Qo%3*8F(YuCui4LIMc|v3wr}OwGp3ib?(VDBc z7M*pckbWkg$`x2IK2S8Xjnl{_lNadj)Vz~D@B6|L5BuLT&1 | FileCheck --check-prefix WEAK-BIND-SET-DYLIB-SPECIAL-IMM %s WEAK-BIND-SET-DYLIB-SPECIAL-IMM: macho-weak-bind-set-dylib-special-imm': truncated or malformed object (BIND_OPCODE_SET_DYLIB_SPECIAL_IMM not allowed in weak bind table for opcode at: 0x2) -RUN: not llvm-objdump --macho --rebase %p/Inputs/macho-rebase-set-type-imm 2>&1 | FileCheck --check-prefix REBASE-SET-TYPE-IMM %s -REBASE-SET-TYPE-IMM: macho-rebase-set-type-imm': truncated or malformed object (for REBASE_OPCODE_SET_TYPE_IMM bad bind type: 5 for opcode at: 0x0) - RUN: not llvm-objdump --macho --rebase %p/Inputs/macho-rebase-uleb-malformed-uleb128 2>&1 | FileCheck --check-prefix REBASE-ULEB-MALFORMED-ULEB128 %s REBASE-ULEB-MALFORMED-ULEB128: macho-rebase-uleb-malformed-uleb128': truncated or malformed object (for REBASE_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB malformed uleb128, extends past end for opcode at: 0x1) From 5f14515676e46bc3fd6848b82b24f62071dbfc63 Mon Sep 17 00:00:00 2001 From: Alessandro Di Federico Date: Wed, 27 Jan 2021 12:40:45 +0100 Subject: [PATCH 015/147] YAML::SequenceTraits: support custom inserter --- llvm/include/llvm/Support/YAMLTraits.h | 31 ++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/llvm/include/llvm/Support/YAMLTraits.h b/llvm/include/llvm/Support/YAMLTraits.h index ef74f38671b5..c2a5a9c86692 100644 --- a/llvm/include/llvm/Support/YAMLTraits.h +++ b/llvm/include/llvm/Support/YAMLTraits.h @@ -1156,16 +1156,42 @@ yamlize(IO &io, T &Val, bool, EmptyContext &Ctx) { char missing_yaml_trait_for_type[sizeof(MissingTrait)]; } +template +class DefaultInserter { +private: + IO &io; + T &Seq; + +public: + DefaultInserter(IO &io, T &Seq) : io(io), Seq(Seq) {} + + auto &preflightElement(unsigned i) { + return SequenceTraits::element(io, Seq, i); + } + + void postflightElement(unsigned i) {}; +}; + +template +auto getInserter(const T &) -> typename SequenceTraits::Inserter { + return std::declval::Inserter>(); +} + +template +DefaultInserter getInserter(...) { return DefaultInserter{}; } + template std::enable_if_t::value, void> yamlize(IO &io, T &Seq, bool, Context &Ctx) { + decltype(getInserter(std::declval())) I(io, Seq); if ( has_FlowTraits< SequenceTraits>::value ) { unsigned incnt = io.beginFlowSequence(); unsigned count = io.outputting() ? SequenceTraits::size(io, Seq) : incnt; for(unsigned i=0; i < count; ++i) { void *SaveInfo; if ( io.preflightFlowElement(i, SaveInfo) ) { - yamlize(io, SequenceTraits::element(io, Seq, i), true, Ctx); + yamlize(io, I.preflightElement(i), true, Ctx); + I.postflightElement(i); io.postflightFlowElement(SaveInfo); } } @@ -1177,7 +1203,8 @@ yamlize(IO &io, T &Seq, bool, Context &Ctx) { for(unsigned i=0; i < count; ++i) { void *SaveInfo; if ( io.preflightElement(i, SaveInfo) ) { - yamlize(io, SequenceTraits::element(io, Seq, i), true, Ctx); + yamlize(io, I.preflightElement(i), true, Ctx); + I.postflightElement(i); io.postflightElement(SaveInfo); } } From fa58fc6eb82bdf6ce1db3e0579a51a50a4ad8605 Mon Sep 17 00:00:00 2001 From: Alessandro Di Federico Date: Wed, 10 Feb 2021 10:32:11 +0100 Subject: [PATCH 016/147] YAMLTraits: support quoted EnumScalar --- llvm/include/llvm/Support/YAMLTraits.h | 28 +++++++++++++++++++------- llvm/lib/Support/YAMLTraits.cpp | 7 ++++--- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/llvm/include/llvm/Support/YAMLTraits.h b/llvm/include/llvm/Support/YAMLTraits.h index c2a5a9c86692..3fb2092ad841 100644 --- a/llvm/include/llvm/Support/YAMLTraits.h +++ b/llvm/include/llvm/Support/YAMLTraits.h @@ -800,7 +800,9 @@ class IO { virtual void endFlowMapping() = 0; virtual void beginEnumScalar() = 0; - virtual bool matchEnumScalar(const char*, bool) = 0; + virtual bool matchEnumScalar(const char*, + bool, + QuotingType=QuotingType::None) = 0; virtual bool matchEnumFallback() = 0; virtual void endEnumScalar() = 0; @@ -818,16 +820,24 @@ class IO { virtual void setAllowUnknownKeys(bool Allow); template - void enumCase(T &Val, const char* Str, const T ConstVal) { - if ( matchEnumScalar(Str, outputting() && Val == ConstVal) ) { + void enumCase(T &Val, + const char* Str, + const T ConstVal, + QuotingType MustQuote=QuotingType::None) { + if ( matchEnumScalar(Str, outputting() && Val == ConstVal, MustQuote) ) { Val = ConstVal; } } // allow anonymous enum values to be used with LLVM_YAML_STRONG_TYPEDEF template - void enumCase(T &Val, const char* Str, const uint32_t ConstVal) { - if ( matchEnumScalar(Str, outputting() && Val == static_cast(ConstVal)) ) { + void enumCase(T &Val, + const char* Str, + const uint32_t ConstVal, + QuotingType MustQuote=QuotingType::None) { + if ( matchEnumScalar(Str, + outputting() && Val == static_cast(ConstVal), + MustQuote) ) { Val = ConstVal; } } @@ -1479,7 +1489,9 @@ class Input : public IO { void postflightFlowElement(void *) override; void endFlowSequence() override; void beginEnumScalar() override; - bool matchEnumScalar(const char*, bool) override; + bool matchEnumScalar(const char*, + bool, + QuotingType=QuotingType::None) override; bool matchEnumFallback() override; void endEnumScalar() override; bool beginBitSetScalar(bool &) override; @@ -1635,7 +1647,9 @@ class Output : public IO { void postflightFlowElement(void *) override; void endFlowSequence() override; void beginEnumScalar() override; - bool matchEnumScalar(const char*, bool) override; + bool matchEnumScalar(const char*, + bool, + QuotingType=QuotingType::None) override; bool matchEnumFallback() override; void endEnumScalar() override; bool beginBitSetScalar(bool &) override; diff --git a/llvm/lib/Support/YAMLTraits.cpp b/llvm/lib/Support/YAMLTraits.cpp index 4eb0b3afd563..d585b5507df1 100644 --- a/llvm/lib/Support/YAMLTraits.cpp +++ b/llvm/lib/Support/YAMLTraits.cpp @@ -271,7 +271,7 @@ void Input::beginEnumScalar() { ScalarMatchFound = false; } -bool Input::matchEnumScalar(const char *Str, bool) { +bool Input::matchEnumScalar(const char *Str, bool, QuotingType MustQuote) { if (ScalarMatchFound) return false; if (ScalarHNode *SN = dyn_cast(CurrentNode)) { @@ -650,10 +650,11 @@ void Output::beginEnumScalar() { EnumerationMatchFound = false; } -bool Output::matchEnumScalar(const char *Str, bool Match) { +bool Output::matchEnumScalar(const char *Str, bool Match, QuotingType MustQuote) { if (Match && !EnumerationMatchFound) { newLineCheck(); - outputUpToEndOfLine(Str); + StringRef String(Str); + scalarString(String, MustQuote); EnumerationMatchFound = true; } return false; From 22ac7be007182409581e1fe47b07aef93a5f4cbd Mon Sep 17 00:00:00 2001 From: Pietro Fezzardi Date: Mon, 24 Feb 2020 14:29:20 +0100 Subject: [PATCH 017/147] Add support to Dominators for FilteredGraphTraits Before this patch, LLVM's DominatorTree and related templates (DominatorTree, DomTreeBuilder) did not support the computation of dominator and post-dominator trees on graphs with markers, such as GraphTraits>. They only supported plain graph traits. This patch restructures all the necessary templates, adding an additional template argument (View), which allows to compute DT and PDT on all the graphs that use GraphTraits, potentially equipped with a View. View is a marker, that allows to provide a different view on a graph that has GraphTraits. A typical example of this is llvm::Inverse. This mechanism of View markers will be used in rev.ng to implement filtered graph traits, to filter the edges of a graph implemented with GraphTraits. The new DT and PDT on the marked version of the GraphTraits will be computed on the same graphs, but treating some edges as if they are seen through the view. For instance, the Post-Dominator Tree of a graph marked with llvm::Inverse view is the Dominator Tree. The previous default behavior of DT and PDT is implemented by means of the DTIdentityView, which simply leaves the node type of the GraphTraits untouched. This is a transparent view that allows all the code in llvm to continue working as before this patch. --- .../llvm/Analysis/IteratedDominanceFrontier.h | 2 +- llvm/include/llvm/Analysis/LoopInfo.h | 11 +- llvm/include/llvm/CodeGen/LiveIntervalCalc.h | 9 +- llvm/include/llvm/CodeGen/LiveRangeCalc.h | 14 +- llvm/include/llvm/CodeGen/MachineDominators.h | 6 +- llvm/include/llvm/CodeGen/MachineSSAContext.h | 2 +- llvm/include/llvm/IR/Dominators.h | 81 +++--- llvm/include/llvm/IR/SSAContext.h | 2 +- llvm/include/llvm/Support/CFGDiff.h | 4 +- llvm/include/llvm/Support/GenericDomTree.h | 251 ++++++++++-------- .../llvm/Support/GenericDomTreeConstruction.h | 88 +++--- .../include/llvm/Transforms/Utils/LoopUtils.h | 10 +- llvm/lib/Analysis/MemorySSAUpdater.cpp | 6 +- llvm/lib/CodeGen/MachineDominators.cpp | 4 +- llvm/lib/CodeGen/MachinePostDominators.cpp | 2 +- llvm/lib/IR/Dominators.cpp | 52 ++-- llvm/lib/Transforms/Vectorize/VPlan.cpp | 2 +- .../IR/DominatorTreeBatchUpdatesTest.cpp | 4 +- 18 files changed, 311 insertions(+), 239 deletions(-) diff --git a/llvm/include/llvm/Analysis/IteratedDominanceFrontier.h b/llvm/include/llvm/Analysis/IteratedDominanceFrontier.h index 542a741ee07e..bcb891fb8d0b 100644 --- a/llvm/include/llvm/Analysis/IteratedDominanceFrontier.h +++ b/llvm/include/llvm/Analysis/IteratedDominanceFrontier.h @@ -73,7 +73,7 @@ ChildrenGetterTy::get(const NodeRef &N) { return {Children.begin(), Children.end()}; } - return GD->template getChildren(N); + return GD->template getChildren(N); } } // end of namespace IDFCalculatorDetail diff --git a/llvm/include/llvm/Analysis/LoopInfo.h b/llvm/include/llvm/Analysis/LoopInfo.h index b77a335d1ee6..e1752a835366 100644 --- a/llvm/include/llvm/Analysis/LoopInfo.h +++ b/llvm/include/llvm/Analysis/LoopInfo.h @@ -63,7 +63,16 @@ class MDNode; class MemorySSAUpdater; class ScalarEvolution; class raw_ostream; -template class DominatorTreeBase; + +template +using DTIdentityView = X; + +template class View> +class DominatorTreeOnView; + +template +using DominatorTreeBase = DominatorTreeOnView; + template class LoopInfoBase; template class LoopBase; diff --git a/llvm/include/llvm/CodeGen/LiveIntervalCalc.h b/llvm/include/llvm/CodeGen/LiveIntervalCalc.h index b8d67ef8f4e1..62485547c2e9 100644 --- a/llvm/include/llvm/CodeGen/LiveIntervalCalc.h +++ b/llvm/include/llvm/CodeGen/LiveIntervalCalc.h @@ -21,7 +21,14 @@ namespace llvm { -template class DomTreeNodeBase; +template +using DTIdentityView = X; + +template class View> +class DomTreeNodeOnView; + +template +using DomTreeNodeBase = DomTreeNodeOnView; using MachineDomTreeNode = DomTreeNodeBase; diff --git a/llvm/include/llvm/CodeGen/LiveRangeCalc.h b/llvm/include/llvm/CodeGen/LiveRangeCalc.h index 895ecff18f89..b941b2dd4312 100644 --- a/llvm/include/llvm/CodeGen/LiveRangeCalc.h +++ b/llvm/include/llvm/CodeGen/LiveRangeCalc.h @@ -35,13 +35,21 @@ namespace llvm { -template class DomTreeNodeBase; +template +using DTIdentityView = X; + +template class View> +class DomTreeNodeOnView; + +template +using DomTreeNodeBase = DomTreeNodeOnView; + +using MachineDomTreeNode = DomTreeNodeBase; + class MachineDominatorTree; class MachineFunction; class MachineRegisterInfo; -using MachineDomTreeNode = DomTreeNodeBase; - class LiveRangeCalc { const MachineFunction *MF = nullptr; const MachineRegisterInfo *MRI = nullptr; diff --git a/llvm/include/llvm/CodeGen/MachineDominators.h b/llvm/include/llvm/CodeGen/MachineDominators.h index 30c18ef410fa..958139a981a0 100644 --- a/llvm/include/llvm/CodeGen/MachineDominators.h +++ b/llvm/include/llvm/CodeGen/MachineDominators.h @@ -37,9 +37,9 @@ inline void DominatorTreeBase::addRoot( this->Roots.push_back(MBB); } -extern template class DomTreeNodeBase; -extern template class DominatorTreeBase; // DomTree -extern template class DominatorTreeBase; // PostDomTree +extern template class DomTreeNodeOnView; +extern template class DominatorTreeOnView; // DomTree +extern template class DominatorTreeOnView; // PostDomTree using MachineDomTree = DomTreeBase; using MachineDomTreeNode = DomTreeNodeBase; diff --git a/llvm/include/llvm/CodeGen/MachineSSAContext.h b/llvm/include/llvm/CodeGen/MachineSSAContext.h index e3b2dc459881..8cf6624e3d20 100644 --- a/llvm/include/llvm/CodeGen/MachineSSAContext.h +++ b/llvm/include/llvm/CodeGen/MachineSSAContext.h @@ -16,6 +16,7 @@ #define LLVM_CODEGEN_MACHINESSACONTEXT_H #include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/Support/GenericDomTree.h" #include "llvm/Support/Printable.h" namespace llvm { @@ -24,7 +25,6 @@ class MachineInstr; class MachineFunction; class Register; template class GenericSSAContext; -template class DominatorTreeBase; inline auto successors(const MachineBasicBlock *BB) { return BB->successors(); } inline auto predecessors(const MachineBasicBlock *BB) { diff --git a/llvm/include/llvm/IR/Dominators.h b/llvm/include/llvm/IR/Dominators.h index c2d080bc2004..52b97b2bc4ea 100644 --- a/llvm/include/llvm/IR/Dominators.h +++ b/llvm/include/llvm/IR/Dominators.h @@ -47,50 +47,57 @@ class Value; class raw_ostream; template struct GraphTraits; -extern template class DomTreeNodeBase; -extern template class DominatorTreeBase; // DomTree -extern template class DominatorTreeBase; // PostDomTree +extern template class DomTreeNodeOnView; +extern template class DominatorTreeOnView; // DomTree +extern template class DominatorTreeOnView; // PostDomTree extern template class cfg::Update; namespace DomTreeBuilder { -using BBDomTree = DomTreeBase; -using BBPostDomTree = PostDomTreeBase; +template class View> +using BBDomTreeOnView = DomTreeOnView; + +using BBDomTree = DomTreeOnView; + +template class View> +using BBPostDomTreeOnView = PostDomTreeOnView; + +using BBPostDomTree = PostDomTreeOnView; using BBUpdates = ArrayRef>; -using BBDomTreeGraphDiff = GraphDiff; -using BBPostDomTreeGraphDiff = GraphDiff; - -extern template void Calculate(BBDomTree &DT); -extern template void CalculateWithUpdates(BBDomTree &DT, - BBUpdates U); - -extern template void Calculate(BBPostDomTree &DT); - -extern template void InsertEdge(BBDomTree &DT, BasicBlock *From, - BasicBlock *To); -extern template void InsertEdge(BBPostDomTree &DT, - BasicBlock *From, - BasicBlock *To); - -extern template void DeleteEdge(BBDomTree &DT, BasicBlock *From, - BasicBlock *To); -extern template void DeleteEdge(BBPostDomTree &DT, - BasicBlock *From, - BasicBlock *To); - -extern template void ApplyUpdates(BBDomTree &DT, - BBDomTreeGraphDiff &, - BBDomTreeGraphDiff *); -extern template void ApplyUpdates(BBPostDomTree &DT, - BBPostDomTreeGraphDiff &, - BBPostDomTreeGraphDiff *); - -extern template bool Verify(const BBDomTree &DT, - BBDomTree::VerificationLevel VL); -extern template bool Verify(const BBPostDomTree &DT, - BBPostDomTree::VerificationLevel VL); +using BBDomTreeGraphDiff = GraphDiff; +using BBPostDomTreeGraphDiff = GraphDiff; + +extern template void Calculate(BBDomTree &DT); +extern template void +CalculateWithUpdates(BBDomTree &DT, BBUpdates U); + +extern template void Calculate(BBPostDomTree &DT); + +extern template void +InsertEdge(BBDomTree &DT, BasicBlock *From, BasicBlock *To); +extern template void +InsertEdge(BBPostDomTree &DT, BasicBlock *From, BasicBlock *To); + +extern template void +DeleteEdge(BBDomTree &DT, BasicBlock *From, BasicBlock *To); +extern template void +DeleteEdge(BBPostDomTree &DT, BasicBlock *From, BasicBlock *To); + +extern template void +ApplyUpdates(BBDomTree &DT, + BBDomTreeGraphDiff &, + BBDomTreeGraphDiff *); +extern template void +ApplyUpdates(BBPostDomTree &DT, + BBPostDomTreeGraphDiff &, + BBPostDomTreeGraphDiff *); + +extern template bool +Verify(const BBDomTree &DT, BBDomTree::VerificationLevel VL); +extern template bool +Verify(const BBPostDomTree &DT, BBPostDomTree::VerificationLevel VL); } // namespace DomTreeBuilder using DomTreeNode = DomTreeNodeBase; diff --git a/llvm/include/llvm/IR/SSAContext.h b/llvm/include/llvm/IR/SSAContext.h index 7551adff1e12..fe5e4885b12a 100644 --- a/llvm/include/llvm/IR/SSAContext.h +++ b/llvm/include/llvm/IR/SSAContext.h @@ -18,6 +18,7 @@ #include "llvm/ADT/GenericSSAContext.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/ModuleSlotTracker.h" +#include "llvm/Support/GenericDomTree.h" #include "llvm/Support/Printable.h" #include @@ -28,7 +29,6 @@ class Function; class Instruction; class Value; template class SmallVectorImpl; -template class DominatorTreeBase; inline auto instrs(const BasicBlock &BB) { return llvm::make_range(BB.begin(), BB.end()); diff --git a/llvm/include/llvm/Support/CFGDiff.h b/llvm/include/llvm/Support/CFGDiff.h index c90b9aca78b5..07cf242d67a8 100644 --- a/llvm/include/llvm/Support/CFGDiff.h +++ b/llvm/include/llvm/Support/CFGDiff.h @@ -132,9 +132,9 @@ template class GraphDiff { } using VectRet = SmallVector; - template VectRet getChildren(NodePtr N) const { + template class View> VectRet getChildren(NodePtr N) const { using DirectedNodeT = - std::conditional_t, NodePtr>; + std::conditional_t>, View>; auto R = children(N); VectRet Res = VectRet(detail::reverse_if(R)); diff --git a/llvm/include/llvm/Support/GenericDomTree.h b/llvm/include/llvm/Support/GenericDomTree.h index 1e5c0ae231d2..9389150adab1 100644 --- a/llvm/include/llvm/Support/GenericDomTree.h +++ b/llvm/include/llvm/Support/GenericDomTree.h @@ -42,44 +42,51 @@ namespace llvm { +template +using DTIdentityView = X; + +template class View> +class DominatorTreeOnView; + template -class DominatorTreeBase; +using DominatorTreeBase = DominatorTreeOnView; namespace DomTreeBuilder { -template -struct SemiNCAInfo; +template class View> +struct SemiNCAInfoOnView; } // namespace DomTreeBuilder /// Base class for the actual dominator tree node. -template class DomTreeNodeBase { +template class View> +class DomTreeNodeOnView { friend class PostDominatorTree; - friend class DominatorTreeBase; - friend class DominatorTreeBase; - friend struct DomTreeBuilder::SemiNCAInfo>; - friend struct DomTreeBuilder::SemiNCAInfo>; + friend class DominatorTreeOnView; + friend class DominatorTreeOnView; + friend struct DomTreeBuilder::SemiNCAInfoOnView; + friend struct DomTreeBuilder::SemiNCAInfoOnView; NodeT *TheBB; - DomTreeNodeBase *IDom; + DomTreeNodeOnView *IDom; unsigned Level; - SmallVector Children; + SmallVector Children; mutable unsigned DFSNumIn = ~0; mutable unsigned DFSNumOut = ~0; public: - DomTreeNodeBase(NodeT *BB, DomTreeNodeBase *iDom) + DomTreeNodeOnView(NodeT *BB, DomTreeNodeOnView *iDom) : TheBB(BB), IDom(iDom), Level(IDom ? IDom->Level + 1 : 0) {} - using iterator = typename SmallVector::iterator; + using iterator = typename SmallVector::iterator; using const_iterator = - typename SmallVector::const_iterator; + typename SmallVector::const_iterator; iterator begin() { return Children.begin(); } iterator end() { return Children.end(); } const_iterator begin() const { return Children.begin(); } const_iterator end() const { return Children.end(); } - DomTreeNodeBase *const &back() const { return Children.back(); } - DomTreeNodeBase *&back() { return Children.back(); } + DomTreeNodeOnView *const &back() const { return Children.back(); } + DomTreeNodeOnView *&back() { return Children.back(); } iterator_range children() { return make_range(begin(), end()); } iterator_range children() const { @@ -87,11 +94,11 @@ template class DomTreeNodeBase { } NodeT *getBlock() const { return TheBB; } - DomTreeNodeBase *getIDom() const { return IDom; } + DomTreeNodeOnView *getIDom() const { return IDom; } unsigned getLevel() const { return Level; } - std::unique_ptr addChild( - std::unique_ptr C) { + std::unique_ptr addChild( + std::unique_ptr C) { Children.push_back(C.get()); return C; } @@ -101,19 +108,19 @@ template class DomTreeNodeBase { void clearAllChildren() { Children.clear(); } - bool compare(const DomTreeNodeBase *Other) const { + bool compare(const DomTreeNodeOnView *Other) const { if (getNumChildren() != Other->getNumChildren()) return true; if (Level != Other->Level) return true; SmallPtrSet OtherChildren; - for (const DomTreeNodeBase *I : *Other) { + for (const DomTreeNodeOnView *I : *Other) { const NodeT *Nd = I->getBlock(); OtherChildren.insert(Nd); } - for (const DomTreeNodeBase *I : *this) { + for (const DomTreeNodeOnView *I : *this) { const NodeT *N = I->getBlock(); if (OtherChildren.count(N) == 0) return true; @@ -121,7 +128,7 @@ template class DomTreeNodeBase { return false; } - void setIDom(DomTreeNodeBase *NewIDom) { + void setIDom(DomTreeNodeOnView *NewIDom) { assert(IDom && "No immediate dominator?"); if (IDom == NewIDom) return; @@ -147,7 +154,7 @@ template class DomTreeNodeBase { private: // Return true if this node is dominated by other. Use this only if DFS info // is valid. - bool DominatedBy(const DomTreeNodeBase *other) const { + bool DominatedBy(const DomTreeNodeOnView *other) const { return this->DFSNumIn >= other->DFSNumIn && this->DFSNumOut <= other->DFSNumOut; } @@ -156,13 +163,13 @@ template class DomTreeNodeBase { assert(IDom); if (Level == IDom->Level + 1) return; - SmallVector WorkStack = {this}; + SmallVector WorkStack = {this}; while (!WorkStack.empty()) { - DomTreeNodeBase *Current = WorkStack.pop_back_val(); + DomTreeNodeOnView *Current = WorkStack.pop_back_val(); Current->Level = Current->IDom->Level + 1; - for (DomTreeNodeBase *C : *Current) { + for (DomTreeNodeOnView *C : *Current) { assert(C->IDom); if (C->Level != C->IDom->Level + 1) WorkStack.push_back(C); } @@ -171,7 +178,10 @@ template class DomTreeNodeBase { }; template -raw_ostream &operator<<(raw_ostream &O, const DomTreeNodeBase *Node) { +using DomTreeNodeBase = DomTreeNodeOnView; + +template class View> +raw_ostream &operator<<(raw_ostream &O, const DomTreeNodeOnView *Node) { if (Node->getBlock()) Node->getBlock()->printAsOperand(O, false); else @@ -183,11 +193,11 @@ raw_ostream &operator<<(raw_ostream &O, const DomTreeNodeBase *Node) { return O; } -template -void PrintDomTree(const DomTreeNodeBase *N, raw_ostream &O, +template class View> +void PrintDomTree(const DomTreeNodeOnView *N, raw_ostream &O, unsigned Lev) { O.indent(2 * Lev) << "[" << Lev << "] " << N; - for (typename DomTreeNodeBase::const_iterator I = N->begin(), + for (typename DomTreeNodeOnView::const_iterator I = N->begin(), E = N->end(); I != E; ++I) PrintDomTree(*I, O, Lev + 1); @@ -195,30 +205,31 @@ void PrintDomTree(const DomTreeNodeBase *N, raw_ostream &O, namespace DomTreeBuilder { // The routines below are provided in a separate header but referenced here. -template -void Calculate(DomTreeT &DT); - -template -void CalculateWithUpdates(DomTreeT &DT, - ArrayRef Updates); - -template -void InsertEdge(DomTreeT &DT, typename DomTreeT::NodePtr From, - typename DomTreeT::NodePtr To); - -template -void DeleteEdge(DomTreeT &DT, typename DomTreeT::NodePtr From, - typename DomTreeT::NodePtr To); - -template -void ApplyUpdates(DomTreeT &DT, - GraphDiff &PreViewCFG, - GraphDiff *PostViewCFG); - -template -bool Verify(const DomTreeT &DT, typename DomTreeT::VerificationLevel VL); +template class View> +void Calculate(DominatorTreeOnView &DT); + +template class View> +void CalculateWithUpdates(DominatorTreeOnView &DT, + ArrayRef::UpdateType> Updates); + +template class View> +void InsertEdge(DominatorTreeOnView &DT, + typename DominatorTreeOnView::NodePtr From, + typename DominatorTreeOnView::NodePtr To); + +template class View> +void DeleteEdge(DominatorTreeOnView &DT, + typename DominatorTreeOnView::NodePtr From, + typename DominatorTreeOnView::NodePtr To); + +template class View> +void ApplyUpdates(DominatorTreeOnView &DT, + GraphDiff::NodePtr, IsPostDom> &PreViewCFG, + GraphDiff::NodePtr, IsPostDom> *PostViewCFG); + +template class View> +bool Verify(const DominatorTreeOnView &DT, + typename DominatorTreeOnView::VerificationLevel VL); } // namespace DomTreeBuilder /// Default DomTreeNode traits for NodeT. The default implementation assume a @@ -239,11 +250,11 @@ template struct DomTreeNodeTraits { /// /// This class is a generic template over graph nodes. It is instantiated for /// various graphs in the LLVM IR or in the code generator. -template -class DominatorTreeBase { +template class View> +class DominatorTreeOnView { public: static_assert(std::is_pointer::NodeRef>::value, - "Currently DominatorTreeBase supports only pointer nodes"); + "Currently DominatorTreeOnView supports only pointer nodes"); using NodeTrait = DomTreeNodeTraits; using NodeType = typename NodeTrait::NodeType; using NodePtr = typename NodeTrait::NodePtr; @@ -265,20 +276,21 @@ class DominatorTreeBase { SmallVector Roots; using DomTreeNodeMapType = - DenseMap>>; + DenseMap>>; DomTreeNodeMapType DomTreeNodes; - DomTreeNodeBase *RootNode = nullptr; + DomTreeNodeOnView *RootNode = nullptr; ParentPtr Parent = nullptr; mutable bool DFSInfoValid = false; mutable unsigned int SlowQueries = 0; - friend struct DomTreeBuilder::SemiNCAInfo; + friend struct DomTreeBuilder::SemiNCAInfoOnView; + friend struct DomTreeBuilder::SemiNCAInfoOnView; public: - DominatorTreeBase() = default; + DominatorTreeOnView() = default; - DominatorTreeBase(DominatorTreeBase &&Arg) + DominatorTreeOnView(DominatorTreeOnView &&Arg) : Roots(std::move(Arg.Roots)), DomTreeNodes(std::move(Arg.DomTreeNodes)), RootNode(Arg.RootNode), @@ -288,7 +300,7 @@ class DominatorTreeBase { Arg.wipe(); } - DominatorTreeBase &operator=(DominatorTreeBase &&RHS) { + DominatorTreeOnView &operator=(DominatorTreeOnView &&RHS) { Roots = std::move(RHS.Roots); DomTreeNodes = std::move(RHS.DomTreeNodes); RootNode = RHS.RootNode; @@ -299,8 +311,8 @@ class DominatorTreeBase { return *this; } - DominatorTreeBase(const DominatorTreeBase &) = delete; - DominatorTreeBase &operator=(const DominatorTreeBase &) = delete; + DominatorTreeOnView(const DominatorTreeOnView &) = delete; + DominatorTreeOnView &operator=(const DominatorTreeOnView &) = delete; /// Iteration over roots. /// @@ -330,7 +342,7 @@ class DominatorTreeBase { /// compare - Return false if the other dominator tree base matches this /// dominator tree base. Otherwise return true. - bool compare(const DominatorTreeBase &Other) const { + bool compare(const DominatorTreeOnView &Other) const { if (Parent != Other.Parent) return true; if (Roots.size() != Other.Roots.size()) @@ -350,8 +362,8 @@ class DominatorTreeBase { if (OI == OtherDomTreeNodes.end()) return true; - DomTreeNodeBase &MyNd = *DomTreeNode.second; - DomTreeNodeBase &OtherNd = *OI->second; + DomTreeNodeOnView &MyNd = *DomTreeNode.second; + DomTreeNodeOnView &OtherNd = *OI->second; if (MyNd.compare(&OtherNd)) return true; @@ -364,7 +376,7 @@ class DominatorTreeBase { /// block. This is the same as using operator[] on this class. The result /// may (but is not required to) be null for a forward (backwards) /// statically unreachable block. - DomTreeNodeBase *getNode(const NodeT *BB) const { + DomTreeNodeOnView *getNode(const NodeT *BB) const { auto I = DomTreeNodes.find(BB); if (I != DomTreeNodes.end()) return I->second.get(); @@ -372,7 +384,7 @@ class DominatorTreeBase { } /// See getNode. - DomTreeNodeBase *operator[](const NodeT *BB) const { + DomTreeNodeOnView *operator[](const NodeT *BB) const { return getNode(BB); } @@ -383,20 +395,20 @@ class DominatorTreeBase { /// post-dominance information must be capable of dealing with this /// possibility. /// - DomTreeNodeBase *getRootNode() { return RootNode; } - const DomTreeNodeBase *getRootNode() const { return RootNode; } + DomTreeNodeOnView *getRootNode() { return RootNode; } + const DomTreeNodeOnView *getRootNode() const { return RootNode; } /// Get all nodes dominated by R, including R itself. void getDescendants(NodeT *R, SmallVectorImpl &Result) const { Result.clear(); - const DomTreeNodeBase *RN = getNode(R); + const DomTreeNodeOnView *RN = getNode(R); if (!RN) return; // If R is unreachable, it will not be present in the DOM tree. - SmallVector *, 8> WL; + SmallVector *, 8> WL; WL.push_back(RN); while (!WL.empty()) { - const DomTreeNodeBase *N = WL.pop_back_val(); + const DomTreeNodeOnView *N = WL.pop_back_val(); Result.push_back(N->getBlock()); WL.append(N->begin(), N->end()); } @@ -405,8 +417,8 @@ class DominatorTreeBase { /// properlyDominates - Returns true iff A dominates B and A != B. /// Note that this is not a constant time operation! /// - bool properlyDominates(const DomTreeNodeBase *A, - const DomTreeNodeBase *B) const { + bool properlyDominates(const DomTreeNodeOnView *A, + const DomTreeNodeOnView *B) const { if (!A || !B) return false; if (A == B) @@ -424,13 +436,13 @@ class DominatorTreeBase { return isReachableFromEntry(getNode(const_cast(A))); } - bool isReachableFromEntry(const DomTreeNodeBase *A) const { return A; } + bool isReachableFromEntry(const DomTreeNodeOnView *A) const { return A; } /// dominates - Returns true iff A dominates B. Note that this is not a /// constant time operation! /// - bool dominates(const DomTreeNodeBase *A, - const DomTreeNodeBase *B) const { + bool dominates(const DomTreeNodeOnView *A, + const DomTreeNodeOnView *B) const { // A node trivially dominates itself. if (B == A) return true; @@ -495,8 +507,8 @@ class DominatorTreeBase { return &Entry; } - DomTreeNodeBase *NodeA = getNode(A); - DomTreeNodeBase *NodeB = getNode(B); + DomTreeNodeOnView *NodeA = getNode(A); + DomTreeNodeOnView *NodeB = getNode(B); assert(NodeA && "A must be in the tree"); assert(NodeB && "B must be in the tree"); @@ -519,7 +531,7 @@ class DominatorTreeBase { const_cast(B)); } - bool isVirtualRoot(const DomTreeNodeBase *A) const { + bool isVirtualRoot(const DomTreeNodeOnView *A) const { return isPostDominator() && !A->getBlock(); } @@ -561,7 +573,7 @@ class DominatorTreeBase { void applyUpdates(ArrayRef Updates) { GraphDiff PreViewCFG( Updates, /*ReverseApplyUpdates=*/true); - DomTreeBuilder::ApplyUpdates(*this, PreViewCFG, nullptr); + DomTreeBuilder::ApplyUpdates(*this, PreViewCFG, nullptr); } /// \param Updates An ordered sequence of updates to perform. The current CFG @@ -573,7 +585,7 @@ class DominatorTreeBase { ArrayRef PostViewUpdates) { if (Updates.empty()) { GraphDiff PostViewCFG(PostViewUpdates); - DomTreeBuilder::ApplyUpdates(*this, PostViewCFG, &PostViewCFG); + DomTreeBuilder::ApplyUpdates(*this, PostViewCFG, &PostViewCFG); } else { // PreViewCFG needs to merge Updates and PostViewCFG. The updates in // Updates need to be reversed, and match the direction in PostViewCFG. @@ -585,7 +597,7 @@ class DominatorTreeBase { GraphDiff PreViewCFG(AllUpdates, /*ReverseApplyUpdates=*/true); GraphDiff PostViewCFG(PostViewUpdates); - DomTreeBuilder::ApplyUpdates(*this, PreViewCFG, &PostViewCFG); + DomTreeBuilder::ApplyUpdates(*this, PreViewCFG, &PostViewCFG); } } @@ -633,9 +645,9 @@ class DominatorTreeBase { /// \param DomBB CFG node that is dominator for BB. /// \returns New dominator tree node that represents new CFG node. /// - DomTreeNodeBase *addNewBlock(NodeT *BB, NodeT *DomBB) { + DomTreeNodeOnView *addNewBlock(NodeT *BB, NodeT *DomBB) { assert(getNode(BB) == nullptr && "Block already in dominator tree!"); - DomTreeNodeBase *IDomNode = getNode(DomBB); + DomTreeNodeOnView *IDomNode = getNode(DomBB); assert(IDomNode && "Not immediate dominator specified for block!"); DFSInfoValid = false; return createChild(BB, IDomNode); @@ -646,12 +658,12 @@ class DominatorTreeBase { /// \param BB New node in CFG. /// \returns New dominator tree node that represents new CFG node. /// - DomTreeNodeBase *setNewRoot(NodeT *BB) { + DomTreeNodeOnView *setNewRoot(NodeT *BB) { assert(getNode(BB) == nullptr && "Block already in dominator tree!"); assert(!this->isPostDominator() && "Cannot change root of post-dominator tree"); DFSInfoValid = false; - DomTreeNodeBase *NewNode = createNode(BB); + DomTreeNodeOnView *NewNode = createNode(BB); if (Roots.empty()) { addRoot(BB); } else { @@ -669,8 +681,8 @@ class DominatorTreeBase { /// changeImmediateDominator - This method is used to update the dominator /// tree information when a node's immediate dominator changes. /// - void changeImmediateDominator(DomTreeNodeBase *N, - DomTreeNodeBase *NewIDom) { + void changeImmediateDominator(DomTreeNodeOnView *N, + DomTreeNodeOnView *NewIDom) { assert(N && NewIDom && "Cannot change null node pointers!"); DFSInfoValid = false; N->setIDom(NewIDom); @@ -684,14 +696,14 @@ class DominatorTreeBase { /// dominate any other blocks. Removes node from its immediate dominator's /// children list. Deletes dominator node associated with basic block BB. void eraseNode(NodeT *BB) { - DomTreeNodeBase *Node = getNode(BB); + DomTreeNodeOnView *Node = getNode(BB); assert(Node && "Removing node that isn't in dominator tree."); assert(Node->isLeaf() && "Node is not a leaf node."); DFSInfoValid = false; // Remove node from immediate dominator's children list. - DomTreeNodeBase *IDom = Node->getIDom(); + DomTreeNodeOnView *IDom = Node->getIDom(); if (IDom) { const auto I = find(IDom->Children, Node); assert(I != IDom->Children.end() && @@ -716,9 +728,9 @@ class DominatorTreeBase { /// tree to reflect this change. void splitBlock(NodeT *NewBB) { if (IsPostDominator) - Split>(NewBB); + Split>>(NewBB); else - Split(NewBB); + Split>(NewBB); } /// print - Convert to human readable form @@ -752,11 +764,11 @@ class DominatorTreeBase { return; } - SmallVector *, - typename DomTreeNodeBase::const_iterator>, + SmallVector *, + typename DomTreeNodeOnView::const_iterator>, 32> WorkStack; - const DomTreeNodeBase *ThisRoot = getRootNode(); + const DomTreeNodeOnView *ThisRoot = getRootNode(); assert((!Parent || ThisRoot) && "Empty constructed DomTree"); if (!ThisRoot) return; @@ -769,7 +781,7 @@ class DominatorTreeBase { ThisRoot->DFSNumIn = DFSNum++; while (!WorkStack.empty()) { - const DomTreeNodeBase *Node = WorkStack.back().first; + const DomTreeNodeOnView *Node = WorkStack.back().first; const auto ChildIt = WorkStack.back().second; // If we visited all of the children of this node, "recurse" back up the @@ -779,7 +791,7 @@ class DominatorTreeBase { WorkStack.pop_back(); } else { // Otherwise, recursively visit this child. - const DomTreeNodeBase *Child = *ChildIt; + const DomTreeNodeOnView *Child = *ChildIt; ++WorkStack.back().second; WorkStack.push_back({Child, Child->begin()}); @@ -832,15 +844,16 @@ class DominatorTreeBase { protected: void addRoot(NodeT *BB) { this->Roots.push_back(BB); } - DomTreeNodeBase *createChild(NodeT *BB, DomTreeNodeBase *IDom) { + DomTreeNodeOnView * + createChild(NodeT *BB, DomTreeNodeOnView *IDom) { return (DomTreeNodes[BB] = IDom->addChild( - std::make_unique>(BB, IDom))) + std::make_unique>(BB, IDom))) .get(); } - DomTreeNodeBase *createNode(NodeT *BB) { + DomTreeNodeOnView *createNode(NodeT *BB) { return (DomTreeNodes[BB] = - std::make_unique>(BB, nullptr)) + std::make_unique>(BB, nullptr)) .get(); } @@ -889,25 +902,25 @@ class DominatorTreeBase { } // Create the new dominator tree node... and set the idom of NewBB. - DomTreeNodeBase *NewBBNode = addNewBlock(NewBB, NewBBIDom); + DomTreeNodeOnView *NewBBNode = addNewBlock(NewBB, NewBBIDom); // If NewBB strictly dominates other blocks, then it is now the immediate // dominator of NewBBSucc. Update the dominator tree as appropriate. if (NewBBDominatesNewBBSucc) { - DomTreeNodeBase *NewBBSuccNode = getNode(NewBBSucc); + DomTreeNodeOnView *NewBBSuccNode = getNode(NewBBSucc); changeImmediateDominator(NewBBSuccNode, NewBBNode); } } private: - bool dominatedBySlowTreeWalk(const DomTreeNodeBase *A, - const DomTreeNodeBase *B) const { + bool dominatedBySlowTreeWalk(const DomTreeNodeOnView *A, + const DomTreeNodeOnView *B) const { assert(A != B); assert(isReachableFromEntry(B)); assert(isReachableFromEntry(A)); const unsigned ALevel = A->getLevel(); - const DomTreeNodeBase *IDom; + const DomTreeNodeOnView *IDom; // Don't walk nodes above A's subtree. When we reach A's level, we must // either find A or be in some other subtree not dominated by A. @@ -928,16 +941,22 @@ class DominatorTreeBase { } }; +template class View> +using DomTreeOnView = DominatorTreeOnView; + template -using DomTreeBase = DominatorTreeBase; +using DomTreeBase = DomTreeOnView; + +template class View> +using PostDomTreeOnView = DominatorTreeOnView; template -using PostDomTreeBase = DominatorTreeBase; +using PostDomTreeBase = PostDomTreeOnView; // These two functions are declared out of line as a workaround for building // with old (< r147295) versions of clang because of pr11642. -template -bool DominatorTreeBase::dominates(const NodeT *A, +template class View> +bool DominatorTreeOnView::dominates(const NodeT *A, const NodeT *B) const { if (A == B) return true; @@ -948,8 +967,8 @@ bool DominatorTreeBase::dominates(const NodeT *A, return dominates(getNode(const_cast(A)), getNode(const_cast(B))); } -template -bool DominatorTreeBase::properlyDominates( +template class View> +bool DominatorTreeOnView::properlyDominates( const NodeT *A, const NodeT *B) const { if (A == B) return false; diff --git a/llvm/include/llvm/Support/GenericDomTreeConstruction.h b/llvm/include/llvm/Support/GenericDomTreeConstruction.h index 2e7716cb0b4b..7c6d10655d19 100644 --- a/llvm/include/llvm/Support/GenericDomTreeConstruction.h +++ b/llvm/include/llvm/Support/GenericDomTreeConstruction.h @@ -52,13 +52,13 @@ namespace llvm { namespace DomTreeBuilder { -template -struct SemiNCAInfo { +template class View> +struct SemiNCAInfoOnView { + using DomTreeT = DominatorTreeOnView; using NodePtr = typename DomTreeT::NodePtr; - using NodeT = typename DomTreeT::NodeType; - using TreeNodePtr = DomTreeNodeBase *; + using TreeNode = DomTreeNodeOnView; + using TreeNodePtr = TreeNode*; using RootsT = decltype(DomTreeT::Roots); - static constexpr bool IsPostDom = DomTreeT::IsPostDominator; using GraphDiffT = GraphDiff; // Information record used by Semi-NCA during tree construction. @@ -96,7 +96,7 @@ struct SemiNCAInfo { using BatchUpdatePtr = BatchUpdateInfo *; // If BUI is a nullptr, then there's no batch update in progress. - SemiNCAInfo(BatchUpdatePtr BUI) : BatchUpdates(BUI) {} + SemiNCAInfoOnView(BatchUpdatePtr BUI) : BatchUpdates(BUI) {} void clear() { NumToNode = {nullptr}; // Restore to initial state with a dummy start node. @@ -108,14 +108,14 @@ struct SemiNCAInfo { template static SmallVector getChildren(NodePtr N, BatchUpdatePtr BUI) { if (BUI) - return BUI->PreViewCFG.template getChildren(N); + return BUI->PreViewCFG.template getChildren(N); return getChildren(N); } template static SmallVector getChildren(NodePtr N) { using DirectedNodeT = - std::conditional_t, NodePtr>; + std::conditional_t>, View>; auto R = children(N); SmallVector Res(detail::reverse_if(R)); @@ -358,7 +358,7 @@ struct SemiNCAInfo { return Roots; } - SemiNCAInfo SNCA(BUI); + SemiNCAInfoOnView SNCA(BUI); // PostDominatorTree always has a virtual root. SNCA.addVirtualRoot(); @@ -509,7 +509,7 @@ struct SemiNCAInfo { assert(IsPostDom && "This function is for postdominators only"); LLVM_DEBUG(dbgs() << "Removing redundant roots\n"); - SemiNCAInfo SNCA(BUI); + SemiNCAInfoOnView SNCA(BUI); for (unsigned i = 0; i < Roots.size(); ++i) { auto &Root = Roots[i]; @@ -570,7 +570,7 @@ struct SemiNCAInfo { } // This is rebuilding the whole tree, not incrementally, but PostViewBUI is // used in case the caller needs a DT update with a CFGView. - SemiNCAInfo SNCA(PostViewBUI); + SemiNCAInfoOnView SNCA(PostViewBUI); // Step #0: Number blocks in depth-first order and initialize variables used // in later stages of the algorithm. @@ -903,7 +903,7 @@ struct SemiNCAInfo { return false; }; - SemiNCAInfo SNCA(BUI); + SemiNCAInfoOnView SNCA(BUI); SNCA.runDFS(Root, 0, UnreachableDescender, 0); SNCA.runSemiNCA(DT); SNCA.attachNewSubtree(DT, Incoming); @@ -996,7 +996,7 @@ struct SemiNCAInfo { LLVM_DEBUG(dbgs() << "\tTop of subtree: " << BlockNamePrinter(ToIDomTN) << "\n"); - SemiNCAInfo SNCA(BUI); + SemiNCAInfoOnView SNCA(BUI); SNCA.runDFS(ToIDom, 0, DescendBelow, 0); LLVM_DEBUG(dbgs() << "\tRunning Semi-NCA\n"); SNCA.runSemiNCA(DT, Level); @@ -1063,7 +1063,7 @@ struct SemiNCAInfo { return false; }; - SemiNCAInfo SNCA(BUI); + SemiNCAInfoOnView SNCA(BUI); unsigned LastDFSNum = SNCA.runDFS(ToTN->getBlock(), 0, DescendAndCollect, 0); @@ -1557,48 +1557,46 @@ struct SemiNCAInfo { } }; -template -void Calculate(DomTreeT &DT) { - SemiNCAInfo::CalculateFromScratch(DT, nullptr); +template class View> +void Calculate(DominatorTreeOnView &DT) { + SemiNCAInfoOnView::CalculateFromScratch(DT, nullptr); } -template -void CalculateWithUpdates(DomTreeT &DT, - ArrayRef Updates) { +template class View> +void CalculateWithUpdates(DominatorTreeOnView &DT, + ArrayRef::UpdateType> Updates) { // FIXME: Updated to use the PreViewCFG and behave the same as until now. // This behavior is however incorrect; this actually needs the PostViewCFG. - GraphDiff PreViewCFG( + GraphDiff::NodePtr, IsPostDom> PreViewCFG( Updates, /*ReverseApplyUpdates=*/true); - typename SemiNCAInfo::BatchUpdateInfo BUI(PreViewCFG); - SemiNCAInfo::CalculateFromScratch(DT, &BUI); + typename SemiNCAInfoOnView::BatchUpdateInfo BUI(PreViewCFG); + SemiNCAInfoOnView::CalculateFromScratch(DT, &BUI); } -template -void InsertEdge(DomTreeT &DT, typename DomTreeT::NodePtr From, - typename DomTreeT::NodePtr To) { +template class View> +void InsertEdge(DominatorTreeOnView &DT, typename DominatorTreeOnView::NodePtr From, + typename DominatorTreeOnView::NodePtr To) { if (DT.isPostDominator()) std::swap(From, To); - SemiNCAInfo::InsertEdge(DT, nullptr, From, To); + SemiNCAInfoOnView::InsertEdge(DT, nullptr, From, To); } -template -void DeleteEdge(DomTreeT &DT, typename DomTreeT::NodePtr From, - typename DomTreeT::NodePtr To) { +template class View> +void DeleteEdge(DominatorTreeOnView &DT, typename DominatorTreeOnView::NodePtr From, + typename DominatorTreeOnView::NodePtr To) { if (DT.isPostDominator()) std::swap(From, To); - SemiNCAInfo::DeleteEdge(DT, nullptr, From, To); + SemiNCAInfoOnView::DeleteEdge(DT, nullptr, From, To); } -template -void ApplyUpdates(DomTreeT &DT, - GraphDiff &PreViewCFG, - GraphDiff *PostViewCFG) { - SemiNCAInfo::ApplyUpdates(DT, PreViewCFG, PostViewCFG); +template class View> +void ApplyUpdates(DominatorTreeOnView &DT, + GraphDiff::NodePtr, IsPostDom> &PreViewCFG, + GraphDiff::NodePtr, IsPostDom> *PostViewCFG) { + SemiNCAInfoOnView::ApplyUpdates(DT, PreViewCFG, PostViewCFG); } -template -bool Verify(const DomTreeT &DT, typename DomTreeT::VerificationLevel VL) { - SemiNCAInfo SNCA(nullptr); +template class View> +bool Verify(const DominatorTreeOnView &DT, typename DominatorTreeOnView::VerificationLevel VL) { + SemiNCAInfoOnView SNCA(nullptr); // Simplist check is to compare against a new tree. This will also // usefully print the old and new trees, if they are different. @@ -1610,12 +1608,12 @@ bool Verify(const DomTreeT &DT, typename DomTreeT::VerificationLevel VL) { !SNCA.VerifyLevels(DT) || !SNCA.VerifyDFSNumbers(DT)) return false; - // Extra checks depending on VerificationLevel. Up to O(N^3). - if (VL == DomTreeT::VerificationLevel::Basic || - VL == DomTreeT::VerificationLevel::Full) + // Extra checks depending on VerificationLevel. Up to O(N^3) + if (VL == DominatorTreeOnView::VerificationLevel::Basic || + VL == DominatorTreeOnView::VerificationLevel::Full) if (!SNCA.verifyParentProperty(DT)) return false; - if (VL == DomTreeT::VerificationLevel::Full) + if (VL == DominatorTreeOnView::VerificationLevel::Full) if (!SNCA.verifySiblingProperty(DT)) return false; diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h index d63bee6fa321..e2af52956db8 100644 --- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h +++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h @@ -19,7 +19,15 @@ namespace llvm { -template class DomTreeNodeBase; +template +using DTIdentityView = X; + +template class View> +class DomTreeNodeOnView; + +template +using DomTreeNodeBase = DomTreeNodeOnView; + using DomTreeNode = DomTreeNodeBase; class AssumptionCache; class StringRef; diff --git a/llvm/lib/Analysis/MemorySSAUpdater.cpp b/llvm/lib/Analysis/MemorySSAUpdater.cpp index 9ad60f774e9f..9512ef50650d 100644 --- a/llvm/lib/Analysis/MemorySSAUpdater.cpp +++ b/llvm/lib/Analysis/MemorySSAUpdater.cpp @@ -877,7 +877,7 @@ void MemorySSAUpdater::applyInsertUpdates(ArrayRef Updates, // Check number of predecessors, we only care if there's more than one. unsigned Count = 0; BasicBlock *Pred = nullptr; - for (auto *Pi : GD->template getChildren(BB)) { + for (auto *Pi : GD->template getChildren(BB)) { Pred = Pi; Count++; if (Count == 2) @@ -971,7 +971,7 @@ void MemorySSAUpdater::applyInsertUpdates(ArrayRef Updates, auto *BB = BBPredPair.first; const auto &AddedBlockSet = BBPredPair.second.Added; auto &PrevBlockSet = BBPredPair.second.Prev; - for (auto *Pi : GD->template getChildren(BB)) { + for (auto *Pi : GD->template getChildren(BB)) { if (!AddedBlockSet.count(Pi)) PrevBlockSet.insert(Pi); EdgeCountMap[{Pi, BB}]++; @@ -1122,7 +1122,7 @@ void MemorySSAUpdater::applyInsertUpdates(ArrayRef Updates, for (unsigned I = 0, E = IDFPhi->getNumIncomingValues(); I < E; ++I) IDFPhi->setIncomingValue(I, GetLastDef(IDFPhi->getIncomingBlock(I))); } else { - for (auto *Pi : GD->template getChildren(BBIDF)) + for (auto *Pi : GD->template getChildren(BBIDF)) IDFPhi->addIncoming(GetLastDef(Pi), Pi); } } diff --git a/llvm/lib/CodeGen/MachineDominators.cpp b/llvm/lib/CodeGen/MachineDominators.cpp index 0632cde9c6f4..62649cfdbcf0 100644 --- a/llvm/lib/CodeGen/MachineDominators.cpp +++ b/llvm/lib/CodeGen/MachineDominators.cpp @@ -35,8 +35,8 @@ static cl::opt VerifyMachineDomInfoX( cl::desc("Verify machine dominator info (time consuming)")); namespace llvm { -template class DomTreeNodeBase; -template class DominatorTreeBase; // DomTreeBase +template class DomTreeNodeOnView; +template class DominatorTreeOnView; // DomTreeBase } char MachineDominatorTree::ID = 0; diff --git a/llvm/lib/CodeGen/MachinePostDominators.cpp b/llvm/lib/CodeGen/MachinePostDominators.cpp index fb96d0efa4d4..c13fdbbc1815 100644 --- a/llvm/lib/CodeGen/MachinePostDominators.cpp +++ b/llvm/lib/CodeGen/MachinePostDominators.cpp @@ -17,7 +17,7 @@ using namespace llvm; namespace llvm { -template class DominatorTreeBase; // PostDomTreeBase +template class DominatorTreeOnView; // PostDomTreeBase extern bool VerifyMachineDomInfo; } // namespace llvm diff --git a/llvm/lib/IR/Dominators.cpp b/llvm/lib/IR/Dominators.cpp index 7c620c3a9331..9330508583aa 100644 --- a/llvm/lib/IR/Dominators.cpp +++ b/llvm/lib/IR/Dominators.cpp @@ -70,43 +70,59 @@ bool BasicBlockEdge::isSingleEdge() const { // //===----------------------------------------------------------------------===// -template class llvm::DomTreeNodeBase; -template class llvm::DominatorTreeBase; // DomTreeBase -template class llvm::DominatorTreeBase; // PostDomTreeBase + +namespace llvm { + +template class View> +class DominatorTreeOnView; + +template +using DominatorTreeBase = DominatorTreeOnView; + +template class View> +class DomTreeNodeOnView; + +template +using DomTreeNodeBase = DomTreeNodeOnView; + +template class DomTreeNodeOnView; +template class DominatorTreeOnView; // DomTree +template class DominatorTreeOnView; // PostDomTree +} // end namespace template class llvm::cfg::Update; -template void llvm::DomTreeBuilder::Calculate( +template void llvm::DomTreeBuilder::Calculate( DomTreeBuilder::BBDomTree &DT); template void -llvm::DomTreeBuilder::CalculateWithUpdates( +llvm::DomTreeBuilder::CalculateWithUpdates( DomTreeBuilder::BBDomTree &DT, BBUpdates U); -template void llvm::DomTreeBuilder::Calculate( +template void llvm::DomTreeBuilder::Calculate( DomTreeBuilder::BBPostDomTree &DT); // No CalculateWithUpdates instantiation, unless a usecase arises. -template void llvm::DomTreeBuilder::InsertEdge( +template void llvm::DomTreeBuilder::InsertEdge( DomTreeBuilder::BBDomTree &DT, BasicBlock *From, BasicBlock *To); -template void llvm::DomTreeBuilder::InsertEdge( +template void llvm::DomTreeBuilder::InsertEdge( DomTreeBuilder::BBPostDomTree &DT, BasicBlock *From, BasicBlock *To); -template void llvm::DomTreeBuilder::DeleteEdge( +template void llvm::DomTreeBuilder::DeleteEdge( DomTreeBuilder::BBDomTree &DT, BasicBlock *From, BasicBlock *To); -template void llvm::DomTreeBuilder::DeleteEdge( +template void llvm::DomTreeBuilder::DeleteEdge( DomTreeBuilder::BBPostDomTree &DT, BasicBlock *From, BasicBlock *To); -template void llvm::DomTreeBuilder::ApplyUpdates( - DomTreeBuilder::BBDomTree &DT, DomTreeBuilder::BBDomTreeGraphDiff &, - DomTreeBuilder::BBDomTreeGraphDiff *); -template void llvm::DomTreeBuilder::ApplyUpdates( - DomTreeBuilder::BBPostDomTree &DT, DomTreeBuilder::BBPostDomTreeGraphDiff &, - DomTreeBuilder::BBPostDomTreeGraphDiff *); +template void +llvm::DomTreeBuilder::ApplyUpdates( + BBDomTree &DT, BBDomTreeGraphDiff &, BBDomTreeGraphDiff *); +template void +llvm::DomTreeBuilder::ApplyUpdates( + BBPostDomTree &DT, BBPostDomTreeGraphDiff &, BBPostDomTreeGraphDiff *); -template bool llvm::DomTreeBuilder::Verify( +template bool llvm::DomTreeBuilder::Verify( const DomTreeBuilder::BBDomTree &DT, DomTreeBuilder::BBDomTree::VerificationLevel VL); -template bool llvm::DomTreeBuilder::Verify( +template bool llvm::DomTreeBuilder::Verify( const DomTreeBuilder::BBPostDomTree &DT, DomTreeBuilder::BBPostDomTree::VerificationLevel VL); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index d554f438c804..5e82a3829868 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -993,7 +993,7 @@ void VPlanIngredient::print(raw_ostream &O) const { #endif -template void DomTreeBuilder::Calculate(VPDominatorTree &DT); +template void DomTreeBuilder::Calculate(VPDominatorTree &DT); void VPValue::replaceAllUsesWith(VPValue *New) { for (unsigned J = 0; J < getNumUsers();) { diff --git a/llvm/unittests/IR/DominatorTreeBatchUpdatesTest.cpp b/llvm/unittests/IR/DominatorTreeBatchUpdatesTest.cpp index 27948c1ad6f1..60af64a7ea09 100644 --- a/llvm/unittests/IR/DominatorTreeBatchUpdatesTest.cpp +++ b/llvm/unittests/IR/DominatorTreeBatchUpdatesTest.cpp @@ -25,8 +25,8 @@ using DomUpdate = DominatorTree::UpdateType; static_assert( std::is_same_v, "Trees differing only in IsPostDom should have the same update types"); -using DomSNCA = DomTreeBuilder::SemiNCAInfo; -using PostDomSNCA = DomTreeBuilder::SemiNCAInfo; +using DomSNCA = DomTreeBuilder::SemiNCAInfoOnView; +using PostDomSNCA = DomTreeBuilder::SemiNCAInfoOnView; const auto Insert = DominatorTree::Insert; const auto Delete = DominatorTree::Delete; From 1eaa6622bc895048a4cee9b8661d721bc86cf816 Mon Sep 17 00:00:00 2001 From: Pietro Fezzardi Date: Thu, 27 May 2021 17:27:49 +0200 Subject: [PATCH 018/147] Enable PDT on GraphTraits using mapped_iterator --- llvm/include/llvm/Support/CFGDiff.h | 5 +++-- llvm/include/llvm/Support/GenericDomTreeConstruction.h | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/llvm/include/llvm/Support/CFGDiff.h b/llvm/include/llvm/Support/CFGDiff.h index 07cf242d67a8..b0a50690d228 100644 --- a/llvm/include/llvm/Support/CFGDiff.h +++ b/llvm/include/llvm/Support/CFGDiff.h @@ -135,8 +135,9 @@ template class GraphDiff { template class View> VectRet getChildren(NodePtr N) const { using DirectedNodeT = std::conditional_t>, View>; - auto R = children(N); - VectRet Res = VectRet(detail::reverse_if(R)); + VectRet Res = VectRet(children(N)); + if (not InverseEdge) + std::reverse(Res.begin(), Res.end()); // Remove nullptr children for clang. llvm::erase_value(Res, nullptr); diff --git a/llvm/include/llvm/Support/GenericDomTreeConstruction.h b/llvm/include/llvm/Support/GenericDomTreeConstruction.h index 7c6d10655d19..d2a826c440e7 100644 --- a/llvm/include/llvm/Support/GenericDomTreeConstruction.h +++ b/llvm/include/llvm/Support/GenericDomTreeConstruction.h @@ -116,8 +116,10 @@ struct SemiNCAInfoOnView { static SmallVector getChildren(NodePtr N) { using DirectedNodeT = std::conditional_t>, View>; - auto R = children(N); - SmallVector Res(detail::reverse_if(R)); + using VectRet = SmallVector; + VectRet Res = VectRet(children(N)); + if (not Inversed) + std::reverse(Res.begin(), Res.end()); // Remove nullptr children for clang. llvm::erase_value(Res, nullptr); From 440a98a73585506c842d47e747fc4db13b55f9ae Mon Sep 17 00:00:00 2001 From: Filippo Cremonese Date: Mon, 7 Mar 2022 17:43:01 +0100 Subject: [PATCH 019/147] Quote ambiguous boolean-like strings YAML v1.1 spec interprets more values as booleans than the YAML v1.2 spec. This commit quotes the ambiguous booleans (e.g. "off"). YAML 1.2 parsers will still correctly parse the quoted values, while YAML 1.1 will be able to correctly parse them as strings and not as booleans. Ref YAML 1.1 spec: https://yaml.org/type/bool.html --- llvm/include/llvm/Support/YAMLTraits.h | 7 +++++- .../MIR/X86/variable-sized-stack-objects.mir | 4 ++-- .../test/DebugInfo/COFF/global-type-hashes.ll | 2 +- .../ObjectYAML/MachO/DWARF-BigEndian.yaml | 4 ++-- .../ObjectYAML/MachO/DWARF-LittleEndian.yaml | 4 ++-- llvm/test/YAMLParser/bool.test | 22 +++++++++++++++++-- llvm/test/tools/llvm-size/common.test | 2 +- 7 files changed, 34 insertions(+), 11 deletions(-) diff --git a/llvm/include/llvm/Support/YAMLTraits.h b/llvm/include/llvm/Support/YAMLTraits.h index 3fb2092ad841..482e059ba16e 100644 --- a/llvm/include/llvm/Support/YAMLTraits.h +++ b/llvm/include/llvm/Support/YAMLTraits.h @@ -665,7 +665,12 @@ inline bool isNull(StringRef S) { inline bool isBool(StringRef S) { // FIXME: using parseBool is causing multiple tests to fail. return S.equals("true") || S.equals("True") || S.equals("TRUE") || - S.equals("false") || S.equals("False") || S.equals("FALSE"); + S.equals("yes") || S.equals("Yes") || S.equals ("YES") || + S.equals("on") || S.equals("On") || S.equals ("ON") || + S.equals("false") || S.equals("False") || S.equals("FALSE") || + S.equals("no") || S.equals("No") || S.equals("NO") || + S.equals("off") || S.equals("Off") || S.equals("OFF") || + S.equals("y") || S.equals("Y") || S.equals("n") || S.equals("N"); } // 5.1. Character Set diff --git a/llvm/test/CodeGen/MIR/X86/variable-sized-stack-objects.mir b/llvm/test/CodeGen/MIR/X86/variable-sized-stack-objects.mir index 81c2b99fc068..e880c03c1a17 100644 --- a/llvm/test/CodeGen/MIR/X86/variable-sized-stack-objects.mir +++ b/llvm/test/CodeGen/MIR/X86/variable-sized-stack-objects.mir @@ -30,11 +30,11 @@ frameInfo: # CHECK-NEXT: - { id: 1, name: '', type: default, offset: -32, size: 8, alignment: 8, # CHECK-NEXT: stack-id: default, callee-saved-register: '', callee-saved-restored: true, # CHECK-NEXT: debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } -# CHECK-NEXT: - { id: 2, name: y, type: variable-sized, offset: -32, alignment: 1, +# CHECK-NEXT: - { id: 2, name: 'y', type: variable-sized, offset: -32, alignment: 1, stack: - { id: 0, offset: -20, size: 4, alignment: 4 } - { id: 1, offset: -32, size: 8, alignment: 8 } - - { id: 2, name: y, type: variable-sized, offset: -32, alignment: 1 } + - { id: 2, name: 'y', type: variable-sized, offset: -32, alignment: 1 } body: | bb.0.entry: MOV32mr $rsp, 1, _, -4, _, $edi diff --git a/llvm/test/DebugInfo/COFF/global-type-hashes.ll b/llvm/test/DebugInfo/COFF/global-type-hashes.ll index 586b97d8fd19..50ad9efb9953 100644 --- a/llvm/test/DebugInfo/COFF/global-type-hashes.ll +++ b/llvm/test/DebugInfo/COFF/global-type-hashes.ll @@ -240,7 +240,7 @@ attributes #3 = { mustprogress noinline nounwind optnone "frame-pointer"="none" ; YAML: Attrs: 3 ; YAML: Type: 116 ; YAML: FieldOffset: 4 -; YAML: Name: Y +; YAML: Name: 'Y' ; YAML: - Kind: LF_ONEMETHOD ; YAML: OneMethod: ; YAML: Type: 4103 diff --git a/llvm/test/ObjectYAML/MachO/DWARF-BigEndian.yaml b/llvm/test/ObjectYAML/MachO/DWARF-BigEndian.yaml index 886597a0129b..092e5c1e9c57 100644 --- a/llvm/test/ObjectYAML/MachO/DWARF-BigEndian.yaml +++ b/llvm/test/ObjectYAML/MachO/DWARF-BigEndian.yaml @@ -272,7 +272,7 @@ DWARF: - long long int - __absvdi2 - a - - N + - 'N' - t debug_abbrev: - Table: @@ -384,7 +384,7 @@ DWARF: #CHECK: - long long int #CHECK: - __absvdi2 #CHECK: - a -#CHECK: - N +#CHECK: - 'N' #CHECK: - t #CHECK: debug_abbrev: #CHECK: - Code: 0x1 diff --git a/llvm/test/ObjectYAML/MachO/DWARF-LittleEndian.yaml b/llvm/test/ObjectYAML/MachO/DWARF-LittleEndian.yaml index 12bf9e70c982..04be63178a24 100644 --- a/llvm/test/ObjectYAML/MachO/DWARF-LittleEndian.yaml +++ b/llvm/test/ObjectYAML/MachO/DWARF-LittleEndian.yaml @@ -261,7 +261,7 @@ DWARF: - long long int - __absvdi2 - a - - N + - 'N' - t debug_abbrev: - Table: @@ -373,7 +373,7 @@ DWARF: #CHECK: - long long int #CHECK: - __absvdi2 #CHECK: - a -#CHECK: - N +#CHECK: - 'N' #CHECK: - t #CHECK: debug_abbrev: #CHECK: - Code: 0x1 diff --git a/llvm/test/YAMLParser/bool.test b/llvm/test/YAMLParser/bool.test index e987a0ec1e32..885c78c6660f 100644 --- a/llvm/test/YAMLParser/bool.test +++ b/llvm/test/YAMLParser/bool.test @@ -1,6 +1,24 @@ # RUN: yaml-bench -canonical %s -- yes -- NO +- true - True +- TRUE +- yes +- Yes +- YES - on +- On +- ON +- false +- False +- FALSE +- no +- No +- NO +- off +- Off +- OFF +- y +- Y +- n +- N diff --git a/llvm/test/tools/llvm-size/common.test b/llvm/test/tools/llvm-size/common.test index ac804778ed76..becc183875b5 100644 --- a/llvm/test/tools/llvm-size/common.test +++ b/llvm/test/tools/llvm-size/common.test @@ -34,7 +34,7 @@ Sections: - Name: .text Type: SHT_PROGBITS Symbols: - - Name: y + - Name: 'y' Type: STT_OBJECT Size: 4 Index: SHN_COMMON From e2f57050dd4129516e81e38f8e633302f56194c2 Mon Sep 17 00:00:00 2001 From: Giacomo Vercesi Date: Wed, 1 Jun 2022 14:15:25 +0200 Subject: [PATCH 020/147] MLIR Dominance: use DominatorTreeOnView This commit fixes a compilation error due to the introduction of DominatorTreeOnView in LLVM, which was not yet adopted in MLIR. In the future, when rebasing rev.ng's out-of-tree patches on new versions of LLVM, this commit could be squashed with the following: 2cca6589f2219e69f62a6fbcc4b16d52872c5e7a Add support to Dominators for FilteredGraphTraits --- mlir/include/mlir/IR/Dominance.h | 5 ++--- mlir/lib/IR/Dominance.cpp | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/mlir/include/mlir/IR/Dominance.h b/mlir/include/mlir/IR/Dominance.h index 30f5674ad71d..49216b80284a 100644 --- a/mlir/include/mlir/IR/Dominance.h +++ b/mlir/include/mlir/IR/Dominance.h @@ -22,9 +22,8 @@ #include "mlir/IR/RegionGraphTraits.h" #include "llvm/Support/GenericDomTree.h" -extern template class llvm::DominatorTreeBase; -extern template class llvm::DominatorTreeBase; -extern template class llvm::DomTreeNodeBase; +extern template class llvm::DominatorTreeOnView; +extern template class llvm::DominatorTreeOnView; namespace mlir { using DominanceInfoNode = llvm::DomTreeNodeBase; diff --git a/mlir/lib/IR/Dominance.cpp b/mlir/lib/IR/Dominance.cpp index beb7f7bfeedf..d44969cf4cb6 100644 --- a/mlir/lib/IR/Dominance.cpp +++ b/mlir/lib/IR/Dominance.cpp @@ -20,9 +20,9 @@ using namespace mlir; using namespace mlir::detail; -template class llvm::DominatorTreeBase; -template class llvm::DominatorTreeBase; -template class llvm::DomTreeNodeBase; +template class llvm::DominatorTreeOnView; +template class llvm::DominatorTreeOnView; +template class llvm::DomTreeNodeOnView; //===----------------------------------------------------------------------===// // DominanceInfoBase From 98ee492cebd5c39a3adc922ccfad3cc2a54dfaf8 Mon Sep 17 00:00:00 2001 From: Giacomo Vercesi Date: Wed, 1 Jun 2022 11:27:05 +0200 Subject: [PATCH 021/147] mlir::raw_indented_ostream: custom indent value Changes raw_indented_ostream to make it possible to specify a custom indentation amount. --- mlir/include/mlir/Support/IndentedOstream.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mlir/include/mlir/Support/IndentedOstream.h b/mlir/include/mlir/Support/IndentedOstream.h index 3d19a1e1ede3..7b19d37c7f38 100644 --- a/mlir/include/mlir/Support/IndentedOstream.h +++ b/mlir/include/mlir/Support/IndentedOstream.h @@ -22,7 +22,7 @@ namespace mlir { /// raw_ostream subclass that simplifies indention a sequence of code. class raw_indented_ostream : public raw_ostream { public: - explicit raw_indented_ostream(llvm::raw_ostream &os) : os(os) { + explicit raw_indented_ostream(llvm::raw_ostream &os, int indentSize = 2) : indentSize(indentSize), os(os) { SetUnbuffered(); } @@ -92,7 +92,7 @@ class raw_indented_ostream : public raw_ostream { uint64_t current_pos() const override { return os.tell(); } /// Constant indent added/removed. - static constexpr int indentSize = 2; + int indentSize; /// Tracker for current indentation. int currentIndent = 0; From 900d80c1244cae631ad4301508c0900ee5a0b12a Mon Sep 17 00:00:00 2001 From: Djordje Todorovic Date: Fri, 21 Oct 2022 14:57:31 +0200 Subject: [PATCH 022/147] [SimplifyIndVar] Make getInsertPointForUses() public --- llvm/include/llvm/Transforms/Utils/SimplifyIndVar.h | 6 ++++++ llvm/lib/Transforms/Utils/SimplifyIndVar.cpp | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/llvm/include/llvm/Transforms/Utils/SimplifyIndVar.h b/llvm/include/llvm/Transforms/Utils/SimplifyIndVar.h index ff60811b6168..cd089574cc7d 100644 --- a/llvm/include/llvm/Transforms/Utils/SimplifyIndVar.h +++ b/llvm/include/llvm/Transforms/Utils/SimplifyIndVar.h @@ -22,12 +22,14 @@ class WeakTrackingVH; template class SmallVectorImpl; class CastInst; class DominatorTree; +class Instruction; class Loop; class LoopInfo; class PHINode; class ScalarEvolution; class SCEVExpander; class TargetTransformInfo; +class Value; /// Interface for visiting interesting IV users that are recognized but not /// simplified by this utility. @@ -58,6 +60,10 @@ bool simplifyLoopIVs(Loop *L, ScalarEvolution *SE, DominatorTree *DT, LoopInfo *LI, const TargetTransformInfo *TTI, SmallVectorImpl &Dead); +/// We make it public so it can be used in our LLVM out-of-tree Passes. +Instruction *getInsertPointForUses(Instruction *User, Value *Def, + DominatorTree *DT, LoopInfo *LI); + /// Collect information about induction variables that are used by sign/zero /// extend operations. This information is recorded by CollectExtend and provides /// the input to WidenIV. diff --git a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp index 4e83d2f6e3c6..7e8b704f0b87 100644 --- a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp @@ -1118,8 +1118,8 @@ class WidenIV { /// common dominator for the incoming blocks. A nullptr can be returned if no /// viable location is found: it may happen if User is a PHI and Def only comes /// to this PHI from unreachable blocks. -static Instruction *getInsertPointForUses(Instruction *User, Value *Def, - DominatorTree *DT, LoopInfo *LI) { +Instruction *llvm::getInsertPointForUses(Instruction *User, Value *Def, + DominatorTree *DT, LoopInfo *LI) { PHINode *PHI = dyn_cast(User); if (!PHI) return User; From 4adf649a0e0eb9fadb5d688c98d9c76d717ad77f Mon Sep 17 00:00:00 2001 From: Alessandro Di Federico Date: Thu, 8 Dec 2022 13:14:24 +0100 Subject: [PATCH 023/147] Make ExitOnError constructor external The lambda in the constructor is a significant source of build time overhead in downstream projects. --- llvm/include/llvm/Support/Error.h | 4 +--- llvm/lib/Support/Error.cpp | 4 ++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/llvm/include/llvm/Support/Error.h b/llvm/include/llvm/Support/Error.h index 8a984db5e681..ece888f247f6 100644 --- a/llvm/include/llvm/Support/Error.h +++ b/llvm/include/llvm/Support/Error.h @@ -1355,9 +1355,7 @@ Error createFileError(const Twine &F, ErrorSuccess) = delete; class ExitOnError { public: /// Create an error on exit helper. - ExitOnError(std::string Banner = "", int DefaultErrorExitCode = 1) - : Banner(std::move(Banner)), - GetExitCode([=](const Error &) { return DefaultErrorExitCode; }) {} + ExitOnError(std::string Banner = "", int DefaultErrorExitCode = 1); /// Set the banner string for any errors caught by operator(). void setBanner(std::string Banner) { this->Banner = std::move(Banner); } diff --git a/llvm/lib/Support/Error.cpp b/llvm/lib/Support/Error.cpp index fbe86f2b59e1..05e7144c06b4 100644 --- a/llvm/lib/Support/Error.cpp +++ b/llvm/lib/Support/Error.cpp @@ -177,3 +177,7 @@ LLVMErrorTypeId LLVMGetStringErrorTypeId() { LLVMErrorRef LLVMCreateStringError(const char *ErrMsg) { return wrap(make_error(ErrMsg, inconvertibleErrorCode())); } + +ExitOnError::ExitOnError(std::string Banner, int DefaultErrorExitCode) + : Banner(std::move(Banner)), + GetExitCode([=](const Error &) { return DefaultErrorExitCode; }) {} From 86725276298450a1a377d050650d26ea6454fb04 Mon Sep 17 00:00:00 2001 From: Pietro Fezzardi Date: Tue, 23 Jun 2020 17:19:19 +0200 Subject: [PATCH 024/147] Triple.cpp: fix getArchTypeForLLVMName for "x86_64" --- llvm/lib/TargetParser/Triple.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/TargetParser/Triple.cpp b/llvm/lib/TargetParser/Triple.cpp index a68035989a93..789839f3d418 100644 --- a/llvm/lib/TargetParser/Triple.cpp +++ b/llvm/lib/TargetParser/Triple.cpp @@ -352,6 +352,7 @@ Triple::ArchType Triple::getArchTypeForLLVMName(StringRef Name) { .Case("x86", x86) .Case("i386", x86) .Case("x86-64", x86_64) + .Case("x86_64", x86_64) .Case("xcore", xcore) .Case("nvptx", nvptx) .Case("nvptx64", nvptx64) From 216b68d0e8fb66f21f81b409705773ecec82f8a4 Mon Sep 17 00:00:00 2001 From: Pietro Fezzardi Date: Tue, 5 Mar 2019 10:27:10 +0100 Subject: [PATCH 025/147] Fix memory leak in `CreateSigAltStack` --- llvm/lib/Support/Unix/Signals.inc | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Support/Unix/Signals.inc b/llvm/lib/Support/Unix/Signals.inc index 05a7335216f4..374022389594 100644 --- a/llvm/lib/Support/Unix/Signals.inc +++ b/llvm/lib/Support/Unix/Signals.inc @@ -259,8 +259,8 @@ static struct { // reported as a leak. We don't make any attempt to remove our alt signal // stack if we remove our signal handlers; that can't be done reliably if // someone else is also trying to do the same thing. -static stack_t OldAltStack; -LLVM_ATTRIBUTE_USED static void *NewAltStackPointer; +thread_local static stack_t OldAltStack; +LLVM_ATTRIBUTE_USED thread_local static void* NewAltStackPointer; static void CreateSigAltStack() { const size_t AltStackSize = MINSIGSTKSZ + 64 * 1024; @@ -276,10 +276,13 @@ static void CreateSigAltStack() { stack_t AltStack = {}; AltStack.ss_sp = static_cast(safe_malloc(AltStackSize)); - NewAltStackPointer = AltStack.ss_sp; // Save to avoid reporting a leak. AltStack.ss_size = AltStackSize; - if (sigaltstack(&AltStack, &OldAltStack) != 0) + if (sigaltstack(&AltStack, &OldAltStack) != 0) { free(AltStack.ss_sp); + } else { + free(NewAltStackPointer); + NewAltStackPointer = AltStack.ss_sp; + } } #else static void CreateSigAltStack() {} From 3307db5e574c681ab64663e69f73f3e39b6f7d9e Mon Sep 17 00:00:00 2001 From: Alessandro Di Federico Date: Wed, 2 Sep 2020 16:40:52 +0200 Subject: [PATCH 026/147] Drop `git remote` from --version --- llvm/cmake/modules/VersionFromVCS.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/cmake/modules/VersionFromVCS.cmake b/llvm/cmake/modules/VersionFromVCS.cmake index 18edbeabe3e4..23e80a91503d 100644 --- a/llvm/cmake/modules/VersionFromVCS.cmake +++ b/llvm/cmake/modules/VersionFromVCS.cmake @@ -5,7 +5,7 @@ function(get_source_info path revision repository) find_package(Git QUIET) - if(GIT_FOUND) + if(FALSE) execute_process(COMMAND ${GIT_EXECUTABLE} rev-parse --git-dir WORKING_DIRECTORY ${path} RESULT_VARIABLE git_result From 3ca701d797e3bd4fe79cfd61bd621d542770ca9f Mon Sep 17 00:00:00 2001 From: Massimo Fioravanti Date: Fri, 1 Jul 2022 18:02:37 +0200 Subject: [PATCH 027/147] Drop: assertion too strict for revng usecase IRMover asserts that a two types into two modules can be pointer wise identical only if they are mapped to each other. This assumption arises from the idea that modules are usually created indipendently from each other. We cannot make that assumption because the pipeline clones and merges back modules from the same origin. --- llvm/lib/Linker/IRMover.cpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/llvm/lib/Linker/IRMover.cpp b/llvm/lib/Linker/IRMover.cpp index 517e2dc8ebe0..462e824f4017 100644 --- a/llvm/lib/Linker/IRMover.cpp +++ b/llvm/lib/Linker/IRMover.cpp @@ -253,13 +253,6 @@ Type *TypeMapTy::get(Type *Ty, SmallPtrSet &Visited) { bool IsUniqued = !isa(Ty) || cast(Ty)->isLiteral(); if (!IsUniqued) { -#ifndef NDEBUG - for (auto &Pair : MappedTypes) { - assert(!(Pair.first != Ty && Pair.second == Ty) && - "mapping to a source type"); - } -#endif - if (!Visited.insert(cast(Ty)).second) { StructType *DTy = StructType::create(Ty->getContext()); return *Entry = DTy; From ea7ad6d29b7404a5f2411fc83875f86ab09628b8 Mon Sep 17 00:00:00 2001 From: Andrea Gussoni Date: Thu, 29 Nov 2018 14:40:28 +0100 Subject: [PATCH 028/147] Customize Doxygen documentation --- clang/docs/doxygen.cfg.in | 18 +++++++++--------- llvm/docs/doxygen.cfg.in | 16 ++++++++-------- mlir/docs/doxygen.cfg.in | 16 ++++++++-------- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/clang/docs/doxygen.cfg.in b/clang/docs/doxygen.cfg.in index 39a346409b93..95dcabca2cbd 100644 --- a/clang/docs/doxygen.cfg.in +++ b/clang/docs/doxygen.cfg.in @@ -1126,7 +1126,7 @@ HTML_TIMESTAMP = YES # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. -HTML_DYNAMIC_SECTIONS = NO +HTML_DYNAMIC_SECTIONS = YES # With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries # shown in the various tree structured indices initially; the user can expand @@ -1153,7 +1153,7 @@ HTML_INDEX_NUM_ENTRIES = 100 # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. -GENERATE_DOCSET = NO +GENERATE_DOCSET = YES # This tag determines the name of the docset feed. A documentation feed provides # an umbrella under which multiple documentation sets from a single provider @@ -1169,7 +1169,7 @@ DOCSET_FEEDNAME = "Doxygen generated docs" # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_DOCSET is set to YES. -DOCSET_BUNDLE_ID = org.doxygen.Project +DOCSET_BUNDLE_ID = clang # The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify # the documentation publisher. This should be a reverse domain-name style @@ -1177,7 +1177,7 @@ DOCSET_BUNDLE_ID = org.doxygen.Project # The default value is: org.doxygen.Publisher. # This tag requires that the tag GENERATE_DOCSET is set to YES. -DOCSET_PUBLISHER_ID = org.doxygen.Publisher +DOCSET_PUBLISHER_ID = Publisher # The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. # The default value is: Publisher. @@ -1252,7 +1252,7 @@ TOC_EXPAND = NO # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. -GENERATE_QHP = @clang_doxygen_generate_qhp@ +GENERATE_QHP = YES # If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify # the file name of the resulting .qch file. The path specified is relative to @@ -1267,7 +1267,7 @@ QCH_FILE = @clang_doxygen_qch_filename@ # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_QHP is set to YES. -QHP_NAMESPACE = @clang_doxygen_qhp_namespace@ +QHP_NAMESPACE = clang # The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt # Help Project output. For more information please see Qt Help Project / Virtual @@ -1337,7 +1337,7 @@ ECLIPSE_DOC_ID = org.doxygen.Project # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. -DISABLE_INDEX = NO +DISABLE_INDEX = YES # The GENERATE_TREEVIEW tag is used to specify whether a tree-like index # structure should be generated to display hierarchical information. If the tag @@ -1468,7 +1468,7 @@ MATHJAX_CODEFILE = # The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. -SEARCHENGINE = @enable_searchengine@ +SEARCHENGINE = NO # When the SERVER_BASED_SEARCH tag is enabled the search engine will be # implemented using a web server instead of a web client using Javascript. There @@ -2090,7 +2090,7 @@ CLASS_GRAPH = YES # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. -COLLABORATION_GRAPH = YES +COLLABORATION_GRAPH = NO # If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for # groups, showing the direct groups dependencies. diff --git a/llvm/docs/doxygen.cfg.in b/llvm/docs/doxygen.cfg.in index b19ca6d21580..a856380fb30b 100644 --- a/llvm/docs/doxygen.cfg.in +++ b/llvm/docs/doxygen.cfg.in @@ -1127,7 +1127,7 @@ HTML_TIMESTAMP = YES # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. -HTML_DYNAMIC_SECTIONS = NO +HTML_DYNAMIC_SECTIONS = YES # With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries # shown in the various tree structured indices initially; the user can expand @@ -1154,7 +1154,7 @@ HTML_INDEX_NUM_ENTRIES = 100 # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. -GENERATE_DOCSET = NO +GENERATE_DOCSET = YES # This tag determines the name of the docset feed. A documentation feed provides # an umbrella under which multiple documentation sets from a single provider @@ -1170,7 +1170,7 @@ DOCSET_FEEDNAME = "Doxygen generated docs" # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_DOCSET is set to YES. -DOCSET_BUNDLE_ID = org.doxygen.Project +DOCSET_BUNDLE_ID = llvm # The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify # the documentation publisher. This should be a reverse domain-name style @@ -1178,7 +1178,7 @@ DOCSET_BUNDLE_ID = org.doxygen.Project # The default value is: org.doxygen.Publisher. # This tag requires that the tag GENERATE_DOCSET is set to YES. -DOCSET_PUBLISHER_ID = org.doxygen.Publisher +DOCSET_PUBLISHER_ID = Publisher # The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. # The default value is: Publisher. @@ -1268,7 +1268,7 @@ QCH_FILE = @llvm_doxygen_qch_filename@ # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_QHP is set to YES. -QHP_NAMESPACE = @llvm_doxygen_qhp_namespace@ +QHP_NAMESPACE = llvm # The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt # Help Project output. For more information please see Qt Help Project / Virtual @@ -1338,7 +1338,7 @@ ECLIPSE_DOC_ID = org.doxygen.Project # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. -DISABLE_INDEX = NO +DISABLE_INDEX = YES # The GENERATE_TREEVIEW tag is used to specify whether a tree-like index # structure should be generated to display hierarchical information. If the tag @@ -1469,7 +1469,7 @@ MATHJAX_CODEFILE = # The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. -SEARCHENGINE = @enable_searchengine@ +SEARCHENGINE = NO # When the SERVER_BASED_SEARCH tag is enabled the search engine will be # implemented using a web server instead of a web client using Javascript. There @@ -2091,7 +2091,7 @@ CLASS_GRAPH = YES # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. -COLLABORATION_GRAPH = YES +COLLABORATION_GRAPH = NO # If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for # groups, showing the direct groups dependencies. diff --git a/mlir/docs/doxygen.cfg.in b/mlir/docs/doxygen.cfg.in index b91b242cffaa..5f817d8c5376 100644 --- a/mlir/docs/doxygen.cfg.in +++ b/mlir/docs/doxygen.cfg.in @@ -1154,7 +1154,7 @@ HTML_INDEX_NUM_ENTRIES = 100 # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. -GENERATE_DOCSET = NO +GENERATE_DOCSET = YES # This tag determines the name of the docset feed. A documentation feed provides # an umbrella under which multiple documentation sets from a single provider @@ -1170,7 +1170,7 @@ DOCSET_FEEDNAME = "Doxygen generated docs" # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_DOCSET is set to YES. -DOCSET_BUNDLE_ID = org.doxygen.Project +DOCSET_BUNDLE_ID = mlir # The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify # the documentation publisher. This should be a reverse domain-name style @@ -1178,7 +1178,7 @@ DOCSET_BUNDLE_ID = org.doxygen.Project # The default value is: org.doxygen.Publisher. # This tag requires that the tag GENERATE_DOCSET is set to YES. -DOCSET_PUBLISHER_ID = org.doxygen.Publisher +DOCSET_PUBLISHER_ID = Publisher # The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. # The default value is: Publisher. @@ -1253,7 +1253,7 @@ TOC_EXPAND = NO # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. -GENERATE_QHP = @llvm_doxygen_generate_qhp@ +GENERATE_QHP = YES # If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify # the file name of the resulting .qch file. The path specified is relative to @@ -1268,7 +1268,7 @@ QCH_FILE = @llvm_doxygen_qch_filename@ # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_QHP is set to YES. -QHP_NAMESPACE = @llvm_doxygen_qhp_namespace@ +QHP_NAMESPACE = mlir # The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt # Help Project output. For more information please see Qt Help Project / Virtual @@ -1338,7 +1338,7 @@ ECLIPSE_DOC_ID = org.doxygen.Project # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. -DISABLE_INDEX = NO +DISABLE_INDEX = YES # The GENERATE_TREEVIEW tag is used to specify whether a tree-like index # structure should be generated to display hierarchical information. If the tag @@ -1469,7 +1469,7 @@ MATHJAX_CODEFILE = # The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. -SEARCHENGINE = @enable_searchengine@ +SEARCHENGINE = NO # When the SERVER_BASED_SEARCH tag is enabled the search engine will be # implemented using a web server instead of a web client using Javascript. There @@ -2091,7 +2091,7 @@ CLASS_GRAPH = YES # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. -COLLABORATION_GRAPH = YES +COLLABORATION_GRAPH = NO # If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for # groups, showing the direct groups dependencies. From a7e8a97b32ff92fc9aceed7218e7ffeccfc5ab21 Mon Sep 17 00:00:00 2001 From: Alessandro Di Federico Date: Wed, 5 Apr 2023 11:01:11 +0200 Subject: [PATCH 029/147] ~ErrorAdapter: add override --- llvm/include/llvm/Support/FormatAdapters.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/include/llvm/Support/FormatAdapters.h b/llvm/include/llvm/Support/FormatAdapters.h index 495205d11748..d84e5f8f5f40 100644 --- a/llvm/include/llvm/Support/FormatAdapters.h +++ b/llvm/include/llvm/Support/FormatAdapters.h @@ -75,7 +75,7 @@ class ErrorAdapter : public FormatAdapter { public: ErrorAdapter(Error &&Item) : FormatAdapter(std::move(Item)) {} ErrorAdapter(ErrorAdapter &&) = default; - ~ErrorAdapter() { consumeError(std::move(Item)); } + ~ErrorAdapter() override { consumeError(std::move(Item)); } void format(llvm::raw_ostream &Stream, StringRef Style) override { Stream << Item; } From bf23ea6ab52d63f4de3f5ea02a1203a0e9259bbd Mon Sep 17 00:00:00 2001 From: Alessandro Di Federico Date: Wed, 5 Apr 2023 11:01:45 +0200 Subject: [PATCH 030/147] GenericDomTreeConstruction: add parentheses For some reason, without these the parser complains. --- llvm/include/llvm/Support/GenericDomTreeConstruction.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/include/llvm/Support/GenericDomTreeConstruction.h b/llvm/include/llvm/Support/GenericDomTreeConstruction.h index d2a826c440e7..4f695eccb4cb 100644 --- a/llvm/include/llvm/Support/GenericDomTreeConstruction.h +++ b/llvm/include/llvm/Support/GenericDomTreeConstruction.h @@ -243,7 +243,7 @@ struct SemiNCAInfoOnView { NodePtr eval(NodePtr V, unsigned LastLinked, SmallVectorImpl &Stack) { InfoRec *VInfo = &NodeToInfo[V]; - if (VInfo->Parent < LastLinked) + if ((VInfo->Parent) < (LastLinked)) return VInfo->Label; // Store ancestors except the last (root of a virtual tree) into a stack. From 5a5775e5ae02e404117c281b1934f5dfe5dfd18b Mon Sep 17 00:00:00 2001 From: Alessandro Di Federico Date: Wed, 5 Apr 2023 11:02:27 +0200 Subject: [PATCH 031/147] ScopedPrinter: mark destructors as override --- llvm/include/llvm/Support/ScopedPrinter.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/include/llvm/Support/ScopedPrinter.h b/llvm/include/llvm/Support/ScopedPrinter.h index b91acb576ba5..613744ee68cb 100644 --- a/llvm/include/llvm/Support/ScopedPrinter.h +++ b/llvm/include/llvm/Support/ScopedPrinter.h @@ -819,7 +819,7 @@ struct DictScope : DelimitedScope { W.objectBegin(); } - ~DictScope() { + ~DictScope() override { if (W) W->objectEnd(); } @@ -838,7 +838,7 @@ struct ListScope : DelimitedScope { W.arrayBegin(); } - ~ListScope() { + ~ListScope() override { if (W) W->arrayEnd(); } From 73c4a674f48db4ff826d430a4d611f11ea9b5b7d Mon Sep 17 00:00:00 2001 From: Alessandro Di Federico Date: Thu, 13 Apr 2023 18:10:00 +0200 Subject: [PATCH 032/147] DIFinder::processInstruction: handle revng.loop --- llvm/lib/IR/DebugInfo.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp index 9198179674bd..136815c7fa78 100644 --- a/llvm/lib/IR/DebugInfo.cpp +++ b/llvm/lib/IR/DebugInfo.cpp @@ -211,6 +211,12 @@ void DebugInfoFinder::processInstruction(const Module &M, if (auto DbgLoc = I.getDebugLoc()) processLocation(M, DbgLoc.get()); + + MDNode *LoopMD = I.getMetadata(LLVMContext::MD_loop); + if (auto *Tuple = cast_or_null(LoopMD)) + for (const MDOperand &Operand : Tuple->operands()) + if (const DILocation *Location = dyn_cast(Operand.get())) + processLocation(M, Location); } void DebugInfoFinder::processLocation(const Module &M, const DILocation *Loc) { From 4f85ee74e8ea2a2a88515683a73d8d3324e0fb78 Mon Sep 17 00:00:00 2001 From: Alessandro Di Federico Date: Mon, 15 May 2023 16:44:00 +0200 Subject: [PATCH 033/147] IntervalMap: suppress enum-related warnings --- llvm/include/llvm/ADT/IntervalMap.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/llvm/include/llvm/ADT/IntervalMap.h b/llvm/include/llvm/ADT/IntervalMap.h index c68d816a8e7d..c2dbc4fffc90 100644 --- a/llvm/include/llvm/ADT/IntervalMap.h +++ b/llvm/include/llvm/ADT/IntervalMap.h @@ -1215,7 +1215,9 @@ IntervalMapImpl::IdxPair IntervalMap:: branchRoot(unsigned Position) { using namespace IntervalMapImpl; // How many external leaf nodes to hold RootLeaf+1? - const unsigned Nodes = RootLeaf::Capacity / Leaf::Capacity + 1; + using CapacityType = decltype(RootLeaf::Capacity); + const unsigned Nodes = (RootLeaf::Capacity + / static_cast(Leaf::Capacity) + 1); // Compute element distribution among new nodes. unsigned size[Nodes]; @@ -1256,7 +1258,9 @@ IntervalMapImpl::IdxPair IntervalMap:: splitRoot(unsigned Position) { using namespace IntervalMapImpl; // How many external leaf nodes to hold RootBranch+1? - const unsigned Nodes = RootBranch::Capacity / Branch::Capacity + 1; + using CapacityType = decltype(RootBranch::Capacity); + const unsigned Nodes = (RootBranch::Capacity + / static_cast(Branch::Capacity) + 1); // Compute element distribution among new nodes. unsigned Size[Nodes]; From edf78cf155d060f4969489d07425419ea840e71e Mon Sep 17 00:00:00 2001 From: Djordje Todorovic Date: Wed, 17 May 2023 12:44:33 +0200 Subject: [PATCH 034/147] clang: mark ~TargetOMPContext as override --- clang/include/clang/AST/OpenMPClause.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h index 627e9025c112..20a1f9012293 100644 --- a/clang/include/clang/AST/OpenMPClause.h +++ b/clang/include/clang/AST/OpenMPClause.h @@ -8851,7 +8851,7 @@ struct TargetOMPContext final : public llvm::omp::OMPContext { const FunctionDecl *CurrentFunctionDecl, ArrayRef ConstructTraits); - virtual ~TargetOMPContext() = default; + virtual ~TargetOMPContext() override = default; /// See llvm::omp::OMPContext::matchesISATrait bool matchesISATrait(StringRef RawString) const override; From 7ed37eac663627dce393ed6e3eb68eccdc014f77 Mon Sep 17 00:00:00 2001 From: Ivan Krysak Date: Tue, 18 Apr 2023 10:42:30 +0300 Subject: [PATCH 035/147] InstPrinter: add missing `imm` tag for AT&T --- llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp index 5a1c4ec81e1b..f1e288923caf 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp @@ -435,7 +435,7 @@ void X86ATTInstPrinter::printMemReference(const MCInst *MI, unsigned Op, if (DispSpec.isImm()) { int64_t DispVal = DispSpec.getImm(); if (DispVal || (!IndexReg.getReg() && !BaseReg.getReg())) - O << formatImm(DispVal); + O << markup(""); } else { assert(DispSpec.isExpr() && "non-immediate displacement for LEA?"); DispSpec.getExpr()->print(O, &MAI); From d2d53e5c4d5d87ea60588979fda4b0469877892c Mon Sep 17 00:00:00 2001 From: Ivan Krysak Date: Wed, 19 Apr 2023 19:27:47 +0300 Subject: [PATCH 036/147] InstPrinter: explicitly mark `pc` as `register` --- llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp index 9f275145adfd..421295e76118 100644 --- a/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp +++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp @@ -372,7 +372,7 @@ void ARMInstPrinter::printThumbLdrLabelOperand(const MCInst *MI, unsigned OpNum, return; } - O << markup("") << ", "; int32_t OffImm = (int32_t)MO1.getImm(); bool isSub = OffImm < 0; From 3f6b01bb8700582b2cee3180e717e577ebc070d2 Mon Sep 17 00:00:00 2001 From: Ivan Krysak Date: Thu, 6 Apr 2023 11:44:41 +0300 Subject: [PATCH 037/147] InstPrinter: introduce address tagging for x86 --- llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp | 10 ++++++++-- .../Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp | 4 ++-- .../Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp | 8 +++++++- llvm/test/MC/Disassembler/X86/marked-up-32.txt | 4 ++-- llvm/test/MC/Disassembler/X86/marked-up.txt | 2 +- 5 files changed, 20 insertions(+), 8 deletions(-) diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp index f1e288923caf..82255dade9d0 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp @@ -434,8 +434,14 @@ void X86ATTInstPrinter::printMemReference(const MCInst *MI, unsigned Op, if (DispSpec.isImm()) { int64_t DispVal = DispSpec.getImm(); - if (DispVal || (!IndexReg.getReg() && !BaseReg.getReg())) - O << markup(""); + if (DispVal || (!IndexReg.getReg() && !BaseReg.getReg())) { + bool IsRIPBased = BaseReg.getReg() == X86::RIP || + BaseReg.getReg() == X86::EIP; + if (IsRIPBased && !IndexReg.getReg()) + O << markup(""); + else + O << markup(""); + } } else { assert(DispSpec.isExpr() && "non-immediate displacement for LEA?"); DispSpec.getExpr()->print(O, &MAI); diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp index f2cb3dfc8ed0..abb2ebb642f7 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp @@ -322,7 +322,7 @@ void X86InstPrinterCommon::printPCRelImm(const MCInst *MI, uint64_t Address, const MCOperand &Op = MI->getOperand(OpNo); if (Op.isImm()) { - O << markup("(Op.getExpr()); int64_t Address; if (BranchTarget && BranchTarget->evaluateAsAbsolute(Address)) { - O << markup(""); + O << markup(""); } else { // Otherwise, just print the expression. Op.getExpr()->print(O, &MAI); diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp index 84da39c4a295..9a8d8f14be2f 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp @@ -419,7 +419,13 @@ void X86IntelInstPrinter::printMemReference(const MCInst *MI, unsigned Op, DispVal = -DispVal; } } - O << markup(""); + + bool IsRIPBased = BaseReg.getReg() == X86::RIP || + BaseReg.getReg() == X86::EIP; + if (IsRIPBased && !IndexReg.getReg()) + O << markup(""); + else + O << markup(""); } } diff --git a/llvm/test/MC/Disassembler/X86/marked-up-32.txt b/llvm/test/MC/Disassembler/X86/marked-up-32.txt index 20fe31538848..363c503a029b 100644 --- a/llvm/test/MC/Disassembler/X86/marked-up-32.txt +++ b/llvm/test/MC/Disassembler/X86/marked-up-32.txt @@ -1,6 +1,6 @@ # RUN: llvm-mc -triple=i386 --mdis %s | FileCheck %s -# CHECK: je +# CHECK: je 0x0f 0x84 0x3c 0x00 0x00 0x00 -# CHECK: jecxz +# CHECK: jecxz 0xe3 0x81 diff --git a/llvm/test/MC/Disassembler/X86/marked-up.txt b/llvm/test/MC/Disassembler/X86/marked-up.txt index f0e51252f8d8..7b1a4952ea87 100644 --- a/llvm/test/MC/Disassembler/X86/marked-up.txt +++ b/llvm/test/MC/Disassembler/X86/marked-up.txt @@ -1,6 +1,6 @@ # RUN: llvm-mc --mdis %s -triple=x86_64-apple-darwin9 2>&1 | FileCheck %s -# CHECK: movq :8>, +# CHECK: movq :>, 0x65 0x48 0x8b 0x0c 0x25 0x08 0x00 0x00 0x00 # CHECK: xorps , 0x0f 0x57 0xd1 From 2e9f5477d024834a9a6a679d865c776993816033 Mon Sep 17 00:00:00 2001 From: Ivan Krysak Date: Thu, 6 Apr 2023 12:11:11 +0300 Subject: [PATCH 038/147] InstPrinter: introduce address tagging for AArch64 --- .../AArch64/MCTargetDesc/AArch64InstPrinter.cpp | 14 ++++++-------- llvm/test/MC/AArch64/basic-a64-instructions.s | 4 ---- llvm/test/MC/Disassembler/AArch64/marked-up.txt | 2 +- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp index 6ff5459fe026..75bf7633e852 100644 --- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp +++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp @@ -1751,12 +1751,11 @@ void AArch64InstPrinter::printAlignedLabel(const MCInst *MI, uint64_t Address, // If the label has already been resolved to an immediate offset (say, when // we're running the disassembler), just print the immediate. if (Op.isImm()) { - O << markup(""); return; } @@ -1766,7 +1765,7 @@ void AArch64InstPrinter::printAlignedLabel(const MCInst *MI, uint64_t Address, dyn_cast(MI->getOperand(OpNum).getExpr()); int64_t TargetAddress; if (BranchTarget && BranchTarget->evaluateAsAbsolute(TargetAddress)) { - O << formatHex((uint64_t)TargetAddress); + O << markup(""); } else { // Otherwise, just print the expression. MI->getOperand(OpNum).getExpr()->print(O, &MAI); @@ -1782,12 +1781,11 @@ void AArch64InstPrinter::printAdrpLabel(const MCInst *MI, uint64_t Address, // If the label has already been resolved to an immediate offset (say, when // we're running the disassembler), just print the immediate. if (Op.isImm()) { - const int64_t Offset = Op.getImm() * 4096; - O << markup(""); return; } diff --git a/llvm/test/MC/AArch64/basic-a64-instructions.s b/llvm/test/MC/AArch64/basic-a64-instructions.s index 89f5944e02c5..f9bba53f2b83 100644 --- a/llvm/test/MC/AArch64/basic-a64-instructions.s +++ b/llvm/test/MC/AArch64/basic-a64-instructions.s @@ -3443,12 +3443,8 @@ _func: adr x9, #1048575 adr x2, #-1048576 - adrp x9, #4294963200 - adrp x20, #-4294967296 // CHECK: adr x9, #1048575 // encoding: [0xe9,0xff,0x7f,0x70] // CHECK: adr x2, #-1048576 // encoding: [0x02,0x00,0x80,0x10] -// CHECK: adrp x9, #4294963200 // encoding: [0xe9,0xff,0x7f,0xf0] -// CHECK: adrp x20, #-4294967296 // encoding: [0x14,0x00,0x80,0x90] //------------------------------------------------------------------------------ // System diff --git a/llvm/test/MC/Disassembler/AArch64/marked-up.txt b/llvm/test/MC/Disassembler/AArch64/marked-up.txt index 212187d2c3d5..137720d68bef 100644 --- a/llvm/test/MC/Disassembler/AArch64/marked-up.txt +++ b/llvm/test/MC/Disassembler/AArch64/marked-up.txt @@ -1,6 +1,6 @@ # RUN: llvm-mc -triple=aarch64 -mattr=+all --mdis %s | FileCheck %s -# CHECK: b.ne +# CHECK: b.ne 0xa1 0x00 0x00 0x54 # CHECK-NEXT: asr , , 0x00 0x7c 0x00 0x13 From a6a37ea3cbc116ae95fa121792854a13d8b2f14f Mon Sep 17 00:00:00 2001 From: Ivan Krysak Date: Thu, 6 Apr 2023 12:20:39 +0300 Subject: [PATCH 039/147] InstPrinter: introduce address tagging for ARM --- llvm/lib/Target/ARM/ARMInstrInfo.td | 4 ++ llvm/lib/Target/ARM/ARMInstrThumb.td | 6 ++ llvm/lib/Target/ARM/ARMInstrThumb2.td | 2 + .../ARM/MCTargetDesc/ARMInstPrinter.cpp | 57 +++++++++++++------ .../Target/ARM/MCTargetDesc/ARMInstPrinter.h | 6 ++ .../MC/Disassembler/ARM/marked-up-thumb.txt | 2 +- 6 files changed, 59 insertions(+), 18 deletions(-) diff --git a/llvm/lib/Target/ARM/ARMInstrInfo.td b/llvm/lib/Target/ARM/ARMInstrInfo.td index f5415c5b5895..15536b7c47f0 100644 --- a/llvm/lib/Target/ARM/ARMInstrInfo.td +++ b/llvm/lib/Target/ARM/ARMInstrInfo.td @@ -556,6 +556,7 @@ def brtarget : Operand { let EncoderMethod = "getBranchTargetOpValue"; let OperandType = "OPERAND_PCREL"; let DecoderMethod = "DecodeT2BROperand"; + let PrintMethod = "printPCRelativeOperand"; } // Branches targeting ARM-mode must be divisible by 4 if they're a raw @@ -574,6 +575,7 @@ def arm_br_target : Operand { let ParserMatchClass = ARMBranchTarget; let EncoderMethod = "getARMBranchTargetOpValue"; let OperandType = "OPERAND_PCREL"; + let PrintMethod = "printPCRelativeOperand"; } // Call target for ARM. Handles conditional/unconditional @@ -582,6 +584,7 @@ def arm_bl_target : Operand { let ParserMatchClass = ARMBranchTarget; let EncoderMethod = "getARMBLTargetOpValue"; let OperandType = "OPERAND_PCREL"; + let PrintMethod = "printPCRelativeOperand"; } // Target for BLX *from* ARM mode. @@ -589,6 +592,7 @@ def arm_blx_target : Operand { let ParserMatchClass = ThumbBranchTarget; let EncoderMethod = "getARMBLXTargetOpValue"; let OperandType = "OPERAND_PCREL"; + let PrintMethod = "printPCRelativeOperand"; } // A list of registers separated by comma. Used by load/store multiple. diff --git a/llvm/lib/Target/ARM/ARMInstrThumb.td b/llvm/lib/Target/ARM/ARMInstrThumb.td index 8f7039a327b3..9e7108c396a7 100644 --- a/llvm/lib/Target/ARM/ARMInstrThumb.td +++ b/llvm/lib/Target/ARM/ARMInstrThumb.td @@ -128,6 +128,7 @@ let OperandType = "OPERAND_PCREL" in { def t_brtarget : Operand { let EncoderMethod = "getThumbBRTargetOpValue"; let DecoderMethod = "DecodeThumbBROperand"; + let PrintMethod = "printPCRelativeOperand"; } // ADR instruction labels. @@ -142,12 +143,14 @@ def thumb_br_target : Operand { let ParserMatchClass = ThumbBranchTarget; let EncoderMethod = "getThumbBranchTargetOpValue"; let OperandType = "OPERAND_PCREL"; + let PrintMethod = "printPCRelativeOperand"; } def thumb_bl_target : Operand { let ParserMatchClass = ThumbBranchTarget; let EncoderMethod = "getThumbBLTargetOpValue"; let DecoderMethod = "DecodeThumbBLTargetOperand"; + let PrintMethod = "printPCRelativeOperand"; } // Target for BLX *from* thumb mode. @@ -155,18 +158,21 @@ def thumb_blx_target : Operand { let ParserMatchClass = ARMBranchTarget; let EncoderMethod = "getThumbBLXTargetOpValue"; let DecoderMethod = "DecodeThumbBLXOffset"; + let PrintMethod = "printPCRelativeOperand"; } def thumb_bcc_target : Operand { let ParserMatchClass = ThumbBranchTarget; let EncoderMethod = "getThumbBCCTargetOpValue"; let DecoderMethod = "DecodeThumbBCCTargetOperand"; + let PrintMethod = "printPCRelativeOperand"; } def thumb_cb_target : Operand { let ParserMatchClass = ThumbBranchTarget; let EncoderMethod = "getThumbCBTargetOpValue"; let DecoderMethod = "DecodeThumbCmpBROperand"; + let PrintMethod = "printPCRelativeOperand"; } } // OperandType = "OPERAND_PCREL" diff --git a/llvm/lib/Target/ARM/ARMInstrThumb2.td b/llvm/lib/Target/ARM/ARMInstrThumb2.td index 521cb0695384..74ac17f1ed38 100644 --- a/llvm/lib/Target/ARM/ARMInstrThumb2.td +++ b/llvm/lib/Target/ARM/ARMInstrThumb2.td @@ -376,6 +376,7 @@ class BFLabelOp"); + let PrintMethod = "printPCRelativeOperand"; } def bflabel_u4 : BFLabelOp<"false", "false", "false", "4", "ARM::fixup_bf_branch">; def bflabel_s12 : BFLabelOp<"true", "false", "true", "12", "ARM::fixup_bfc_target">; @@ -407,6 +408,7 @@ def bfafter_target : Operand { let EncoderMethod = "getBFAfterTargetOpValue"; let OperandType = "OPERAND_PCREL"; let DecoderMethod = "DecodeBFAfterTargetOperand"; + let PrintMethod = "printPCRelativeOperand"; } // pred operand excluding AL diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp index 421295e76118..da8e7f03abd2 100644 --- a/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp +++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp @@ -349,16 +349,23 @@ void ARMInstPrinter::printOperand(const MCInst *MI, unsigned OpNo, } } -void ARMInstPrinter::printOperand(const MCInst *MI, uint64_t Address, - unsigned OpNum, const MCSubtargetInfo &STI, - raw_ostream &O) { +void ARMInstPrinter::printPCRelativeOperand(const MCInst *MI, uint64_t Address, + unsigned OpNum, + const MCSubtargetInfo &STI, + raw_ostream &O) { const MCOperand &Op = MI->getOperand(OpNum); - if (!Op.isImm() || !PrintBranchImmAsAddress || getUseMarkup()) + if (!Op.isImm() || getUseMarkup()) return printOperand(MI, OpNum, STI, O); + + if (!PrintBranchImmAsAddress) { + O << markup(""); + return; + } + uint64_t Target = ARM_MC::evaluateBranchTarget(MII.get(MI->getOpcode()), Address, Op.getImm()); Target &= 0xffffffff; - O << formatHex(Target); + O << markup(""); if (CommentStream) *CommentStream << "imm = #" << formatImm(Op.getImm()) << '\n'; } @@ -381,9 +388,9 @@ void ARMInstPrinter::printThumbLdrLabelOperand(const MCInst *MI, unsigned OpNum, if (OffImm == INT32_MIN) OffImm = 0; if (isSub) { - O << markup(""); + O << markup(""); } else { - O << markup(""); + O << markup(""); } O << "]" << markup(">"); } @@ -1171,6 +1178,10 @@ void ARMInstPrinter::printT2SOOperand(const MCInst *MI, unsigned OpNum, ARM_AM::getSORegOffset(MO2.getImm()), UseMarkup); } +StringRef ARMInstPrinter::immediateOffsetMarkup(const MCOperand &Operand) { + return markup(Operand.getReg() == ARM::PC ? " void ARMInstPrinter::printAddrModeImm12Operand(const MCInst *MI, unsigned OpNum, const MCSubtargetInfo &STI, @@ -1191,11 +1202,15 @@ void ARMInstPrinter::printAddrModeImm12Operand(const MCInst *MI, unsigned OpNum, // Special value for #-0. All others are normal. if (OffImm == INT32_MIN) OffImm = 0; + if (isSub) { - O << ", " << markup(""); + O << ", " << immediateOffsetMarkup(MO1); + O << "#-" << formatImm(-OffImm) << markup(">"); } else if (AlwaysPrintImm0 || OffImm > 0) { - O << ", " << markup(""); + O << ", " << immediateOffsetMarkup(MO1); + O << "#" << formatImm(OffImm) << markup(">"); } + O << "]" << markup(">"); } @@ -1215,11 +1230,15 @@ void ARMInstPrinter::printT2AddrModeImm8Operand(const MCInst *MI, // Don't print +0. if (OffImm == INT32_MIN) OffImm = 0; + if (isSub) { - O << ", " << markup(""); + O << ", " << immediateOffsetMarkup(MO1); + O << "#-" << -OffImm << markup(">"); } else if (AlwaysPrintImm0 || OffImm > 0) { - O << ", " << markup(""); + O << ", " << immediateOffsetMarkup(MO1); + O << "#" << OffImm << markup(">"); } + O << "]" << markup(">"); } @@ -1247,11 +1266,15 @@ void ARMInstPrinter::printT2AddrModeImm8s4Operand(const MCInst *MI, // Don't print +0. if (OffImm == INT32_MIN) OffImm = 0; + if (isSub) { - O << ", " << markup(""); + O << ", " << immediateOffsetMarkup(MO1); + O << "#-" << -OffImm << markup(">"); } else if (AlwaysPrintImm0 || OffImm > 0) { - O << ", " << markup(""); + O << ", " << immediateOffsetMarkup(MO1); + O << "#" << OffImm << markup(">"); } + O << "]" << markup(">"); } @@ -1264,8 +1287,8 @@ void ARMInstPrinter::printT2AddrModeImm0_1020s4Operand( O << markup(""); + O << ", " << markup(MO1.getReg() == ARM::PC ? ""); } O << "]" << markup(">"); } @@ -1275,7 +1298,7 @@ void ARMInstPrinter::printT2AddrModeImm8OffsetOperand( raw_ostream &O) { const MCOperand &MO1 = MI->getOperand(OpNum); int32_t OffImm = (int32_t)MO1.getImm(); - O << ", " << markup(", ]> +# CHECK: ldr , , ]> 0x08 0x4c # CHECK: push {, , } 0x86 0xb4 From 1fb411122ce44342e5f14efb516567fac3dec8cd Mon Sep 17 00:00:00 2001 From: Ivan Krysak Date: Thu, 6 Apr 2023 16:49:23 +0300 Subject: [PATCH 040/147] InstPrinter: introduce address tagging for Mips --- llvm/lib/Target/Mips/MCTargetDesc/MipsInstPrinter.cpp | 8 ++++---- llvm/test/MC/Disassembler/Mips/marked-up.txt | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsInstPrinter.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsInstPrinter.cpp index 72590ab81a3e..5cbbc712fffd 100644 --- a/llvm/lib/Target/Mips/MCTargetDesc/MipsInstPrinter.cpp +++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsInstPrinter.cpp @@ -149,9 +149,9 @@ void MipsInstPrinter::printJumpOperand(const MCInst *MI, unsigned OpNo, return printOperand(MI, OpNo, STI, O); if (PrintBranchImmAsAddress) - O << markup(""); + O << markup(""); else - O << markup(""); + O << markup(""); } void MipsInstPrinter::printBranchOperand(const MCInst *MI, uint64_t Address, @@ -168,9 +168,9 @@ void MipsInstPrinter::printBranchOperand(const MCInst *MI, uint64_t Address, Target &= 0xffffffff; else if (STI.hasFeature(Mips::FeatureMips16)) Target &= 0xffff; - O << markup(""); + O << markup(""); } else { - O << markup(""); + O << markup(""); } } diff --git a/llvm/test/MC/Disassembler/Mips/marked-up.txt b/llvm/test/MC/Disassembler/Mips/marked-up.txt index ef805c0a1cde..94935276af7e 100644 --- a/llvm/test/MC/Disassembler/Mips/marked-up.txt +++ b/llvm/test/MC/Disassembler/Mips/marked-up.txt @@ -1,6 +1,6 @@ # RUN: llvm-mc --mdis %s -triple=mips-unknown-linux 2>&1 | FileCheck %s -# CHECK: j +# CHECK: j 0x09 0x33 0x00 0x2a # CHECK: addi , , 0x21 0x2d 0x66 0xd2 From f55424568cb6223c3d68b5adde285dd7607039e5 Mon Sep 17 00:00:00 2001 From: Ivan Krysak Date: Wed, 19 Apr 2023 12:21:35 +0300 Subject: [PATCH 041/147] InstPrinter: introduce address tagging for SystemZ --- llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinter.cpp | 2 +- llvm/test/MC/Disassembler/SystemZ/marked-up.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinter.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinter.cpp index 3e0e385b25c4..8c72675ed8e3 100644 --- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinter.cpp +++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinter.cpp @@ -165,7 +165,7 @@ void SystemZInstPrinter::printPCRelOperand(const MCInst *MI, int OpNum, raw_ostream &O) { const MCOperand &MO = MI->getOperand(OpNum); if (MO.isImm()) { - O << markup(""); } else diff --git a/llvm/test/MC/Disassembler/SystemZ/marked-up.txt b/llvm/test/MC/Disassembler/SystemZ/marked-up.txt index c75f4e60c829..f3413e3091a5 100644 --- a/llvm/test/MC/Disassembler/SystemZ/marked-up.txt +++ b/llvm/test/MC/Disassembler/SystemZ/marked-up.txt @@ -2,7 +2,7 @@ # CHECK: blr 0x07 0x4a -# CHECK: lrl , +# CHECK: lrl , 0xc4 0xfd 0x00 0x00 0x00 0x47 # CHECK: stc , 0x42 0x00 0x0f 0xff From 431b015da19c1c5e6f9fad76efd3e27dad3d823d Mon Sep 17 00:00:00 2001 From: Teresa Johnson Date: Thu, 16 Feb 2023 14:37:07 -0800 Subject: [PATCH 042/147] New SetOperations and unittesting for all SetOperations New set operations split out of D140908 as suggested, and I have added unit testing for all set operations. This adds a set_intersection, which returns the intersection instead of updating the first set like set_intersect (using a different name analogous to set_difference vs set_subtract). Also adds a variant of set_subtract that updates two additional set arguments to note which members of the subtrahend were removed from the minuend and which were not. Differential Revision: https://reviews.llvm.org/D144220 --- llvm/include/llvm/ADT/SetOperations.h | 32 +++ llvm/unittests/ADT/CMakeLists.txt | 1 + llvm/unittests/ADT/SetOperationsTest.cpp | 266 +++++++++++++++++++++++ 3 files changed, 299 insertions(+) create mode 100644 llvm/unittests/ADT/SetOperationsTest.cpp diff --git a/llvm/include/llvm/ADT/SetOperations.h b/llvm/include/llvm/ADT/SetOperations.h index c9462f077dc8..52aced706893 100644 --- a/llvm/include/llvm/ADT/SetOperations.h +++ b/llvm/include/llvm/ADT/SetOperations.h @@ -45,6 +45,25 @@ void set_intersect(S1Ty &S1, const S2Ty &S2) { } } +template +S1Ty set_intersection_impl(const S1Ty &S1, const S2Ty &S2) { + S1Ty Result; + for (typename S1Ty::const_iterator SI = S1.begin(), SE = S1.end(); SI != SE; + ++SI) + if (S2.count(*SI)) + Result.insert(*SI); + return Result; +} + +/// set_intersection(A, B) - Return A ^ B +template +S1Ty set_intersection(const S1Ty &S1, const S2Ty &S2) { + if (S1.size() < S2.size()) + return set_intersection_impl(S1, S2); + else + return set_intersection_impl(S2, S1); +} + /// set_difference(A, B) - Return A - B /// template @@ -66,6 +85,19 @@ void set_subtract(S1Ty &S1, const S2Ty &S2) { S1.erase(*SI); } +/// set_subtract(A, B, C, D) - Compute A := A - B, set C to the elements of B +/// removed from A (A ^ B), and D to the elements of B not found in and removed +/// from A (B - A). +template +void set_subtract(S1Ty &S1, const S2Ty &S2, S1Ty &Removed, S1Ty &Remaining) { + for (typename S2Ty::const_iterator SI = S2.begin(), SE = S2.end(); SI != SE; + ++SI) + if (S1.erase(*SI)) + Removed.insert(*SI); + else + Remaining.insert(*SI); +} + /// set_is_subset(A, B) - Return true iff A in B /// template diff --git a/llvm/unittests/ADT/CMakeLists.txt b/llvm/unittests/ADT/CMakeLists.txt index f20bc2eb7dcc..900294d4216e 100644 --- a/llvm/unittests/ADT/CMakeLists.txt +++ b/llvm/unittests/ADT/CMakeLists.txt @@ -62,6 +62,7 @@ add_llvm_unittest(ADTTests STLForwardCompatTest.cpp ScopeExitTest.cpp SequenceTest.cpp + SetOperationsTest.cpp SetVectorTest.cpp SimpleIListTest.cpp SmallPtrSetTest.cpp diff --git a/llvm/unittests/ADT/SetOperationsTest.cpp b/llvm/unittests/ADT/SetOperationsTest.cpp new file mode 100644 index 000000000000..982ea819fd80 --- /dev/null +++ b/llvm/unittests/ADT/SetOperationsTest.cpp @@ -0,0 +1,266 @@ +//===- SetOperations.cpp - Unit tests for set operations ------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/SetOperations.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include + +using namespace llvm; + +using testing::IsEmpty; + +namespace { + +TEST(SetOperationsTest, SetUnion) { + std::set Set1 = {1, 2, 3, 4}; + std::set Set2 = {5, 6, 7, 8}; + // Set1 should be the union of input sets Set1 and Set2. + std::set ExpectedSet1 = {1, 2, 3, 4, 5, 6, 7, 8}; + // Set2 should not be touched. + std::set ExpectedSet2 = Set2; + + set_union(Set1, Set2); + EXPECT_EQ(ExpectedSet1, Set1); + EXPECT_EQ(ExpectedSet2, Set2); + + Set1.clear(); + Set2 = {1, 2}; + // Set1 should be the union of input sets Set1 and Set2, which in this case + // will be Set2. + ExpectedSet1 = Set2; + // Set2 should not be touched. + ExpectedSet2 = Set2; + + set_union(Set1, Set2); + EXPECT_EQ(ExpectedSet1, Set1); + EXPECT_EQ(ExpectedSet2, Set2); +} + +TEST(SetOperationsTest, SetIntersect) { + std::set Set1 = {1, 2, 3, 4}; + std::set Set2 = {3, 4, 5, 6}; + // Set1 should be the intersection of sets Set1 and Set2. + std::set ExpectedSet1 = {3, 4}; + // Set2 should not be touched. + std::set ExpectedSet2 = Set2; + + set_intersect(Set1, Set2); + EXPECT_EQ(ExpectedSet1, Set1); + EXPECT_EQ(ExpectedSet2, Set2); + + Set1 = {1, 2, 3, 4}; + Set2 = {5, 6}; + // Set2 should not be touched. + ExpectedSet2 = Set2; + + set_intersect(Set1, Set2); + // Set1 should be the intersection of sets Set1 and Set2, which + // is empty as they are non-overlapping. + EXPECT_THAT(Set1, IsEmpty()); + EXPECT_EQ(ExpectedSet2, Set2); +} + +TEST(SetOperationsTest, SetIntersection) { + std::set Set1 = {1, 2, 3, 4}; + std::set Set2 = {3, 4, 5, 6}; + std::set Result; + // Result should be the intersection of sets Set1 and Set2. + std::set ExpectedResult = {3, 4}; + // Set1 and Set2 should not be touched. + std::set ExpectedSet1 = Set1; + std::set ExpectedSet2 = Set2; + + Result = set_intersection(Set1, Set2); + EXPECT_EQ(ExpectedResult, Result); + EXPECT_EQ(ExpectedSet1, Set1); + EXPECT_EQ(ExpectedSet2, Set2); + + Set1 = {1, 2, 3, 4}; + Set2 = {5, 6}; + // Set1 and Set2 should not be touched. + ExpectedSet1 = Set1; + ExpectedSet2 = Set2; + + Result = set_intersection(Set1, Set2); + // Result should be the intersection of sets Set1 and Set2, which + // is empty as they are non-overlapping. + EXPECT_THAT(Result, IsEmpty()); + EXPECT_EQ(ExpectedSet1, Set1); + EXPECT_EQ(ExpectedSet2, Set2); + + Set1 = {5, 6}; + Set2 = {1, 2, 3, 4}; + // Set1 and Set2 should not be touched. + ExpectedSet1 = Set1; + ExpectedSet2 = Set2; + + Result = set_intersection(Set1, Set2); + // Result should be the intersection of sets Set1 and Set2, which + // is empty as they are non-overlapping. Test this again with the input sets + // reversed, since the code takes a different path depending on which input + // set is smaller. + EXPECT_THAT(Result, IsEmpty()); + EXPECT_EQ(ExpectedSet1, Set1); + EXPECT_EQ(ExpectedSet2, Set2); +} + +TEST(SetOperationsTest, SetDifference) { + std::set Set1 = {1, 2, 3, 4}; + std::set Set2 = {3, 4, 5, 6}; + std::set Result; + // Result should be Set1 - Set2, leaving only {1, 2}. + std::set ExpectedResult = {1, 2}; + // Set1 and Set2 should not be touched. + std::set ExpectedSet1 = Set1; + std::set ExpectedSet2 = Set2; + + Result = set_difference(Set1, Set2); + EXPECT_EQ(ExpectedResult, Result); + EXPECT_EQ(ExpectedSet1, Set1); + EXPECT_EQ(ExpectedSet2, Set2); + + Set1 = {1, 2, 3, 4}; + Set2 = {1, 2, 3, 4}; + // Set1 and Set2 should not be touched. + ExpectedSet1 = Set1; + ExpectedSet2 = Set2; + + Result = set_difference(Set1, Set2); + // Result should be Set1 - Set2, which should be empty. + EXPECT_THAT(Result, IsEmpty()); + EXPECT_EQ(ExpectedSet1, Set1); + EXPECT_EQ(ExpectedSet2, Set2); + + Set1 = {1, 2, 3, 4}; + Set2 = {5, 6}; + // Result should be Set1 - Set2, which should be Set1 as they are + // non-overlapping. + ExpectedResult = Set1; + // Set1 and Set2 should not be touched. + ExpectedSet1 = Set1; + ExpectedSet2 = Set2; + + Result = set_difference(Set1, Set2); + EXPECT_EQ(ExpectedResult, Result); + EXPECT_EQ(ExpectedSet1, Set1); + EXPECT_EQ(ExpectedSet2, Set2); +} + +TEST(SetOperationsTest, SetSubtract) { + std::set Set1 = {1, 2, 3, 4}; + std::set Set2 = {3, 4, 5, 6}; + // Set1 should get Set1 - Set2, leaving only {1, 2}. + std::set ExpectedSet1 = {1, 2}; + // Set2 should not be touched. + std::set ExpectedSet2 = Set2; + + set_subtract(Set1, Set2); + EXPECT_EQ(ExpectedSet1, Set1); + EXPECT_EQ(ExpectedSet2, Set2); + + Set1 = {1, 2, 3, 4}; + Set2 = {1, 2, 3, 4}; + // Set2 should not be touched. + ExpectedSet2 = Set2; + + set_subtract(Set1, Set2); + // Set1 should get Set1 - Set2, which should be empty. + EXPECT_THAT(Set1, IsEmpty()); + EXPECT_EQ(ExpectedSet2, Set2); + + Set1 = {1, 2, 3, 4}; + Set2 = {5, 6}; + // Set1 should get Set1 - Set2, which should be Set1 as they are + // non-overlapping. + ExpectedSet1 = Set1; + // Set2 should not be touched. + ExpectedSet2 = Set2; + + set_subtract(Set1, Set2); + EXPECT_EQ(ExpectedSet1, Set1); + EXPECT_EQ(ExpectedSet2, Set2); +} + +TEST(SetOperationsTest, SetSubtractRemovedRemaining) { + std::set Removed, Remaining; + + std::set Set1 = {1, 2, 3, 4}; + std::set Set2 = {3, 4, 5, 6}; + // Set1 should get Set1 - Set2, leaving only {1, 2}. + std::set ExpectedSet1 = {1, 2}; + // Set2 should not be touched. + std::set ExpectedSet2 = Set2; + // We should get back that {3, 4} from Set2 were removed from Set1, and {5, 6} + // were not removed from Set1. + std::set ExpectedRemoved = {3, 4}; + std::set ExpectedRemaining = {5, 6}; + + set_subtract(Set1, Set2, Removed, Remaining); + EXPECT_EQ(ExpectedSet1, Set1); + EXPECT_EQ(ExpectedSet2, Set2); + EXPECT_EQ(ExpectedRemoved, Removed); + EXPECT_EQ(ExpectedRemaining, Remaining); + + Set1 = {1, 2, 3, 4}; + Set2 = {1, 2, 3, 4}; + Removed.clear(); + Remaining.clear(); + // Set2 should not be touched. + ExpectedSet2 = Set2; + // Set should get back that all of Set2 was removed from Set1, and nothing + // left in Set2 was not removed from Set1. + ExpectedRemoved = Set2; + + set_subtract(Set1, Set2, Removed, Remaining); + // Set1 should get Set1 - Set2, which should be empty. + EXPECT_THAT(Set1, IsEmpty()); + EXPECT_EQ(ExpectedSet2, Set2); + EXPECT_EQ(ExpectedRemoved, Removed); + EXPECT_THAT(Remaining, IsEmpty()); + + Set1 = {1, 2, 3, 4}; + Set2 = {5, 6}; + Removed.clear(); + Remaining.clear(); + // Set1 should get Set1 - Set2, which should be Set1 as they are + // non-overlapping. + ExpectedSet1 = {1, 2, 3, 4}; + // Set2 should not be touched. + ExpectedSet2 = Set2; + // Set should get back that none of Set2 was removed from Set1, and all + // of Set2 was not removed from Set1. + ExpectedRemaining = Set2; + + set_subtract(Set1, Set2, Removed, Remaining); + EXPECT_EQ(ExpectedSet1, Set1); + EXPECT_EQ(ExpectedSet2, Set2); + EXPECT_THAT(Removed, IsEmpty()); + EXPECT_EQ(ExpectedRemaining, Remaining); +} + +TEST(SetOperationsTest, SetIsSubset) { + std::set Set1 = {1, 2, 3, 4}; + std::set Set2 = {3, 4}; + EXPECT_FALSE(set_is_subset(Set1, Set2)); + + Set1 = {1, 2, 3, 4}; + Set2 = {1, 2, 3, 4}; + EXPECT_TRUE(set_is_subset(Set1, Set2)); + + Set1 = {1, 2}; + Set2 = {1, 2, 3, 4}; + EXPECT_TRUE(set_is_subset(Set1, Set2)); + + Set1 = {1, 2}; + Set2 = {3, 4}; + EXPECT_FALSE(set_is_subset(Set1, Set2)); +} + +} // namespace From 210d7c137179063e07599a5143fe04681b67c126 Mon Sep 17 00:00:00 2001 From: Adrian Vogelsgesang Date: Thu, 4 May 2023 12:20:29 +0000 Subject: [PATCH 043/147] [ADT][NFC] Fix compilation of headers under C++23 `DoubleAPFloat` has a `unique_ptr` member. In `DoubleAPFloat::operator=` and `DoubleAPFloat::get{First,Second}`, the methods of this unique_ptr are getting instantiated. At that point `APFloat` is still only a forward declaration. This triggers undefined behavior. So far, we were probaly just lucky and the code compiled fine. However, with C++23 `std::unique_ptr` became constexpr, and clang (and other compilers) are now diagnosing this latent bug as an error. This commit fixes the issue by moving the function definitions out of the class definition of `DoubleAPFloat`, after the declaration of `APFloat`. A similar issue exists in `ModuleSummaryIndex.h`, the fix is pretty much identical. Fixes #59784 Differential Revision: https://reviews.llvm.org/D149854 --- llvm/include/llvm/ADT/APFloat.h | 38 ++++++++++++++++------- llvm/include/llvm/IR/ModuleSummaryIndex.h | 4 ++- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/llvm/include/llvm/ADT/APFloat.h b/llvm/include/llvm/ADT/APFloat.h index c0e2d13c2939..b86a03c42433 100644 --- a/llvm/include/llvm/ADT/APFloat.h +++ b/llvm/include/llvm/ADT/APFloat.h @@ -640,21 +640,14 @@ class DoubleAPFloat final : public APFloatBase { DoubleAPFloat(DoubleAPFloat &&RHS); DoubleAPFloat &operator=(const DoubleAPFloat &RHS); - - DoubleAPFloat &operator=(DoubleAPFloat &&RHS) { - if (this != &RHS) { - this->~DoubleAPFloat(); - new (this) DoubleAPFloat(std::move(RHS)); - } - return *this; - } + inline DoubleAPFloat &operator=(DoubleAPFloat &&RHS); bool needsCleanup() const { return Floats != nullptr; } - APFloat &getFirst() { return Floats[0]; } - const APFloat &getFirst() const { return Floats[0]; } - APFloat &getSecond() { return Floats[1]; } - const APFloat &getSecond() const { return Floats[1]; } + inline APFloat &getFirst(); + inline const APFloat &getFirst() const; + inline APFloat &getSecond(); + inline const APFloat &getSecond() const; opStatus add(const DoubleAPFloat &RHS, roundingMode RM); opStatus subtract(const DoubleAPFloat &RHS, roundingMode RM); @@ -1358,6 +1351,27 @@ inline APFloat maximum(const APFloat &A, const APFloat &B) { return A < B ? B : A; } +// We want the following functions to be available in the header for inlining. +// We cannot define them inline in the class definition of `DoubleAPFloat` +// because doing so would instantiate `std::unique_ptr` before +// `APFloat` is defined, and that would be undefined behavior. +namespace detail { + +DoubleAPFloat &DoubleAPFloat::operator=(DoubleAPFloat &&RHS) { + if (this != &RHS) { + this->~DoubleAPFloat(); + new (this) DoubleAPFloat(std::move(RHS)); + } + return *this; +} + +APFloat &DoubleAPFloat::getFirst() { return Floats[0]; } +const APFloat &DoubleAPFloat::getFirst() const { return Floats[0]; } +APFloat &DoubleAPFloat::getSecond() { return Floats[1]; } +const APFloat &DoubleAPFloat::getSecond() const { return Floats[1]; } + +} // namespace detail + } // namespace llvm #undef APFLOAT_DISPATCH_ON_SEMANTICS diff --git a/llvm/include/llvm/IR/ModuleSummaryIndex.h b/llvm/include/llvm/IR/ModuleSummaryIndex.h index e5236523a522..4e249b16eeb6 100644 --- a/llvm/include/llvm/IR/ModuleSummaryIndex.h +++ b/llvm/include/llvm/IR/ModuleSummaryIndex.h @@ -147,7 +147,7 @@ struct alignas(8) GlobalValueSummaryInfo { StringRef Name; } U; - GlobalValueSummaryInfo(bool HaveGVs) : U(HaveGVs) {} + inline GlobalValueSummaryInfo(bool HaveGVs); /// List of global value summary structures for a particular value held /// in the GlobalValueMap. Requires a vector in the case of multiple @@ -519,6 +519,8 @@ class GlobalValueSummary { friend class ModuleSummaryIndex; }; +GlobalValueSummaryInfo::GlobalValueSummaryInfo(bool HaveGVs) : U(HaveGVs) {} + /// Alias summary information. class AliasSummary : public GlobalValueSummary { ValueInfo AliaseeValueInfo; From 5b57095140571eb2cedb278432193aa4d423a26a Mon Sep 17 00:00:00 2001 From: Ivan Krysak Date: Tue, 16 May 2023 12:12:29 +0300 Subject: [PATCH 044/147] [llvm] Fix compilation of headers under C++23 Thanks to stricter `std::unique_ptr` checks some of the headers do not compile in c++23 mode. This can be easily solved by making sure all the virtual interfaces used with `std::unique_ptr` are defined before they are used as opposed to only being declared. --- llvm/include/llvm/Analysis/AliasAnalysis.h | 136 +++++++++--------- .../llvm/DebugInfo/PDB/Native/InputFile.h | 2 +- llvm/include/llvm/Support/ScopedPrinter.h | 16 +-- 3 files changed, 75 insertions(+), 79 deletions(-) diff --git a/llvm/include/llvm/Analysis/AliasAnalysis.h b/llvm/include/llvm/Analysis/AliasAnalysis.h index 953e15e358f1..1a8a87d9e360 100644 --- a/llvm/include/llvm/Analysis/AliasAnalysis.h +++ b/llvm/include/llvm/Analysis/AliasAnalysis.h @@ -588,7 +588,73 @@ class AAResults { MemoryEffects getMemoryEffects(const CallBase *Call, AAQueryInfo &AAQI); private: - class Concept; + /// A private abstract base class describing the concept of an individual alias + /// analysis implementation. + /// + /// This interface is implemented by any \c Model instantiation. It is also the + /// interface which a type used to instantiate the model must provide. + /// + /// All of these methods model methods by the same name in the \c + /// AAResults class. Only differences and specifics to how the + /// implementations are called are documented here. + class Concept { + public: + virtual ~Concept() = 0; + + //===--------------------------------------------------------------------===// + /// \name Alias Queries + /// @{ + + /// The main low level interface to the alias analysis implementation. + /// Returns an AliasResult indicating whether the two pointers are aliased to + /// each other. This is the interface that must be implemented by specific + /// alias analysis implementations. + virtual AliasResult alias(const MemoryLocation &LocA, + const MemoryLocation &LocB, AAQueryInfo &AAQI, + const Instruction *CtxI) = 0; + + /// @} + //===--------------------------------------------------------------------===// + /// \name Simple mod/ref information + /// @{ + + /// Returns a bitmask that should be unconditionally applied to the ModRef + /// info of a memory location. This allows us to eliminate Mod and/or Ref from + /// the ModRef info based on the knowledge that the memory location points to + /// constant and/or locally-invariant memory. + virtual ModRefInfo getModRefInfoMask(const MemoryLocation &Loc, + AAQueryInfo &AAQI, + bool IgnoreLocals) = 0; + + /// Get the ModRef info associated with a pointer argument of a callsite. The + /// result's bits are set to indicate the allowed aliasing ModRef kinds. Note + /// that these bits do not necessarily account for the overall behavior of + /// the function, but rather only provide additional per-argument + /// information. + virtual ModRefInfo getArgModRefInfo(const CallBase *Call, + unsigned ArgIdx) = 0; + + /// Return the behavior of the given call site. + virtual MemoryEffects getMemoryEffects(const CallBase *Call, + AAQueryInfo &AAQI) = 0; + + /// Return the behavior when calling the given function. + virtual MemoryEffects getMemoryEffects(const Function *F) = 0; + + /// getModRefInfo (for call sites) - Return information about whether + /// a particular call site modifies or reads the specified memory location. + virtual ModRefInfo getModRefInfo(const CallBase *Call, + const MemoryLocation &Loc, + AAQueryInfo &AAQI) = 0; + + /// Return information about whether two call sites may refer to the same set + /// of memory locations. See the AA documentation for details: + /// http://llvm.org/docs/AliasAnalysis.html#ModRefInfo + virtual ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2, + AAQueryInfo &AAQI) = 0; + + /// @} + }; template class Model; @@ -665,74 +731,6 @@ class BatchAAResults { /// pointer or reference. using AliasAnalysis = AAResults; -/// A private abstract base class describing the concept of an individual alias -/// analysis implementation. -/// -/// This interface is implemented by any \c Model instantiation. It is also the -/// interface which a type used to instantiate the model must provide. -/// -/// All of these methods model methods by the same name in the \c -/// AAResults class. Only differences and specifics to how the -/// implementations are called are documented here. -class AAResults::Concept { -public: - virtual ~Concept() = 0; - - //===--------------------------------------------------------------------===// - /// \name Alias Queries - /// @{ - - /// The main low level interface to the alias analysis implementation. - /// Returns an AliasResult indicating whether the two pointers are aliased to - /// each other. This is the interface that must be implemented by specific - /// alias analysis implementations. - virtual AliasResult alias(const MemoryLocation &LocA, - const MemoryLocation &LocB, AAQueryInfo &AAQI, - const Instruction *CtxI) = 0; - - /// @} - //===--------------------------------------------------------------------===// - /// \name Simple mod/ref information - /// @{ - - /// Returns a bitmask that should be unconditionally applied to the ModRef - /// info of a memory location. This allows us to eliminate Mod and/or Ref from - /// the ModRef info based on the knowledge that the memory location points to - /// constant and/or locally-invariant memory. - virtual ModRefInfo getModRefInfoMask(const MemoryLocation &Loc, - AAQueryInfo &AAQI, - bool IgnoreLocals) = 0; - - /// Get the ModRef info associated with a pointer argument of a callsite. The - /// result's bits are set to indicate the allowed aliasing ModRef kinds. Note - /// that these bits do not necessarily account for the overall behavior of - /// the function, but rather only provide additional per-argument - /// information. - virtual ModRefInfo getArgModRefInfo(const CallBase *Call, - unsigned ArgIdx) = 0; - - /// Return the behavior of the given call site. - virtual MemoryEffects getMemoryEffects(const CallBase *Call, - AAQueryInfo &AAQI) = 0; - - /// Return the behavior when calling the given function. - virtual MemoryEffects getMemoryEffects(const Function *F) = 0; - - /// getModRefInfo (for call sites) - Return information about whether - /// a particular call site modifies or reads the specified memory location. - virtual ModRefInfo getModRefInfo(const CallBase *Call, - const MemoryLocation &Loc, - AAQueryInfo &AAQI) = 0; - - /// Return information about whether two call sites may refer to the same set - /// of memory locations. See the AA documentation for details: - /// http://llvm.org/docs/AliasAnalysis.html#ModRefInfo - virtual ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2, - AAQueryInfo &AAQI) = 0; - - /// @} -}; - /// A private class template which derives from \c Concept and wraps some other /// type. /// diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/InputFile.h b/llvm/include/llvm/DebugInfo/PDB/Native/InputFile.h index 834cd96b77b4..582e613c30a2 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/InputFile.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/InputFile.h @@ -16,6 +16,7 @@ #include "llvm/DebugInfo/CodeView/StringsAndChecksums.h" #include "llvm/DebugInfo/PDB/Native/LinePrinter.h" #include "llvm/DebugInfo/PDB/Native/ModuleDebugStream.h" +#include "llvm/DebugInfo/PDB/Native/NativeSession.h" #include "llvm/Object/Binary.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/Error.h" @@ -32,7 +33,6 @@ namespace pdb { class InputFile; class LinePrinter; class PDBFile; -class NativeSession; class SymbolGroupIterator; class SymbolGroup; diff --git a/llvm/include/llvm/Support/ScopedPrinter.h b/llvm/include/llvm/Support/ScopedPrinter.h index 613744ee68cb..d21fc5ca7737 100644 --- a/llvm/include/llvm/Support/ScopedPrinter.h +++ b/llvm/include/llvm/Support/ScopedPrinter.h @@ -519,7 +519,13 @@ ScopedPrinter::printHex(StringRef Label, startLine() << Label << ": " << hex(Value) << "\n"; } -struct DelimitedScope; +struct DelimitedScope { + DelimitedScope(ScopedPrinter &W) : W(&W) {} + DelimitedScope() : W(nullptr) {} + virtual ~DelimitedScope() = default; + virtual void setPrinter(ScopedPrinter &W) = 0; + ScopedPrinter *W; +}; class JSONScopedPrinter : public ScopedPrinter { private: @@ -798,14 +804,6 @@ class JSONScopedPrinter : public ScopedPrinter { } }; -struct DelimitedScope { - DelimitedScope(ScopedPrinter &W) : W(&W) {} - DelimitedScope() : W(nullptr) {} - virtual ~DelimitedScope() = default; - virtual void setPrinter(ScopedPrinter &W) = 0; - ScopedPrinter *W; -}; - struct DictScope : DelimitedScope { explicit DictScope() = default; explicit DictScope(ScopedPrinter &W) : DelimitedScope(W) { W.objectBegin(); } From 36f49989b61c75307482744e64f3b76872c3134d Mon Sep 17 00:00:00 2001 From: Andrea Gussoni Date: Tue, 16 May 2023 16:30:24 +0200 Subject: [PATCH 045/147] MLIR: explicit instantiation for `DomTreeBuilder` --- mlir/lib/IR/Dominance.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mlir/lib/IR/Dominance.cpp b/mlir/lib/IR/Dominance.cpp index d44969cf4cb6..aabf9093892b 100644 --- a/mlir/lib/IR/Dominance.cpp +++ b/mlir/lib/IR/Dominance.cpp @@ -24,6 +24,13 @@ template class llvm::DominatorTreeOnView; template class llvm::DomTreeNodeOnView; +// Explicit instantiation for the `llvm::DomTreeBuilder::DeleteEdge` method, +template void llvm::DomTreeBuilder::DeleteEdge( + DomTreeOnView &DT, Block *From, Block *To); + +template void llvm::DomTreeBuilder::DeleteEdge( + PostDomTreeOnView &DT, Block *From, Block *To); + //===----------------------------------------------------------------------===// // DominanceInfoBase //===----------------------------------------------------------------------===// From 1a5116c62ef05d9b9d2c94f30e608176f288f413 Mon Sep 17 00:00:00 2001 From: Andrea Gussoni Date: Wed, 1 Mar 2023 14:10:45 +0100 Subject: [PATCH 046/147] ViewOpGraph: introduce CFG edges printing Previously, the `ViewOpGraph` pass could only print edges representing the dataflow, and the control-flow between `Operation`s contained in the same `Block` (even though this control-flow is very peculiar, since it is just a straight line connecting all the `Operation`s in a sequential manner. In this commit, we implement `Region` control-flow, i.e., control flow connecting the various `Block`s in a `Region`, connecting the `Block`s with their successors. It's use is enabled with the `print-region-control-flow-edges` option. Since the `ViewOpGraph` pass uses DOT clusters for representing `Block`s, and edges are not allowed between clusters, we use the first and last `Operation`s in a `Block`, to represent control-flow between `Blocks`. --- mlir/include/mlir/Transforms/Passes.td | 4 ++- mlir/lib/Transforms/ViewOpGraph.cpp | 38 +++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/mlir/include/mlir/Transforms/Passes.td b/mlir/include/mlir/Transforms/Passes.td index e2a15934e3cf..335ae7e2519b 100644 --- a/mlir/include/mlir/Transforms/Passes.td +++ b/mlir/include/mlir/Transforms/Passes.td @@ -277,7 +277,9 @@ def ViewOpGraph : Pass<"view-op-graph"> { Option<"printDataFlowEdges", "print-data-flow-edges", "bool", /*default=*/"true", "Print data flow edges">, Option<"printResultTypes", "print-result-types", "bool", - /*default=*/"true", "Print result types of operations"> + /*default=*/"true", "Print result types of operations">, + Option<"printRegionControlFlowEdges", "print-region-control-flow-edges", + "bool", /*default=*/"false", "Print region control flow edges"> ]; let constructor = "mlir::createPrintOpGraphPass()"; } diff --git a/mlir/lib/Transforms/ViewOpGraph.cpp b/mlir/lib/Transforms/ViewOpGraph.cpp index 186118b2e067..bf7ae4926f06 100644 --- a/mlir/lib/Transforms/ViewOpGraph.cpp +++ b/mlir/lib/Transforms/ViewOpGraph.cpp @@ -16,6 +16,7 @@ #include "llvm/ADT/StringMap.h" #include "llvm/Support/Format.h" #include "llvm/Support/GraphWriter.h" +#include "llvm/Support/raw_ostream.h" #include #include @@ -28,6 +29,7 @@ using namespace mlir; static const StringRef kLineStyleControlFlow = "dashed"; static const StringRef kLineStyleDataFlow = "solid"; +static const StringRef kLineStyleRegionControlFlow = "bold"; static const StringRef kShapeNode = "ellipse"; static const StringRef kShapeNone = "plain"; @@ -82,6 +84,10 @@ struct Node { /// Note: See https://www.graphviz.org/doc/info/lang.html for more information /// about the Graphviz DOT language. class PrintOpPass : public impl::ViewOpGraphBase { +private: + llvm::DenseMap blockFirstNodeMap; + llvm::DenseMap blockLastNodeMap; + public: PrintOpPass(raw_ostream &os) : os(os) {} PrintOpPass(const PrintOpPass &o) : PrintOpPass(o.os.getOStream()) {} @@ -250,6 +256,11 @@ class PrintOpPass : public impl::ViewOpGraphBase { /// Process a block. Emit a cluster and one node per block argument and /// operation inside the cluster. void processBlock(Block &block) { + + // Prepare the name for the block node + std::string nodeName; + llvm::raw_string_ostream blockNameStr(nodeName); + block.printAsOperand(blockNameStr); emitClusterStmt([&]() { for (BlockArgument &blockArg : block.getArguments()) valueToNode[blockArg] = emitNodeStmt(getLabel(blockArg)); @@ -261,9 +272,18 @@ class PrintOpPass : public impl::ViewOpGraphBase { if (printControlFlowEdges && prevNode) emitEdgeStmt(*prevNode, nextNode, /*label=*/"", kLineStyleControlFlow); + + // If we are at first iteration, save the first operand node for the + // incoming edges. + if (not prevNode) { + blockFirstNodeMap[&block] = nextNode; + } prevNode = nextNode; } - }); + + // Save the last operation for the outgoing edges. + blockLastNodeMap[&block] = *prevNode; + }, nodeName); } /// Process an operation. If the operation has regions, emit a cluster. @@ -301,6 +321,22 @@ class PrintOpPass : public impl::ViewOpGraphBase { void processRegion(Region ®ion) { for (Block &block : region.getBlocks()) processBlock(block); + + // Print control flow edges between blocks if the option is activated. + if (printRegionControlFlowEdges) { + for (Block &block : region.getBlocks()) { + unsigned numSuccessors = block.getNumSuccessors(); + for (unsigned i = 0; i < numSuccessors; i++) { + Block *successor = block.getSuccessor(i); + assert(blockLastNodeMap.count(&block) == 1); + Node blockNode = blockLastNodeMap[&block]; + assert(blockFirstNodeMap.count(successor) == 1); + Node successorNode = blockFirstNodeMap[successor]; + emitEdgeStmt(blockNode, successorNode, + /*label=*/numSuccessors == 1 ? "" : std::to_string(i), kLineStyleRegionControlFlow); + } + } + } } /// Truncate long strings. From 0a9750c11f55263bd322091b32ec4056b7fb77ab Mon Sep 17 00:00:00 2001 From: Andrea Gussoni Date: Mon, 17 Apr 2023 11:38:38 +0200 Subject: [PATCH 047/147] ViewOpGraph: option to print first and last ops Introduce the `only-entry-exit-ops` option, used to limit the serialization of DOT graphs by the `ViewOpGraph` pass to print only the first and last `Operation` for each `Block`. This option is useful to limit the dimension of the output graphs, facilitating its understanding in some situations. --- mlir/include/mlir/Transforms/Passes.td | 4 +- mlir/lib/Transforms/ViewOpGraph.cpp | 54 +++++++++++++++++++------- 2 files changed, 44 insertions(+), 14 deletions(-) diff --git a/mlir/include/mlir/Transforms/Passes.td b/mlir/include/mlir/Transforms/Passes.td index 335ae7e2519b..f52cbfc482c5 100644 --- a/mlir/include/mlir/Transforms/Passes.td +++ b/mlir/include/mlir/Transforms/Passes.td @@ -279,7 +279,9 @@ def ViewOpGraph : Pass<"view-op-graph"> { Option<"printResultTypes", "print-result-types", "bool", /*default=*/"true", "Print result types of operations">, Option<"printRegionControlFlowEdges", "print-region-control-flow-edges", - "bool", /*default=*/"false", "Print region control flow edges"> + "bool", /*default=*/"false", "Print region control flow edges">, + Option<"onlyEntryAndExitOperations", "only-entry-exit-ops", + "bool", /*default=*/"false", "Print only entry and exit operations"> ]; let constructor = "mlir::createPrintOpGraphPass()"; } diff --git a/mlir/lib/Transforms/ViewOpGraph.cpp b/mlir/lib/Transforms/ViewOpGraph.cpp index bf7ae4926f06..85e7c87b4c7b 100644 --- a/mlir/lib/Transforms/ViewOpGraph.cpp +++ b/mlir/lib/Transforms/ViewOpGraph.cpp @@ -9,6 +9,7 @@ #include "mlir/Transforms/ViewOpGraph.h" #include "mlir/IR/Block.h" +#include "mlir/IR/BuiltinOps.h" #include "mlir/IR/BuiltinTypes.h" #include "mlir/IR/Operation.h" #include "mlir/Pass/Pass.h" @@ -257,6 +258,12 @@ class PrintOpPass : public impl::ViewOpGraphBase { /// operation inside the cluster. void processBlock(Block &block) { + // If we are processing the `Block` containing the functions, we should not + // skip all the intermediate operations (the `LLVMFuncOp` themself). We can + // check this by disabling the feature if the parent `Operation` of the + // current block is the `Module` itself. + bool IsModuleBlock = llvm::isa(block.getParentOp()); + // Prepare the name for the block node std::string nodeName; llvm::raw_string_ostream blockNameStr(nodeName); @@ -267,22 +274,43 @@ class PrintOpPass : public impl::ViewOpGraphBase { // Emit a node for each operation. std::optional prevNode; - for (Operation &op : block) { - Node nextNode = processOperation(&op); - if (printControlFlowEdges && prevNode) - emitEdgeStmt(*prevNode, nextNode, /*label=*/"", - kLineStyleControlFlow); - // If we are at first iteration, save the first operand node for the - // incoming edges. - if (not prevNode) { - blockFirstNodeMap[&block] = nextNode; + // Separately handle the `onlyEntryAndExitOperations` option, by only + // printing the first and last operation in a `mlir::Block`. + if (!IsModuleBlock and onlyEntryAndExitOperations + and block.getOperations().size() > 2) { + Operation &firstOp = block.front(); + Node firstNode = processOperation(&firstOp); + blockFirstNodeMap[&block] = firstNode; + + Operation &lastOp = block.back(); + Node lastNode = processOperation(&lastOp); + blockLastNodeMap[&block] = lastNode; + + if (printControlFlowEdges) { + emitEdgeStmt(firstNode, lastNode, /*label=*/"", + kLineStyleControlFlow); + } + } else { + + // In all the other cases, we loop over all the operations. + for (Operation &op : block) { + Node nextNode = processOperation(&op); + if (printControlFlowEdges && prevNode) + emitEdgeStmt(*prevNode, nextNode, /*label=*/"", + kLineStyleControlFlow); + + // If we are at first iteration, save the first operand node for the + // incoming edges. + if (not prevNode) { + blockFirstNodeMap[&block] = nextNode; + } + prevNode = nextNode; } - prevNode = nextNode; - } - // Save the last operation for the outgoing edges. - blockLastNodeMap[&block] = *prevNode; + // Save the last operation for the outgoing edges. + blockLastNodeMap[&block] = *prevNode; + } }, nodeName); } From ca5212db1722f44a725820397d1b5b16a0d45866 Mon Sep 17 00:00:00 2001 From: Andrea Gussoni Date: Wed, 19 Jul 2023 12:04:48 +0200 Subject: [PATCH 048/147] SetVector: implement `takeSet` method Implement the `takeSet` method, specular to the `takeVector` one, in order to extract the inner `DenseSet` from the `SetVector` container. --- llvm/include/llvm/ADT/SetVector.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/llvm/include/llvm/ADT/SetVector.h b/llvm/include/llvm/ADT/SetVector.h index 37509e28f891..5aa8771044d1 100644 --- a/llvm/include/llvm/ADT/SetVector.h +++ b/llvm/include/llvm/ADT/SetVector.h @@ -68,6 +68,12 @@ class SetVector { return std::move(vector_); } + /// Clear the SetVector and return the underlying set. + Set takeSet() { + vector_.clear(); + return std::move(set_); + } + /// Determine if the SetVector is empty or not. bool empty() const { return vector_.empty(); From 92fa38a4cd2d005de5b52e1a8c5eb9dc7c771de8 Mon Sep 17 00:00:00 2001 From: Alessandro Di Federico Date: Tue, 1 Aug 2023 10:30:13 +0200 Subject: [PATCH 049/147] mlir: do not force C++17 --- mlir/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/mlir/CMakeLists.txt b/mlir/CMakeLists.txt index c9b0d53bc3e9..ab690b16a4fb 100644 --- a/mlir/CMakeLists.txt +++ b/mlir/CMakeLists.txt @@ -22,7 +22,6 @@ endif() # Must go below project(..) include(GNUInstallDirs) -set(CMAKE_CXX_STANDARD 17) if(MLIR_STANDALONE_BUILD) find_package(LLVM CONFIG REQUIRED) From 5fd1326d07957240e338e6d4c18e4fedc0f75c49 Mon Sep 17 00:00:00 2001 From: Alessandro Di Federico Date: Sun, 15 Jan 2023 15:32:50 +0100 Subject: [PATCH 050/147] llvm::get_threadid: use pthread_self on Linux On certain architectures, `pthread_self` is much more efficient than performing a syscall. For instance, on x86-64, the thread ID is obtained through the `fs` register. --- llvm/lib/Support/Unix/Threading.inc | 2 -- 1 file changed, 2 deletions(-) diff --git a/llvm/lib/Support/Unix/Threading.inc b/llvm/lib/Support/Unix/Threading.inc index 819748db4ec2..81176f0a5822 100644 --- a/llvm/lib/Support/Unix/Threading.inc +++ b/llvm/lib/Support/Unix/Threading.inc @@ -127,8 +127,6 @@ uint64_t llvm::get_threadid() { #elif defined(__ANDROID__) return uint64_t(gettid()); #elif defined(__linux__) - return uint64_t(syscall(SYS_gettid)); -#else return uint64_t(pthread_self()); #endif } From 5ce658209926645fa25aa7cd9a2d214575af7a17 Mon Sep 17 00:00:00 2001 From: Alessandro Di Federico Date: Sun, 15 Jan 2023 17:26:21 +0100 Subject: [PATCH 051/147] Introduce llvm::Progress --- llvm/include/llvm/ADT/STLExtras.h | 4 + llvm/include/llvm/IR/LegacyPassManagers.h | 2 + llvm/include/llvm/IR/PassManager.h | 14 + llvm/include/llvm/Pass.h | 2 + llvm/include/llvm/Support/Progress.h | 255 ++++++++++++++++++ llvm/lib/Analysis/CallGraphSCCPass.cpp | 17 +- llvm/lib/IR/LegacyPassManager.cpp | 17 +- llvm/lib/IR/Pass.cpp | 4 + llvm/lib/Support/CMakeLists.txt | 1 + llvm/lib/Support/Progress.cpp | 116 ++++++++ llvm/lib/Transforms/IPO/SCCP.cpp | 4 + .../InstCombine/InstructionCombining.cpp | 4 + llvm/lib/Transforms/Scalar/SCCP.cpp | 1 - 13 files changed, 438 insertions(+), 3 deletions(-) create mode 100644 llvm/include/llvm/Support/Progress.h create mode 100644 llvm/lib/Support/Progress.cpp diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h index 79b145632d5a..c38864b5147e 100644 --- a/llvm/include/llvm/ADT/STLExtras.h +++ b/llvm/include/llvm/ADT/STLExtras.h @@ -1468,6 +1468,10 @@ template auto make_second_range(ContainerTy &&c) { }); } +template auto make_address_range(R &&Range) { + return map_range(Range, [](auto &Element) { return ∈ }); +} + //===----------------------------------------------------------------------===// // Extra additions to //===----------------------------------------------------------------------===// diff --git a/llvm/include/llvm/IR/LegacyPassManagers.h b/llvm/include/llvm/IR/LegacyPassManagers.h index 41c11d26aa45..1dd94b93432d 100644 --- a/llvm/include/llvm/IR/LegacyPassManagers.h +++ b/llvm/include/llvm/IR/LegacyPassManagers.h @@ -498,6 +498,8 @@ class FPPassManager : public ModulePass, public PMDataManager { StringRef getPassName() const override { return "Function Pass Manager"; } + bool isSingleTask() const override { return true; } + FunctionPass *getContainedPass(unsigned N) { assert ( N < PassVector.size() && "Pass number out of range!"); FunctionPass *FP = static_cast(PassVector[N]); diff --git a/llvm/include/llvm/IR/PassManager.h b/llvm/include/llvm/IR/PassManager.h index 21a0af64da4e..6712e7a7fad1 100644 --- a/llvm/include/llvm/IR/PassManager.h +++ b/llvm/include/llvm/IR/PassManager.h @@ -46,6 +46,7 @@ #include "llvm/IR/Module.h" #include "llvm/IR/PassInstrumentation.h" #include "llvm/IR/PassManagerInternal.h" +#include "llvm/Support/Progress.h" #include "llvm/Support/TimeProfiler.h" #include "llvm/Support/TypeName.h" #include @@ -60,6 +61,11 @@ namespace llvm { +template +concept HasName = requires(const T &Object) { + { Object.getName() } -> std::same_as; +}; + /// A special type used by analysis passes to provide an address that /// identifies that particular analysis pass type. /// @@ -507,7 +513,15 @@ class PassManager : public PassInfoMixin< detail::getAnalysisResult( AM, IR, std::tuple(ExtraArgs...)); + std::string Description = "LLVM passes"; + if constexpr (HasName) { + Description += " on " + IR.getName().str(); + } + + Task T(Passes.size(), Description); + for (auto &Pass : Passes) { + T.advance(Pass->name()); // Check the PassInstrumentation's BeforePass callbacks before running the // pass, skip its execution completely if asked to (callback returns // false). diff --git a/llvm/include/llvm/Pass.h b/llvm/include/llvm/Pass.h index 6445e16ab68f..5e3a87d2f9d9 100644 --- a/llvm/include/llvm/Pass.h +++ b/llvm/include/llvm/Pass.h @@ -106,6 +106,8 @@ class Pass { /// Registration templates, but can be overloaded directly. virtual StringRef getPassName() const; + virtual bool isSingleTask() const; + /// getPassID - Return the PassID number that corresponds to this pass. AnalysisID getPassID() const { return PassID; diff --git a/llvm/include/llvm/Support/Progress.h b/llvm/include/llvm/Support/Progress.h new file mode 100644 index 000000000000..4336af97c61d --- /dev/null +++ b/llvm/include/llvm/Support/Progress.h @@ -0,0 +1,255 @@ +//===-- Progress - Generic infrastructure to report progress ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_PROGRESS_H +#define LLVM_SUPPORT_PROGRESS_H + +extern "C" { +#include +} + +#include +#include +#include +#include +#include + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/Threading.h" + +namespace llvm { + +class Progress; +class TaskStack; + +/// Track advancement of a task +/// +/// A task can have a name and optionally a set of maximum steps. +class Task { +protected: + TaskStack &Stack; + int64_t StepIndex = -1; + std::optional TotalSteps; + +private: + std::string TaskName; + std::string StepName; + bool SubtaskCreated = false; + bool Completed = false; + bool CurrentStepHasSingleSubtask = false; + +public: + Task(std::optional TotalSteps, const llvm::Twine &TaskName = ""); + + Task(const Task &) = delete; + Task(Task &&) = delete; + + ~Task(); + +public: + llvm::StringRef name() const { return TaskName; } + llvm::StringRef stepName() const { return StepName; } + /// \return -1 if no step (i.e., call to advance) has been initiated, + /// the index of the last initiated step otherwise. + int64_t stepIndex() const { return StepIndex; } + std::optional totalSteps() const { return TotalSteps; } + bool subtaskCreated() const { return SubtaskCreated; } + bool currentStepHasSingleSubtask() const { + return CurrentStepHasSingleSubtask; + } + bool completed() const { return Completed; } + size_t index() const; + + TaskStack &stack() { return Stack; } + const TaskStack &stack() const { return Stack; } + +public: + /// \param NewStepName the name of the new step starting after this advance + /// invocation. + /// \param SingleSubtask whether we can expect 0 or 1 subtask being created + /// during this step. If true, will assert in case multiple subtasks + /// are create during this step. + void advance(const llvm::Twine &NewStepName = "", bool SingleSubtask = false); + + void setSubtaskCreated(); + + void complete(); +}; + +/// Represents the set of tasks started but not completed in an execution thread +class TaskStack { +private: + Progress &Parent; + unsigned SuspendRequests = 0; + +public: + llvm::SmallVector Tasks; + +public: + TaskStack(Progress &Parent) : Parent(Parent) {} + + TaskStack(TaskStack &&) = delete; + TaskStack(const TaskStack &) = delete; + TaskStack &operator=(const TaskStack &) = delete; + TaskStack &operator=(TaskStack &&) = delete; + +public: + void registerTask(Task *T); + void unregisterTask(Task *T); + void advanceTask(Task *T, llvm::StringRef PreviousStepName); + +public: + bool isSuspended() const { return SuspendRequests > 0; } + void resumeTracking() { + assert(isSuspended()); + --SuspendRequests; + } + + void suspendTracking() { ++SuspendRequests; } +}; + +/// Class to monitor progress of a Progress instance +class ProgressListener { +public: + virtual ~ProgressListener() = default; + + /// Invoked to notify the creation of a new task + /// + /// \note \p T will always be the last task of its stack. + /// + /// \note This method could be called by multiple threads in parallel. + virtual void handleNewTask(const Task *T) = 0; + + /// Invoked to notify the completion of a task + /// + /// \note \p T will always be the last task of its stack. + /// + /// \note This method could be called by multiple threads in parallel. + virtual void handleTaskCompleted(const Task *T) = 0; + + /// Invoked to notify of the advancement of a certain task to a new step + /// + /// \note \p T will always be the last task of its stack. + /// + /// \note This method could be called by multiple threads in parallel. + virtual void handleTaskAdvancement(const Task *T, + llvm::StringRef PreviousStepName) = 0; +}; + +/// Class to monitor progress of task staks over multiple threads +class Progress { +private: + struct RegistryEntry { + bool AllThreads = false; + std::unique_ptr Listener; + }; + +private: + uint64_t MainThreadID; + llvm::SmallVector Registry; + +public: + Progress(); + +public: + /// \note call this method from the main thread only. + template + void registerListener(Types &&...Args) { + assert(isMainThread()); + Registry.push_back({T::AllThreads, std::make_unique(Args...)}); + } + +public: + bool isMainThread() const; + +public: + void handleNewTask(const Task *T); + void handleTaskCompleted(const Task *T); + void handleTaskAdvancement(const Task *T, llvm::StringRef PreviousStepName); +}; + +/// The default instance of Progress each Task will use +extern llvm::ManagedStatic ProgressReport; + +/// Similar to Task, but gets a list of objects at construction times that +/// represent elements the task iterates on. +/// Each invocation of advance accepts a similar object: if the passed object is +/// not in the list of expected objects, the monitor of progress will be +/// suspended until the next step. +template , + llvm::SmallPtrSet, + std::set>> +class TaskOnSet : public Task { +private: + Set Predicted; + bool Suspended = false; + +public: + template + TaskOnSet(R &&Elements, const llvm::Twine &TaskName = "") + : Task({}, TaskName) { + unsigned Count = 0; + for (const auto &Element : Elements) { + Predicted.insert(Element); + ++Count; + } + + TotalSteps = Predicted.size(); + + assert(TotalSteps == Count); + } + + ~TaskOnSet() { + if (Suspended) + Stack.resumeTracking(); + } + +public: + void advance(const ValueType &Element, const llvm::Twine &NewStepName = "", + bool SingleSubtask = false) { + + if (Predicted.count(Element) != 0) { + if (Suspended) { + Stack.resumeTracking(); + Suspended = false; + } + + // This is an expected element, proceed + Task::advance(NewStepName, SingleSubtask); + + } else { + if (not Suspended) { + // Ignore all unexpected elements + Stack.suspendTracking(); + Suspended = true; + } + } + } +}; + +template +auto make_task_on_set(R &&Elements, const llvm::Twine &TaskName = "") { + return TaskOnSet().begin())>>( + Elements, TaskName); +} + +template +auto make_task_on_set(TaskStack &Stack, R &&Elements, + const llvm::Twine &TaskName = "") { + return TaskOnSet().begin())>>( + Stack, Elements, TaskName); +} + +} // namespace llvm + +#endif diff --git a/llvm/lib/Analysis/CallGraphSCCPass.cpp b/llvm/lib/Analysis/CallGraphSCCPass.cpp index d66f1e261780..564056c4ad4a 100644 --- a/llvm/lib/Analysis/CallGraphSCCPass.cpp +++ b/llvm/lib/Analysis/CallGraphSCCPass.cpp @@ -31,6 +31,7 @@ #include "llvm/Pass.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/Progress.h" #include "llvm/Support/Timer.h" #include "llvm/Support/raw_ostream.h" #include @@ -81,6 +82,8 @@ class CGPassManager : public ModulePass, public PMDataManager { StringRef getPassName() const override { return "CallGraph Pass Manager"; } + bool isSingleTask() const override { return true; } + PMDataManager *getAsPMDataManager() override { return this; } Pass *getAsPass() override { return this; } @@ -171,14 +174,18 @@ bool CGPassManager::RunPassOnSCC(Pass *P, CallGraphSCC &CurSCC, FPPassManager *FPP = (FPPassManager*)P; // Run pass P on all functions in the current SCC. + Task T(CurSCC.size(), "Run function pass manager on SCC"); for (CallGraphNode *CGN : CurSCC) { if (Function *F = CGN->getFunction()) { + T.advance(F->getName(), true); dumpPassInfo(P, EXECUTION_MSG, ON_FUNCTION_MSG, F->getName()); { TimeRegion PassTimer(getPassTimer(FPP)); Changed |= FPP->runOnFunction(*F); } F->getContext().yield(); + } else { + T.advance(); } } @@ -436,10 +443,13 @@ bool CGPassManager::RunAllPassesOnSCC(CallGraphSCC &CurSCC, CallGraph &CG, // the callgraph when we need to run a CGSCCPass again. bool CallGraphUpToDate = true; + Task T(getNumContainedPasses(), "Run passes on SCC"); + // Run all passes on current SCC. for (unsigned PassNo = 0, e = getNumContainedPasses(); PassNo != e; ++PassNo) { Pass *P = getContainedPass(PassNo); + T.advance(P->getPassName(), P->isSingleTask()); // If we're in -debug-pass=Executions mode, construct the SCC node list, // otherwise avoid constructing this string as it is expensive. @@ -502,8 +512,10 @@ bool CGPassManager::runOnModule(Module &M) { bool Changed = doInitialization(CG); // Walk the callgraph in bottom-up SCC order. - scc_iterator CGI = scc_begin(&CG); + auto T = make_task_on_set(make_range(scc_begin(&CG), scc_end(&CG)), + "LLVM SCC Passes"); + scc_iterator CGI = scc_begin(&CG); CallGraphSCC CurSCC(CG, &CGI); while (!CGI.isAtEnd()) { // Copy the current SCC and increment past it so that the pass can hack @@ -511,6 +523,7 @@ bool CGPassManager::runOnModule(Module &M) { const std::vector &NodeVec = *CGI; CurSCC.initialize(NodeVec); ++CGI; + T.advance(NodeVec, "", true); // At the top level, we run all the passes in this pass manager on the // functions in this SCC. However, we support iterative compilation in the @@ -524,9 +537,11 @@ bool CGPassManager::runOnModule(Module &M) { // This only happens in the case of a devirtualized call, so we only burn // compile time in the case that we're making progress. We also have a hard // iteration count limit in case there is crazy code. + Task SCCT({}, "Fixed-point SCC visit"); unsigned Iteration = 0; bool DevirtualizedCall = false; do { + SCCT.advance("", true); LLVM_DEBUG(if (Iteration) dbgs() << " SCCPASSMGR: Re-visiting SCC, iteration #" << Iteration << '\n'); diff --git a/llvm/lib/IR/LegacyPassManager.cpp b/llvm/lib/IR/LegacyPassManager.cpp index ef3465177647..e0e20eef30de 100644 --- a/llvm/lib/IR/LegacyPassManager.cpp +++ b/llvm/lib/IR/LegacyPassManager.cpp @@ -12,6 +12,7 @@ #include "llvm/IR/LegacyPassManager.h" #include "llvm/ADT/MapVector.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/IRPrintingPasses.h" #include "llvm/IR/LLVMContext.h" @@ -24,6 +25,7 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/Progress.h" #include "llvm/Support/TimeProfiler.h" #include "llvm/Support/Timer.h" #include "llvm/Support/raw_ostream.h" @@ -358,7 +360,9 @@ bool FunctionPassManagerImpl::run(Function &F) { bool Changed = false; initializeAllAnalysisInfo(); + Task T(getNumContainedManagers(), "Run function pass managers"); for (unsigned Index = 0; Index < getNumContainedManagers(); ++Index) { + T.advance(); Changed |= getContainedManager(Index)->runOnFunction(F); F.getContext().yield(); } @@ -423,6 +427,8 @@ class MPPassManager : public Pass, public PMDataManager { StringRef getPassName() const override { return "Module Pass Manager"; } + bool isSingleTask() const override { return true; } + PMDataManager *getAsPMDataManager() override { return this; } Pass *getAsPass() override { return this; } @@ -531,7 +537,9 @@ bool PassManagerImpl::run(Module &M) { Changed |= ImPass->doInitialization(M); initializeAllAnalysisInfo(); + Task T(getNumContainedManagers(), "Run module pass managers"); for (unsigned Index = 0; Index < getNumContainedManagers(); ++Index) { + T.advance(); Changed |= getContainedManager(Index)->runOnModule(M); M.getContext().yield(); } @@ -1410,8 +1418,10 @@ bool FPPassManager::runOnFunction(Function &F) { llvm::TimeTraceScope FunctionScope("OptFunction", F.getName()); + Task T(getNumContainedPasses(), "Run function passes"); for (unsigned Index = 0; Index < getNumContainedPasses(); ++Index) { FunctionPass *FP = getContainedPass(Index); + T.advance(FP->getPassName(), FP->isSingleTask()); bool LocalChanged = false; llvm::TimeTraceScope PassScope("RunPass", FP->getPassName()); @@ -1472,8 +1482,11 @@ bool FPPassManager::runOnFunction(Function &F) { bool FPPassManager::runOnModule(Module &M) { bool Changed = false; - for (Function &F : M) + auto T = make_task_on_set(make_address_range(M), "Run function pass manager"); + for (Function &F : M) { + T.advance(&F, F.getName()); Changed |= runOnFunction(F); + } return Changed; } @@ -1525,8 +1538,10 @@ MPPassManager::runOnModule(Module &M) { if (EmitICRemark) InstrCount = initSizeRemarkInfo(M, FunctionToInstrCount); + Task T(getNumContainedPasses(), "Run module passes"); for (unsigned Index = 0; Index < getNumContainedPasses(); ++Index) { ModulePass *MP = getContainedPass(Index); + T.advance(MP->getPassName(), MP->isSingleTask()); bool LocalChanged = false; dumpPassInfo(MP, EXECUTION_MSG, ON_MODULE_MSG, M.getModuleIdentifier()); diff --git a/llvm/lib/IR/Pass.cpp b/llvm/lib/IR/Pass.cpp index 716d9d546f4f..018131dbe381 100644 --- a/llvm/lib/IR/Pass.cpp +++ b/llvm/lib/IR/Pass.cpp @@ -86,6 +86,10 @@ StringRef Pass::getPassName() const { return "Unnamed pass: implement Pass::getPassName()"; } +bool Pass::isSingleTask() const { + return false; +} + void Pass::preparePassManager(PMStack &) { // By default, don't do anything. } diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt index 4cbc3b79f3bb..1058f4848601 100644 --- a/llvm/lib/Support/CMakeLists.txt +++ b/llvm/lib/Support/CMakeLists.txt @@ -198,6 +198,7 @@ add_llvm_component_library(LLVMSupport Parallel.cpp PluginLoader.cpp PrettyStackTrace.cpp + Progress.cpp RandomNumberGenerator.cpp Regex.cpp RISCVAttributes.cpp diff --git a/llvm/lib/Support/Progress.cpp b/llvm/lib/Support/Progress.cpp new file mode 100644 index 000000000000..ea7f74b8e9bb --- /dev/null +++ b/llvm/lib/Support/Progress.cpp @@ -0,0 +1,116 @@ +//===-- Progress - Generic infrastructure to report progress ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the core logic for progress reporting. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/Progress.h" + +using namespace llvm; + +ManagedStatic llvm::ProgressReport; + +static thread_local TaskStack CurrentStack(*ProgressReport); + +Progress::Progress() : MainThreadID(get_threadid()) {} + +bool Progress::isMainThread() const { return MainThreadID == get_threadid(); } + +void Progress::handleNewTask(const Task *T) { + bool IsMainThread = isMainThread(); + for (auto &[AllThreads, Listener] : Registry) + if (IsMainThread or AllThreads) + Listener->handleNewTask(T); +} + +void Progress::handleTaskCompleted(const Task *T) { + bool IsMainThread = isMainThread(); + for (auto &[AllThreads, Listener] : Registry) + if (IsMainThread or AllThreads) + Listener->handleTaskCompleted(T); +} + +void Progress::handleTaskAdvancement(const Task *T, + StringRef PreviousStepName) { + bool IsMainThread = isMainThread(); + for (auto &[AllThreads, Listener] : Registry) + if (IsMainThread or AllThreads) + Listener->handleTaskAdvancement(T, PreviousStepName); +} + +Task::Task(std::optional TotalSteps, const Twine &TaskName) + : Stack(CurrentStack), TotalSteps(TotalSteps), TaskName(TaskName.str()) { + Stack.registerTask(this); +} + +void Task::complete() { + if (not Completed) { + Completed = true; + Stack.unregisterTask(this); + } +} + +Task::~Task() { complete(); } + +void Task::advance(const Twine &NewStepName, bool SingleSubtask) { + int64_t NewStepIndex = StepIndex + 1; + if (TotalSteps.has_value()) { + assert(NewStepIndex < *TotalSteps); + } + StepIndex = NewStepIndex; + std::string PreviousStepName = std::move(StepName); + StepName = NewStepName.str(); + SubtaskCreated = false; + CurrentStepHasSingleSubtask = SingleSubtask; + + Stack.advanceTask(this, PreviousStepName); +} + +void Task::setSubtaskCreated() { SubtaskCreated = true; } + +size_t Task::index() const { + auto &Stack = stack(); + auto Begin = Stack.Tasks.begin(); + auto End = Stack.Tasks.end(); + auto It = std::find(Begin, End, this); + assert(It != End); + return It - Begin; +} + +void TaskStack::registerTask(Task *T) { + if (isSuspended()) + return; + + if (Tasks.size() > 0) { + if (T->currentStepHasSingleSubtask()) { + assert(not Tasks.back()->subtaskCreated()); + } + Tasks.back()->setSubtaskCreated(); + } + + Tasks.push_back(T); + + Parent.handleNewTask(T); +} + +void TaskStack::unregisterTask(Task *T) { + if (isSuspended()) + return; + + assert(Tasks.back() == T); + Parent.handleTaskCompleted(T); + Tasks.pop_back(); +} + +void TaskStack::advanceTask(Task *T, StringRef PreviousStepName) { + if (isSuspended()) + return; + + Parent.handleTaskAdvancement(T, PreviousStepName); +} diff --git a/llvm/lib/Transforms/IPO/SCCP.cpp b/llvm/lib/Transforms/IPO/SCCP.cpp index 5c1582ddfdae..b42904ab65ad 100644 --- a/llvm/lib/Transforms/IPO/SCCP.cpp +++ b/llvm/lib/Transforms/IPO/SCCP.cpp @@ -114,11 +114,15 @@ static bool runIPSCCP( function_ref getAnalysis, bool IsFuncSpecEnabled) { SCCPSolver Solver(DL, GetTLI, M.getContext()); + auto T = make_task_on_set(make_address_range(M), "LLVM SCC Passes"); + FunctionSpecializer Specializer(Solver, M, FAM, GetTLI, GetTTI, GetAC); // Loop over all functions, marking arguments to those with their addresses // taken or that are external as overdefined. for (Function &F : M) { + T.advance(&F, F.getName()); + if (F.isDeclaration()) continue; diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp index fb6f4f96ea48..200ff3fc2027 100644 --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -4609,12 +4609,16 @@ static bool combineInstructionsOverFunction( RemoveRedundantDbgInstrs(&BB); } + Task T({}, "InstCombine"); + // Iterate while there is work to do. unsigned Iteration = 0; while (true) { ++NumWorklistIterations; ++Iteration; + T.advance(); + if (Iteration > InfiniteLoopDetectionThreshold) { report_fatal_error( "Instruction Combining seems stuck in an infinite loop after " + diff --git a/llvm/lib/Transforms/Scalar/SCCP.cpp b/llvm/lib/Transforms/Scalar/SCCP.cpp index 7b396c6ee074..55012911016c 100644 --- a/llvm/lib/Transforms/Scalar/SCCP.cpp +++ b/llvm/lib/Transforms/Scalar/SCCP.cpp @@ -186,4 +186,3 @@ INITIALIZE_PASS_END(SCCPLegacyPass, "sccp", // createSCCPPass - This is the public interface to this file. FunctionPass *llvm::createSCCPPass() { return new SCCPLegacyPass(); } - From 4e4c6768d278dd98fbd5df4b6ca4f56d216bcb1d Mon Sep 17 00:00:00 2001 From: Pietro Fezzardi Date: Tue, 29 Aug 2023 12:04:56 +0200 Subject: [PATCH 052/147] Improve messages in progress bars Before this commit, the affected progress bar messages were just "null". This was effectively useless information and it also made impossible to distinguish between the various "null" cases, which were more than one. This commit adds minimal but distinguishable progress bar messages. --- llvm/lib/IR/LegacyPassManager.cpp | 4 ++-- llvm/lib/Transforms/InstCombine/InstructionCombining.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/lib/IR/LegacyPassManager.cpp b/llvm/lib/IR/LegacyPassManager.cpp index e0e20eef30de..f097161120fd 100644 --- a/llvm/lib/IR/LegacyPassManager.cpp +++ b/llvm/lib/IR/LegacyPassManager.cpp @@ -362,7 +362,7 @@ bool FunctionPassManagerImpl::run(Function &F) { initializeAllAnalysisInfo(); Task T(getNumContainedManagers(), "Run function pass managers"); for (unsigned Index = 0; Index < getNumContainedManagers(); ++Index) { - T.advance(); + T.advance(Twine("FPM index: ") + Twine(Index)); Changed |= getContainedManager(Index)->runOnFunction(F); F.getContext().yield(); } @@ -539,7 +539,7 @@ bool PassManagerImpl::run(Module &M) { initializeAllAnalysisInfo(); Task T(getNumContainedManagers(), "Run module pass managers"); for (unsigned Index = 0; Index < getNumContainedManagers(); ++Index) { - T.advance(); + T.advance(Twine("MPM index: ") + Twine(Index)); Changed |= getContainedManager(Index)->runOnModule(M); M.getContext().yield(); } diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp index 200ff3fc2027..9c6a57650ed5 100644 --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -4617,7 +4617,7 @@ static bool combineInstructionsOverFunction( ++NumWorklistIterations; ++Iteration; - T.advance(); + T.advance(Twine("Iteration: ") + Twine(Iteration)); if (Iteration > InfiniteLoopDetectionThreshold) { report_fatal_error( From 46f10685709a419ada397d53ebd759a4c0721482 Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Sun, 5 Mar 2023 22:08:31 +0100 Subject: [PATCH 053/147] [mlir][LLVM] Add param attr verifiers This commit introduces unified parameter attribute verifiers to the LLVM dialect and removes according checks in the export. As LLVM does not verify the validity of certain attributes on return values, this commit unifies the handling of argument and result attributes wherever possible. Depends on D142212 Reviewed By: gysit Differential Revision: https://reviews.llvm.org/D142372 (cherry picked from commit be4b87353e4230e8dab4173d5200e8d88f6ab032) --- .../include/mlir/Dialect/LLVMIR/LLVMOpBase.td | 7 + .../mlir/Target/LLVMIR/ModuleTranslation.h | 7 +- mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 236 ++++++++++-------- mlir/lib/Target/LLVMIR/ModuleTranslation.cpp | 185 ++++---------- mlir/test/Dialect/LLVMIR/func.mlir | 56 +---- mlir/test/Dialect/LLVMIR/invalid.mlir | 14 -- .../LLVMIR/parameter-attrs-invalid.mlir | 188 ++++++++++++++ mlir/test/Target/LLVMIR/llvmir-invalid.mlir | 97 ------- 8 files changed, 384 insertions(+), 406 deletions(-) create mode 100644 mlir/test/Dialect/LLVMIR/parameter-attrs-invalid.mlir diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td index eda30e36086f..9166ae78bd11 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td @@ -54,6 +54,7 @@ def LLVM_Dialect : Dialect { static StringRef getSExtAttrName() { return "llvm.signext"; } static StringRef getZExtAttrName() { return "llvm.zeroext"; } static StringRef getTBAAAttrName() { return "llvm.tbaa"; } + static StringRef getNestAttrName() { return "llvm.nest"; } /// Verifies if the attribute is a well-formed value for "llvm.struct_attrs" static LogicalResult verifyStructAttr( @@ -87,6 +88,12 @@ def LLVM_Dialect : Dialect { void printType(Type, DialectAsmPrinter &p) const override; private: + /// Verifies a parameter attribute attached to a parameter of type + /// paramType. + LogicalResult verifyParameterAttribute(Operation *op, + Type paramType, + NamedAttribute paramAttr); + /// Register all types. void registerTypes(); diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h index 33f8ab7d066d..4e85dcf6ab34 100644 --- a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h +++ b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h @@ -15,8 +15,8 @@ #define MLIR_TARGET_LLVMIR_MODULETRANSLATION_H #include "mlir/IR/Operation.h" -#include "mlir/IR/Value.h" #include "mlir/IR/SymbolTable.h" +#include "mlir/IR/Value.h" #include "mlir/Target/LLVMIR/Export.h" #include "mlir/Target/LLVMIR/LLVMTranslationInterface.h" #include "mlir/Target/LLVMIR/TypeToLLVM.h" @@ -273,7 +273,7 @@ class ModuleTranslation { ModuleTranslation &moduleTranslation; }; - SymbolTableCollection& symbolTable() { return symbolTableCollection; } + SymbolTableCollection &symbolTable() { return symbolTableCollection; } private: ModuleTranslation(Operation *module, @@ -306,6 +306,9 @@ class ModuleTranslation { /// Translates dialect attributes attached to the given operation. LogicalResult convertDialectAttributes(Operation *op); + /// Translates parameter attributes and adds them to the returned AttrBuilder. + llvm::AttrBuilder convertParameterAttrs(DictionaryAttr paramAttrs); + /// Original and translated module. Operation *mlirModule; std::unique_ptr llvmModule; diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp index 4f301ff00c95..a381efc189a6 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -3120,123 +3120,157 @@ LogicalResult LLVMDialect::verifyStructAttr(Operation *op, Attribute attr, return success(); } -static LogicalResult verifyFuncOpInterfaceStructAttr( - Operation *op, Attribute attr, - const std::function &getAnnotatedType) { - if (auto funcOp = dyn_cast(op)) - return LLVMDialect::verifyStructAttr(op, attr, getAnnotatedType(funcOp)); +static LogicalResult verifyFuncOpInterfaceStructAttr(Operation *op, + Attribute attr, + Type annotatedType) { + if (isa(op)) + return LLVMDialect::verifyStructAttr(op, attr, annotatedType); return op->emitError() << "expected '" << LLVMDialect::getStructAttrsAttrName() << "' to be used on function-like operations"; } +LogicalResult LLVMDialect::verifyParameterAttribute(Operation *op, + Type paramType, + NamedAttribute paramAttr) { + // LLVM attribute may be attached to a result of operation that has not been + // converted to LLVM dialect yet, so the result may have a type with unknown + // representation in LLVM dialect type space. In this case we cannot verify + // whether the attribute may be + bool verifyValueType = isCompatibleType(paramType); + StringAttr name = paramAttr.getName(); + + auto checkUnitAttrType = [&]() -> LogicalResult { + if (!paramAttr.getValue().isa()) + return op->emitError() << name << " should be a unit attribute"; + return success(); + }; + auto checkTypeAttrType = [&]() -> LogicalResult { + if (!paramAttr.getValue().isa()) + return op->emitError() << name << " should be a type attribute"; + return success(); + }; + auto checkIntegerAttrType = [&]() -> LogicalResult { + if (!paramAttr.getValue().isa()) + return op->emitError() << name << " should be an integer attribute"; + return success(); + }; + auto checkPointerType = [&]() -> LogicalResult { + if (!paramType.isa()) + return op->emitError() + << name << " attribute attached to non-pointer LLVM type"; + return success(); + }; + auto checkIntegerType = [&]() -> LogicalResult { + if (!paramType.isa()) + return op->emitError() + << name << " attribute attached to non-integer LLVM type"; + return success(); + }; + auto checkPointerTypeMatches = [&]() -> LogicalResult { + if (failed(checkPointerType())) + return failure(); + auto ptrType = paramType.cast(); + auto typeAttr = paramAttr.getValue().cast(); + + if (!ptrType.isOpaque() && ptrType.getElementType() != typeAttr.getValue()) + return op->emitError() + << name + << " attribute attached to LLVM pointer argument of " + "different type"; + return success(); + }; + + // Note: The struct parameter attributes are not lowered to LLVM IR. + if (name == LLVMDialect::getStructAttrsAttrName()) + return verifyFuncOpInterfaceStructAttr(op, paramAttr.getValue(), paramType); + + // Check a unit attribute that is attached to a pointer value. + if (name == LLVMDialect::getNoAliasAttrName() || + name == LLVMDialect::getReadonlyAttrName() || + name == LLVMDialect::getNestAttrName()) { + if (failed(checkUnitAttrType())) + return failure(); + if (verifyValueType && failed(checkPointerType())) + return failure(); + return success(); + } + + // Check a type attribute that is attached to a pointer value. + if (name == LLVMDialect::getStructRetAttrName() || + name == LLVMDialect::getByValAttrName() || + name == LLVMDialect::getByRefAttrName() || + name == LLVMDialect::getInAllocaAttrName()) { + if (failed(checkTypeAttrType())) + return failure(); + if (verifyValueType && failed(checkPointerTypeMatches())) + return failure(); + return success(); + } + + // Check a unit attribute that is attached to an integer value. + if (name == LLVMDialect::getSExtAttrName() || + name == LLVMDialect::getZExtAttrName()) { + if (failed(checkUnitAttrType())) + return failure(); + if (verifyValueType && failed(checkIntegerType())) + return failure(); + return success(); + } + + // Check an integer attribute that is attached to a pointer value. + if (name == LLVMDialect::getAlignAttrName()) { + if (failed(checkIntegerAttrType())) + return failure(); + if (verifyValueType && failed(checkPointerType())) + return failure(); + return success(); + } + + if (name == LLVMDialect::getNoUndefAttrName()) + return checkUnitAttrType(); + return success(); +} + /// Verify LLVMIR function argument attributes. LogicalResult LLVMDialect::verifyRegionArgAttribute(Operation *op, unsigned regionIdx, unsigned argIdx, NamedAttribute argAttr) { - // Check that llvm.noalias is a unit attribute. - if (argAttr.getName() == LLVMDialect::getNoAliasAttrName() && - !argAttr.getValue().isa()) - return op->emitError() - << "expected llvm.noalias argument attribute to be a unit attribute"; - // Check that llvm.align is an integer attribute. - if (argAttr.getName() == LLVMDialect::getAlignAttrName() && - !argAttr.getValue().isa()) - return op->emitError() - << "llvm.align argument attribute of non integer type"; - if (argAttr.getName() == LLVMDialect::getStructAttrsAttrName()) { - return verifyFuncOpInterfaceStructAttr( - op, argAttr.getValue(), [argIdx](FunctionOpInterface funcOp) { - return funcOp.getArgumentTypes()[argIdx]; - }); - } - return success(); + auto funcOp = dyn_cast(op); + if (!funcOp) + return success(); + Type argType = funcOp.getArgumentTypes()[argIdx]; + + return verifyParameterAttribute(op, argType, argAttr); } LogicalResult LLVMDialect::verifyRegionResultAttribute(Operation *op, unsigned regionIdx, unsigned resIdx, NamedAttribute resAttr) { - StringAttr name = resAttr.getName(); - if (name == LLVMDialect::getStructAttrsAttrName()) { - return verifyFuncOpInterfaceStructAttr( - op, resAttr.getValue(), [resIdx](FunctionOpInterface funcOp) { - return funcOp.getResultTypes()[resIdx]; - }); - } - if (auto funcOp = dyn_cast(op)) { - mlir::Type resTy = funcOp.getResultTypes()[resIdx]; - - // Check to see if this function has a void return with a result attribute - // to it. It isn't clear what semantics we would assign to that. - if (resTy.isa()) - return op->emitError() << "cannot attach result attributes to functions " - "with a void return"; - - // LLVM attribute may be attached to a result of operation - // that has not been converted to LLVM dialect yet, so the result - // may have a type with unknown representation in LLVM dialect type - // space. In this case we cannot verify whether the attribute may be - // attached to a result of such type. - bool verifyValueType = isCompatibleType(resTy); - Attribute attrValue = resAttr.getValue(); - - // TODO: get rid of code duplication here and in verifyRegionArgAttribute(). - if (name == LLVMDialect::getAlignAttrName()) { - if (!attrValue.isa()) - return op->emitError() << "expected llvm.align result attribute to be " - "an integer attribute"; - if (verifyValueType && !resTy.isa()) - return op->emitError() - << "llvm.align attribute attached to non-pointer result"; - return success(); - } - if (name == LLVMDialect::getNoAliasAttrName()) { - if (!attrValue.isa()) - return op->emitError() << "expected llvm.noalias result attribute to " - "be a unit attribute"; - if (verifyValueType && !resTy.isa()) - return op->emitError() - << "llvm.noalias attribute attached to non-pointer result"; - return success(); - } - if (name == LLVMDialect::getReadonlyAttrName()) { - if (!attrValue.isa()) - return op->emitError() << "expected llvm.readonly result attribute to " - "be a unit attribute"; - if (verifyValueType && !resTy.isa()) - return op->emitError() - << "llvm.readonly attribute attached to non-pointer result"; - return success(); - } - if (name == LLVMDialect::getNoUndefAttrName()) { - if (!attrValue.isa()) - return op->emitError() << "expected llvm.noundef result attribute to " - "be a unit attribute"; - return success(); - } - if (name == LLVMDialect::getSExtAttrName()) { - if (!attrValue.isa()) - return op->emitError() << "expected llvm.signext result attribute to " - "be a unit attribute"; - if (verifyValueType && !resTy.isa()) - return op->emitError() - << "llvm.signext attribute attached to non-integer result"; - return success(); - } - if (name == LLVMDialect::getZExtAttrName()) { - if (!attrValue.isa()) - return op->emitError() << "expected llvm.zeroext result attribute to " - "be a unit attribute"; - if (verifyValueType && !resTy.isa()) - return op->emitError() - << "llvm.zeroext attribute attached to non-integer result"; - return success(); - } - } - - return success(); + auto funcOp = dyn_cast(op); + if (!funcOp) + return success(); + Type resType = funcOp.getResultTypes()[resIdx]; + + // Check to see if this function has a void return with a result attribute + // to it. It isn't clear what semantics we would assign to that. + if (resType.isa()) + return op->emitError() << "cannot attach result attributes to functions " + "with a void return"; + + // Check to see if this attribute is allowed as a result attribute. Only + // explicitly forbidden LLVM attributes will cause an error. + auto name = resAttr.getName(); + if (name == LLVMDialect::getReadonlyAttrName() || + name == LLVMDialect::getNestAttrName() || + name == LLVMDialect::getStructRetAttrName() || + name == LLVMDialect::getByValAttrName() || + name == LLVMDialect::getByRefAttrName() || + name == LLVMDialect::getInAllocaAttrName()) + return op->emitError() << name << " is not a valid result attribute"; + return verifyParameterAttribute(op, resType, resAttr); } //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp index ecb9908c6cb6..b5323de221d7 100644 --- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp @@ -896,6 +896,48 @@ static void convertFunctionAttributes(LLVMFuncOp func, llvmFunc->setMemoryEffects(newMemEffects); } +llvm::AttrBuilder +ModuleTranslation::convertParameterAttrs(DictionaryAttr paramAttrs) { + llvm::AttrBuilder attrBuilder(llvmModule->getContext()); + if (auto attr = paramAttrs.getAs(LLVMDialect::getNoAliasAttrName())) + attrBuilder.addAttribute(llvm::Attribute::AttrKind::NoAlias); + + if (auto attr = + paramAttrs.getAs(LLVMDialect::getReadonlyAttrName())) + attrBuilder.addAttribute(llvm::Attribute::AttrKind::ReadOnly); + + if (auto attr = + paramAttrs.getAs(LLVMDialect::getAlignAttrName())) + attrBuilder.addAlignmentAttr(llvm::Align(attr.getInt())); + + if (auto attr = + paramAttrs.getAs(LLVMDialect::getStructRetAttrName())) + attrBuilder.addStructRetAttr(convertType(attr.getValue())); + + if (auto attr = paramAttrs.getAs(LLVMDialect::getByValAttrName())) + attrBuilder.addByValAttr(convertType(attr.getValue())); + + if (auto attr = paramAttrs.getAs(LLVMDialect::getByRefAttrName())) + attrBuilder.addByRefAttr(convertType(attr.getValue())); + + if (auto attr = + paramAttrs.getAs(LLVMDialect::getInAllocaAttrName())) + attrBuilder.addInAllocaAttr(convertType(attr.getValue())); + + if (auto attr = paramAttrs.getAs(LLVMDialect::getNestAttrName())) + attrBuilder.addAttribute(llvm::Attribute::Nest); + + if (auto attr = paramAttrs.getAs(LLVMDialect::getNoUndefAttrName())) + attrBuilder.addAttribute(llvm::Attribute::NoUndef); + + if (auto attr = paramAttrs.getAs(LLVMDialect::getSExtAttrName())) + attrBuilder.addAttribute(llvm::Attribute::SExt); + + if (auto attr = paramAttrs.getAs(LLVMDialect::getZExtAttrName())) + attrBuilder.addAttribute(llvm::Attribute::ZExt); + return attrBuilder; +} + LogicalResult ModuleTranslation::convertFunctionSignatures() { // Declare all functions first because there may be function calls that form a // call graph with cycles, or global initializers that reference functions. @@ -918,149 +960,16 @@ LogicalResult ModuleTranslation::convertFunctionSignatures() { // Convert result attributes. if (ArrayAttr allResultAttrs = function.getAllResultAttrs()) { - llvm::AttrBuilder retAttrs(llvmFunc->getContext()); DictionaryAttr resultAttrs = allResultAttrs[0].cast(); - for (const NamedAttribute &attr : resultAttrs) { - StringAttr name = attr.getName(); - if (name == LLVMDialect::getAlignAttrName()) { - auto alignAmount = attr.getValue().cast(); - retAttrs.addAlignmentAttr(llvm::Align(alignAmount.getInt())); - } else if (name == LLVMDialect::getNoAliasAttrName()) { - retAttrs.addAttribute(llvm::Attribute::NoAlias); - } else if (name == LLVMDialect::getNoUndefAttrName()) { - retAttrs.addAttribute(llvm::Attribute::NoUndef); - } else if (name == LLVMDialect::getSExtAttrName()) { - retAttrs.addAttribute(llvm::Attribute::SExt); - } else if (name == LLVMDialect::getZExtAttrName()) { - retAttrs.addAttribute(llvm::Attribute::ZExt); - } - } - llvmFunc->addRetAttrs(retAttrs); + llvmFunc->addRetAttrs(convertParameterAttrs(resultAttrs)); } // Convert argument attributes. - unsigned int argIdx = 0; - for (auto [mlirArgTy, llvmArg] : - llvm::zip(function.getArgumentTypes(), llvmFunc->args())) { - if (auto attr = function.getArgAttrOfType( - argIdx, LLVMDialect::getNoAliasAttrName())) { - // NB: Attribute already verified to be boolean, so check if we can - // indeed attach the attribute to this argument, based on its type. - if (!mlirArgTy.isa()) - return function.emitError( - "llvm.noalias attribute attached to LLVM non-pointer argument"); - llvmArg.addAttr(llvm::Attribute::AttrKind::NoAlias); - } - if (auto attr = function.getArgAttrOfType( - argIdx, LLVMDialect::getReadonlyAttrName())) { - if (!mlirArgTy.isa()) - return function.emitError( - "llvm.readonly attribute attached to LLVM non-pointer argument"); - llvmArg.addAttr(llvm::Attribute::AttrKind::ReadOnly); - } - - if (auto attr = function.getArgAttrOfType( - argIdx, LLVMDialect::getAlignAttrName())) { - // NB: Attribute already verified to be int, so check if we can indeed - // attach the attribute to this argument, based on its type. - if (!mlirArgTy.isa()) - return function.emitError( - "llvm.align attribute attached to LLVM non-pointer argument"); - llvmArg.addAttrs(llvm::AttrBuilder(llvmArg.getContext()) - .addAlignmentAttr(llvm::Align(attr.getInt()))); - } - - if (auto attr = function.getArgAttrOfType( - argIdx, LLVMDialect::getStructRetAttrName())) { - auto argTy = mlirArgTy.dyn_cast(); - if (!argTy) - return function.emitError( - "llvm.sret attribute attached to LLVM non-pointer argument"); - if (!argTy.isOpaque() && argTy.getElementType() != attr.getValue()) - return function.emitError( - "llvm.sret attribute attached to LLVM pointer " - "argument of a different type"); - llvmArg.addAttrs(llvm::AttrBuilder(llvmArg.getContext()) - .addStructRetAttr(convertType(attr.getValue()))); - } - - if (auto attr = function.getArgAttrOfType( - argIdx, LLVMDialect::getByValAttrName())) { - auto argTy = mlirArgTy.dyn_cast(); - if (!argTy) - return function.emitError( - "llvm.byval attribute attached to LLVM non-pointer argument"); - if (!argTy.isOpaque() && argTy.getElementType() != attr.getValue()) - return function.emitError( - "llvm.byval attribute attached to LLVM pointer " - "argument of a different type"); - llvmArg.addAttrs(llvm::AttrBuilder(llvmArg.getContext()) - .addByValAttr(convertType(attr.getValue()))); - } - - if (auto attr = function.getArgAttrOfType( - argIdx, LLVMDialect::getByRefAttrName())) { - auto argTy = mlirArgTy.dyn_cast(); - if (!argTy) - return function.emitError( - "llvm.byref attribute attached to LLVM non-pointer argument"); - if (!argTy.isOpaque() && argTy.getElementType() != attr.getValue()) - return function.emitError( - "llvm.byref attribute attached to LLVM pointer " - "argument of a different type"); - llvmArg.addAttrs(llvm::AttrBuilder(llvmArg.getContext()) - .addByRefAttr(convertType(attr.getValue()))); - } - - if (auto attr = function.getArgAttrOfType( - argIdx, LLVMDialect::getInAllocaAttrName())) { - auto argTy = mlirArgTy.dyn_cast(); - if (!argTy) - return function.emitError( - "llvm.inalloca attribute attached to LLVM non-pointer argument"); - if (!argTy.isOpaque() && argTy.getElementType() != attr.getValue()) - return function.emitError( - "llvm.inalloca attribute attached to LLVM pointer " - "argument of a different type"); - llvmArg.addAttrs(llvm::AttrBuilder(llvmArg.getContext()) - .addInAllocaAttr(convertType(attr.getValue()))); - } - - if (auto attr = - function.getArgAttrOfType(argIdx, "llvm.nest")) { - if (!mlirArgTy.isa()) - return function.emitError( - "llvm.nest attribute attached to LLVM non-pointer argument"); - llvmArg.addAttrs(llvm::AttrBuilder(llvmArg.getContext()) - .addAttribute(llvm::Attribute::Nest)); - } - - if (auto attr = function.getArgAttrOfType( - argIdx, LLVMDialect::getNoUndefAttrName())) { - // llvm.noundef can be added to any argument type. - llvmArg.addAttrs(llvm::AttrBuilder(llvmArg.getContext()) - .addAttribute(llvm::Attribute::NoUndef)); - } - if (auto attr = function.getArgAttrOfType( - argIdx, LLVMDialect::getSExtAttrName())) { - // llvm.signext can be added to any integer argument type. - if (!mlirArgTy.isa()) - return function.emitError( - "llvm.signext attribute attached to LLVM non-integer argument"); - llvmArg.addAttrs(llvm::AttrBuilder(llvmArg.getContext()) - .addAttribute(llvm::Attribute::SExt)); + for (auto [argIdx, llvmArg] : llvm::enumerate(llvmFunc->args())) { + if (DictionaryAttr argAttrs = function.getArgAttrDict(argIdx)) { + llvm::AttrBuilder attrBuilder = convertParameterAttrs(argAttrs); + llvmArg.addAttrs(attrBuilder); } - if (auto attr = function.getArgAttrOfType( - argIdx, LLVMDialect::getZExtAttrName())) { - // llvm.zeroext can be added to any integer argument type. - if (!mlirArgTy.isa()) - return function.emitError( - "llvm.zeroext attribute attached to LLVM non-integer argument"); - llvmArg.addAttrs(llvm::AttrBuilder(llvmArg.getContext()) - .addAttribute(llvm::Attribute::ZExt)); - } - - ++argIdx; } // Forward the pass-through attributes to LLVM. diff --git a/mlir/test/Dialect/LLVMIR/func.mlir b/mlir/test/Dialect/LLVMIR/func.mlir index c4afd42e91bb..4ad8f445c784 100644 --- a/mlir/test/Dialect/LLVMIR/func.mlir +++ b/mlir/test/Dialect/LLVMIR/func.mlir @@ -89,8 +89,8 @@ module { llvm.return } - // CHECK: llvm.func @byvalattr(%{{.*}}: !llvm.ptr {llvm.byval}) - llvm.func @byvalattr(%arg0: !llvm.ptr {llvm.byval}) { + // CHECK: llvm.func @byvalattr(%{{.*}}: !llvm.ptr {llvm.byval = i32}) + llvm.func @byvalattr(%arg0: !llvm.ptr {llvm.byval = i32}) { llvm.return } @@ -267,58 +267,6 @@ module { // ----- -module { - // expected-error@+1 {{cannot attach result attributes to functions with a void return}} - llvm.func @variadic_def() -> (!llvm.void {llvm.noundef}) -} - -// ----- - -// expected-error @below{{expected llvm.align result attribute to be an integer attribute}} -llvm.func @alignattr_ret() -> (!llvm.ptr {llvm.align = 1.0 : f32}) - -// ----- - -// expected-error @below{{llvm.align attribute attached to non-pointer result}} -llvm.func @alignattr_ret() -> (i32 {llvm.align = 4}) - -// ----- - -// expected-error @below{{expected llvm.noalias result attribute to be a unit attribute}} -llvm.func @noaliasattr_ret() -> (!llvm.ptr {llvm.noalias = 1}) - -// ----- - -// expected-error @below{{llvm.noalias attribute attached to non-pointer result}} -llvm.func @noaliasattr_ret() -> (i32 {llvm.noalias}) - -// ----- - -// expected-error @below{{expected llvm.noundef result attribute to be a unit attribute}} -llvm.func @noundefattr_ret() -> (!llvm.ptr {llvm.noundef = 1}) - -// ----- - -// expected-error @below{{expected llvm.signext result attribute to be a unit attribute}} -llvm.func @signextattr_ret() -> (i32 {llvm.signext = 1}) - -// ----- - -// expected-error @below{{llvm.signext attribute attached to non-integer result}} -llvm.func @signextattr_ret() -> (f32 {llvm.signext}) - -// ----- - -// expected-error @below{{expected llvm.zeroext result attribute to be a unit attribute}} -llvm.func @zeroextattr_ret() -> (i32 {llvm.zeroext = 1}) - -// ----- - -// expected-error @below{{llvm.zeroext attribute attached to non-integer result}} -llvm.func @zeroextattr_ret() -> (f32 {llvm.zeroext}) - -// ----- - module { // expected-error@+1 {{variadic arguments must be in the end of the argument list}} llvm.func @variadic_inside(%arg0: i32, ..., %arg1: i32) diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir index fe76979c9808..aa2c43ff42b7 100644 --- a/mlir/test/Dialect/LLVMIR/invalid.mlir +++ b/mlir/test/Dialect/LLVMIR/invalid.mlir @@ -33,20 +33,6 @@ llvm.func @dtor() // expected-error@+1{{'dtor' does not have a definition}} llvm.mlir.global_dtors {dtors = [@dtor], priorities = [0 : i32]} -// ----- - -// expected-error@+1{{expected llvm.noalias argument attribute to be a unit attribute}} -func.func @invalid_noalias(%arg0: i32 {llvm.noalias = 3}) { - "llvm.return"() : () -> () -} - -// ----- - -// expected-error@+1{{llvm.align argument attribute of non integer type}} -func.func @invalid_align(%arg0: i32 {llvm.align = "foo"}) { - "llvm.return"() : () -> () -} - //////////////////////////////////////////////////////////////////////////////// // Check that parser errors are properly produced and do not crash the compiler. diff --git a/mlir/test/Dialect/LLVMIR/parameter-attrs-invalid.mlir b/mlir/test/Dialect/LLVMIR/parameter-attrs-invalid.mlir new file mode 100644 index 000000000000..f6305f171013 --- /dev/null +++ b/mlir/test/Dialect/LLVMIR/parameter-attrs-invalid.mlir @@ -0,0 +1,188 @@ +// RUN: mlir-opt %s -split-input-file -verify-diagnostics + +// Argument attributes + +// expected-error@below {{"llvm.noalias" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_noalias_arg_type(%0 : i32 {llvm.noalias}) + +// ----- + +// expected-error@below {{"llvm.noalias" should be a unit attribute}} +llvm.func @invalid_noalias_attr_type(%0 : !llvm.ptr {llvm.noalias = 10 : i32}) + +// ----- + +// expected-error@below {{"llvm.readonly" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_readonly_arg_type(%0 : i32 {llvm.readonly}) + +// ----- + +// expected-error@below {{"llvm.readonly" should be a unit attribute}} +llvm.func @invalid_readonly_attr_type(%0 : i32 {llvm.readonly = i32}) + +// ----- + +// expected-error@below {{"llvm.nest" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_nest_arg_type(%0 : i32 {llvm.nest}) + +// ----- + +// expected-error@below {{"llvm.nest" should be a unit attribute}} +llvm.func @invalid_nest_attr_type(%0 : i32 {llvm.nest = "foo"}) + +// ----- + +// expected-error@below {{"llvm.align" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_align_arg_type(%0 : i32 {llvm.align = 10 : i32}) + +// ----- + +// expected-error@below {{"llvm.align" should be an integer attribute}} +llvm.func @invalid_align_attr_type(%0 : i32 {llvm.align = "foo"}) + +// ----- + +// expected-error@below {{"llvm.sret" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_sret_arg_type(%0 : i32 {llvm.sret = !llvm.struct<(i32)>}) + +// ----- + +// expected-error@below {{"llvm.sret" attribute attached to LLVM pointer argument of different type}} +llvm.func @invalid_sret_attr_type(%0 : !llvm.ptr {llvm.sret = !llvm.struct<(i32)>}) + +// ----- + +// expected-error@below {{"llvm.byval" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_byval_arg_type(%0 : i32 {llvm.byval = !llvm.struct<(i32)>}) + +// ----- + +// expected-error@below {{"llvm.byval" attribute attached to LLVM pointer argument of different type}} +llvm.func @invalid_byval_attr_type(%0 : !llvm.ptr> {llvm.byval = !llvm.struct<(i32)>}) + +// ----- + +// expected-error@below {{"llvm.byref" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_byref_arg_type(%0 : i32 {llvm.byref = !llvm.struct<(i32)>}) + +// ----- + +// expected-error@below {{"llvm.byref" attribute attached to LLVM pointer argument of different type}} +llvm.func @invalid_byref_attr_type(%0 : !llvm.ptr> {llvm.byref = !llvm.struct<(i32)>}) + +// ----- + +// expected-error@below {{"llvm.inalloca" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_inalloca_arg_type(%0 : i32 {llvm.inalloca = !llvm.struct<(i32)>}) + +// ----- + +// expected-error@below {{"llvm.inalloca" attribute attached to LLVM pointer argument of different type}} +llvm.func @invalid_inalloca_attr_type(%0 : !llvm.ptr> {llvm.inalloca = !llvm.struct<(i32)>}) + +// ----- + +// expected-error@below {{"llvm.signext" attribute attached to non-integer LLVM type}} +llvm.func @invalid_signext_arg_type(%0 : f32 {llvm.signext}) + +// ----- + +// expected-error@below {{"llvm.signext" should be a unit attribute}} +llvm.func @invalid_signext_attr_type(%0 : i32 {llvm.signext = !llvm.struct<(i32)>}) + +// ----- + +// expected-error@below {{"llvm.zeroext" attribute attached to non-integer LLVM type}} +llvm.func @invalid_zeroext_arg_type(%0 : f32 {llvm.zeroext}) + +// ----- + +// expected-error@below {{"llvm.zeroext" should be a unit attribute}} +llvm.func @invalid_zeroext_attr_type(%0 : i32 {llvm.zeroext = !llvm.struct<(i32)>}) + +// ----- + +// expected-error@below {{"llvm.noundef" should be a unit attribute}} +llvm.func @invalid_noundef_attr_type(%0 : i32 {llvm.noundef = !llvm.ptr}) + +// ----- + +// Result attributes + +// expected-error@below {{cannot attach result attributes to functions with a void return}} +llvm.func @void_def() -> (!llvm.void {llvm.noundef}) + +// ----- + +// expected-error @below{{"llvm.align" should be an integer attribute}} +llvm.func @alignattr_ret() -> (!llvm.ptr {llvm.align = 1.0 : f32}) + +// ----- + +// expected-error @below{{"llvm.align" attribute attached to non-pointer LLVM type}} +llvm.func @alignattr_ret() -> (i32 {llvm.align = 4}) + +// ----- + +// expected-error @below{{"llvm.noalias" should be a unit attribute}} +llvm.func @noaliasattr_ret() -> (!llvm.ptr {llvm.noalias = 1}) + +// ----- + +// expected-error @below{{"llvm.noalias" attribute attached to non-pointer LLVM type}} +llvm.func @noaliasattr_ret() -> (i32 {llvm.noalias}) + +// ----- + +// expected-error @below{{"llvm.noundef" should be a unit attribute}} +llvm.func @noundefattr_ret() -> (!llvm.ptr {llvm.noundef = 1}) + +// ----- + +// expected-error @below{{"llvm.signext" should be a unit attribute}} +llvm.func @signextattr_ret() -> (i32 {llvm.signext = 1}) + +// ----- + +// expected-error @below{{"llvm.signext" attribute attached to non-integer LLVM type}} +llvm.func @signextattr_ret() -> (f32 {llvm.signext}) + +// ----- + +// expected-error @below{{"llvm.zeroext" should be a unit attribute}} +llvm.func @zeroextattr_ret() -> (i32 {llvm.zeroext = 1}) + +// ----- + +// expected-error @below{{"llvm.zeroext" attribute attached to non-integer LLVM type}} +llvm.func @zeroextattr_ret() -> (f32 {llvm.zeroext}) + +// ----- + +// expected-error @below{{"llvm.readonly" is not a valid result attribute}} +llvm.func @readonly_ret() -> (f32 {llvm.readonly}) + +// ----- + +// expected-error @below{{"llvm.nest" is not a valid result attribute}} +llvm.func @nest_ret() -> (f32 {llvm.nest}) + +// ----- + +// expected-error @below{{"llvm.sret" is not a valid result attribute}} +llvm.func @sret_ret() -> (!llvm.ptr {llvm.sret = i64}) + +// ----- + +// expected-error @below{{"llvm.byval" is not a valid result attribute}} +llvm.func @byval_ret() -> (!llvm.ptr {llvm.byval = i64}) + +// ----- + +// expected-error @below{{"llvm.byref" is not a valid result attribute}} +llvm.func @byref_ret() -> (!llvm.ptr {llvm.byref = i64}) + +// ----- + +// expected-error @below{{"llvm.inalloca" is not a valid result attribute}} +llvm.func @inalloca_ret() -> (!llvm.ptr {llvm.inalloca = i64}) diff --git a/mlir/test/Target/LLVMIR/llvmir-invalid.mlir b/mlir/test/Target/LLVMIR/llvmir-invalid.mlir index e8571a992e5c..19e0b1501e09 100644 --- a/mlir/test/Target/LLVMIR/llvmir-invalid.mlir +++ b/mlir/test/Target/LLVMIR/llvmir-invalid.mlir @@ -7,96 +7,6 @@ func.func @foo() { // ----- -// expected-error @below{{llvm.noalias attribute attached to LLVM non-pointer argument}} -llvm.func @invalid_noalias(%arg0 : f32 {llvm.noalias}) -> f32 { - llvm.return %arg0 : f32 -} - -// ----- - -// expected-error @below{{llvm.sret attribute attached to LLVM non-pointer argument}} -llvm.func @invalid_sret(%arg0 : f32 {llvm.sret = f32}) -> f32 { - llvm.return %arg0 : f32 -} - -// ----- - -// expected-error @below{{llvm.sret attribute attached to LLVM pointer argument of a different type}} -llvm.func @invalid_sret(%arg0 : !llvm.ptr {llvm.sret = i32}) -> !llvm.ptr { - llvm.return %arg0 : !llvm.ptr -} - -// ----- - -// expected-error @below{{llvm.nest attribute attached to LLVM non-pointer argument}} -llvm.func @invalid_nest(%arg0 : f32 {llvm.nest}) -> f32 { - llvm.return %arg0 : f32 -} -// ----- - -// expected-error @below{{llvm.byval attribute attached to LLVM non-pointer argument}} -llvm.func @invalid_byval(%arg0 : f32 {llvm.byval = f32}) -> f32 { - llvm.return %arg0 : f32 -} - -// ----- - -// expected-error @below{{llvm.byval attribute attached to LLVM pointer argument of a different type}} -llvm.func @invalid_sret(%arg0 : !llvm.ptr {llvm.byval = i32}) -> !llvm.ptr { - llvm.return %arg0 : !llvm.ptr -} - -// ----- - -// expected-error @below{{llvm.byref attribute attached to LLVM non-pointer argument}} -llvm.func @invalid_byval(%arg0 : f32 {llvm.byref = f32}) -> f32 { - llvm.return %arg0 : f32 -} - -// ----- - -// expected-error @below{{llvm.byref attribute attached to LLVM pointer argument of a different type}} -llvm.func @invalid_sret(%arg0 : !llvm.ptr {llvm.byref = i32}) -> !llvm.ptr { - llvm.return %arg0 : !llvm.ptr -} - -// ----- - -// expected-error @below{{llvm.inalloca attribute attached to LLVM non-pointer argument}} -llvm.func @invalid_byval(%arg0 : f32 {llvm.inalloca = f32}) -> f32 { - llvm.return %arg0 : f32 -} - -// ----- - -// expected-error @below{{llvm.inalloca attribute attached to LLVM pointer argument of a different type}} -llvm.func @invalid_sret(%arg0 : !llvm.ptr {llvm.inalloca = i32}) -> !llvm.ptr { - llvm.return %arg0 : !llvm.ptr -} - -// ----- - -// expected-error @below{{llvm.align attribute attached to LLVM non-pointer argument}} -llvm.func @invalid_align(%arg0 : f32 {llvm.align = 4}) -> f32 { - llvm.return %arg0 : f32 -} - -// ----- - -// expected-error @below{{llvm.signext attribute attached to LLVM non-integer argument}} -llvm.func @invalid_signext(%arg0: f32 {llvm.signext}) { - "llvm.return"() : () -> () -} - -// ----- - -// expected-error @below{{llvm.zeroext attribute attached to LLVM non-integer argument}} -llvm.func @invalid_zeroext(%arg0: f32 {llvm.zeroext}) { - "llvm.return"() : () -> () -} - -// ----- - llvm.func @no_non_complex_struct() -> !llvm.array<2 x array<2 x array<2 x struct<(i32)>>>> { // expected-error @below{{expected struct type to be a complex number}} %0 = llvm.mlir.constant(dense<[[[1, 2], [3, 4]], [[42, 43], [44, 45]]]> : tensor<2x2x2xi32>) : !llvm.array<2 x array<2 x array<2 x struct<(i32)>>>> @@ -348,10 +258,3 @@ llvm.func @stepvector_intr_wrong_type() -> vector<7xf32> { %0 = llvm.intr.experimental.stepvector : vector<7xf32> llvm.return %0 : vector<7xf32> } - -// ----- - -// expected-error @below{{llvm.readonly attribute attached to LLVM non-pointer argument}} -llvm.func @wrong_readonly_attribute(%vec : f32 {llvm.readonly}) { - llvm.return -} From 3bc0a9833c8878cffb005d76b3f43faf34dca30d Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Mon, 6 Mar 2023 08:56:20 +0100 Subject: [PATCH 054/147] [mlir][LLVM] Add result attribute import support This commit introduces support for importing result attributes. Depends on D142372 Reviewed By: gysit Differential Revision: https://reviews.llvm.org/D142476 (cherry picked from commit 56880fd257e22cdc4846e9674ada9fa1d70b3ecb) --- .../include/mlir/Target/LLVMIR/ModuleImport.h | 8 ++ mlir/lib/Target/LLVMIR/ModuleImport.cpp | 85 ++++++++++++------- .../LLVMIR/Import/function-attributes.ll | 40 ++++++++- 3 files changed, 103 insertions(+), 30 deletions(-) diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h index 62a60a4c0d46..58b8ae39f301 100644 --- a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h +++ b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h @@ -210,6 +210,14 @@ class ModuleImport { LogicalResult convertCallTypeAndOperands(llvm::CallBase *callInst, SmallVectorImpl &types, SmallVectorImpl &operands); + /// Converts the parameter attributes attached to `func` and adds them to the + /// `funcOp`. + void convertParameterAttributes(llvm::Function *func, LLVMFuncOp funcOp, + OpBuilder &builder); + /// Converts the AttributeSet of one parameter in LLVM IR to a corresponding + /// DictionaryAttr for the LLVM dialect. + DictionaryAttr convertParameterAttribute(llvm::AttributeSet llvmParamAttrs, + OpBuilder &builder); /// Returns the builtin type equivalent to be used in attributes for the given /// LLVM IR dialect type. Type getStdTypeForAttr(Type type); diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp index 17cdd5bb8855..979762e794bc 100644 --- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp @@ -1482,6 +1482,61 @@ void ModuleImport::processFunctionAttributes(llvm::Function *func, processPassthroughAttrs(func, funcOp); } +DictionaryAttr +ModuleImport::convertParameterAttribute(llvm::AttributeSet llvmParamAttrs, + OpBuilder &builder) { + using ElemTy = std::pair; + // Mapping from llvm attribute kinds to their corresponding MLIR name. + static const SmallVector kindNamePairs = { + {llvm::Attribute::AttrKind::NoAlias, LLVMDialect::getNoAliasAttrName()}, + {llvm::Attribute::AttrKind::ReadOnly, LLVMDialect::getReadonlyAttrName()}, + {llvm::Attribute::AttrKind::Nest, LLVMDialect::getNestAttrName()}, + {llvm::Attribute::AttrKind::SExt, LLVMDialect::getSExtAttrName()}, + {llvm::Attribute::AttrKind::ZExt, LLVMDialect::getZExtAttrName()}, + {llvm::Attribute::AttrKind::NoUndef, LLVMDialect::getNoUndefAttrName()}, + {llvm::Attribute::AttrKind::StructRet, + LLVMDialect::getStructRetAttrName()}, + {llvm::Attribute::AttrKind::ByVal, LLVMDialect::getByValAttrName()}, + {llvm::Attribute::AttrKind::ByRef, LLVMDialect::getByRefAttrName()}, + {llvm::Attribute::AttrKind::InAlloca, LLVMDialect::getInAllocaAttrName()}, + {llvm::Attribute::AttrKind::Alignment, LLVMDialect::getAlignAttrName()}}; + + SmallVector paramAttrs; + for (auto [llvmKind, mlirName] : kindNamePairs) { + auto llvmAttr = llvmParamAttrs.getAttribute(llvmKind); + // Skip attributes that are not attached. + if (!llvmAttr.isValid()) + continue; + Attribute mlirAttr; + if (llvmAttr.isTypeAttribute()) + mlirAttr = TypeAttr::get(convertType(llvmAttr.getValueAsType())); + else if (llvmAttr.isIntAttribute()) + mlirAttr = builder.getI64IntegerAttr(llvmAttr.getValueAsInt()); + else if (llvmAttr.isEnumAttribute()) + mlirAttr = builder.getUnitAttr(); + else + llvm_unreachable("unexpected parameter attribute kind"); + paramAttrs.push_back(builder.getNamedAttr(mlirName, mlirAttr)); + } + + return builder.getDictionaryAttr(paramAttrs); +} + +void ModuleImport::convertParameterAttributes(llvm::Function *func, + LLVMFuncOp funcOp, + OpBuilder &builder) { + auto llvmAttrs = func->getAttributes(); + for (size_t i = 0, e = funcOp.getNumArguments(); i < e; ++i) { + llvm::AttributeSet llvmArgAttrs = llvmAttrs.getParamAttrs(i); + funcOp.setArgAttrs(i, convertParameterAttribute(llvmArgAttrs, builder)); + } + // Convert the result attributes and attach them wrapped in an ArrayAttribute + // to the funcOp. + llvm::AttributeSet llvmResAttr = llvmAttrs.getRetAttrs(); + funcOp.setResAttrsAttr( + builder.getArrayAttr(convertParameterAttribute(llvmResAttr, builder))); +} + LogicalResult ModuleImport::processFunction(llvm::Function *func) { clearBlockAndValueMapping(); @@ -1505,35 +1560,7 @@ LogicalResult ModuleImport::processFunction(llvm::Function *func) { // Set the function debug information if available. debugImporter->translate(func, funcOp); - for (const auto &it : llvm::enumerate(functionType.getParams())) { - llvm::SmallVector argAttrs; - if (auto *type = func->getParamByValType(it.index())) { - Type mlirType = convertType(type); - argAttrs.push_back( - NamedAttribute(builder.getStringAttr(LLVMDialect::getByValAttrName()), - TypeAttr::get(mlirType))); - } - if (auto *type = func->getParamByRefType(it.index())) { - Type mlirType = convertType(type); - argAttrs.push_back( - NamedAttribute(builder.getStringAttr(LLVMDialect::getByRefAttrName()), - TypeAttr::get(mlirType))); - } - if (auto *type = func->getParamStructRetType(it.index())) { - Type mlirType = convertType(type); - argAttrs.push_back(NamedAttribute( - builder.getStringAttr(LLVMDialect::getStructRetAttrName()), - TypeAttr::get(mlirType))); - } - if (auto *type = func->getParamInAllocaType(it.index())) { - Type mlirType = convertType(type); - argAttrs.push_back(NamedAttribute( - builder.getStringAttr(LLVMDialect::getInAllocaAttrName()), - TypeAttr::get(mlirType))); - } - - funcOp.setArgAttrs(it.index(), argAttrs); - } + convertParameterAttributes(func, funcOp, builder); if (FlatSymbolRefAttr personality = getPersonalityAsAttr(func)) funcOp.setPersonalityAttr(personality); diff --git a/mlir/test/Target/LLVMIR/Import/function-attributes.ll b/mlir/test/Target/LLVMIR/Import/function-attributes.ll index 163020049c79..ed38a160afa6 100644 --- a/mlir/test/Target/LLVMIR/Import/function-attributes.ll +++ b/mlir/test/Target/LLVMIR/Import/function-attributes.ll @@ -31,14 +31,52 @@ attributes #0 = { readnone } ; CHECK-SAME: !llvm.ptr {llvm.byref = i64} ; CHECK-SAME: !llvm.ptr {llvm.sret = i64} ; CHECK-SAME: !llvm.ptr {llvm.inalloca = i64} +; CHECK-SAME: !llvm.ptr {llvm.noalias} +; CHECK-SAME: !llvm.ptr {llvm.readonly} +; CHECK-SAME: !llvm.ptr {llvm.nest} +; CHECK-SAME: i32 {llvm.signext} +; CHECK-SAME: i64 {llvm.zeroext} +; CHECK-SAME: !llvm.ptr {llvm.align = 64 : i64, llvm.noundef} define void @func_arg_attrs( ptr byval(i64) %arg0, ptr byref(i64) %arg1, ptr sret(i64) %arg2, - ptr inalloca(i64) %arg3) { + ptr inalloca(i64) %arg3, + ptr noalias %arg4, + ptr readonly %arg5, + ptr nest %arg6, + i32 signext %arg7, + i64 zeroext %arg8, + ptr align(64) noundef %arg9) { ret void } +; // ----- + +; CHECK-LABEL: @func_res_attr_align +; CHECK-SAME: !llvm.ptr {llvm.align = 16 : i64} +declare align(16) ptr @func_res_attr_align() + +; // ----- + +; CHECK-LABEL: @func_res_attr_noalias +; CHECK-SAME: !llvm.ptr {llvm.noalias} +declare noalias ptr @func_res_attr_noalias() + +; // ----- + +; CHECK-LABEL: @func_res_attr_signext +; CHECK-DAG: llvm.noundef +; CHECK-DAG: llvm.signext +declare noundef signext i32 @func_res_attr_signext() + +; // ----- + +; CHECK-LABEL: @func_res_attr_zeroext +; CHECK-SAME: i32 {llvm.zeroext} +declare zeroext i32 @func_res_attr_zeroext() + + ; // ----- ; CHECK-LABEL: @entry_count From f51b5384ed4ae0eccbcd7edf3d81b637da2ac3f5 Mon Sep 17 00:00:00 2001 From: Tobias Gysi Date: Mon, 6 Mar 2023 09:21:55 +0100 Subject: [PATCH 055/147] [mlir][llvm] Import access group metadata. The revision adds support to import access group metadata from LLVM IR. It closely follows the design of the TBAA metadata import with an up-front conversion of the metadata nodes to operations stored in the body of a module-level metadata operation. The revision chooses to use only one module-level metadata operation for all kinds of metadata. This design ensures there is only one metadata operation that pollutes the user namespace. The import of loop metadata, which will use the access groups, is left to a follow up revision. Reviewed By: Dinistro Differential Revision: https://reviews.llvm.org/D142605 (cherry picked from commit af3a5ef171f9d6a125bde43d8b4d09a8cffc33e0) --- .../include/mlir/Target/LLVMIR/ModuleImport.h | 63 ++++++----- .../LLVMIR/LLVMIRToLLVMTranslation.cpp | 54 +++++++--- mlir/lib/Target/LLVMIR/ModuleImport.cpp | 100 +++++++++++++----- .../Target/LLVMIR/Import/import-failure.ll | 23 ++++ .../Target/LLVMIR/Import/metadata-loop.ll | 27 +++++ ...ling-metadata.ll => metadata-profiling.ll} | 0 .../Import/{tbaa.ll => metadata-tbaa.ll} | 12 +-- 7 files changed, 202 insertions(+), 77 deletions(-) create mode 100644 mlir/test/Target/LLVMIR/Import/metadata-loop.ll rename mlir/test/Target/LLVMIR/Import/{profiling-metadata.ll => metadata-profiling.ll} (100%) rename mlir/test/Target/LLVMIR/Import/{tbaa.ll => metadata-tbaa.ll} (88%) diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h index 58b8ae39f301..a19123e9ee2f 100644 --- a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h +++ b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h @@ -149,17 +149,27 @@ class ModuleImport { /// implement the fastmath interface. void setFastmathFlagsAttr(llvm::Instruction *inst, Operation *op) const; - /// Converts LLVM metadata to corresponding MLIR representation, - /// e.g. metadata nodes referenced via !tbaa are converted to - /// TBAA operations hosted inside a MetadataOp. + /// Converts all LLVM metadata nodes that translate to operations nested in a + /// global metadata operation, such as alias analysis or access group + /// metadata, and builds a map from the metadata nodes to the symbols pointing + /// to the converted operations. Returns success if all conversions succeed + /// and failure otherwise. + // Note: All metadata is nested inside a single global metadata operation to + // minimize the number of symbols that pollute the global namespace. LogicalResult convertMetadata(); - /// Returns SymbolRefAttr representing TBAA metadata `node` - /// in `tbaaMapping`. - SymbolRefAttr lookupTBAAAttr(const llvm::MDNode *node) { + /// Returns the MLIR symbol reference mapped to the given LLVM TBAA + /// metadata `node`. + SymbolRefAttr lookupTBAAAttr(const llvm::MDNode *node) const { return tbaaMapping.lookup(node); } + /// Returns the MLIR symbol reference mapped to the given LLVM access + /// group metadata `node`. + SymbolRefAttr lookupAccessGroupAttr(const llvm::MDNode *node) const { + return accessGroupMapping.lookup(node); + } + private: /// Clears the block and value mapping before processing a new region. void clearBlockAndValueMapping() { @@ -237,25 +247,21 @@ class ModuleImport { /// them fails. All operations are inserted at the start of the current /// function entry block. FailureOr convertConstantExpr(llvm::Constant *constant); - /// Returns symbol name to be used for MetadataOp containing - /// TBAA metadata operations. It must not conflict with the user - /// name space. - StringRef getTBAAMetadataOpName() const { return "__tbaa"; } - /// Returns a terminated MetadataOp into which TBAA metadata - /// operations can be placed. The MetadataOp is created - /// on the first invocation of this function. - MetadataOp getTBAAMetadataOp(); + /// Returns a global metadata operation that serves as a container for LLVM + /// metadata that converts to MLIR operations. Creates the global metadata + /// operation on the first invocation. + MetadataOp getGlobalMetadataOp(); /// Performs conversion of LLVM TBAA metadata starting from /// `node`. On exit from this function all nodes reachable /// from `node` are converted, and tbaaMapping map is updated /// (unless all dependencies have been converted by a previous /// invocation of this function). LogicalResult processTBAAMetadata(const llvm::MDNode *node); - /// Returns unique string name of a symbol that may be used - /// for a TBAA metadata operation. The name will contain - /// the provided `basename` and will be uniqued via - /// tbaaNodeCounter (see below). - std::string getNewTBAANodeName(StringRef basename); + /// Converts all LLVM access groups starting from `node` to MLIR access group + /// operations and stores a mapping from every nested access group node to the + /// symbol pointing to the translated operation. Returns success if all + /// conversions succeed and failure otherwise. + LogicalResult processAccessGroupMetadata(const llvm::MDNode *node); /// Builder pointing at where the next instruction should be generated. OpBuilder builder; @@ -265,6 +271,8 @@ class ModuleImport { Operation *constantInsertionOp = nullptr; /// Operation to insert the next global after. Operation *globalInsertionOp = nullptr; + /// Operation to insert metadata operations into. + MetadataOp globalMetadataOp = nullptr; /// The current context. MLIRContext *context; /// The MLIR module being created. @@ -284,20 +292,17 @@ class ModuleImport { /// operations for all operations that return no result. All operations that /// return a result have a valueMapping entry instead. DenseMap noResultOpMapping; - /// The stateful type translator (contains named structs). - LLVM::TypeFromLLVMIRTranslator typeTranslator; - /// Stateful debug information importer. - std::unique_ptr debugImporter; - /// A terminated MetadataOp where TBAA metadata operations - /// can be inserted. - MetadataOp tbaaMetadataOp{}; /// Mapping between LLVM TBAA metadata nodes and symbol references /// to the LLVMIR dialect TBAA operations corresponding to these /// nodes. DenseMap tbaaMapping; - /// A counter to be used as a unique suffix for symbols - /// defined by TBAA operations. - unsigned tbaaNodeCounter = 0; + /// Mapping between original LLVM access group metadata nodes and the symbol + /// references pointing to the imported MLIR access group operations. + DenseMap accessGroupMapping; + /// The stateful type translator (contains named structs). + LLVM::TypeFromLLVMIRTranslator typeTranslator; + /// Stateful debug information importer. + std::unique_ptr debugImporter; }; } // namespace LLVM diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp index 7d55eae86cdd..da15cd9b6624 100644 --- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp @@ -71,17 +71,17 @@ static LogicalResult convertIntrinsicImpl(OpBuilder &odsBuilder, /// dialect attributes. static ArrayRef getSupportedMetadataImpl() { static const SmallVector convertibleMetadata = { - llvm::LLVMContext::MD_prof, // profiling metadata - llvm::LLVMContext::MD_tbaa}; + llvm::LLVMContext::MD_prof, llvm::LLVMContext::MD_tbaa, + llvm::LLVMContext::MD_access_group}; return convertibleMetadata; } -/// Attaches the given profiling metadata to the imported operation if a -/// conversion to an MLIR profiling attribute exists and succeeds. Returns -/// failure otherwise. -static LogicalResult setProfilingAttrs(OpBuilder &builder, llvm::MDNode *node, - Operation *op, - LLVM::ModuleImport &moduleImport) { +/// Converts the given profiling metadata `node` to an MLIR profiling attribute +/// and attaches it to the imported operation if the translation succeeds. +/// Returns failure otherwise. +static LogicalResult setProfilingAttr(OpBuilder &builder, llvm::MDNode *node, + Operation *op, + LLVM::ModuleImport &moduleImport) { // Return success for empty metadata nodes since there is nothing to import. if (!node->getNumOperands()) return success(); @@ -127,11 +127,11 @@ static LogicalResult setProfilingAttrs(OpBuilder &builder, llvm::MDNode *node, .Default([](auto) { return failure(); }); } -/// Attaches the given TBAA metadata `node` to the imported operation. -/// Returns success, if the metadata has been converted and the attachment -/// succeeds, failure - otherwise. -static LogicalResult setTBAAAttrs(const llvm::MDNode *node, Operation *op, - LLVM::ModuleImport &moduleImport) { +/// Searches the symbol reference pointing to the metadata operation that +/// maps to the given TBAA metadata `node` and attaches it to the imported +/// operation if the lookup succeeds. Returns failure otherwise. +static LogicalResult setTBAAAttr(const llvm::MDNode *node, Operation *op, + LLVM::ModuleImport &moduleImport) { SymbolRefAttr tbaaTagSym = moduleImport.lookupTBAAAttr(node); if (!tbaaTagSym) return failure(); @@ -141,6 +141,28 @@ static LogicalResult setTBAAAttrs(const llvm::MDNode *node, Operation *op, return success(); } +/// Searches the symbol references pointing to the access group operations that +/// map to the access group nodes starting from the access group metadata +/// `node`, and attaches all of them to the imported operation if the lookups +/// succeed. Returns failure otherwise. +static LogicalResult setAccessGroupAttr(const llvm::MDNode *node, Operation *op, + LLVM::ModuleImport &moduleImport) { + // An access group node is either access group or an access group list. + SmallVector accessGroups; + if (!node->getNumOperands()) + accessGroups.push_back(moduleImport.lookupAccessGroupAttr(node)); + for (const llvm::MDOperand &operand : node->operands()) { + auto *node = cast(operand.get()); + accessGroups.push_back(moduleImport.lookupAccessGroupAttr(node)); + } + // Exit if one of the access group node lookups failed. + if (llvm::is_contained(accessGroups, nullptr)) + return failure(); + + op->setAttr(LLVMDialect::getAccessGroupsAttrName(), + ArrayAttr::get(op->getContext(), accessGroups)); + return success(); +} namespace { /// Implementation of the dialect interface that converts operations belonging @@ -164,9 +186,11 @@ class LLVMDialectLLVMIRImportInterface : public LLVMImportDialectInterface { LLVM::ModuleImport &moduleImport) const final { // Call metadata specific handlers. if (kind == llvm::LLVMContext::MD_prof) - return setProfilingAttrs(builder, node, op, moduleImport); + return setProfilingAttr(builder, node, op, moduleImport); if (kind == llvm::LLVMContext::MD_tbaa) - return setTBAAAttrs(node, op, moduleImport); + return setTBAAAttr(node, op, moduleImport); + if (kind == llvm::LLVMContext::MD_access_group) + return setAccessGroupAttr(node, op, moduleImport); // A handler for a supported metadata kind is missing. llvm_unreachable("unknown metadata type"); diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp index 979762e794bc..3e1e94c9e92e 100644 --- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp @@ -71,6 +71,12 @@ static constexpr StringRef getGlobalDtorsVarName() { return "llvm.global_dtors"; } +/// Returns the symbol name for the module-level metadata operation. It must not +/// conflict with the user namespace. +static constexpr StringRef getGlobalMetadataOpName() { + return "__llvm_global_metadata"; +} + /// Returns a supported MLIR floating point type of the given bit width or null /// if the bit width is not supported. static FloatType getDLFloatType(MLIRContext &ctx, int32_t bitwidth) { @@ -359,23 +365,14 @@ ModuleImport::ModuleImport(ModuleOp mlirModule, builder.setInsertionPointToStart(mlirModule.getBody()); } -MetadataOp ModuleImport::getTBAAMetadataOp() { - if (tbaaMetadataOp) - return tbaaMetadataOp; +MetadataOp ModuleImport::getGlobalMetadataOp() { + if (globalMetadataOp) + return globalMetadataOp; OpBuilder::InsertionGuard guard(builder); - Location loc = mlirModule.getLoc(); - builder.setInsertionPointToEnd(mlirModule.getBody()); - tbaaMetadataOp = builder.create(loc, getTBAAMetadataOpName()); - - return tbaaMetadataOp; -} - -std::string ModuleImport::getNewTBAANodeName(StringRef basename) { - return (Twine("tbaa_") + Twine(basename) + Twine('_') + - Twine(tbaaNodeCounter++)) - .str(); + return globalMetadataOp = builder.create( + mlirModule.getLoc(), getGlobalMetadataOpName()); } LogicalResult ModuleImport::processTBAAMetadata(const llvm::MDNode *node) { @@ -534,10 +531,18 @@ LogicalResult ModuleImport::processTBAAMetadata(const llvm::MDNode *node) { return true; }; + // Helper to compute a unique symbol name that includes the given `baseName`. + // Uses the size of the mapping to unique the symbol name. + auto getUniqueSymbolName = [&](StringRef baseName) { + return (Twine("tbaa_") + Twine(baseName) + Twine('_') + + Twine(tbaaMapping.size())) + .str(); + }; + // Insert new operations at the end of the MetadataOp. OpBuilder::InsertionGuard guard(builder); - builder.setInsertionPointToEnd(&getTBAAMetadataOp().getBody().back()); - StringAttr metadataOpName = SymbolTable::getSymbolName(getTBAAMetadataOp()); + builder.setInsertionPointToEnd(&getGlobalMetadataOp().getBody().back()); + StringAttr metadataOpName = SymbolTable::getSymbolName(getGlobalMetadataOp()); // On the first walk, create SymbolRefAttr's and map them // to nodes in `nodesToConvert`. @@ -550,7 +555,7 @@ LogicalResult ModuleImport::processTBAAMetadata(const llvm::MDNode *node) { // The root nodes do not have operands, so we can create // the TBAARootMetadataOp on the first walk. auto rootNode = builder.create( - loc, getNewTBAANodeName("root"), identity.value()); + loc, getUniqueSymbolName("root"), identity.value()); tbaaMapping.try_emplace(current, FlatSymbolRefAttr::get(rootNode)); continue; } @@ -559,7 +564,7 @@ LogicalResult ModuleImport::processTBAAMetadata(const llvm::MDNode *node) { return failure(); tbaaMapping.try_emplace( current, FlatSymbolRefAttr::get(builder.getContext(), - getNewTBAANodeName("type_desc"))); + getUniqueSymbolName("type_desc"))); continue; } if (std::optional isValid = isTagNode(current)) { @@ -571,7 +576,7 @@ LogicalResult ModuleImport::processTBAAMetadata(const llvm::MDNode *node) { current, SymbolRefAttr::get( builder.getContext(), metadataOpName, FlatSymbolRefAttr::get(builder.getContext(), - getNewTBAANodeName("tag")))); + getUniqueSymbolName("tag")))); continue; } return emitError(loc) << "unsupported TBAA node format: " @@ -611,21 +616,62 @@ LogicalResult ModuleImport::processTBAAMetadata(const llvm::MDNode *node) { return success(); } +LogicalResult +ModuleImport::processAccessGroupMetadata(const llvm::MDNode *node) { + // An access group node is either access group or an access group list. Start + // by collecting all access groups to translate. + SmallVector accessGroups; + if (!node->getNumOperands()) + accessGroups.push_back(node); + for (const llvm::MDOperand &operand : node->operands()) + accessGroups.push_back(cast(operand.get())); + + // Convert all entries of the access group list to access group operations. + for (const llvm::MDNode *accessGroup : accessGroups) { + if (accessGroupMapping.count(accessGroup)) + continue; + // Verify the access group node is distinct and empty. + Location loc = mlirModule.getLoc(); + if (accessGroup->getNumOperands() != 0 || !accessGroup->isDistinct()) + return emitError(loc) << "unsupported access group node: " + << diagMD(accessGroup, llvmModule.get()); + + MetadataOp metadataOp = getGlobalMetadataOp(); + OpBuilder::InsertionGuard guard(builder); + builder.setInsertionPointToEnd(&metadataOp.getBody().back()); + auto groupOp = builder.create( + loc, (Twine("group_") + Twine(accessGroupMapping.size())).str()); + // Add a mapping from the access group node to the symbol reference pointing + // to the newly created operation. + accessGroupMapping[accessGroup] = SymbolRefAttr::get( + builder.getContext(), metadataOp.getSymName(), + FlatSymbolRefAttr::get(builder.getContext(), groupOp.getSymName())); + } + return success(); +} + LogicalResult ModuleImport::convertMetadata() { OpBuilder::InsertionGuard guard(builder); builder.setInsertionPointToEnd(mlirModule.getBody()); - for (const llvm::Function &func : llvmModule->functions()) + for (const llvm::Function &func : llvmModule->functions()) { for (const llvm::Instruction &inst : llvm::instructions(func)) { - llvm::AAMDNodes nodes = inst.getAAMetadata(); - if (!nodes) - continue; + // Convert access group metadata nodes. + if (llvm::MDNode *node = + inst.getMetadata(llvm::LLVMContext::MD_access_group)) + if (failed(processAccessGroupMetadata(node))) + return failure(); - if (const llvm::MDNode *tbaaMD = nodes.TBAA) - if (failed(processTBAAMetadata(tbaaMD))) + // Convert alias analysis metadata nodes. + llvm::AAMDNodes aliasAnalysisNodes = inst.getAAMetadata(); + if (!aliasAnalysisNodes) + continue; + if (aliasAnalysisNodes.TBAA) + if (failed(processTBAAMetadata(aliasAnalysisNodes.TBAA))) return failure(); - // TODO: only TBAA metadata is currently supported. - } + // TODO: Support noalias and scope metadata nodes. + } + } return success(); } diff --git a/mlir/test/Target/LLVMIR/Import/import-failure.ll b/mlir/test/Target/LLVMIR/Import/import-failure.ll index 0570fe4bdeeb..107d886d697b 100644 --- a/mlir/test/Target/LLVMIR/Import/import-failure.ll +++ b/mlir/test/Target/LLVMIR/Import/import-failure.ll @@ -239,3 +239,26 @@ define dso_local void @tbaa(ptr %0) { !3 = !{!4, i64 4, !"int"} !4 = !{!5, i64 1, !"omnipotent char"} !5 = !{!"Simple C++ TBAA"} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: error: unsupported access group node: !0 = !{} +define void @access_group(ptr %arg1) { + %1 = load i32, ptr %arg1, !llvm.access.group !0 + ret void +} + +!0 = !{} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: error: unsupported access group node: !1 = distinct !{!"unsupported access group"} +define void @access_group(ptr %arg1) { + %1 = load i32, ptr %arg1, !llvm.access.group !0 + ret void +} + +!0 = !{!1} +!1 = distinct !{!"unsupported access group"} diff --git a/mlir/test/Target/LLVMIR/Import/metadata-loop.ll b/mlir/test/Target/LLVMIR/Import/metadata-loop.ll new file mode 100644 index 000000000000..87ac08ecd7e8 --- /dev/null +++ b/mlir/test/Target/LLVMIR/Import/metadata-loop.ll @@ -0,0 +1,27 @@ +; RUN: mlir-translate -import-llvm -split-input-file %s | FileCheck %s + +; CHECK: llvm.metadata @__llvm_global_metadata { +; CHECK: llvm.access_group @[[$GROUP0:.*]] +; CHECK: llvm.access_group @[[$GROUP1:.*]] +; CHECK: llvm.access_group @[[$GROUP2:.*]] +; CHECK: llvm.access_group @[[$GROUP3:.*]] +; CHECK: } + +; CHECK-LABEL: llvm.func @access_group +; CHECK-SAME: %[[ARG1:[a-zA-Z0-9]+]] +define void @access_group(ptr %arg1) { + ; CHECK: llvm.load %[[ARG1]] {access_groups = [@__llvm_global_metadata::@[[$GROUP0]], @__llvm_global_metadata::@[[$GROUP1]]]} + %1 = load i32, ptr %arg1, !llvm.access.group !0 + ; CHECK: llvm.load %[[ARG1]] {access_groups = [@__llvm_global_metadata::@[[$GROUP2]], @__llvm_global_metadata::@[[$GROUP0]]]} + %2 = load i32, ptr %arg1, !llvm.access.group !1 + ; CHECK: llvm.load %[[ARG1]] {access_groups = [@__llvm_global_metadata::@[[$GROUP3]]]} + %3 = load i32, ptr %arg1, !llvm.access.group !2 + ret void +} + +!0 = !{!3, !4} +!1 = !{!5, !3} +!2 = distinct !{} +!3 = distinct !{} +!4 = distinct !{} +!5 = distinct !{} diff --git a/mlir/test/Target/LLVMIR/Import/profiling-metadata.ll b/mlir/test/Target/LLVMIR/Import/metadata-profiling.ll similarity index 100% rename from mlir/test/Target/LLVMIR/Import/profiling-metadata.ll rename to mlir/test/Target/LLVMIR/Import/metadata-profiling.ll diff --git a/mlir/test/Target/LLVMIR/Import/tbaa.ll b/mlir/test/Target/LLVMIR/Import/metadata-tbaa.ll similarity index 88% rename from mlir/test/Target/LLVMIR/Import/tbaa.ll rename to mlir/test/Target/LLVMIR/Import/metadata-tbaa.ll index 7daf7d2112be..c0cbe7b9bdd0 100644 --- a/mlir/test/Target/LLVMIR/Import/tbaa.ll +++ b/mlir/test/Target/LLVMIR/Import/metadata-tbaa.ll @@ -2,7 +2,7 @@ // ----- -; CHECK-LABEL: llvm.metadata @__tbaa { +; CHECK-LABEL: llvm.metadata @__llvm_global_metadata { ; CHECK-NEXT: llvm.tbaa_root @[[R0:tbaa_root_[0-9]+]] {id = "Simple C/C++ TBAA"} ; CHECK-NEXT: llvm.tbaa_tag @[[T0:tbaa_tag_[0-9]+]] {access_type = @[[R0]], base_type = @[[R0]], offset = 0 : i64} ; CHECK-NEXT: llvm.tbaa_root @[[R1:tbaa_root_[0-9]+]] {id = "Other language TBAA"} @@ -10,10 +10,10 @@ ; CHECK-NEXT: } ; CHECK: llvm.func @tbaa1 ; CHECK: llvm.store %{{.*}}, %{{.*}} { -; CHECK-SAME: tbaa = [@__tbaa::@[[T0]]] +; CHECK-SAME: tbaa = [@__llvm_global_metadata::@[[T0]]] ; CHECK-SAME: } : i8, !llvm.ptr ; CHECK: llvm.store %{{.*}}, %{{.*}} { -; CHECK-SAME: tbaa = [@__tbaa::@[[T1]]] +; CHECK-SAME: tbaa = [@__llvm_global_metadata::@[[T1]]] ; CHECK-SAME: } : i8, !llvm.ptr define dso_local void @tbaa1(ptr %0, ptr %1) { store i8 1, ptr %0, align 4, !tbaa !1 @@ -28,7 +28,7 @@ define dso_local void @tbaa1(ptr %0, ptr %1) { // ----- -; CHECK-LABEL: llvm.metadata @__tbaa { +; CHECK-LABEL: llvm.metadata @__llvm_global_metadata { ; CHECK-NEXT: llvm.tbaa_root @[[R0:tbaa_root_[0-9]+]] {id = "Simple C/C++ TBAA"} ; CHECK-NEXT: llvm.tbaa_tag @[[T0:tbaa_tag_[0-9]+]] {access_type = @[[D1:tbaa_type_desc_[0-9]+]], base_type = @[[D2:tbaa_type_desc_[0-9]+]], offset = 8 : i64} ; CHECK-NEXT: llvm.tbaa_type_desc @[[D1]] {id = "long long", members = {<@[[D0:tbaa_type_desc_[0-9]+]], 0>}} @@ -40,10 +40,10 @@ define dso_local void @tbaa1(ptr %0, ptr %1) { ; CHECK-NEXT: } ; CHECK: llvm.func @tbaa2 ; CHECK: llvm.load %{{.*}} { -; CHECK-SAME: tbaa = [@__tbaa::@[[T0]]] +; CHECK-SAME: tbaa = [@__llvm_global_metadata::@[[T0]]] ; CHECK-SAME: } : !llvm.ptr -> i64 ; CHECK: llvm.store %{{.*}}, %{{.*}} { -; CHECK-SAME: tbaa = [@__tbaa::@[[T1]]] +; CHECK-SAME: tbaa = [@__llvm_global_metadata::@[[T1]]] ; CHECK-SAME: } : i32, !llvm.ptr %struct.agg2_t = type { i64, i64 } %struct.agg1_t = type { i32, i32 } From c3727e8f27981fdba40e25637139ab3097a70c47 Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Mon, 6 Mar 2023 10:00:56 +0100 Subject: [PATCH 056/147] Reland "[mlir][LLVM] Add all LLVM parameter attributes" This change was reverted because it introduced a linking issue due to duplicated symbols. Making sure that the detail helper only has a static header implementation fixes this issue. Reviewed By: gysit Differential Revision: https://reviews.llvm.org/D142635 (cherry picked from commit 5212058405ba9ed1292920f075c2ae06f67cc5f7) --- .../include/mlir/Dialect/LLVMIR/LLVMOpBase.td | 40 +++-- mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 38 ++++- mlir/lib/Target/LLVMIR/AttrKindDetail.h | 66 ++++++++ mlir/lib/Target/LLVMIR/ModuleImport.cpp | 19 +-- mlir/lib/Target/LLVMIR/ModuleTranslation.cpp | 53 +++---- .../LLVMIR/parameter-attrs-invalid.mlir | 149 ++++++++++++++++-- .../LLVMIR/Import/function-attributes.ll | 70 +++++++- mlir/test/Target/LLVMIR/llvmir.mlir | 49 ++++++ 8 files changed, 395 insertions(+), 89 deletions(-) create mode 100644 mlir/lib/Target/LLVMIR/AttrKindDetail.h diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td index 9166ae78bd11..3f99dcbcccaa 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td @@ -36,9 +36,6 @@ def LLVM_Dialect : Dialect { let extraClassDeclaration = [{ /// Name of the data layout attributes. static StringRef getDataLayoutAttrName() { return "llvm.data_layout"; } - static StringRef getAlignAttrName() { return "llvm.align"; } - static StringRef getNoAliasAttrName() { return "llvm.noalias"; } - static StringRef getReadonlyAttrName() { return "llvm.readonly"; } static StringRef getNoAliasScopesAttrName() { return "noalias_scopes"; } static StringRef getAliasScopesAttrName() { return "alias_scopes"; } static StringRef getLoopAttrName() { return "llvm.loop"; } @@ -46,15 +43,37 @@ def LLVM_Dialect : Dialect { static StringRef getLoopOptionsAttrName() { return "options"; } static StringRef getAccessGroupsAttrName() { return "access_groups"; } static StringRef getStructAttrsAttrName() { return "llvm.struct_attrs"; } + static StringRef getTBAAAttrName() { return "llvm.tbaa"; } + + /// Names of llvm parameter attributes. + static StringRef getAlignAttrName() { return "llvm.align"; } + static StringRef getAllocAlignAttrName() { return "llvm.allocalign"; } + static StringRef getAllocatedPointerAttrName() { return "llvm.allocptr"; } static StringRef getByValAttrName() { return "llvm.byval"; } static StringRef getByRefAttrName() { return "llvm.byref"; } - static StringRef getStructRetAttrName() { return "llvm.sret"; } - static StringRef getInAllocaAttrName() { return "llvm.inalloca"; } static StringRef getNoUndefAttrName() { return "llvm.noundef"; } + static StringRef getDereferenceableAttrName() { return "llvm.dereferenceable"; } + static StringRef getDereferenceableOrNullAttrName() { return "llvm.dereferenceable_or_null"; } + static StringRef getInAllocaAttrName() { return "llvm.inalloca"; } + static StringRef getInRegAttrName() { return "llvm.inreg"; } + static StringRef getNestAttrName() { return "llvm.nest"; } + static StringRef getNoAliasAttrName() { return "llvm.noalias"; } + static StringRef getNoCaptureAttrName() { return "llvm.nocapture"; } + static StringRef getNoFreeAttrName() { return "llvm.nofree"; } + static StringRef getNonNullAttrName() { return "llvm.nonnull"; } + static StringRef getPreallocatedAttrName() { return "llvm.preallocated"; } + static StringRef getReadonlyAttrName() { return "llvm.readonly"; } + static StringRef getReturnedAttrName() { return "llvm.returned"; } static StringRef getSExtAttrName() { return "llvm.signext"; } + static StringRef getStackAlignmentAttrName() { return "llvm.alignstack"; } + static StringRef getStructRetAttrName() { return "llvm.sret"; } + static StringRef getWriteOnlyAttrName() { return "llvm.writeonly"; } static StringRef getZExtAttrName() { return "llvm.zeroext"; } - static StringRef getTBAAAttrName() { return "llvm.tbaa"; } - static StringRef getNestAttrName() { return "llvm.nest"; } + // TODO Restrict the usage of this to parameter attributes once there is an + // alternative way of modeling memory effects on FunctionOpInterface. + /// Name of the attribute that will cause the creation of a readnone memory + /// effect when lowering to the LLVMDialect. + static StringRef getReadnoneAttrName() { return "llvm.readnone"; } /// Verifies if the attribute is a well-formed value for "llvm.struct_attrs" static LogicalResult verifyStructAttr( @@ -76,13 +95,6 @@ def LLVM_Dialect : Dialect { /// Returns `true` if the given type is compatible with the LLVM dialect. static bool isCompatibleType(Type); - /// TODO Remove this once there is an alternative way of modeling memory - /// effects on FunctionOpInterface. - /// Name of the attribute that will cause the creation of a readnone memory - /// effect when lowering to the LLVMDialect. - static StringRef getReadnoneAttrName() { - return "llvm.readnone"; - } Type parseType(DialectAsmParser &p) const override; void printType(Type, DialectAsmPrinter &p) const override; diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp index a381efc189a6..4546411155f1 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -3188,7 +3188,12 @@ LogicalResult LLVMDialect::verifyParameterAttribute(Operation *op, // Check a unit attribute that is attached to a pointer value. if (name == LLVMDialect::getNoAliasAttrName() || name == LLVMDialect::getReadonlyAttrName() || - name == LLVMDialect::getNestAttrName()) { + name == LLVMDialect::getReadnoneAttrName() || + name == LLVMDialect::getWriteOnlyAttrName() || + name == LLVMDialect::getNestAttrName() || + name == LLVMDialect::getNoCaptureAttrName() || + name == LLVMDialect::getNoFreeAttrName() || + name == LLVMDialect::getNonNullAttrName()) { if (failed(checkUnitAttrType())) return failure(); if (verifyValueType && failed(checkPointerType())) @@ -3200,7 +3205,8 @@ LogicalResult LLVMDialect::verifyParameterAttribute(Operation *op, if (name == LLVMDialect::getStructRetAttrName() || name == LLVMDialect::getByValAttrName() || name == LLVMDialect::getByRefAttrName() || - name == LLVMDialect::getInAllocaAttrName()) { + name == LLVMDialect::getInAllocaAttrName() || + name == LLVMDialect::getPreallocatedAttrName()) { if (failed(checkTypeAttrType())) return failure(); if (verifyValueType && failed(checkPointerTypeMatches())) @@ -3219,7 +3225,10 @@ LogicalResult LLVMDialect::verifyParameterAttribute(Operation *op, } // Check an integer attribute that is attached to a pointer value. - if (name == LLVMDialect::getAlignAttrName()) { + if (name == LLVMDialect::getAlignAttrName() || + name == LLVMDialect::getDereferenceableAttrName() || + name == LLVMDialect::getDereferenceableOrNullAttrName() || + name == LLVMDialect::getStackAlignmentAttrName()) { if (failed(checkIntegerAttrType())) return failure(); if (verifyValueType && failed(checkPointerType())) @@ -3227,8 +3236,12 @@ LogicalResult LLVMDialect::verifyParameterAttribute(Operation *op, return success(); } - if (name == LLVMDialect::getNoUndefAttrName()) + // Check a unit attribute that can be attached to arbitrary types. + if (name == LLVMDialect::getNoUndefAttrName() || + name == LLVMDialect::getInRegAttrName() || + name == LLVMDialect::getReturnedAttrName()) return checkUnitAttrType(); + return success(); } @@ -3263,12 +3276,21 @@ LogicalResult LLVMDialect::verifyRegionResultAttribute(Operation *op, // Check to see if this attribute is allowed as a result attribute. Only // explicitly forbidden LLVM attributes will cause an error. auto name = resAttr.getName(); - if (name == LLVMDialect::getReadonlyAttrName() || - name == LLVMDialect::getNestAttrName() || - name == LLVMDialect::getStructRetAttrName() || + if (name == LLVMDialect::getAllocAlignAttrName() || + name == LLVMDialect::getAllocatedPointerAttrName() || name == LLVMDialect::getByValAttrName() || name == LLVMDialect::getByRefAttrName() || - name == LLVMDialect::getInAllocaAttrName()) + name == LLVMDialect::getInAllocaAttrName() || + name == LLVMDialect::getNestAttrName() || + name == LLVMDialect::getNoCaptureAttrName() || + name == LLVMDialect::getNoFreeAttrName() || + name == LLVMDialect::getPreallocatedAttrName() || + name == LLVMDialect::getReadnoneAttrName() || + name == LLVMDialect::getReadonlyAttrName() || + name == LLVMDialect::getReturnedAttrName() || + name == LLVMDialect::getStackAlignmentAttrName() || + name == LLVMDialect::getStructRetAttrName() || + name == LLVMDialect::getWriteOnlyAttrName()) return op->emitError() << name << " is not a valid result attribute"; return verifyParameterAttribute(op, resType, resAttr); } diff --git a/mlir/lib/Target/LLVMIR/AttrKindDetail.h b/mlir/lib/Target/LLVMIR/AttrKindDetail.h new file mode 100644 index 000000000000..7f81777886f5 --- /dev/null +++ b/mlir/lib/Target/LLVMIR/AttrKindDetail.h @@ -0,0 +1,66 @@ +//===- AttrKindDetail.h - AttrKind conversion details -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef ATTRKINDDETAIL_H_ +#define ATTRKINDDETAIL_H_ + +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "llvm/IR/Attributes.h" + +namespace mlir { +namespace LLVM { +namespace detail { + +/// Returns a list of pairs that each hold a mapping from LLVM attribute kinds +/// to their corresponding string name in LLVM IR dialect. +static llvm::ArrayRef> +getAttrKindToNameMapping() { + using ElemTy = std::pair; + // Mapping from llvm attribute kinds to their corresponding MLIR name. + static const llvm::SmallVector kindNamePairs = { + {llvm::Attribute::AttrKind::Alignment, LLVMDialect::getAlignAttrName()}, + {llvm::Attribute::AttrKind::AllocAlign, + LLVMDialect::getAllocAlignAttrName()}, + {llvm::Attribute::AttrKind::AllocatedPointer, + LLVMDialect::getAllocatedPointerAttrName()}, + {llvm::Attribute::AttrKind::ByVal, LLVMDialect::getByValAttrName()}, + {llvm::Attribute::AttrKind::ByRef, LLVMDialect::getByRefAttrName()}, + {llvm::Attribute::AttrKind::NoUndef, LLVMDialect::getNoUndefAttrName()}, + {llvm::Attribute::AttrKind::Dereferenceable, + LLVMDialect::getDereferenceableAttrName()}, + {llvm::Attribute::AttrKind::DereferenceableOrNull, + LLVMDialect::getDereferenceableOrNullAttrName()}, + {llvm::Attribute::AttrKind::InAlloca, LLVMDialect::getInAllocaAttrName()}, + {llvm::Attribute::AttrKind::InReg, LLVMDialect::getInRegAttrName()}, + {llvm::Attribute::AttrKind::Nest, LLVMDialect::getNestAttrName()}, + {llvm::Attribute::AttrKind::NoAlias, LLVMDialect::getNoAliasAttrName()}, + {llvm::Attribute::AttrKind::NoCapture, + LLVMDialect::getNoCaptureAttrName()}, + {llvm::Attribute::AttrKind::NoFree, LLVMDialect::getNoFreeAttrName()}, + {llvm::Attribute::AttrKind::NonNull, LLVMDialect::getNonNullAttrName()}, + {llvm::Attribute::AttrKind::Preallocated, + LLVMDialect::getPreallocatedAttrName()}, + {llvm::Attribute::AttrKind::ReadOnly, LLVMDialect::getReadonlyAttrName()}, + {llvm::Attribute::AttrKind::ReadNone, LLVMDialect::getReadnoneAttrName()}, + {llvm::Attribute::AttrKind::Returned, LLVMDialect::getReturnedAttrName()}, + {llvm::Attribute::AttrKind::SExt, LLVMDialect::getSExtAttrName()}, + {llvm::Attribute::AttrKind::StackAlignment, + LLVMDialect::getStackAlignmentAttrName()}, + {llvm::Attribute::AttrKind::StructRet, + LLVMDialect::getStructRetAttrName()}, + {llvm::Attribute::AttrKind::WriteOnly, + LLVMDialect::getWriteOnlyAttrName()}, + {llvm::Attribute::AttrKind::ZExt, LLVMDialect::getZExtAttrName()}}; + return kindNamePairs; +} + +} // namespace detail +} // namespace LLVM +} // namespace mlir + +#endif // ATTRKINDDETAIL_H_ diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp index 3e1e94c9e92e..697ccb80971b 100644 --- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp @@ -14,6 +14,7 @@ #include "mlir/Target/LLVMIR/ModuleImport.h" #include "mlir/Target/LLVMIR/Import.h" +#include "AttrKindDetail.h" #include "DebugImporter.h" #include "mlir/Dialect/DLTI/DLTI.h" @@ -1531,24 +1532,8 @@ void ModuleImport::processFunctionAttributes(llvm::Function *func, DictionaryAttr ModuleImport::convertParameterAttribute(llvm::AttributeSet llvmParamAttrs, OpBuilder &builder) { - using ElemTy = std::pair; - // Mapping from llvm attribute kinds to their corresponding MLIR name. - static const SmallVector kindNamePairs = { - {llvm::Attribute::AttrKind::NoAlias, LLVMDialect::getNoAliasAttrName()}, - {llvm::Attribute::AttrKind::ReadOnly, LLVMDialect::getReadonlyAttrName()}, - {llvm::Attribute::AttrKind::Nest, LLVMDialect::getNestAttrName()}, - {llvm::Attribute::AttrKind::SExt, LLVMDialect::getSExtAttrName()}, - {llvm::Attribute::AttrKind::ZExt, LLVMDialect::getZExtAttrName()}, - {llvm::Attribute::AttrKind::NoUndef, LLVMDialect::getNoUndefAttrName()}, - {llvm::Attribute::AttrKind::StructRet, - LLVMDialect::getStructRetAttrName()}, - {llvm::Attribute::AttrKind::ByVal, LLVMDialect::getByValAttrName()}, - {llvm::Attribute::AttrKind::ByRef, LLVMDialect::getByRefAttrName()}, - {llvm::Attribute::AttrKind::InAlloca, LLVMDialect::getInAllocaAttrName()}, - {llvm::Attribute::AttrKind::Alignment, LLVMDialect::getAlignAttrName()}}; - SmallVector paramAttrs; - for (auto [llvmKind, mlirName] : kindNamePairs) { + for (auto [llvmKind, mlirName] : getAttrKindToNameMapping()) { auto llvmAttr = llvmParamAttrs.getAttribute(llvmKind); // Skip attributes that are not attached. if (!llvmAttr.isValid()) diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp index b5323de221d7..5f4833279eb5 100644 --- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp @@ -13,6 +13,7 @@ #include "mlir/Target/LLVMIR/ModuleTranslation.h" +#include "AttrKindDetail.h" #include "DebugTranslation.h" #include "mlir/Dialect/DLTI/DLTI.h" #include "mlir/Dialect/LLVMIR/LLVMDialect.h" @@ -29,6 +30,7 @@ #include "llvm/ADT/PostOrderIterator.h" #include "llvm/ADT/SetVector.h" +#include "llvm/ADT/TypeSwitch.h" #include "llvm/Frontend/OpenMP/OMPIRBuilder.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" @@ -899,42 +901,27 @@ static void convertFunctionAttributes(LLVMFuncOp func, llvm::AttrBuilder ModuleTranslation::convertParameterAttrs(DictionaryAttr paramAttrs) { llvm::AttrBuilder attrBuilder(llvmModule->getContext()); - if (auto attr = paramAttrs.getAs(LLVMDialect::getNoAliasAttrName())) - attrBuilder.addAttribute(llvm::Attribute::AttrKind::NoAlias); - - if (auto attr = - paramAttrs.getAs(LLVMDialect::getReadonlyAttrName())) - attrBuilder.addAttribute(llvm::Attribute::AttrKind::ReadOnly); - - if (auto attr = - paramAttrs.getAs(LLVMDialect::getAlignAttrName())) - attrBuilder.addAlignmentAttr(llvm::Align(attr.getInt())); - - if (auto attr = - paramAttrs.getAs(LLVMDialect::getStructRetAttrName())) - attrBuilder.addStructRetAttr(convertType(attr.getValue())); - - if (auto attr = paramAttrs.getAs(LLVMDialect::getByValAttrName())) - attrBuilder.addByValAttr(convertType(attr.getValue())); - if (auto attr = paramAttrs.getAs(LLVMDialect::getByRefAttrName())) - attrBuilder.addByRefAttr(convertType(attr.getValue())); - - if (auto attr = - paramAttrs.getAs(LLVMDialect::getInAllocaAttrName())) - attrBuilder.addInAllocaAttr(convertType(attr.getValue())); - - if (auto attr = paramAttrs.getAs(LLVMDialect::getNestAttrName())) - attrBuilder.addAttribute(llvm::Attribute::Nest); - - if (auto attr = paramAttrs.getAs(LLVMDialect::getNoUndefAttrName())) - attrBuilder.addAttribute(llvm::Attribute::NoUndef); + for (auto [llvmKind, mlirName] : getAttrKindToNameMapping()) { + Attribute attr = paramAttrs.get(mlirName); + // Skip attributes that are not present. + if (!attr) + continue; - if (auto attr = paramAttrs.getAs(LLVMDialect::getSExtAttrName())) - attrBuilder.addAttribute(llvm::Attribute::SExt); + // NOTE: C++17 does not support capturing structured bindings. + llvm::Attribute::AttrKind llvmKindCap = llvmKind; + + llvm::TypeSwitch(attr) + .Case([&](auto typeAttr) { + attrBuilder.addTypeAttr(llvmKindCap, + convertType(typeAttr.getValue())); + }) + .Case([&](auto intAttr) { + attrBuilder.addRawIntAttr(llvmKindCap, intAttr.getInt()); + }) + .Case([&](auto) { attrBuilder.addAttribute(llvmKindCap); }); + } - if (auto attr = paramAttrs.getAs(LLVMDialect::getZExtAttrName())) - attrBuilder.addAttribute(llvm::Attribute::ZExt); return attrBuilder; } diff --git a/mlir/test/Dialect/LLVMIR/parameter-attrs-invalid.mlir b/mlir/test/Dialect/LLVMIR/parameter-attrs-invalid.mlir index f6305f171013..72bf45052ef1 100644 --- a/mlir/test/Dialect/LLVMIR/parameter-attrs-invalid.mlir +++ b/mlir/test/Dialect/LLVMIR/parameter-attrs-invalid.mlir @@ -107,6 +107,97 @@ llvm.func @invalid_noundef_attr_type(%0 : i32 {llvm.noundef = !llvm.ptr}) // ----- +// expected-error@below {{"llvm.dereferenceable" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_dereferenceable_arg_type(%0 : f32 {llvm.dereferenceable = 12 : i64}) + +// ----- + +// expected-error@below {{"llvm.dereferenceable" should be an integer attribute}} +llvm.func @invalid_dereferenceable_attr_type(%0 : !llvm.ptr {llvm.dereferenceable = !llvm.struct<(i32)>}) + +// ----- + +// expected-error@below {{"llvm.dereferenceable_or_null" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_dereferenceable_or_null_arg_type(%0 : f32 {llvm.dereferenceable_or_null = 12 : i64}) + +// ----- + +// expected-error@below {{"llvm.dereferenceable_or_null" should be an integer attribute}} +llvm.func @invalid_dereferenceable_or_null_attr_type(%0 : !llvm.ptr {llvm.dereferenceable_or_null = !llvm.struct<(i32)>}) + +// ----- + +// expected-error@below {{"llvm.inreg" should be a unit attribute}} +llvm.func @invalid_inreg_attr_type(%0 : i32 {llvm.inreg = !llvm.ptr}) + +// ----- + +// expected-error@below {{"llvm.nocapture" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_nocapture_arg_type(%0 : f32 {llvm.nocapture}) + +// ----- + +// expected-error@below {{"llvm.nocapture" should be a unit attribute}} +llvm.func @invalid_nocapture_attr_type(%0 : !llvm.ptr {llvm.nocapture = f32}) + +// ----- + +// expected-error@below {{"llvm.nofree" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_nofree_arg_type(%0 : f32 {llvm.nofree}) + +// ----- + +// expected-error@below {{"llvm.nofree" should be a unit attribute}} +llvm.func @invalid_nofree_attr_type(%0 : !llvm.ptr {llvm.nofree = f32}) + +// ----- + +// expected-error@below {{"llvm.nonnull" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_nonnull_arg_type(%0 : f32 {llvm.nonnull}) + +// ----- + +// expected-error@below {{"llvm.nonnull" should be a unit attribute}} +llvm.func @invalid_nonnull_attr_type(%0 : !llvm.ptr {llvm.nonnull = f32}) + +// ----- + +// expected-error@below {{"llvm.preallocated" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_preallocated_arg_type(%0 : f32 {llvm.preallocated = i64}) + +// ----- + +// expected-error@below {{"llvm.preallocated" should be a type attribute}} +llvm.func @invalid_preallocated_attr_type(%0 : !llvm.ptr {llvm.preallocated}) + +// ----- + +// expected-error@below {{"llvm.returned" should be a unit attribute}} +llvm.func @invalid_returned_attr_type(%0 : i32 {llvm.returned = !llvm.ptr}) + +// ----- + +// expected-error@below {{"llvm.alignstack" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_alignstack_arg_type(%0 : i32 {llvm.alignstack = 10 : i32}) + +// ----- + +// expected-error@below {{"llvm.alignstack" should be an integer attribute}} +llvm.func @invalid_alignstack_attr_type(%0 : i32 {llvm.alignstack = "foo"}) + +// ----- + +// expected-error@below {{"llvm.writeonly" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_writeonly_arg_type(%0 : i32 {llvm.writeonly}) + +// ----- + +// expected-error@below {{"llvm.writeonly" should be a unit attribute}} +llvm.func @invalid_writeonly_attr_type(%0 : i32 {llvm.writeonly = i32}) + +// ----- + + // Result attributes // expected-error@below {{cannot attach result attributes to functions with a void return}} @@ -159,18 +250,13 @@ llvm.func @zeroextattr_ret() -> (f32 {llvm.zeroext}) // ----- -// expected-error @below{{"llvm.readonly" is not a valid result attribute}} -llvm.func @readonly_ret() -> (f32 {llvm.readonly}) +// expected-error @below{{"llvm.allocalign" is not a valid result attribute}} +llvm.func @allocalign_ret() -> (f32 {llvm.allocalign}) // ----- -// expected-error @below{{"llvm.nest" is not a valid result attribute}} -llvm.func @nest_ret() -> (f32 {llvm.nest}) - -// ----- - -// expected-error @below{{"llvm.sret" is not a valid result attribute}} -llvm.func @sret_ret() -> (!llvm.ptr {llvm.sret = i64}) +// expected-error @below{{"llvm.allocptr" is not a valid result attribute}} +llvm.func @allocptr_ret() -> (!llvm.ptr {llvm.allocptr}) // ----- @@ -186,3 +272,48 @@ llvm.func @byref_ret() -> (!llvm.ptr {llvm.byref = i64}) // expected-error @below{{"llvm.inalloca" is not a valid result attribute}} llvm.func @inalloca_ret() -> (!llvm.ptr {llvm.inalloca = i64}) + +// ----- + +// expected-error @below{{"llvm.nest" is not a valid result attribute}} +llvm.func @nest_ret() -> (!llvm.ptr {llvm.nest}) + +// ----- + +// expected-error @below{{"llvm.nocapture" is not a valid result attribute}} +llvm.func @nocapture_ret() -> (!llvm.ptr {llvm.nocapture}) + +// ----- + +// expected-error @below{{"llvm.nofree" is not a valid result attribute}} +llvm.func @nofree_ret() -> (!llvm.ptr {llvm.nofree}) + +// ----- + +// expected-error @below{{"llvm.preallocated" is not a valid result attribute}} +llvm.func @preallocated_ret() -> (!llvm.ptr {llvm.preallocated = i64}) + +// ----- + +// expected-error @below{{"llvm.readnone" is not a valid result attribute}} +llvm.func @readnone_ret() -> (!llvm.ptr {llvm.readnone}) + +// ----- + +// expected-error @below{{"llvm.readonly" is not a valid result attribute}} +llvm.func @readonly_ret() -> (!llvm.ptr {llvm.readonly}) + +// ----- + +// expected-error @below{{"llvm.alignstack" is not a valid result attribute}} +llvm.func @alignstack_ret() -> (!llvm.ptr {llvm.alignstack = 16 : i64}) + +// ----- + +// expected-error @below{{"llvm.sret" is not a valid result attribute}} +llvm.func @sret_ret() -> (!llvm.ptr {llvm.sret = i64}) + +// ----- + +// expected-error @below{{"llvm.writeonly" is not a valid result attribute}} +llvm.func @writeonly_ret() -> (!llvm.ptr {llvm.writeonly}) diff --git a/mlir/test/Target/LLVMIR/Import/function-attributes.ll b/mlir/test/Target/LLVMIR/Import/function-attributes.ll index ed38a160afa6..1d21b708bd30 100644 --- a/mlir/test/Target/LLVMIR/Import/function-attributes.ll +++ b/mlir/test/Target/LLVMIR/Import/function-attributes.ll @@ -37,7 +37,17 @@ attributes #0 = { readnone } ; CHECK-SAME: i32 {llvm.signext} ; CHECK-SAME: i64 {llvm.zeroext} ; CHECK-SAME: !llvm.ptr {llvm.align = 64 : i64, llvm.noundef} -define void @func_arg_attrs( +; CHECK-SAME: !llvm.ptr {llvm.dereferenceable = 12 : i64} +; CHECK-SAME: !llvm.ptr {llvm.dereferenceable_or_null = 42 : i64} +; CHECK-SAME: f64 {llvm.inreg} +; CHECK-SAME: !llvm.ptr {llvm.nocapture} +; CHECK-SAME: !llvm.ptr {llvm.nofree} +; CHECK-SAME: !llvm.ptr {llvm.nonnull} +; CHECK-SAME: !llvm.ptr {llvm.preallocated = f64} +; CHECK-SAME: !llvm.ptr {llvm.returned} +; CHECK-SAME: !llvm.ptr {llvm.alignstack = 32 : i64} +; CHECK-SAME: !llvm.ptr {llvm.writeonly} +define ptr @func_arg_attrs( ptr byval(i64) %arg0, ptr byref(i64) %arg1, ptr sret(i64) %arg2, @@ -47,15 +57,24 @@ define void @func_arg_attrs( ptr nest %arg6, i32 signext %arg7, i64 zeroext %arg8, - ptr align(64) noundef %arg9) { - ret void + ptr align(64) noundef %arg9, + ptr dereferenceable(12) %arg10, + ptr dereferenceable_or_null(42) %arg11, + double inreg %arg12, + ptr nocapture %arg13, + ptr nofree %arg14, + ptr nonnull %arg15, + ptr preallocated(double) %arg16, + ptr returned %arg17, + ptr alignstack(32) %arg18, + ptr writeonly %arg19) { + ret ptr %arg17 } -; // ----- - -; CHECK-LABEL: @func_res_attr_align -; CHECK-SAME: !llvm.ptr {llvm.align = 16 : i64} -declare align(16) ptr @func_res_attr_align() +; CHECK-LABEL: @allocator +; CHECK-SAME: i64 {llvm.allocalign} +; CHECK-SAME: ptr {llvm.allocptr} +declare ptr @allocator(i64 allocalign, ptr allocptr) ; // ----- @@ -65,6 +84,12 @@ declare noalias ptr @func_res_attr_noalias() ; // ----- +; CHECK-LABEL: @func_res_attr_nonnull +; CHECK-SAME: !llvm.ptr {llvm.nonnull} +declare nonnull ptr @func_res_attr_nonnull() + +; // ----- + ; CHECK-LABEL: @func_res_attr_signext ; CHECK-DAG: llvm.noundef ; CHECK-DAG: llvm.signext @@ -76,6 +101,35 @@ declare noundef signext i32 @func_res_attr_signext() ; CHECK-SAME: i32 {llvm.zeroext} declare zeroext i32 @func_res_attr_zeroext() +; // ----- + +; CHECK-LABEL: @func_res_attr_align +; CHECK-SAME: !llvm.ptr {llvm.align = 16 : i64} +declare align(16) ptr @func_res_attr_align() + +; // ----- + +; CHECK-LABEL: @func_res_attr_noundef +; CHECK-SAME: !llvm.ptr {llvm.noundef} +declare noundef ptr @func_res_attr_noundef() + +; // ----- + +; CHECK-LABEL: @func_res_attr_dereferenceable +; CHECK-SAME: !llvm.ptr {llvm.dereferenceable = 42 : i64} +declare dereferenceable(42) ptr @func_res_attr_dereferenceable() + +; // ----- + +; CHECK-LABEL: @func_res_attr_dereferenceable_or_null +; CHECK-SAME: !llvm.ptr {llvm.dereferenceable_or_null = 42 : i64} +declare dereferenceable_or_null(42) ptr @func_res_attr_dereferenceable_or_null() + +; // ----- + +; CHECK-LABEL: @func_res_attr_inreg +; CHECK-SAME: !llvm.ptr {llvm.inreg} +declare inreg ptr @func_res_attr_inreg() ; // ----- diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir index dde4bb3d95e7..4c4e85ce08ad 100644 --- a/mlir/test/Target/LLVMIR/llvmir.mlir +++ b/mlir/test/Target/LLVMIR/llvmir.mlir @@ -1124,17 +1124,66 @@ llvm.func @zeroextattr(%arg0: i1 {llvm.zeroext}) { // CHECK-LABEL: declare void @zeroextattr_decl(i1 zeroext) llvm.func @zeroextattr_decl(i1 {llvm.zeroext}) +// CHECK-LABEL: declare void @alignattr_decl(ptr align 64) +llvm.func @alignattr_decl(!llvm.ptr {llvm.align = 64 : i64}) + +// CHECK-LABEL: declare void @dereferenceableattr_decl(ptr dereferenceable(32)) +llvm.func @dereferenceableattr_decl(!llvm.ptr {llvm.dereferenceable = 32 : i64}) + +// CHECK-LABEL: declare void @dereferenceableornullattr_decl(ptr dereferenceable_or_null(32)) +llvm.func @dereferenceableornullattr_decl(!llvm.ptr {llvm.dereferenceable_or_null = 32 : i64}) + +// CHECK-LABEL: declare void @inregattr_decl(ptr inreg) +llvm.func @inregattr_decl(!llvm.ptr {llvm.inreg}) + +// CHECK-LABEL: declare void @nocaptureattr_decl(ptr nocapture) +llvm.func @nocaptureattr_decl(!llvm.ptr {llvm.nocapture}) + +// CHECK-LABEL: declare void @nofreeattr_decl(ptr nofree) +llvm.func @nofreeattr_decl(!llvm.ptr {llvm.nofree}) + +// CHECK-LABEL: declare void @nonnullattr_decl(ptr nonnull) +llvm.func @nonnullattr_decl(!llvm.ptr {llvm.nonnull}) + +// CHECK-LABEL: declare void @preallocatedattr_decl(ptr preallocated(float)) +llvm.func @preallocatedattr_decl(!llvm.ptr {llvm.preallocated = f32}) + +// CHECK-LABEL: declare ptr @returnedattr_decl(ptr returned) +llvm.func @returnedattr_decl(!llvm.ptr {llvm.returned}) -> !llvm.ptr + +// CHECK-LABEL: declare void @alignstackattr_decl(ptr alignstack(32)) +llvm.func @alignstackattr_decl(!llvm.ptr {llvm.alignstack = 32 : i64}) + +// CHECK-LABEL: declare void @writeonlyattr_decl(ptr writeonly) +llvm.func @writeonlyattr_decl(!llvm.ptr {llvm.writeonly}) + // CHECK-LABEL: declare align 4 ptr @alignattr_ret_decl() llvm.func @alignattr_ret_decl() -> (!llvm.ptr {llvm.align = 4}) + // CHECK-LABEL: declare noalias ptr @noaliasattr_ret_decl() llvm.func @noaliasattr_ret_decl() -> (!llvm.ptr {llvm.noalias}) + // CHECK-LABEL: declare noundef ptr @noundefattr_ret_decl() llvm.func @noundefattr_ret_decl() -> (!llvm.ptr {llvm.noundef}) + // CHECK-LABEL: declare signext i1 @signextattr_ret_decl() llvm.func @signextattr_ret_decl() -> (i1 {llvm.signext}) + // CHECK-LABEL: declare zeroext i1 @zeroextattr_ret_decl() llvm.func @zeroextattr_ret_decl() -> (i1 {llvm.zeroext}) +// CHECK-LABEL: declare nonnull ptr @nonnullattr_ret_decl() +llvm.func @nonnullattr_ret_decl() -> (!llvm.ptr {llvm.nonnull}) + +// CHECK-LABEL: declare dereferenceable(32) ptr @dereferenceableattr_ret_decl() +llvm.func @dereferenceableattr_ret_decl() -> (!llvm.ptr {llvm.dereferenceable = 32 : i64}) + +// CHECK-LABEL: declare dereferenceable_or_null(16) ptr @dereferenceableornullattr_ret_decl() +llvm.func @dereferenceableornullattr_ret_decl() -> (!llvm.ptr {llvm.dereferenceable_or_null = 16 : i64}) + +// CHECK-LABEL: declare inreg ptr @inregattr_ret_decl() +llvm.func @inregattr_ret_decl() -> (!llvm.ptr {llvm.inreg}) + // CHECK-LABEL: @llvm_varargs(...) llvm.func @llvm_varargs(...) From ce6122b5382eca9305279c935ba8bebd23bd20e5 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Mon, 6 Mar 2023 10:02:39 +0100 Subject: [PATCH 057/147] [MLIR] Convert remaining tests to opaque pointers (NFC) These were the final tests using -opaque-pointers=0 in mlir/. (cherry picked from commit 14d99ea16c4433a6b6dbaf6a7e3b9f14984a9417) --- mlir/test/Target/LLVMIR/Import/basic.ll | 90 ++++--- mlir/test/Target/LLVMIR/Import/constant.ll | 110 ++++----- .../Import/incorrect-constant-caching.ll | 16 +- .../incorrect-constexpr-inst-caching.ll | 16 +- .../Import/incorrect-scalable-vector-check.ll | 8 +- mlir/test/Target/LLVMIR/Import/intrinsic.ll | 228 +++++++++--------- .../Target/LLVMIR/Import/zeroinitializer.ll | 13 +- 7 files changed, 232 insertions(+), 249 deletions(-) diff --git a/mlir/test/Target/LLVMIR/Import/basic.ll b/mlir/test/Target/LLVMIR/Import/basic.ll index 050197c90c83..3b9b1c2909c1 100644 --- a/mlir/test/Target/LLVMIR/Import/basic.ll +++ b/mlir/test/Target/LLVMIR/Import/basic.ll @@ -1,5 +1,5 @@ -; RUN: mlir-translate -opaque-pointers=0 -import-llvm %s | FileCheck %s -; RUN: mlir-translate -opaque-pointers=0 -import-llvm -mlir-print-debuginfo %s | FileCheck %s --check-prefix=CHECK-DBG +; RUN: mlir-translate -import-llvm %s | FileCheck %s +; RUN: mlir-translate -import-llvm -mlir-print-debuginfo %s | FileCheck %s --check-prefix=CHECK-DBG ; CHECK-DBG: #[[MODULELOC:.+]] = loc({{.*}}basic.ll{{.*}}:0:0) @@ -16,16 +16,16 @@ declare float @fe(i32) ; CHECK: %[[c42:[0-9]+]] = llvm.mlir.constant(42 : i32) : i32 define internal dso_local i32 @f1(i64 %a) norecurse { entry: -; CHECK: %{{[0-9]+}} = llvm.inttoptr %arg0 : i64 to !llvm.ptr - %aa = inttoptr i64 %a to i64* -; CHECK-DBG: llvm.mlir.addressof @global : !llvm.ptr loc(#[[MODULELOC]]) -; %[[addrof:[0-9]+]] = llvm.mlir.addressof @global : !llvm.ptr -; %[[addrof2:[0-9]+]] = llvm.mlir.addressof @global : !llvm.ptr -; %{{[0-9]+}} = llvm.inttoptr %arg0 : i64 to !llvm.ptr -; %{{[0-9]+}} = llvm.ptrtoint %[[addrof2]] : !llvm.ptr to i64 -; %{{[0-9]+}} = llvm.getelementptr %[[addrof]][%3] : (!llvm.ptr, i32) -> !llvm.ptr - %bb = ptrtoint double* @global to i64 - %cc = getelementptr double, double* @global, i32 3 +; CHECK: %{{[0-9]+}} = llvm.inttoptr %arg0 : i64 to !llvm.ptr + %aa = inttoptr i64 %a to ptr +; CHECK-DBG: llvm.mlir.addressof @global : !llvm.ptr loc(#[[MODULELOC]]) +; %[[addrof:[0-9]+]] = llvm.mlir.addressof @global : !llvm.ptr +; %[[addrof2:[0-9]+]] = llvm.mlir.addressof @global : !llvm.ptr +; %{{[0-9]+}} = llvm.inttoptr %arg0 : i64 to !llvm.ptr +; %{{[0-9]+}} = llvm.ptrtoint %[[addrof2]] : !llvm.ptr to i64 +; %{{[0-9]+}} = llvm.getelementptr %[[addrof]][%3] : (!llvm.ptr, i32) -> !llvm.ptr + %bb = ptrtoint ptr @global to i64 + %cc = getelementptr double, ptr @global, i32 3 ; CHECK: %[[b:[0-9]+]] = llvm.trunc %arg0 : i64 to i32 ; CHECK-DBG: llvm.trunc %arg0 : i64 to i32 loc(#[[MODULELOC]]) %b = trunc i64 %a to i32 @@ -54,35 +54,35 @@ if.end: ; CHECK-DBG: } loc(#[[MODULELOC]]) -@_ZTIi = external dso_local constant i8* -@_ZTIii= external dso_local constant i8** -declare void @foo(i8*) -declare i8* @bar(i8*) +@_ZTIi = external dso_local constant ptr +@_ZTIii= external dso_local constant ptr +declare void @foo(ptr) +declare ptr @bar(ptr) declare i32 @__gxx_personality_v0(...) ; CHECK-LABEL: @invokeLandingpad -define i32 @invokeLandingpad() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { - ; CHECK: %[[a1:[0-9]+]] = llvm.bitcast %{{[0-9]+}} : !llvm.ptr>> to !llvm.ptr - ; CHECK: %[[a3:[0-9]+]] = llvm.alloca %{{[0-9]+}} x i8 {alignment = 1 : i64} : (i32) -> !llvm.ptr +define i32 @invokeLandingpad() personality ptr @__gxx_personality_v0 { + ; CHECK: %[[a1:[0-9]+]] = llvm.mlir.addressof @_ZTIii : !llvm.ptr + ; CHECK: %[[a3:[0-9]+]] = llvm.alloca %{{[0-9]+}} x i8 {alignment = 1 : i64} : (i32) -> !llvm.ptr %1 = alloca i8 - ; CHECK: llvm.invoke @foo(%[[a3]]) to ^bb2 unwind ^bb1 : (!llvm.ptr) -> () - invoke void @foo(i8* %1) to label %4 unwind label %2 + ; CHECK: llvm.invoke @foo(%[[a3]]) to ^bb2 unwind ^bb1 : (!llvm.ptr) -> () + invoke void @foo(ptr %1) to label %4 unwind label %2 ; CHECK: ^bb1: - ; CHECK: %{{[0-9]+}} = llvm.landingpad (catch %{{[0-9]+}} : !llvm.ptr>) (catch %[[a1]] : !llvm.ptr) (filter %{{[0-9]+}} : !llvm.array<1 x i8>) : !llvm.struct<(ptr, i32)> - %3 = landingpad { i8*, i32 } catch i8** @_ZTIi catch i8* bitcast (i8*** @_ZTIii to i8*) + ; CHECK: %{{[0-9]+}} = llvm.landingpad (catch %{{[0-9]+}} : !llvm.ptr) (catch %[[a1]] : !llvm.ptr) (filter %{{[0-9]+}} : !llvm.array<1 x i8>) : !llvm.struct<(ptr, i32)> + %3 = landingpad { ptr, i32 } catch ptr @_ZTIi catch ptr @_ZTIii ; FIXME: Change filter to a constant array once they are handled. ; Currently, even though it parses this, LLVM module is broken filter [1 x i8] [i8 1] - resume { i8*, i32 } %3 + resume { ptr, i32 } %3 ; CHECK: ^bb2: ; CHECK: llvm.return %{{[0-9]+}} : i32 ret i32 1 ; CHECK: ^bb3: - ; CHECK: %{{[0-9]+}} = llvm.invoke @bar(%[[a3]]) to ^bb2 unwind ^bb1 : (!llvm.ptr) -> !llvm.ptr - %6 = invoke i8* @bar(i8* %1) to label %4 unwind label %2 + ; CHECK: %{{[0-9]+}} = llvm.invoke @bar(%[[a3]]) to ^bb2 unwind ^bb1 : (!llvm.ptr) -> !llvm.ptr + %6 = invoke ptr @bar(ptr %1) to label %4 unwind label %2 ; CHECK: ^bb4: ; CHECK: llvm.return %{{[0-9]+}} : i32 @@ -107,32 +107,28 @@ define i32 @useFreezeOp(i32 %x) { } ; Varadic function definition -%struct.va_list = type { i8* } +%struct.va_list = type { ptr } -declare void @llvm.va_start(i8*) -declare void @llvm.va_copy(i8*, i8*) -declare void @llvm.va_end(i8*) +declare void @llvm.va_start(ptr) +declare void @llvm.va_copy(ptr, ptr) +declare void @llvm.va_end(ptr) ; CHECK-LABEL: llvm.func @variadic_function define void @variadic_function(i32 %X, ...) { - ; CHECK: %[[ALLOCA0:.+]] = llvm.alloca %{{.*}} x !llvm.struct<"struct.va_list", (ptr)> {{.*}} : (i32) -> !llvm.ptr)>> + ; CHECK: %[[ALLOCA0:.+]] = llvm.alloca %{{.*}} x !llvm.struct<"struct.va_list", (ptr)> {{.*}} : (i32) -> !llvm.ptr %ap = alloca %struct.va_list - ; CHECK: %[[CAST0:.+]] = llvm.bitcast %[[ALLOCA0]] : !llvm.ptr)>> to !llvm.ptr - %ap2 = bitcast %struct.va_list* %ap to i8* - ; CHECK: llvm.intr.vastart %[[CAST0]] - call void @llvm.va_start(i8* %ap2) - - ; CHECK: %[[ALLOCA1:.+]] = llvm.alloca %{{.*}} x !llvm.ptr {{.*}} : (i32) -> !llvm.ptr> - %aq = alloca i8* - ; CHECK: %[[CAST1:.+]] = llvm.bitcast %[[ALLOCA1]] : !llvm.ptr> to !llvm.ptr - %aq2 = bitcast i8** %aq to i8* - ; CHECK: llvm.intr.vacopy %[[CAST0]] to %[[CAST1]] - call void @llvm.va_copy(i8* %aq2, i8* %ap2) - ; CHECK: llvm.intr.vaend %[[CAST1]] - call void @llvm.va_end(i8* %aq2) - - ; CHECK: llvm.intr.vaend %[[CAST0]] - call void @llvm.va_end(i8* %ap2) + ; CHECK: llvm.intr.vastart %[[ALLOCA0]] + call void @llvm.va_start(ptr %ap) + + ; CHECK: %[[ALLOCA1:.+]] = llvm.alloca %{{.*}} x !llvm.ptr {{.*}} : (i32) -> !llvm.ptr + %aq = alloca ptr + ; CHECK: llvm.intr.vacopy %[[ALLOCA0]] to %[[ALLOCA1]] + call void @llvm.va_copy(ptr %aq, ptr %ap) + ; CHECK: llvm.intr.vaend %[[ALLOCA1]] + call void @llvm.va_end(ptr %aq) + + ; CHECK: llvm.intr.vaend %[[ALLOCA0]] + call void @llvm.va_end(ptr %ap) ; CHECK: llvm.return ret void } diff --git a/mlir/test/Target/LLVMIR/Import/constant.ll b/mlir/test/Target/LLVMIR/Import/constant.ll index 8ff836d066c4..12175fbc9cb6 100644 --- a/mlir/test/Target/LLVMIR/Import/constant.ll +++ b/mlir/test/Target/LLVMIR/Import/constant.ll @@ -1,4 +1,4 @@ -; RUN: mlir-translate -opaque-pointers=0 -import-llvm -split-input-file %s | FileCheck %s +; RUN: mlir-translate -import-llvm -split-input-file %s | FileCheck %s ; CHECK-LABEL: @int_constants define void @int_constants(i16 %arg0, i32 %arg1, i1 %arg2) { @@ -48,22 +48,10 @@ define void @undef_constant(i32 %arg0) { ; // ----- ; CHECK-LABEL: @null_constant -define i32* @null_constant() { - ; CHECK: %[[NULL:[0-9]+]] = llvm.mlir.null : !llvm.ptr - ; CHECK: llvm.return %[[NULL]] : !llvm.ptr - ret i32* bitcast (double* null to i32*) -} - -; // ----- - -@global = external global double, align 8 - -; CHECK-LABEL: @bitcast_const_expr -define i32* @bitcast_const_expr() { - ; CHECK: %[[VAL0:.*]] = llvm.mlir.addressof @global : !llvm.ptr - ; CHECK: %[[VAL1:.*]] = llvm.bitcast %[[VAL0]] : !llvm.ptr to !llvm.ptr - ; CHECK: llvm.return %[[VAL1]] : !llvm.ptr - ret i32* bitcast (double* @global to i32*) +define ptr @null_constant() { + ; CHECK: %[[NULL:[0-9]+]] = llvm.mlir.null : !llvm.ptr + ; CHECK: llvm.return %[[NULL]] : !llvm.ptr + ret ptr null } ; // ----- @@ -71,12 +59,12 @@ define i32* @bitcast_const_expr() { @global = external global i32, align 8 ; CHECK-LABEL: @gep_const_expr -define i32* @gep_const_expr() { - ; CHECK: %[[ADDR:[0-9]+]] = llvm.mlir.addressof @global : !llvm.ptr +define ptr @gep_const_expr() { + ; CHECK: %[[ADDR:[0-9]+]] = llvm.mlir.addressof @global : !llvm.ptr ; CHECK: %[[IDX:[0-9]+]] = llvm.mlir.constant(2 : i32) : i32 - ; CHECK: %[[GEP:[0-9]+]] = llvm.getelementptr %[[ADDR]][%[[IDX]]] : (!llvm.ptr, i32) -> !llvm.ptr - ; CHECK: llvm.return %[[GEP]] : !llvm.ptr - ret i32* getelementptr (i32, i32* @global, i32 2) + ; CHECK: %[[GEP:[0-9]+]] = llvm.getelementptr %[[ADDR]][%[[IDX]]] : (!llvm.ptr, i32) -> !llvm.ptr + ; CHECK: llvm.return %[[GEP]] : !llvm.ptr + ret ptr getelementptr (i32, ptr @global, i32 2) } ; // ----- @@ -85,16 +73,16 @@ define i32* @gep_const_expr() { ; CHECK-LABEL: @const_expr_with_duplicate define i64 @const_expr_with_duplicate() { - ; CHECK: %[[ADDR:[0-9]+]] = llvm.mlir.addressof @global : !llvm.ptr + ; CHECK: %[[ADDR:[0-9]+]] = llvm.mlir.addressof @global : !llvm.ptr ; CHECK: %[[IDX:[0-9]+]] = llvm.mlir.constant(7 : i32) : i32 - ; CHECK: %[[GEP:[0-9]+]] = llvm.getelementptr %[[ADDR]][%[[IDX]]] : (!llvm.ptr, i32) -> !llvm.ptr - ; CHECK: %[[DUP:[0-9]+]] = llvm.ptrtoint %[[GEP]] : !llvm.ptr to i64 + ; CHECK: %[[GEP:[0-9]+]] = llvm.getelementptr %[[ADDR]][%[[IDX]]] : (!llvm.ptr, i32) -> !llvm.ptr + ; CHECK: %[[DUP:[0-9]+]] = llvm.ptrtoint %[[GEP]] : !llvm.ptr to i64 ; Verify the duplicate sub expression is converted only once. ; CHECK: %[[SUM:[0-9]+]] = llvm.add %[[DUP]], %[[DUP]] : i64 ; CHECK: llvm.return %[[SUM]] : i64 - ret i64 add (i64 ptrtoint (i32* getelementptr (i32, i32* @global, i32 7) to i64), - i64 ptrtoint (i32* getelementptr (i32, i32* @global, i32 7) to i64)) + ret i64 add (i64 ptrtoint (ptr getelementptr (i32, ptr @global, i32 7) to i64), + i64 ptrtoint (ptr getelementptr (i32, ptr @global, i32 7) to i64)) } ; // ----- @@ -105,10 +93,10 @@ define i64 @const_expr_with_duplicate() { define i64 @const_expr_with_aggregate() { ; Compute the vector elements. ; CHECK: %[[VAL1:[0-9]+]] = llvm.mlir.constant(33 : i64) : i64 - ; CHECK: %[[ADDR:[0-9]+]] = llvm.mlir.addressof @global : !llvm.ptr + ; CHECK: %[[ADDR:[0-9]+]] = llvm.mlir.addressof @global : !llvm.ptr ; CHECK: %[[IDX1:[0-9]+]] = llvm.mlir.constant(7 : i32) : i32 - ; CHECK: %[[GEP1:[0-9]+]] = llvm.getelementptr %[[ADDR]][%[[IDX1]]] : (!llvm.ptr, i32) -> !llvm.ptr - ; CHECK: %[[VAL2:[0-9]+]] = llvm.ptrtoint %[[GEP1]] : !llvm.ptr to i64 + ; CHECK: %[[GEP1:[0-9]+]] = llvm.getelementptr %[[ADDR]][%[[IDX1]]] : (!llvm.ptr, i32) -> !llvm.ptr + ; CHECK: %[[VAL2:[0-9]+]] = llvm.ptrtoint %[[GEP1]] : !llvm.ptr to i64 ; Fill the vector. ; CHECK: %[[VEC1:[0-9]+]] = llvm.mlir.undef : vector<2xi64> @@ -119,15 +107,15 @@ define i64 @const_expr_with_aggregate() { ; CHECK: %[[IDX4:[0-9]+]] = llvm.mlir.constant(42 : i32) : i32 ; Compute the extract index. - ; CHECK: %[[GEP2:[0-9]+]] = llvm.getelementptr %[[ADDR]][%[[IDX4]]] : (!llvm.ptr, i32) -> !llvm.ptr - ; CHECK: %[[IDX5:[0-9]+]] = llvm.ptrtoint %[[GEP2]] : !llvm.ptr to i64 + ; CHECK: %[[GEP2:[0-9]+]] = llvm.getelementptr %[[ADDR]][%[[IDX4]]] : (!llvm.ptr, i32) -> !llvm.ptr + ; CHECK: %[[IDX5:[0-9]+]] = llvm.ptrtoint %[[GEP2]] : !llvm.ptr to i64 ; Extract the vector element. ; CHECK: %[[ELEM:[0-9]+]] = llvm.extractelement %[[VEC3]][%[[IDX5]] : i64] : vector<2xi64> ; CHECK: llvm.return %[[ELEM]] : i64 ret i64 extractelement ( - <2 x i64> , - i64 ptrtoint (i32* getelementptr (i32, i32* @global, i32 42) to i64)) + <2 x i64> , + i64 ptrtoint (ptr getelementptr (i32, ptr @global, i32 42) to i64)) } ; // ----- @@ -137,12 +125,12 @@ define i64 @const_expr_with_aggregate() { ; Calling a function that has not been defined yet. ; CHECK-LABEL: @function_address_before_def define i32 @function_address_before_def() { - %1 = alloca i32 ()* - ; CHECK: %[[FUN:.*]] = llvm.mlir.addressof @callee : !llvm.ptr> - ; CHECK: llvm.store %[[FUN]], %[[PTR:.*]] - store i32 ()* @callee, i32 ()** %1 - ; CHECK: %[[INDIR:.*]] = llvm.load %[[PTR]] - %2 = load i32 ()*, i32 ()** %1 + %1 = alloca ptr + ; CHECK: %[[FUN:.*]] = llvm.mlir.addressof @callee : !llvm.ptr + ; CHECK: llvm.store %[[FUN]], %[[PTR:.*]] : !llvm.ptr, !llvm.ptr + store ptr @callee, ptr %1 + ; CHECK: %[[INDIR:.*]] = llvm.load %[[PTR]] : !llvm.ptr -> !llvm.ptr + %2 = load ptr, ptr %1 ; CHECK: llvm.call %[[INDIR]]() %3 = call i32 %2() ret i32 %3 @@ -155,12 +143,12 @@ define i32 @callee() { ; Calling a function that has been defined. ; CHECK-LABEL: @function_address_after_def define i32 @function_address_after_def() { - %1 = alloca i32 ()* - ; CHECK: %[[FUN:.*]] = llvm.mlir.addressof @callee : !llvm.ptr> - ; CHECK: llvm.store %[[FUN]], %[[PTR:.*]] - store i32 ()* @callee, i32 ()** %1 - ; CHECK: %[[INDIR:.*]] = llvm.load %[[PTR]] - %2 = load i32 ()*, i32 ()** %1 + %1 = alloca ptr + ; CHECK: %[[FUN:.*]] = llvm.mlir.addressof @callee : !llvm.ptr + ; CHECK: llvm.store %[[FUN]], %[[PTR:.*]] : !llvm.ptr, !llvm.ptr + store ptr @callee, ptr %1 + ; CHECK: %[[INDIR:.*]] = llvm.load %[[PTR]] : !llvm.ptr -> !llvm.ptr + %2 = load ptr, ptr %1 ; CHECK: llvm.call %[[INDIR]]() %3 = call i32 %2() ret i32 %3 @@ -192,22 +180,22 @@ define i32 @function_address_after_def() { ; CHECK: %[[CHAIN1:.+]] = llvm.insertvalue %[[C2]], %[[CHAIN0]][1] ; CHECK: %[[CHAIN2:.+]] = llvm.insertvalue %[[C3]], %[[CHAIN1]][2] ; CHECK: %[[CHAIN3:.+]] = llvm.insertvalue %[[C4]], %[[CHAIN2]][3] -; CHECK: %[[NULL:.+]] = llvm.mlir.null : !llvm.ptr> -; CHECK: %[[ROOT:.+]] = llvm.mlir.undef : !llvm.struct<"nested_agg_type", (struct<"simple_agg_type", (i32, i8, i16, i32)>, ptr>)> +; CHECK: %[[NULL:.+]] = llvm.mlir.null : !llvm.ptr +; CHECK: %[[ROOT:.+]] = llvm.mlir.undef : !llvm.struct<"nested_agg_type", (struct<"simple_agg_type", (i32, i8, i16, i32)>, ptr)> ; CHECK: %[[CHAIN4:.+]] = llvm.insertvalue %[[CHAIN3]], %[[ROOT]][0] ; CHECK: %[[CHAIN5:.+]] = llvm.insertvalue %[[NULL]], %[[CHAIN4]][1] ; CHECK: llvm.return %[[CHAIN5]] -%nested_agg_type = type {%simple_agg_type, %simple_agg_type*} -@nested_agg = global %nested_agg_type { %simple_agg_type{i32 1, i8 2, i16 3, i32 4}, %simple_agg_type* null } +%nested_agg_type = type {%simple_agg_type, ptr} +@nested_agg = global %nested_agg_type { %simple_agg_type{i32 1, i8 2, i16 3, i32 4}, ptr null } -; CHECK: %[[NULL:.+]] = llvm.mlir.null : !llvm.ptr> -; CHECK: %[[ROOT:.+]] = llvm.mlir.undef : !llvm.vec<2 x ptr>> +; CHECK: %[[NULL:.+]] = llvm.mlir.null : !llvm.ptr +; CHECK: %[[ROOT:.+]] = llvm.mlir.undef : !llvm.vec<2 x ptr> ; CHECK: %[[P0:.+]] = llvm.mlir.constant(0 : i32) : i32 -; CHECK: %[[CHAIN0:.+]] = llvm.insertelement %[[NULL]], %[[ROOT]][%[[P0]] : i32] : !llvm.vec<2 x ptr>> +; CHECK: %[[CHAIN0:.+]] = llvm.insertelement %[[NULL]], %[[ROOT]][%[[P0]] : i32] : !llvm.vec<2 x ptr> ; CHECK: %[[P1:.+]] = llvm.mlir.constant(1 : i32) : i32 -; CHECK: %[[CHAIN1:.+]] = llvm.insertelement %[[NULL]], %[[CHAIN0]][%[[P1]] : i32] : !llvm.vec<2 x ptr>> -; CHECK: llvm.return %[[CHAIN1]] : !llvm.vec<2 x ptr>> -@vector_agg = global <2 x %simple_agg_type*> <%simple_agg_type* null, %simple_agg_type* null> +; CHECK: %[[CHAIN1:.+]] = llvm.insertelement %[[NULL]], %[[CHAIN0]][%[[P1]] : i32] : !llvm.vec<2 x ptr> +; CHECK: llvm.return %[[CHAIN1]] : !llvm.vec<2 x ptr> +@vector_agg = global <2 x ptr> ; // ----- @@ -217,12 +205,12 @@ define i32 @function_address_after_def() { ; CHECK-LABEL: @const_exprs_with_duplicate define i64 @const_exprs_with_duplicate() { - ; CHECK: %[[ADDR:.+]] = llvm.mlir.addressof @global : !llvm.ptr - ; CHECK: llvm.getelementptr %[[ADDR]][%{{.*}}] : (!llvm.ptr, i32) -> !llvm.ptr - %1 = add i64 1, ptrtoint (i32* getelementptr (i32, i32* @global, i32 7) to i64) + ; CHECK: %[[ADDR:.+]] = llvm.mlir.addressof @global : !llvm.ptr + ; CHECK: llvm.getelementptr %[[ADDR]][%{{.*}}] : (!llvm.ptr, i32) -> !llvm.ptr + %1 = add i64 1, ptrtoint (ptr getelementptr (i32, ptr @global, i32 7) to i64) ; Verify the address value is reused. - ; CHECK: llvm.getelementptr %[[ADDR]][%{{.*}}] : (!llvm.ptr, i32) -> !llvm.ptr - %2 = add i64 %1, ptrtoint (i32* getelementptr (i32, i32* @global, i32 42) to i64) + ; CHECK: llvm.getelementptr %[[ADDR]][%{{.*}}] : (!llvm.ptr, i32) -> !llvm.ptr + %2 = add i64 %1, ptrtoint (ptr getelementptr (i32, ptr @global, i32 42) to i64) ret i64 %2 } diff --git a/mlir/test/Target/LLVMIR/Import/incorrect-constant-caching.ll b/mlir/test/Target/LLVMIR/Import/incorrect-constant-caching.ll index e430283ed36b..1953ecd68372 100644 --- a/mlir/test/Target/LLVMIR/Import/incorrect-constant-caching.ll +++ b/mlir/test/Target/LLVMIR/Import/incorrect-constant-caching.ll @@ -1,4 +1,4 @@ -; RUN: mlir-translate -opaque-pointers=0 --import-llvm %s | FileCheck %s +; RUN: mlir-translate --import-llvm %s | FileCheck %s ; Testing the fix for issue where llvm.getelementptr translated from the second ; ConstantExpr-GEP tried to use llvm.constant(0: i32)-s that were below itself, @@ -7,24 +7,24 @@ ; This test is primarily used to make sure an verification is passed. Thus, we ; only wrote minimum level of checks. -%my_struct = type {i32, i8*} +%my_struct = type {i32, ptr} ; CHECK: llvm.mlir.constant(8 : i32) : i32 -; CHECK: llvm.mlir.addressof @str0 : !llvm.ptr> +; CHECK: llvm.mlir.addressof @str0 : !llvm.ptr ; CHECK: llvm.mlir.constant(0 : i32) : i32 ; CHECK: llvm.getelementptr -; CHECK: llvm.mlir.undef : !llvm.struct<"my_struct", (i32, ptr)> +; CHECK: llvm.mlir.undef : !llvm.struct<"my_struct", (i32, ptr)> ; CHECK: llvm.insertvalue ; CHECK: llvm.insertvalue ; CHECK: llvm.mlir.constant(7 : i32) : i32 -; CHECK: llvm.mlir.addressof @str1 : !llvm.ptr> +; CHECK: llvm.mlir.addressof @str1 : !llvm.ptr ; CHECK: llvm.getelementptr -; CHECK: llvm.mlir.undef : !llvm.struct<"my_struct", (i32, ptr)> +; CHECK: llvm.mlir.undef : !llvm.struct<"my_struct", (i32, ptr)> ; CHECK: llvm.insertvalue ; CHECK: llvm.insertvalue -; CHECK: llvm.mlir.undef : !llvm.array<2 x struct<"my_struct", (i32, ptr)>> +; CHECK: llvm.mlir.undef : !llvm.array<2 x struct<"my_struct", (i32, ptr)>> ; CHECK: llvm.insertvalue ; CHECK: llvm.insertvalue ; CHECK: llvm.return @str0 = private unnamed_addr constant [5 x i8] c"aaaa\00" @str1 = private unnamed_addr constant [5 x i8] c"bbbb\00" -@g = global [2 x %my_struct] [%my_struct {i32 8, i8* getelementptr ([5 x i8], [5 x i8]* @str0, i32 0, i32 0)}, %my_struct {i32 7, i8* getelementptr ([5 x i8], [5 x i8]* @str1, i32 0, i32 0)}] +@g = global [2 x %my_struct] [%my_struct {i32 8, ptr getelementptr ([5 x i8], ptr @str0, i32 0, i32 1)}, %my_struct {i32 7, ptr getelementptr ([5 x i8], ptr @str1, i32 0, i32 1)}] diff --git a/mlir/test/Target/LLVMIR/Import/incorrect-constexpr-inst-caching.ll b/mlir/test/Target/LLVMIR/Import/incorrect-constexpr-inst-caching.ll index 43f6377c9555..15957050e079 100644 --- a/mlir/test/Target/LLVMIR/Import/incorrect-constexpr-inst-caching.ll +++ b/mlir/test/Target/LLVMIR/Import/incorrect-constexpr-inst-caching.ll @@ -1,30 +1,30 @@ -; RUN: mlir-translate -opaque-pointers=0 --import-llvm %s | FileCheck %s +; RUN: mlir-translate --import-llvm %s | FileCheck %s ; REQUIRES: asserts ; This test is primarily used to make sure an assertion is not triggered. ; Thus, we only wrote minimum level of checks. -%my_struct = type {i32, i8*} +%my_struct = type {i32, ptr} ; CHECK: llvm.mlir.constant(8 : i32) : i32 -; CHECK: llvm.mlir.addressof @str0 : !llvm.ptr> +; CHECK: llvm.mlir.addressof @str0 : !llvm.ptr ; CHECK: llvm.mlir.constant(0 : i32) : i32 ; CHECK: llvm.mlir.constant(1 : i32) : i32 ; CHECK: llvm.getelementptr -; CHECK: llvm.mlir.undef : !llvm.struct<"my_struct", (i32, ptr)> +; CHECK: llvm.mlir.undef : !llvm.struct<"my_struct", (i32, ptr)> ; CHECK: llvm.insertvalue ; CHECK: llvm.insertvalue ; CHECK: llvm.mlir.constant(7 : i32) : i32 -; CHECK: llvm.mlir.addressof @str1 : !llvm.ptr> +; CHECK: llvm.mlir.addressof @str1 : !llvm.ptr ; CHECK: llvm.mlir.constant(2 : i32) : i32 ; CHECK: llvm.mlir.constant(3 : i32) : i32 ; CHECK: llvm.getelementptr -; CHECK: llvm.mlir.undef : !llvm.struct<"my_struct", (i32, ptr)> +; CHECK: llvm.mlir.undef : !llvm.struct<"my_struct", (i32, ptr)> ; CHECK: llvm.insertvalue ; CHECK: llvm.insertvalue -; CHECK: llvm.mlir.undef : !llvm.array<2 x struct<"my_struct", (i32, ptr)>> +; CHECK: llvm.mlir.undef : !llvm.array<2 x struct<"my_struct", (i32, ptr)>> ; CHECK: llvm.insertvalue ; CHECK: llvm.insertvalue ; CHECK: llvm.return @str0 = private unnamed_addr constant [5 x i8] c"aaaa\00" @str1 = private unnamed_addr constant [5 x i8] c"bbbb\00" -@g = global [2 x %my_struct] [%my_struct {i32 8, i8* getelementptr ([5 x i8], [5 x i8]* @str0, i32 0, i32 1)}, %my_struct {i32 7, i8* getelementptr ([5 x i8], [5 x i8]* @str1, i32 2, i32 3)}] +@g = global [2 x %my_struct] [%my_struct {i32 8, ptr getelementptr ([5 x i8], ptr @str0, i32 0, i32 1)}, %my_struct {i32 7, ptr getelementptr ([5 x i8], ptr @str1, i32 2, i32 3)}] diff --git a/mlir/test/Target/LLVMIR/Import/incorrect-scalable-vector-check.ll b/mlir/test/Target/LLVMIR/Import/incorrect-scalable-vector-check.ll index 7de99158cc02..6bf7572e7e79 100644 --- a/mlir/test/Target/LLVMIR/Import/incorrect-scalable-vector-check.ll +++ b/mlir/test/Target/LLVMIR/Import/incorrect-scalable-vector-check.ll @@ -1,8 +1,8 @@ -; RUN: mlir-translate -opaque-pointers=0 --import-llvm %s | FileCheck %s +; RUN: mlir-translate --import-llvm %s | FileCheck %s ; CHECK: llvm.func @shufflevector_crash -define void @shufflevector_crash(<2 x i32*> %arg0) { - ; CHECK: llvm.shufflevector %{{.+}}, %{{.+}} [1, 0] : !llvm.vec<2 x ptr> - %1 = shufflevector <2 x i32*> %arg0, <2 x i32*> undef, <2 x i32> +define void @shufflevector_crash(<2 x ptr> %arg0) { + ; CHECK: llvm.shufflevector %{{.+}}, %{{.+}} [1, 0] : !llvm.vec<2 x ptr> + %1 = shufflevector <2 x ptr> %arg0, <2 x ptr> undef, <2 x i32> ret void } diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll index 7f225a3e2b44..1bcafb7d5c23 100644 --- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll +++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll @@ -1,7 +1,7 @@ -; RUN: mlir-translate -opaque-pointers=0 -import-llvm %s | FileCheck %s +; RUN: mlir-translate -import-llvm %s | FileCheck %s ; CHECK-LABEL: llvm.func @fmuladd_test -define void @fmuladd_test(float %0, float %1, <8 x float> %2, i8* %3) { +define void @fmuladd_test(float %0, float %1, <8 x float> %2, ptr %3) { ; CHECK: llvm.intr.fmuladd(%{{.*}}, %{{.*}}, %{{.*}}) : (f32, f32, f32) -> f32 %5 = call float @llvm.fmuladd.f32(float %0, float %1, float %0) ; CHECK: llvm.intr.fmuladd(%{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xf32>, vector<8xf32>, vector<8xf32>) -> vector<8xf32> @@ -10,8 +10,8 @@ define void @fmuladd_test(float %0, float %1, <8 x float> %2, i8* %3) { %7 = call float @llvm.fma.f32(float %0, float %1, float %0) ; CHECK: llvm.intr.fma(%{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xf32>, vector<8xf32>, vector<8xf32>) -> vector<8xf32> %8 = call <8 x float> @llvm.fma.v8f32(<8 x float> %2, <8 x float> %2, <8 x float> %2) - ; CHECK: "llvm.intr.prefetch"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, i32, i32, i32) -> () - call void @llvm.prefetch.p0i8(i8* %3, i32 0, i32 3, i32 1) + ; CHECK: "llvm.intr.prefetch"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, i32, i32, i32) -> () + call void @llvm.prefetch.p0(ptr %3, i32 0, i32 3, i32 1) ret void } @@ -265,7 +265,7 @@ define void @vector_reductions(float %0, <8 x float> %1, <8 x i32> %2) { ; CHECK-SAME: %[[VEC2:[a-zA-Z0-9]+]] ; CHECK-SAME: %[[PTR:[a-zA-Z0-9]+]] ; CHECK-SAME: %[[STRIDE:[a-zA-Z0-9]+]] -define void @matrix_intrinsics(<64 x float> %vec1, <48 x float> %vec2, float* %ptr, i64 %stride) { +define void @matrix_intrinsics(<64 x float> %vec1, <48 x float> %vec2, ptr %ptr, i64 %stride) { ; CHECK: llvm.intr.matrix.multiply %[[VEC1]], %[[VEC2]] ; CHECK-SAME: {lhs_columns = 16 : i32, lhs_rows = 4 : i32, rhs_columns = 3 : i32} %1 = call <12 x float> @llvm.matrix.multiply.v12f32.v64f32.v48f32(<64 x float> %vec1, <48 x float> %vec2, i32 4, i32 16, i32 3) @@ -274,10 +274,10 @@ define void @matrix_intrinsics(<64 x float> %vec1, <48 x float> %vec2, float* %p %2 = call <48 x float> @llvm.matrix.transpose.v48f32(<48 x float> %vec2, i32 3, i32 16) ; CHECK: %[[VAL1:.+]] = llvm.intr.matrix.column.major.load %[[PTR]], ; CHECK-SAME: {columns = 16 : i32, isVolatile = false, rows = 3 : i32} - %3 = call <48 x float> @llvm.matrix.column.major.load.v48f32.i64(float* align 4 %ptr, i64 %stride, i1 false, i32 3, i32 16) + %3 = call <48 x float> @llvm.matrix.column.major.load.v48f32.i64(ptr align 4 %ptr, i64 %stride, i1 false, i32 3, i32 16) ; CHECK: llvm.intr.matrix.column.major.store %[[VAL1]], %[[PTR]], ; CHECK-SAME: {columns = 16 : i32, isVolatile = true, rows = 3 : i32} - call void @llvm.matrix.column.major.store.v48f32.i64(<48 x float> %3, float* align 4 %ptr, i64 %stride, i1 true, i32 3, i32 16) + call void @llvm.matrix.column.major.store.v48f32.i64(<48 x float> %3, ptr align 4 %ptr, i64 %stride, i1 true, i32 3, i32 16) ret void } @@ -291,68 +291,68 @@ define <7 x i1> @get_active_lane_mask(i64 %0, i64 %1) { ; CHECK-LABEL: @masked_load_store_intrinsics ; CHECK-SAME: %[[VEC:[a-zA-Z0-9]+]] ; CHECK-SAME: %[[MASK:[a-zA-Z0-9]+]] -define void @masked_load_store_intrinsics(<7 x float>* %vec, <7 x i1> %mask) { +define void @masked_load_store_intrinsics(ptr %vec, <7 x i1> %mask) { ; CHECK: %[[UNDEF:.+]] = llvm.mlir.undef ; CHECK: %[[VAL1:.+]] = llvm.intr.masked.load %[[VEC]], %[[MASK]], %[[UNDEF]] {alignment = 1 : i32} - ; CHECK-SAME: (!llvm.ptr>, vector<7xi1>, vector<7xf32>) -> vector<7xf32> - %1 = call <7 x float> @llvm.masked.load.v7f32.p0v7f32(<7 x float>* %vec, i32 1, <7 x i1> %mask, <7 x float> undef) + ; CHECK-SAME: (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32> + %1 = call <7 x float> @llvm.masked.load.v7f32.p0(ptr %vec, i32 1, <7 x i1> %mask, <7 x float> undef) ; CHECK: %[[VAL2:.+]] = llvm.intr.masked.load %[[VEC]], %[[MASK]], %[[VAL1]] {alignment = 4 : i32} - %2 = call <7 x float> @llvm.masked.load.v7f32.p0v7f32(<7 x float>* %vec, i32 4, <7 x i1> %mask, <7 x float> %1) + %2 = call <7 x float> @llvm.masked.load.v7f32.p0(ptr %vec, i32 4, <7 x i1> %mask, <7 x float> %1) ; CHECK: llvm.intr.masked.store %[[VAL2]], %[[VEC]], %[[MASK]] {alignment = 8 : i32} - ; CHECK-SAME: vector<7xf32>, vector<7xi1> into !llvm.ptr> - call void @llvm.masked.store.v7f32.p0v7f32(<7 x float> %2, <7 x float>* %vec, i32 8, <7 x i1> %mask) + ; CHECK-SAME: vector<7xf32>, vector<7xi1> into !llvm.ptr + call void @llvm.masked.store.v7f32.p0(<7 x float> %2, ptr %vec, i32 8, <7 x i1> %mask) ret void } ; CHECK-LABEL: @masked_gather_scatter_intrinsics ; CHECK-SAME: %[[VEC:[a-zA-Z0-9]+]] ; CHECK-SAME: %[[MASK:[a-zA-Z0-9]+]] -define void @masked_gather_scatter_intrinsics(<7 x float*> %vec, <7 x i1> %mask) { +define void @masked_gather_scatter_intrinsics(<7 x ptr> %vec, <7 x i1> %mask) { ; CHECK: %[[UNDEF:.+]] = llvm.mlir.undef ; CHECK: %[[VAL1:.+]] = llvm.intr.masked.gather %[[VEC]], %[[MASK]], %[[UNDEF]] {alignment = 1 : i32} - ; CHECK-SAME: (!llvm.vec<7 x ptr>, vector<7xi1>, vector<7xf32>) -> vector<7xf32> - %1 = call <7 x float> @llvm.masked.gather.v7f32.v7p0f32(<7 x float*> %vec, i32 1, <7 x i1> %mask, <7 x float> undef) + ; CHECK-SAME: (!llvm.vec<7 x ptr>, vector<7xi1>, vector<7xf32>) -> vector<7xf32> + %1 = call <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr> %vec, i32 1, <7 x i1> %mask, <7 x float> undef) ; CHECK: %[[VAL2:.+]] = llvm.intr.masked.gather %[[VEC]], %[[MASK]], %[[VAL1]] {alignment = 4 : i32} - %2 = call <7 x float> @llvm.masked.gather.v7f32.v7p0f32(<7 x float*> %vec, i32 4, <7 x i1> %mask, <7 x float> %1) + %2 = call <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr> %vec, i32 4, <7 x i1> %mask, <7 x float> %1) ; CHECK: llvm.intr.masked.scatter %[[VAL2]], %[[VEC]], %[[MASK]] {alignment = 8 : i32} - ; CHECK-SAME: vector<7xf32>, vector<7xi1> into !llvm.vec<7 x ptr> - call void @llvm.masked.scatter.v7f32.v7p0f32(<7 x float> %2, <7 x float*> %vec, i32 8, <7 x i1> %mask) + ; CHECK-SAME: vector<7xf32>, vector<7xi1> into !llvm.vec<7 x ptr> + call void @llvm.masked.scatter.v7f32.v7p0(<7 x float> %2, <7 x ptr> %vec, i32 8, <7 x i1> %mask) ret void } ; CHECK-LABEL: llvm.func @masked_expand_compress_intrinsics -define void @masked_expand_compress_intrinsics(float* %0, <7 x i1> %1, <7 x float> %2) { - ; CHECK: %[[val1:.+]] = "llvm.intr.masked.expandload"(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32> - %4 = call <7 x float> @llvm.masked.expandload.v7f32(float* %0, <7 x i1> %1, <7 x float> %2) - ; CHECK: "llvm.intr.masked.compressstore"(%[[val1]], %{{.*}}, %{{.*}}) : (vector<7xf32>, !llvm.ptr, vector<7xi1>) -> () - call void @llvm.masked.compressstore.v7f32(<7 x float> %4, float* %0, <7 x i1> %1) +define void @masked_expand_compress_intrinsics(ptr %0, <7 x i1> %1, <7 x float> %2) { + ; CHECK: %[[val1:.+]] = "llvm.intr.masked.expandload"(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32> + %4 = call <7 x float> @llvm.masked.expandload.v7f32(ptr %0, <7 x i1> %1, <7 x float> %2) + ; CHECK: "llvm.intr.masked.compressstore"(%[[val1]], %{{.*}}, %{{.*}}) : (vector<7xf32>, !llvm.ptr, vector<7xi1>) -> () + call void @llvm.masked.compressstore.v7f32(<7 x float> %4, ptr %0, <7 x i1> %1) ret void } ; CHECK-LABEL: llvm.func @memcpy_test -define void @memcpy_test(i32 %0, i8* %1, i8* %2) { +define void @memcpy_test(i32 %0, ptr %1, ptr %2) { ; CHECK: %[[FALSE:.+]] = llvm.mlir.constant(false) : i1 ; CHECK: %[[CST:.+]] = llvm.mlir.constant(10 : i64) : i64 - ; CHECK: "llvm.intr.memcpy"(%{{.*}}, %{{.*}}, %{{.*}}, %[[FALSE]]) : (!llvm.ptr, !llvm.ptr, i32, i1) -> () - call void @llvm.memcpy.p0i8.p0i8.i32(i8* %1, i8* %2, i32 %0, i1 false) - ; CHECK: "llvm.intr.memcpy.inline"(%{{.*}}, %{{.*}}, %[[CST]], %[[FALSE]]) : (!llvm.ptr, !llvm.ptr, i64, i1) -> () - call void @llvm.memcpy.inline.p0i8.p0i8.i64(i8* %1, i8* %2, i64 10, i1 false) + ; CHECK: "llvm.intr.memcpy"(%{{.*}}, %{{.*}}, %{{.*}}, %[[FALSE]]) : (!llvm.ptr, !llvm.ptr, i32, i1) -> () + call void @llvm.memcpy.p0.p0.i32(ptr %1, ptr %2, i32 %0, i1 false) + ; CHECK: "llvm.intr.memcpy.inline"(%{{.*}}, %{{.*}}, %[[CST]], %[[FALSE]]) : (!llvm.ptr, !llvm.ptr, i64, i1) -> () + call void @llvm.memcpy.inline.p0.p0.i64(ptr %1, ptr %2, i64 10, i1 false) ret void } ; CHECK-LABEL: llvm.func @memmove_test -define void @memmove_test(i32 %0, i8* %1, i8* %2) { +define void @memmove_test(i32 %0, ptr %1, ptr %2) { ; CHECK: %[[falseval:.+]] = llvm.mlir.constant(false) : i1 - ; CHECK: "llvm.intr.memmove"(%{{.*}}, %{{.*}}, %{{.*}}, %[[falseval]]) : (!llvm.ptr, !llvm.ptr, i32, i1) -> () - call void @llvm.memmove.p0i8.p0i8.i32(i8* %1, i8* %2, i32 %0, i1 false) + ; CHECK: "llvm.intr.memmove"(%{{.*}}, %{{.*}}, %{{.*}}, %[[falseval]]) : (!llvm.ptr, !llvm.ptr, i32, i1) -> () + call void @llvm.memmove.p0.p0.i32(ptr %1, ptr %2, i32 %0, i1 false) ret void } ; CHECK-LABEL: llvm.func @memset_test -define void @memset_test(i32 %0, i8* %1, i8 %2) { +define void @memset_test(i32 %0, ptr %1, i8 %2) { ; CHECK: %[[falseval:.+]] = llvm.mlir.constant(false) : i1 - ; CHECK: "llvm.intr.memset"(%{{.*}}, %{{.*}}, %{{.*}}, %[[falseval]]) : (!llvm.ptr, i8, i32, i1) -> () - call void @llvm.memset.p0i8.i32(i8* %1, i8 %2, i32 %0, i1 false) + ; CHECK: "llvm.intr.memset"(%{{.*}}, %{{.*}}, %{{.*}}, %[[falseval]]) : (!llvm.ptr, i8, i32, i1) -> () + call void @llvm.memset.p0.i32(ptr %1, i8 %2, i32 %0, i1 false) ret void } @@ -411,13 +411,13 @@ define void @umul_with_overflow_test(i32 %0, i32 %1, <8 x i32> %2, <8 x i32> %3) } ; CHECK-LABEL: llvm.func @va_intrinsics_test -define void @va_intrinsics_test(i8* %0, i8* %1) { +define void @va_intrinsics_test(ptr %0, ptr %1) { ; CHECK: llvm.intr.vastart %{{.*}} - call void @llvm.va_start(i8* %0) + call void @llvm.va_start(ptr %0) ; CHECK: llvm.intr.vacopy %{{.*}} to %{{.*}} - call void @llvm.va_copy(i8* %1, i8* %0) + call void @llvm.va_copy(ptr %1, ptr %0) ; CHECK: llvm.intr.vaend %{{.*}} - call void @llvm.va_end(i8* %0) + call void @llvm.va_end(ptr %0) ret void } @@ -430,18 +430,18 @@ define void @assume(i1 %true) { } ; CHECK-LABEL: llvm.func @coro_id -define void @coro_id(i32 %0, i8* %1) { +define void @coro_id(i32 %0, ptr %1) { ; CHECK: llvm.intr.coro.id %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : !llvm.token - %3 = call token @llvm.coro.id(i32 %0, i8* %1, i8* %1, i8* null) + %3 = call token @llvm.coro.id(i32 %0, ptr %1, ptr %1, ptr null) ret void } ; CHECK-LABEL: llvm.func @coro_begin -define void @coro_begin(i32 %0, i8* %1) { +define void @coro_begin(i32 %0, ptr %1) { ; CHECK: llvm.intr.coro.id %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : !llvm.token - %3 = call token @llvm.coro.id(i32 %0, i8* %1, i8* %1, i8* null) - ; CHECK: llvm.intr.coro.begin %{{.*}}, %{{.*}} : !llvm.ptr - %4 = call i8* @llvm.coro.begin(token %3, i8* %1) + %3 = call token @llvm.coro.id(i32 %0, ptr %1, ptr %1, ptr null) + ; CHECK: llvm.intr.coro.begin %{{.*}}, %{{.*}} : !llvm.ptr + %4 = call ptr @llvm.coro.begin(token %3, ptr %1) ret void } @@ -463,76 +463,76 @@ define void @coro_align() { } ; CHECK-LABEL: llvm.func @coro_save -define void @coro_save(i8* %0) { +define void @coro_save(ptr %0) { ; CHECK: llvm.intr.coro.save %{{.*}} : !llvm.token - %2 = call token @llvm.coro.save(i8* %0) + %2 = call token @llvm.coro.save(ptr %0) ret void } ; CHECK-LABEL: llvm.func @coro_suspend -define void @coro_suspend(i32 %0, i1 %1, i8* %2) { +define void @coro_suspend(i32 %0, i1 %1, ptr %2) { ; CHECK: llvm.intr.coro.id %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : !llvm.token - %4 = call token @llvm.coro.id(i32 %0, i8* %2, i8* %2, i8* null) + %4 = call token @llvm.coro.id(i32 %0, ptr %2, ptr %2, ptr null) ; CHECK: llvm.intr.coro.suspend %{{.*}}, %{{.*}} : i8 %5 = call i8 @llvm.coro.suspend(token %4, i1 %1) ret void } ; CHECK-LABEL: llvm.func @coro_end -define void @coro_end(i8* %0, i1 %1) { +define void @coro_end(ptr %0, i1 %1) { ; CHECK: llvm.intr.coro.end - call i1 @llvm.coro.end(i8* %0, i1 %1) + call i1 @llvm.coro.end(ptr %0, i1 %1) ret void } ; CHECK-LABEL: llvm.func @coro_free -define void @coro_free(i32 %0, i8* %1) { +define void @coro_free(i32 %0, ptr %1) { ; CHECK: llvm.intr.coro.id %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : !llvm.token - %3 = call token @llvm.coro.id(i32 %0, i8* %1, i8* %1, i8* null) - ; CHECK: llvm.intr.coro.free %{{.*}}, %{{.*}} : !llvm.ptr - %4 = call i8* @llvm.coro.free(token %3, i8* %1) + %3 = call token @llvm.coro.id(i32 %0, ptr %1, ptr %1, ptr null) + ; CHECK: llvm.intr.coro.free %{{.*}}, %{{.*}} : !llvm.ptr + %4 = call ptr @llvm.coro.free(token %3, ptr %1) ret void } ; CHECK-LABEL: llvm.func @coro_resume -define void @coro_resume(i8* %0) { +define void @coro_resume(ptr %0) { ; CHECK: llvm.intr.coro.resume %{{.*}} - call void @llvm.coro.resume(i8* %0) + call void @llvm.coro.resume(ptr %0) ret void } ; CHECK-LABEL: llvm.func @eh_typeid_for -define void @eh_typeid_for(i8* %0) { +define void @eh_typeid_for(ptr %0) { ; CHECK: llvm.intr.eh.typeid.for %{{.*}} : i32 - %2 = call i32 @llvm.eh.typeid.for(i8* %0) + %2 = call i32 @llvm.eh.typeid.for(ptr %0) ret void } ; CHECK-LABEL: llvm.func @stack_save() { define void @stack_save() { - ; CHECK: llvm.intr.stacksave : !llvm.ptr - %1 = call i8* @llvm.stacksave() + ; CHECK: llvm.intr.stacksave : !llvm.ptr + %1 = call ptr @llvm.stacksave() ret void } ; CHECK-LABEL: llvm.func @stack_restore -define void @stack_restore(i8* %0) { +define void @stack_restore(ptr %0) { ; CHECK: llvm.intr.stackrestore %{{.*}} - call void @llvm.stackrestore(i8* %0) + call void @llvm.stackrestore(ptr %0) ret void } ; CHECK-LABEL: llvm.func @lifetime -define void @lifetime(i8* %0) { - ; CHECK: llvm.intr.lifetime.start 16, %{{.*}} : !llvm.ptr - call void @llvm.lifetime.start.p0i8(i64 16, i8* %0) - ; CHECK: llvm.intr.lifetime.end 32, %{{.*}} : !llvm.ptr - call void @llvm.lifetime.end.p0i8(i64 32, i8* %0) +define void @lifetime(ptr %0) { + ; CHECK: llvm.intr.lifetime.start 16, %{{.*}} : !llvm.ptr + call void @llvm.lifetime.start.p0(i64 16, ptr %0) + ; CHECK: llvm.intr.lifetime.end 32, %{{.*}} : !llvm.ptr + call void @llvm.lifetime.end.p0(i64 32, ptr %0) ret void } ; CHECK-LABEL: llvm.func @vector_predication_intrinsics -define void @vector_predication_intrinsics(<8 x i32> %0, <8 x i32> %1, <8 x float> %2, <8 x float> %3, <8 x i64> %4, <8 x double> %5, <8 x i32*> %6, i32 %7, float %8, i32* %9, float* %10, <8 x i1> %11, i32 %12) { +define void @vector_predication_intrinsics(<8 x i32> %0, <8 x i32> %1, <8 x float> %2, <8 x float> %3, <8 x i64> %4, <8 x double> %5, <8 x ptr> %6, i32 %7, float %8, ptr %9, ptr %10, <8 x i1> %11, i32 %12) { ; CHECK: "llvm.intr.vp.add"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi32>, vector<8xi32>, vector<8xi1>, i32) -> vector<8xi32> %14 = call <8 x i32> @llvm.vp.add.v8i32(<8 x i32> %0, <8 x i32> %1, <8 x i1> %11, i32 %12) ; CHECK: "llvm.intr.vp.sub"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi32>, vector<8xi32>, vector<8xi1>, i32) -> vector<8xi32> @@ -603,14 +603,14 @@ define void @vector_predication_intrinsics(<8 x i32> %0, <8 x i32> %1, <8 x floa %47 = call <8 x i32> @llvm.vp.select.v8i32(<8 x i1> %11, <8 x i32> %0, <8 x i32> %1, i32 %12) ; CHECK: "llvm.intr.vp.merge"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi1>, vector<8xi32>, vector<8xi32>, i32) -> vector<8xi32> %48 = call <8 x i32> @llvm.vp.merge.v8i32(<8 x i1> %11, <8 x i32> %0, <8 x i32> %1, i32 %12) - ; CHECK: "llvm.intr.vp.store"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi32>, !llvm.ptr, vector<8xi1>, i32) -> () - call void @llvm.vp.store.v8i32.p0i32(<8 x i32> %0, i32* %9, <8 x i1> %11, i32 %12) - ; CHECK: "llvm.intr.vp.load"(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<8xi1>, i32) -> vector<8xi32> - %49 = call <8 x i32> @llvm.vp.load.v8i32.p0i32(i32* %9, <8 x i1> %11, i32 %12) - ; CHECK: "llvm.intr.experimental.vp.strided.store"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi32>, !llvm.ptr, i32, vector<8xi1>, i32) -> () - call void @llvm.experimental.vp.strided.store.v8i32.p0i32.i32(<8 x i32> %0, i32* %9, i32 %7, <8 x i1> %11, i32 %12) - ; CHECK: "llvm.intr.experimental.vp.strided.load"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, i32, vector<8xi1>, i32) -> vector<8xi32> - %50 = call <8 x i32> @llvm.experimental.vp.strided.load.v8i32.p0i32.i32(i32* %9, i32 %7, <8 x i1> %11, i32 %12) + ; CHECK: "llvm.intr.vp.store"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi32>, !llvm.ptr, vector<8xi1>, i32) -> () + call void @llvm.vp.store.v8i32.p0(<8 x i32> %0, ptr %9, <8 x i1> %11, i32 %12) + ; CHECK: "llvm.intr.vp.load"(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<8xi1>, i32) -> vector<8xi32> + %49 = call <8 x i32> @llvm.vp.load.v8i32.p0(ptr %9, <8 x i1> %11, i32 %12) + ; CHECK: "llvm.intr.experimental.vp.strided.store"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi32>, !llvm.ptr, i32, vector<8xi1>, i32) -> () + call void @llvm.experimental.vp.strided.store.v8i32.p0.i32(<8 x i32> %0, ptr %9, i32 %7, <8 x i1> %11, i32 %12) + ; CHECK: "llvm.intr.experimental.vp.strided.load"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, i32, vector<8xi1>, i32) -> vector<8xi32> + %50 = call <8 x i32> @llvm.experimental.vp.strided.load.v8i32.p0.i32(ptr %9, i32 %7, <8 x i1> %11, i32 %12) ; CHECK: "llvm.intr.vp.trunc"(%{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi64>, vector<8xi1>, i32) -> vector<8xi32> %51 = call <8 x i32> @llvm.vp.trunc.v8i32.v8i64(<8 x i64> %4, <8 x i1> %11, i32 %12) ; CHECK: "llvm.intr.vp.zext"(%{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi32>, vector<8xi1>, i32) -> vector<8xi64> @@ -625,10 +625,10 @@ define void @vector_predication_intrinsics(<8 x i32> %0, <8 x i32> %1, <8 x floa %56 = call <8 x i64> @llvm.vp.fptoui.v8i64.v8f64(<8 x double> %5, <8 x i1> %11, i32 %12) ; CHECK: "llvm.intr.vp.fptosi"(%{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xf64>, vector<8xi1>, i32) -> vector<8xi64> %57 = call <8 x i64> @llvm.vp.fptosi.v8i64.v8f64(<8 x double> %5, <8 x i1> %11, i32 %12) - ; CHECK: "llvm.intr.vp.ptrtoint"(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.vec<8 x ptr>, vector<8xi1>, i32) -> vector<8xi64> - %58 = call <8 x i64> @llvm.vp.ptrtoint.v8i64.v8p0i32(<8 x i32*> %6, <8 x i1> %11, i32 %12) - ; CHECK: "llvm.intr.vp.inttoptr"(%{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi64>, vector<8xi1>, i32) -> !llvm.vec<8 x ptr> - %59 = call <8 x i32*> @llvm.vp.inttoptr.v8p0i32.v8i64(<8 x i64> %4, <8 x i1> %11, i32 %12) + ; CHECK: "llvm.intr.vp.ptrtoint"(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.vec<8 x ptr>, vector<8xi1>, i32) -> vector<8xi64> + %58 = call <8 x i64> @llvm.vp.ptrtoint.v8i64.v8p0(<8 x ptr> %6, <8 x i1> %11, i32 %12) + ; CHECK: "llvm.intr.vp.inttoptr"(%{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi64>, vector<8xi1>, i32) -> !llvm.vec<8 x ptr> + %59 = call <8 x ptr> @llvm.vp.inttoptr.v8p0.v8i64(<8 x i64> %4, <8 x i1> %11, i32 %12) ret void } @@ -636,7 +636,7 @@ declare float @llvm.fmuladd.f32(float, float, float) declare <8 x float> @llvm.fmuladd.v8f32(<8 x float>, <8 x float>, <8 x float>) declare float @llvm.fma.f32(float, float, float) declare <8 x float> @llvm.fma.v8f32(<8 x float>, <8 x float>, <8 x float>) -declare void @llvm.prefetch.p0i8(i8* nocapture readonly, i32 immarg, i32 immarg, i32) +declare void @llvm.prefetch.p0(ptr nocapture readonly, i32 immarg, i32 immarg, i32) declare float @llvm.exp.f32(float) declare <8 x float> @llvm.exp.v8f32(<8 x float>) declare float @llvm.exp2.f32(float) @@ -700,19 +700,19 @@ declare float @llvm.vector.reduce.fmul.v8f32(float, <8 x float>) declare i32 @llvm.vector.reduce.xor.v8i32(<8 x i32>) declare <12 x float> @llvm.matrix.multiply.v12f32.v64f32.v48f32(<64 x float>, <48 x float>, i32 immarg, i32 immarg, i32 immarg) declare <48 x float> @llvm.matrix.transpose.v48f32(<48 x float>, i32 immarg, i32 immarg) -declare <48 x float> @llvm.matrix.column.major.load.v48f32.i64(float* nocapture, i64, i1 immarg, i32 immarg, i32 immarg) -declare void @llvm.matrix.column.major.store.v48f32.i64(<48 x float>, float* nocapture writeonly, i64, i1 immarg, i32 immarg, i32 immarg) +declare <48 x float> @llvm.matrix.column.major.load.v48f32.i64(ptr nocapture, i64, i1 immarg, i32 immarg, i32 immarg) +declare void @llvm.matrix.column.major.store.v48f32.i64(<48 x float>, ptr nocapture writeonly, i64, i1 immarg, i32 immarg, i32 immarg) declare <7 x i1> @llvm.get.active.lane.mask.v7i1.i64(i64, i64) -declare <7 x float> @llvm.masked.load.v7f32.p0v7f32(<7 x float>*, i32 immarg, <7 x i1>, <7 x float>) -declare void @llvm.masked.store.v7f32.p0v7f32(<7 x float>, <7 x float>*, i32 immarg, <7 x i1>) -declare <7 x float> @llvm.masked.gather.v7f32.v7p0f32(<7 x float*>, i32 immarg, <7 x i1>, <7 x float>) -declare void @llvm.masked.scatter.v7f32.v7p0f32(<7 x float>, <7 x float*>, i32 immarg, <7 x i1>) -declare <7 x float> @llvm.masked.expandload.v7f32(float*, <7 x i1>, <7 x float>) -declare void @llvm.masked.compressstore.v7f32(<7 x float>, float*, <7 x i1>) -declare void @llvm.memcpy.p0i8.p0i8.i32(i8* noalias nocapture writeonly, i8* noalias nocapture readonly, i32, i1 immarg) -declare void @llvm.memcpy.inline.p0i8.p0i8.i64(i8* noalias nocapture writeonly, i8* noalias nocapture readonly, i64 immarg, i1 immarg) -declare void @llvm.memmove.p0i8.p0i8.i32(i8* nocapture writeonly, i8* nocapture readonly, i32, i1 immarg) -declare void @llvm.memset.p0i8.i32(i8* nocapture writeonly, i8, i32, i1 immarg) +declare <7 x float> @llvm.masked.load.v7f32.p0(ptr, i32 immarg, <7 x i1>, <7 x float>) +declare void @llvm.masked.store.v7f32.p0(<7 x float>, ptr, i32 immarg, <7 x i1>) +declare <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr>, i32 immarg, <7 x i1>, <7 x float>) +declare void @llvm.masked.scatter.v7f32.v7p0(<7 x float>, <7 x ptr>, i32 immarg, <7 x i1>) +declare <7 x float> @llvm.masked.expandload.v7f32(ptr, <7 x i1>, <7 x float>) +declare void @llvm.masked.compressstore.v7f32(<7 x float>, ptr, <7 x i1>) +declare void @llvm.memcpy.p0.p0.i32(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i32, i1 immarg) +declare void @llvm.memcpy.inline.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64 immarg, i1 immarg) +declare void @llvm.memmove.p0.p0.i32(ptr nocapture writeonly, ptr nocapture readonly, i32, i1 immarg) +declare void @llvm.memset.p0.i32(ptr nocapture writeonly, i8, i32, i1 immarg) declare { i32, i1 } @llvm.sadd.with.overflow.i32(i32, i32) declare { <8 x i32>, <8 x i1> } @llvm.sadd.with.overflow.v8i32(<8 x i32>, <8 x i32>) declare { i32, i1 } @llvm.uadd.with.overflow.i32(i32, i32) @@ -725,23 +725,23 @@ declare { i32, i1 } @llvm.smul.with.overflow.i32(i32, i32) declare { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32>, <8 x i32>) declare { i32, i1 } @llvm.umul.with.overflow.i32(i32, i32) declare { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32>, <8 x i32>) -declare token @llvm.coro.id(i32, i8* readnone, i8* nocapture readonly, i8*) -declare i8* @llvm.coro.begin(token, i8* writeonly) +declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) +declare ptr @llvm.coro.begin(token, ptr writeonly) declare i64 @llvm.coro.size.i64() declare i32 @llvm.coro.size.i32() declare i64 @llvm.coro.align.i64() declare i32 @llvm.coro.align.i32() -declare token @llvm.coro.save(i8*) +declare token @llvm.coro.save(ptr) declare i8 @llvm.coro.suspend(token, i1) -declare i1 @llvm.coro.end(i8*, i1) -declare i8* @llvm.coro.free(token, i8* nocapture readonly) -declare void @llvm.coro.resume(i8*) -declare i32 @llvm.eh.typeid.for(i8*) -declare i8* @llvm.stacksave() -declare void @llvm.stackrestore(i8*) -declare void @llvm.va_start(i8*) -declare void @llvm.va_copy(i8*, i8*) -declare void @llvm.va_end(i8*) +declare i1 @llvm.coro.end(ptr, i1) +declare ptr @llvm.coro.free(token, ptr nocapture readonly) +declare void @llvm.coro.resume(ptr) +declare i32 @llvm.eh.typeid.for(ptr) +declare ptr @llvm.stacksave() +declare void @llvm.stackrestore(ptr) +declare void @llvm.va_start(ptr) +declare void @llvm.va_copy(ptr, ptr) +declare void @llvm.va_end(ptr) declare <8 x i32> @llvm.vp.add.v8i32(<8 x i32>, <8 x i32>, <8 x i1>, i32) declare <8 x i32> @llvm.vp.sub.v8i32(<8 x i32>, <8 x i32>, <8 x i1>, i32) declare <8 x i32> @llvm.vp.mul.v8i32(<8 x i32>, <8 x i32>, <8 x i1>, i32) @@ -777,10 +777,10 @@ declare float @llvm.vp.reduce.fmax.v8f32(float, <8 x float>, <8 x i1>, i32) declare float @llvm.vp.reduce.fmin.v8f32(float, <8 x float>, <8 x i1>, i32) declare <8 x i32> @llvm.vp.select.v8i32(<8 x i1>, <8 x i32>, <8 x i32>, i32) declare <8 x i32> @llvm.vp.merge.v8i32(<8 x i1>, <8 x i32>, <8 x i32>, i32) -declare void @llvm.vp.store.v8i32.p0i32(<8 x i32>, i32* nocapture, <8 x i1>, i32) -declare <8 x i32> @llvm.vp.load.v8i32.p0i32(i32* nocapture, <8 x i1>, i32) -declare void @llvm.experimental.vp.strided.store.v8i32.p0i32.i32(<8 x i32>, i32* nocapture, i32, <8 x i1>, i32) -declare <8 x i32> @llvm.experimental.vp.strided.load.v8i32.p0i32.i32(i32* nocapture, i32, <8 x i1>, i32) +declare void @llvm.vp.store.v8i32.p0(<8 x i32>, ptr nocapture, <8 x i1>, i32) +declare <8 x i32> @llvm.vp.load.v8i32.p0(ptr nocapture, <8 x i1>, i32) +declare void @llvm.experimental.vp.strided.store.v8i32.p0.i32(<8 x i32>, ptr nocapture, i32, <8 x i1>, i32) +declare <8 x i32> @llvm.experimental.vp.strided.load.v8i32.p0.i32(ptr nocapture, i32, <8 x i1>, i32) declare <8 x i32> @llvm.vp.trunc.v8i32.v8i64(<8 x i64>, <8 x i1>, i32) declare <8 x i64> @llvm.vp.zext.v8i64.v8i32(<8 x i32>, <8 x i1>, i32) declare <8 x i64> @llvm.vp.sext.v8i64.v8i32(<8 x i32>, <8 x i1>, i32) @@ -788,8 +788,8 @@ declare <8 x float> @llvm.vp.fptrunc.v8f32.v8f64(<8 x double>, <8 x i1>, i32) declare <8 x double> @llvm.vp.fpext.v8f64.v8f32(<8 x float>, <8 x i1>, i32) declare <8 x i64> @llvm.vp.fptoui.v8i64.v8f64(<8 x double>, <8 x i1>, i32) declare <8 x i64> @llvm.vp.fptosi.v8i64.v8f64(<8 x double>, <8 x i1>, i32) -declare <8 x i64> @llvm.vp.ptrtoint.v8i64.v8p0i32(<8 x i32*>, <8 x i1>, i32) -declare <8 x i32*> @llvm.vp.inttoptr.v8p0i32.v8i64(<8 x i64>, <8 x i1>, i32) -declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) -declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) +declare <8 x i64> @llvm.vp.ptrtoint.v8i64.v8p0(<8 x ptr>, <8 x i1>, i32) +declare <8 x ptr> @llvm.vp.inttoptr.v8p0.v8i64(<8 x i64>, <8 x i1>, i32) +declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) +declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) declare void @llvm.assume(i1) diff --git a/mlir/test/Target/LLVMIR/Import/zeroinitializer.ll b/mlir/test/Target/LLVMIR/Import/zeroinitializer.ll index 27a940cd4e85..41933dcbe7cb 100644 --- a/mlir/test/Target/LLVMIR/Import/zeroinitializer.ll +++ b/mlir/test/Target/LLVMIR/Import/zeroinitializer.ll @@ -1,13 +1,12 @@ -; RUN: mlir-translate -opaque-pointers=0 --import-llvm %s | FileCheck %s +; RUN: mlir-translate --import-llvm %s | FileCheck %s -%Domain = type { %Domain**, %Domain* } +%Domain = type { ptr, ptr } ; CHECK: llvm.mlir.global external @D() -; CHECK-SAME: !llvm.struct<"Domain", (ptr>>, ptr>)> -; CHECK: %[[E0:.+]] = llvm.mlir.null : !llvm.ptr>>, ptr>)>>> -; CHECK: %[[E1:.+]] = llvm.mlir.null : !llvm.ptr>>, ptr>)>> -; CHECK: %[[ROOT:.+]] = llvm.mlir.undef : !llvm.struct<"Domain", (ptr>>, ptr>)> +; CHECK-SAME: !llvm.struct<"Domain", (ptr, ptr)> +; CHECK: %[[E0:.+]] = llvm.mlir.null : !llvm.ptr +; CHECK: %[[ROOT:.+]] = llvm.mlir.undef : !llvm.struct<"Domain", (ptr, ptr)> ; CHECK: %[[CHAIN:.+]] = llvm.insertvalue %[[E0]], %[[ROOT]][0] -; CHECK: %[[RES:.+]] = llvm.insertvalue %[[E1]], %[[CHAIN]][1] +; CHECK: %[[RES:.+]] = llvm.insertvalue %[[E0]], %[[CHAIN]][1] ; CHECK: llvm.return %[[RES]] @D = global %Domain zeroinitializer From b7e52bbf1a3f1bfec93b6deaca6e7ff33cd0dbae Mon Sep 17 00:00:00 2001 From: Tobias Gysi Date: Mon, 6 Mar 2023 10:16:14 +0100 Subject: [PATCH 058/147] [mlir][llvm] Opaque pointer support for atomic and call ops. This revision adapts the printers and parsers of the LLVM Dialect AtomicRMWOp, AtomicCmpXchgOp, CallOp, and InvokeOp to support both opaque and typed pointers by printing the pointer types explicitly. Previously, the printers and parser of these operations silently assumed typed pointers. This assumption is problematic if a lowering or the LLVM IR import produce LLVM Dialect with opaque pointers and the IR is then printed and parsed, for example, when running mlir-translate. In LLVM IR itself all tests with typed pointers are already gone. It is thus important to start switching to opaque pointers. This revision can be seen as a preparation step for the switch of the LLVM Dialect to opaque pointers. Once printing and parsing works seamlessly, all lowerings to LLVM Dialect can be switched to produce opaque pointers. After a transition period, LLVM Dialect itself can by simplified to support opaque pointers only. Reviewed By: ftynse, Dinistro Differential Revision: https://reviews.llvm.org/D142884 (cherry picked from commit d8153ae29959e8e27a1458fab4107e93b5861240) --- mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td | 46 +- mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 660 +++++++----------- .../LLVMIR/LLVMToLLVMIRTranslation.cpp | 32 +- .../Conversion/FuncToLLVM/convert-funcs.mlir | 4 +- .../MemRefToLLVM/memref-to-llvm.mlir | 2 +- mlir/test/Dialect/LLVMIR/invalid.mlir | 113 +-- mlir/test/Dialect/LLVMIR/roundtrip.mlir | 30 +- mlir/test/Dialect/OpenMP/invalid.mlir | 4 +- mlir/test/Dialect/OpenMP/ops.mlir | 2 +- mlir/test/Target/LLVMIR/Import/constant.ll | 4 +- .../test/Target/LLVMIR/Import/instructions.ll | 55 +- mlir/test/Target/LLVMIR/llvmir.mlir | 51 +- mlir/test/Target/LLVMIR/openmp-reduction.mlir | 10 +- 13 files changed, 484 insertions(+), 529 deletions(-) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td index c31245cf6f03..f318173549aa 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td @@ -513,6 +513,9 @@ def LLVM_LandingpadOp : LLVM_Op<"landingpad"> { // CallOp //===----------------------------------------------------------------------===// +// FIXME: Add a type attribute that carries the LLVM function type to support +// indirect calls to variadic functions. The type attribute is necessary to +// distinguish normal and variadic arguments. def LLVM_CallOp : LLVM_Op<"call", [DeclareOpInterfaceMethods, DeclareOpInterfaceMethods, @@ -527,9 +530,10 @@ def LLVM_CallOp : LLVM_Op<"call", The `call` instruction supports both direct and indirect calls. Direct calls start with a function name (`@`-prefixed) and indirect calls start with an SSA value (`%`-prefixed). The direct callee, if present, is stored as a - function attribute `callee`. The trailing type of the instruction is always - the MLIR function type, which may be different from the indirect callee that - has the wrapped LLVM IR function type. + function attribute `callee`. The trailing type list contains the optional + indirect callee type and the MLIR function type, which differs from the + LLVM function type that uses a explicit void type to model functions that do + not return a value. Examples: @@ -541,7 +545,7 @@ def LLVM_CallOp : LLVM_Op<"call", llvm.call @bar(%0) : (f32) -> () // Indirect call with an argument and without a result. - llvm.call %1(%0) : (f32) -> () + llvm.call %1(%0) : !llvm.ptr, (f32) -> () ``` }]; @@ -1652,13 +1656,18 @@ def LLVM_AtomicRMWType : AnyTypeOf<[LLVM_AnyFloat, AnyInteger]>; // FIXME: Need to add alignment and syncscope attribute to MLIR atomicrmw // operation. -def LLVM_AtomicRMWOp : LLVM_Op<"atomicrmw"> { +def LLVM_AtomicRMWOp : LLVM_Op<"atomicrmw", [ + TypesMatchWith<"result #0 and operand #1 have the same type", + "val", "res", "$_self">]> { let arguments = (ins AtomicBinOp:$bin_op, LLVM_PointerTo:$ptr, LLVM_AtomicRMWType:$val, AtomicOrdering:$ordering); let results = (outs LLVM_AtomicRMWType:$res); - let hasCustomAssemblyFormat = 1; let hasVerifier = 1; + let assemblyFormat = [{ + $bin_op $ptr `,` $val $ordering + attr-dict `:` qualified(type($ptr)) `,` type($val) + }]; string llvmInstName = "AtomicRMW"; string llvmBuilder = [{ $res = builder.CreateAtomicRMW(getLLVMAtomicBinOp($bin_op), $ptr, $val, @@ -1676,27 +1685,24 @@ def LLVM_AtomicRMWOp : LLVM_Op<"atomicrmw"> { } def LLVM_AtomicCmpXchgType : AnyTypeOf<[AnyInteger, LLVM_AnyPointer]>; -def LLVM_AtomicCmpXchgResultType : Type().getBody().size() == 2">, - SubstLeaves<"$_self", - "$_self.cast<::mlir::LLVM::LLVMStructType>().getBody()[0]", - LLVM_AtomicCmpXchgType.predicate>, - SubstLeaves<"$_self", - "$_self.cast<::mlir::LLVM::LLVMStructType>().getBody()[1]", - I1.predicate>]>, - "an LLVM struct type with any integer or pointer followed by a single-bit " - "integer">; // FIXME: Need to add alignment attribute to MLIR cmpxchg operation. -def LLVM_AtomicCmpXchgOp : LLVM_Op<"cmpxchg"> { +def LLVM_AtomicCmpXchgOp : LLVM_Op<"cmpxchg", [ + TypesMatchWith<"operand #1 and operand #2 have the same type", + "val", "cmp", "$_self">, + TypesMatchWith<"result #0 has an LLVM struct type consisting of " + "the type of operand #2 and a bool", + "val", "res", "getValAndBoolStructType($_self)">]> { let arguments = (ins LLVM_PointerTo:$ptr, LLVM_AtomicCmpXchgType:$cmp, LLVM_AtomicCmpXchgType:$val, AtomicOrdering:$success_ordering, AtomicOrdering:$failure_ordering); - let results = (outs LLVM_AtomicCmpXchgResultType:$res); - let hasCustomAssemblyFormat = 1; + let results = (outs LLVM_AnyStruct:$res); let hasVerifier = 1; + let assemblyFormat = [{ + $ptr `,` $cmp `,` $val $success_ordering $failure_ordering + attr-dict `:` qualified(type($ptr)) `,` type($val) + }]; string llvmInstName = "AtomicCmpXchg"; string llvmBuilder = [{ $res = builder.CreateAtomicCmpXchg($ptr, $cmp, $val, llvm::MaybeAlign(), diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp index 4546411155f1..04f91755314d 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -891,6 +891,250 @@ ParseResult StoreOp::parse(OpAsmParser &parser, OperationState &result) { return success(); } +//===----------------------------------------------------------------------===// +// CallOp +//===----------------------------------------------------------------------===// + +void CallOp::build(OpBuilder &builder, OperationState &state, TypeRange results, + StringRef callee, ValueRange args) { + build(builder, state, results, builder.getStringAttr(callee), args); +} + +void CallOp::build(OpBuilder &builder, OperationState &state, TypeRange results, + StringAttr callee, ValueRange args) { + build(builder, state, results, SymbolRefAttr::get(callee), args, nullptr, + nullptr); +} + +void CallOp::build(OpBuilder &builder, OperationState &state, TypeRange results, + FlatSymbolRefAttr callee, ValueRange args) { + build(builder, state, results, callee, args, nullptr, nullptr); +} + +void CallOp::build(OpBuilder &builder, OperationState &state, LLVMFuncOp func, + ValueRange args) { + SmallVector results; + Type resultType = func.getFunctionType().getReturnType(); + if (!resultType.isa()) + results.push_back(resultType); + build(builder, state, results, SymbolRefAttr::get(func), args, nullptr, + nullptr); +} + +CallInterfaceCallable CallOp::getCallableForCallee() { + // Direct call. + if (FlatSymbolRefAttr calleeAttr = getCalleeAttr()) + return calleeAttr; + // Indirect call, callee Value is the first operand. + return getOperand(0); +} + +Operation::operand_range CallOp::getArgOperands() { + return getOperands().drop_front(getCallee().has_value() ? 0 : 1); +} + +LogicalResult CallOp::verifySymbolUses(SymbolTableCollection &symbolTable) { + if (getNumResults() > 1) + return emitOpError("must have 0 or 1 result"); + + // Type for the callee, we'll get it differently depending if it is a direct + // or indirect call. + Type fnType; + + bool isIndirect = false; + + // If this is an indirect call, the callee attribute is missing. + FlatSymbolRefAttr calleeName = getCalleeAttr(); + if (!calleeName) { + isIndirect = true; + if (!getNumOperands()) + return emitOpError( + "must have either a `callee` attribute or at least an operand"); + auto ptrType = getOperand(0).getType().dyn_cast(); + if (!ptrType) + return emitOpError("indirect call expects a pointer as callee: ") + << getOperand(0).getType(); + + if (ptrType.isOpaque()) + return success(); + + fnType = ptrType.getElementType(); + } else { + Operation *callee = + symbolTable.lookupNearestSymbolFrom(*this, calleeName.getAttr()); + if (!callee) + return emitOpError() + << "'" << calleeName.getValue() + << "' does not reference a symbol in the current scope"; + auto fn = dyn_cast(callee); + if (!fn) + return emitOpError() << "'" << calleeName.getValue() + << "' does not reference a valid LLVM function"; + + fnType = fn.getFunctionType(); + } + + LLVMFunctionType funcType = fnType.dyn_cast(); + if (!funcType) + return emitOpError("callee does not have a functional type: ") << fnType; + + // Indirect variadic function calls are not supported since the translation to + // LLVM IR reconstructs the LLVM function type from the argument and result + // types. An additional type attribute that stores the LLVM function type + // would be needed to distinguish normal and variadic function arguments. + // TODO: Support indirect calls to variadic function pointers. + if (isIndirect && funcType.isVarArg()) + return emitOpError() + << "indirect calls to variadic functions are not supported"; + + // Verify that the operand and result types match the callee. + + if (!funcType.isVarArg() && + funcType.getNumParams() != (getNumOperands() - isIndirect)) + return emitOpError() << "incorrect number of operands (" + << (getNumOperands() - isIndirect) + << ") for callee (expecting: " + << funcType.getNumParams() << ")"; + + if (funcType.getNumParams() > (getNumOperands() - isIndirect)) + return emitOpError() << "incorrect number of operands (" + << (getNumOperands() - isIndirect) + << ") for varargs callee (expecting at least: " + << funcType.getNumParams() << ")"; + + for (unsigned i = 0, e = funcType.getNumParams(); i != e; ++i) + if (getOperand(i + isIndirect).getType() != funcType.getParamType(i)) + return emitOpError() << "operand type mismatch for operand " << i << ": " + << getOperand(i + isIndirect).getType() + << " != " << funcType.getParamType(i); + + if (getNumResults() == 0 && + !funcType.getReturnType().isa()) + return emitOpError() << "expected function call to produce a value"; + + if (getNumResults() != 0 && + funcType.getReturnType().isa()) + return emitOpError() + << "calling function with void result must not produce values"; + + if (getNumResults() > 1) + return emitOpError() + << "expected LLVM function call to produce 0 or 1 result"; + + if (getNumResults() && getResult().getType() != funcType.getReturnType()) + return emitOpError() << "result type mismatch: " << getResult().getType() + << " != " << funcType.getReturnType(); + + return success(); +} + +void CallOp::print(OpAsmPrinter &p) { + auto callee = getCallee(); + bool isDirect = callee.has_value(); + + // Print the direct callee if present as a function attribute, or an indirect + // callee (first operand) otherwise. + p << ' '; + if (isDirect) + p.printSymbolName(callee.value()); + else + p << getOperand(0); + + auto args = getOperands().drop_front(isDirect ? 0 : 1); + p << '(' << args << ')'; + p.printOptionalAttrDict(processFMFAttr((*this)->getAttrs()), {"callee"}); + + p << " : "; + if (!isDirect) + p << getOperand(0).getType() << ", "; + + // Reconstruct the function MLIR function type from operand and result types. + p.printFunctionalType(args.getTypes(), getResultTypes()); +} + +/// Parses the type of a call operation and resolves the operands if the parsing +/// succeeds. Returns failure otherwise. +static ParseResult parseCallTypeAndResolveOperands( + OpAsmParser &parser, OperationState &result, bool isDirect, + ArrayRef operands) { + SMLoc trailingTypesLoc = parser.getCurrentLocation(); + SmallVector types; + if (parser.parseColonTypeList(types)) + return failure(); + + if (isDirect && types.size() != 1) + return parser.emitError(trailingTypesLoc, + "expected direct call to have 1 trailing type"); + if (!isDirect && types.size() != 2) + return parser.emitError(trailingTypesLoc, + "expected indirect call to have 2 trailing types"); + + auto funcType = types.pop_back_val().dyn_cast(); + if (!funcType) + return parser.emitError(trailingTypesLoc, + "expected trailing function type"); + if (funcType.getNumResults() > 1) + return parser.emitError(trailingTypesLoc, + "expected function with 0 or 1 result"); + if (funcType.getNumResults() == 1 && + funcType.getResult(0).isa()) + return parser.emitError(trailingTypesLoc, + "expected a non-void result type"); + + // The head element of the types list matches the callee type for + // indirect calls, while the types list is emtpy for direct calls. + // Append the function input types to resolve the call operation + // operands. + llvm::append_range(types, funcType.getInputs()); + if (parser.resolveOperands(operands, types, parser.getNameLoc(), + result.operands)) + return failure(); + if (funcType.getNumResults() != 0) + result.addTypes(funcType.getResults()); + + return success(); +} + +/// Parses an optional function pointer operand before the call argument list +/// for indirect calls, or stops parsing at the function identifier otherwise. +static ParseResult parseOptionalCallFuncPtr( + OpAsmParser &parser, + SmallVectorImpl &operands) { + OpAsmParser::UnresolvedOperand funcPtrOperand; + OptionalParseResult parseResult = parser.parseOptionalOperand(funcPtrOperand); + if (parseResult.has_value()) { + if (failed(*parseResult)) + return *parseResult; + operands.push_back(funcPtrOperand); + } + return success(); +} + +// ::= `llvm.call` (function-id | ssa-use)`(` ssa-use-list `)` +// attribute-dict? `:` (type `,`)? function-type +ParseResult CallOp::parse(OpAsmParser &parser, OperationState &result) { + SymbolRefAttr funcAttr; + SmallVector operands; + + // Parse a function pointer for indirect calls. + if (parseOptionalCallFuncPtr(parser, operands)) + return failure(); + bool isDirect = operands.empty(); + + // Parse a function identifier for direct calls. + if (isDirect) + if (parser.parseAttribute(funcAttr, "callee", result.attributes)) + return failure(); + + // Parse the function arguments. + if (parser.parseOperandList(operands, OpAsmParser::Delimiter::Paren) || + parser.parseOptionalAttrDict(result.attributes)) + return failure(); + + // Parse the trailing type list and resolve the operands. + return parseCallTypeAndResolveOperands(parser, result, isDirect, operands); +} + ///===---------------------------------------------------------------------===// /// LLVM::InvokeOp ///===---------------------------------------------------------------------===// @@ -949,93 +1193,48 @@ void InvokeOp::print(OpAsmPrinter &p) { p.printOptionalAttrDict((*this)->getAttrs(), {InvokeOp::getOperandSegmentSizeAttr(), "callee"}); + p << " : "; + if (!isDirect) + p << getOperand(0).getType() << ", "; p.printFunctionalType(llvm::drop_begin(getOperandTypes(), isDirect ? 0 : 1), getResultTypes()); } -/// ::= `llvm.invoke` (function-id | ssa-use) `(` ssa-use-list `)` -/// `to` bb-id (`[` ssa-use-and-type-list `]`)? -/// `unwind` bb-id (`[` ssa-use-and-type-list `]`)? -/// attribute-dict? `:` function-type +// ::= `llvm.invoke` (function-id | ssa-use) +// `(` ssa-use-list `)` +// `to` bb-id (`[` ssa-use-and-type-list `]`)? +// `unwind` bb-id (`[` ssa-use-and-type-list `]`)? +// attribute-dict? `:` (type `,`)? function-type ParseResult InvokeOp::parse(OpAsmParser &parser, OperationState &result) { SmallVector operands; - FunctionType funcType; SymbolRefAttr funcAttr; - SMLoc trailingTypeLoc; Block *normalDest, *unwindDest; SmallVector normalOperands, unwindOperands; Builder &builder = parser.getBuilder(); - // Parse an operand list that will, in practice, contain 0 or 1 operand. In - // case of an indirect call, there will be 1 operand before `(`. In case of a - // direct call, there will be no operands and the parser will stop at the - // function identifier without complaining. - if (parser.parseOperandList(operands)) + // Parse a function pointer for indirect calls. + if (parseOptionalCallFuncPtr(parser, operands)) return failure(); bool isDirect = operands.empty(); - // Optionally parse a function identifier. + // Parse a function identifier for direct calls. if (isDirect && parser.parseAttribute(funcAttr, "callee", result.attributes)) return failure(); + // Parse the function arguments. if (parser.parseOperandList(operands, OpAsmParser::Delimiter::Paren) || parser.parseKeyword("to") || parser.parseSuccessorAndUseList(normalDest, normalOperands) || parser.parseKeyword("unwind") || parser.parseSuccessorAndUseList(unwindDest, unwindOperands) || - parser.parseOptionalAttrDict(result.attributes) || parser.parseColon() || - parser.getCurrentLocation(&trailingTypeLoc) || parser.parseType(funcType)) + parser.parseOptionalAttrDict(result.attributes)) return failure(); - if (isDirect) { - // Make sure types match. - if (parser.resolveOperands(operands, funcType.getInputs(), - parser.getNameLoc(), result.operands)) - return failure(); - result.addTypes(funcType.getResults()); - } else { - // Construct the LLVM IR Dialect function type that the first operand - // should match. - if (funcType.getNumResults() > 1) - return parser.emitError(trailingTypeLoc, - "expected function with 0 or 1 result"); - - Type llvmResultType; - if (funcType.getNumResults() == 0) { - llvmResultType = LLVM::LLVMVoidType::get(builder.getContext()); - } else { - llvmResultType = funcType.getResult(0); - if (!isCompatibleType(llvmResultType)) - return parser.emitError(trailingTypeLoc, - "expected result to have LLVM type"); - } - - SmallVector argTypes; - argTypes.reserve(funcType.getNumInputs()); - for (Type ty : funcType.getInputs()) { - if (isCompatibleType(ty)) - argTypes.push_back(ty); - else - return parser.emitError(trailingTypeLoc, - "expected LLVM types as inputs"); - } - - auto llvmFuncType = LLVM::LLVMFunctionType::get(llvmResultType, argTypes); - auto wrappedFuncType = LLVM::LLVMPointerType::get(llvmFuncType); - - auto funcArguments = llvm::ArrayRef(operands).drop_front(); - - // Make sure that the first operand (indirect callee) matches the wrapped - // LLVM IR function type, and that the types of the other call operands - // match the types of the function arguments. - if (parser.resolveOperand(operands[0], wrappedFuncType, result.operands) || - parser.resolveOperands(funcArguments, funcType.getInputs(), - parser.getNameLoc(), result.operands)) - return failure(); + // Parse the trailing type list and resolve the function operands. + if (parseCallTypeAndResolveOperands(parser, result, isDirect, operands)) + return failure(); - result.addTypes(llvmResultType); - } result.addSuccessors({normalDest, unwindDest}); result.addOperands(normalOperands); result.addOperands(unwindOperands); @@ -1108,8 +1307,8 @@ void LandingpadOp::print(OpAsmPrinter &p) { p << ": " << getType(); } -/// ::= `llvm.landingpad` `cleanup`? -/// ((`catch` | `filter`) operand-type ssa-use)* attribute-dict? +// ::= `llvm.landingpad` `cleanup`? +// ((`catch` | `filter`) operand-type ssa-use)* attribute-dict? ParseResult LandingpadOp::parse(OpAsmParser &parser, OperationState &result) { // Check for cleanup if (succeeded(parser.parseOptionalKeyword("cleanup"))) @@ -1136,237 +1335,6 @@ ParseResult LandingpadOp::parse(OpAsmParser &parser, OperationState &result) { return success(); } -//===----------------------------------------------------------------------===// -// CallOp -//===----------------------------------------------------------------------===// - -void CallOp::build(OpBuilder &builder, OperationState &state, TypeRange results, - StringRef callee, ValueRange args) { - build(builder, state, results, builder.getStringAttr(callee), args); -} - -void CallOp::build(OpBuilder &builder, OperationState &state, TypeRange results, - StringAttr callee, ValueRange args) { - build(builder, state, results, SymbolRefAttr::get(callee), args, nullptr, - nullptr); -} - -void CallOp::build(OpBuilder &builder, OperationState &state, TypeRange results, - FlatSymbolRefAttr callee, ValueRange args) { - build(builder, state, results, callee, args, nullptr, nullptr); -} - -void CallOp::build(OpBuilder &builder, OperationState &state, LLVMFuncOp func, - ValueRange args) { - SmallVector results; - Type resultType = func.getFunctionType().getReturnType(); - if (!resultType.isa()) - results.push_back(resultType); - build(builder, state, results, SymbolRefAttr::get(func), args, nullptr, - nullptr); -} - -CallInterfaceCallable CallOp::getCallableForCallee() { - // Direct call. - if (FlatSymbolRefAttr calleeAttr = getCalleeAttr()) - return calleeAttr; - // Indirect call, callee Value is the first operand. - return getOperand(0); -} - -Operation::operand_range CallOp::getArgOperands() { - return getOperands().drop_front(getCallee().has_value() ? 0 : 1); -} - -LogicalResult CallOp::verifySymbolUses(SymbolTableCollection &symbolTable) { - if (getNumResults() > 1) - return emitOpError("must have 0 or 1 result"); - - // Type for the callee, we'll get it differently depending if it is a direct - // or indirect call. - Type fnType; - - bool isIndirect = false; - - // If this is an indirect call, the callee attribute is missing. - FlatSymbolRefAttr calleeName = getCalleeAttr(); - if (!calleeName) { - isIndirect = true; - if (!getNumOperands()) - return emitOpError( - "must have either a `callee` attribute or at least an operand"); - auto ptrType = getOperand(0).getType().dyn_cast(); - if (!ptrType) - return emitOpError("indirect call expects a pointer as callee: ") - << ptrType; - - if (ptrType.isOpaque()) - return success(); - - fnType = ptrType.getElementType(); - } else { - Operation *callee = - symbolTable.lookupNearestSymbolFrom(*this, calleeName.getAttr()); - if (!callee) - return emitOpError() - << "'" << calleeName.getValue() - << "' does not reference a symbol in the current scope"; - auto fn = dyn_cast(callee); - if (!fn) - return emitOpError() << "'" << calleeName.getValue() - << "' does not reference a valid LLVM function"; - - fnType = fn.getFunctionType(); - } - - LLVMFunctionType funcType = fnType.dyn_cast(); - if (!funcType) - return emitOpError("callee does not have a functional type: ") << fnType; - - // Verify that the operand and result types match the callee. - - if (!funcType.isVarArg() && - funcType.getNumParams() != (getNumOperands() - isIndirect)) - return emitOpError() << "incorrect number of operands (" - << (getNumOperands() - isIndirect) - << ") for callee (expecting: " - << funcType.getNumParams() << ")"; - - if (funcType.getNumParams() > (getNumOperands() - isIndirect)) - return emitOpError() << "incorrect number of operands (" - << (getNumOperands() - isIndirect) - << ") for varargs callee (expecting at least: " - << funcType.getNumParams() << ")"; - - for (unsigned i = 0, e = funcType.getNumParams(); i != e; ++i) - if (getOperand(i + isIndirect).getType() != funcType.getParamType(i)) - return emitOpError() << "operand type mismatch for operand " << i << ": " - << getOperand(i + isIndirect).getType() - << " != " << funcType.getParamType(i); - - if (getNumResults() == 0 && - !funcType.getReturnType().isa()) - return emitOpError() << "expected function call to produce a value"; - - if (getNumResults() != 0 && - funcType.getReturnType().isa()) - return emitOpError() - << "calling function with void result must not produce values"; - - if (getNumResults() > 1) - return emitOpError() - << "expected LLVM function call to produce 0 or 1 result"; - - if (getNumResults() && getResult().getType() != funcType.getReturnType()) - return emitOpError() << "result type mismatch: " << getResult().getType() - << " != " << funcType.getReturnType(); - - return success(); -} - -void CallOp::print(OpAsmPrinter &p) { - auto callee = getCallee(); - bool isDirect = callee.has_value(); - - // Print the direct callee if present as a function attribute, or an indirect - // callee (first operand) otherwise. - p << ' '; - if (isDirect) - p.printSymbolName(callee.value()); - else - p << getOperand(0); - - auto args = getOperands().drop_front(isDirect ? 0 : 1); - p << '(' << args << ')'; - p.printOptionalAttrDict(processFMFAttr((*this)->getAttrs()), {"callee"}); - - // Reconstruct the function MLIR function type from operand and result types. - p << " : "; - p.printFunctionalType(args.getTypes(), getResultTypes()); -} - -// ::= `llvm.call` (function-id | ssa-use) `(` ssa-use-list `)` -// attribute-dict? `:` function-type -ParseResult CallOp::parse(OpAsmParser &parser, OperationState &result) { - SmallVector operands; - Type type; - SymbolRefAttr funcAttr; - SMLoc trailingTypeLoc; - - // Parse an operand list that will, in practice, contain 0 or 1 operand. In - // case of an indirect call, there will be 1 operand before `(`. In case of a - // direct call, there will be no operands and the parser will stop at the - // function identifier without complaining. - if (parser.parseOperandList(operands)) - return failure(); - bool isDirect = operands.empty(); - - // Optionally parse a function identifier. - if (isDirect) - if (parser.parseAttribute(funcAttr, "callee", result.attributes)) - return failure(); - - if (parser.parseOperandList(operands, OpAsmParser::Delimiter::Paren) || - parser.parseOptionalAttrDict(result.attributes) || parser.parseColon() || - parser.getCurrentLocation(&trailingTypeLoc) || parser.parseType(type)) - return failure(); - - auto funcType = type.dyn_cast(); - if (!funcType) - return parser.emitError(trailingTypeLoc, "expected function type"); - if (funcType.getNumResults() > 1) - return parser.emitError(trailingTypeLoc, - "expected function with 0 or 1 result"); - if (isDirect) { - // Make sure types match. - if (parser.resolveOperands(operands, funcType.getInputs(), - parser.getNameLoc(), result.operands)) - return failure(); - if (funcType.getNumResults() != 0 && - !funcType.getResult(0).isa()) - result.addTypes(funcType.getResults()); - } else { - Builder &builder = parser.getBuilder(); - Type llvmResultType; - if (funcType.getNumResults() == 0) { - llvmResultType = LLVM::LLVMVoidType::get(builder.getContext()); - } else { - llvmResultType = funcType.getResult(0); - if (!isCompatibleType(llvmResultType)) - return parser.emitError(trailingTypeLoc, - "expected result to have LLVM type"); - } - - SmallVector argTypes; - argTypes.reserve(funcType.getNumInputs()); - for (int i = 0, e = funcType.getNumInputs(); i < e; ++i) { - auto argType = funcType.getInput(i); - if (!isCompatibleType(argType)) - return parser.emitError(trailingTypeLoc, - "expected LLVM types as inputs"); - argTypes.push_back(argType); - } - auto llvmFuncType = LLVM::LLVMFunctionType::get(llvmResultType, argTypes); - auto wrappedFuncType = LLVM::LLVMPointerType::get(llvmFuncType); - - auto funcArguments = - ArrayRef(operands).drop_front(); - - // Make sure that the first operand (indirect callee) matches the wrapped - // LLVM IR function type, and that the types of the other call operands - // match the types of the function arguments. - if (parser.resolveOperand(operands[0], wrappedFuncType, result.operands) || - parser.resolveOperands(funcArguments, funcType.getInputs(), - parser.getNameLoc(), result.operands)) - return failure(); - - if (!llvmResultType.isa()) - result.addTypes(llvmResultType); - } - - return success(); -} - //===----------------------------------------------------------------------===// // ExtractValueOp //===----------------------------------------------------------------------===// @@ -2262,32 +2230,6 @@ OpFoldResult LLVM::ConstantOp::fold(FoldAdaptor) { return getValue(); } // Utility functions for parsing atomic ops //===----------------------------------------------------------------------===// -// Helper function to parse a keyword into the specified attribute named by -// `attrName`. The keyword must match one of the string values defined by the -// AtomicBinOp enum. The resulting I64 attribute is added to the `result` -// state. -static ParseResult parseAtomicBinOp(OpAsmParser &parser, OperationState &result, - StringRef attrName) { - SMLoc loc; - StringRef keyword; - if (parser.getCurrentLocation(&loc) || parser.parseKeyword(&keyword)) - return failure(); - - // Replace the keyword `keyword` with an integer attribute. - auto kind = symbolizeAtomicBinOp(keyword); - if (!kind) { - return parser.emitError(loc) - << "'" << keyword << "' is an incorrect value of the '" << attrName - << "' attribute"; - } - - auto value = static_cast(*kind); - auto attr = parser.getBuilder().getI64IntegerAttr(value); - result.addAttribute(attrName, attr); - - return success(); -} - // Helper function to parse a keyword into the specified attribute named by // `attrName`. The keyword must match one of the string values defined by the // AtomicOrdering enum. The resulting I64 attribute is added to the `result` @@ -2316,45 +2258,15 @@ static ParseResult parseAtomicOrdering(OpAsmParser &parser, } //===----------------------------------------------------------------------===// -// Printer, parser and verifier for LLVM::AtomicRMWOp. +// Verifier for LLVM::AtomicRMWOp. //===----------------------------------------------------------------------===// -void AtomicRMWOp::print(OpAsmPrinter &p) { - p << ' ' << stringifyAtomicBinOp(getBinOp()) << ' ' << getPtr() << ", " - << getVal() << ' ' << stringifyAtomicOrdering(getOrdering()) << ' '; - p.printOptionalAttrDict((*this)->getAttrs(), {"bin_op", "ordering"}); - p << " : " << getRes().getType(); -} - -// ::= `llvm.atomicrmw` keyword ssa-use `,` ssa-use keyword -// attribute-dict? `:` type -ParseResult AtomicRMWOp::parse(OpAsmParser &parser, OperationState &result) { - Type type; - OpAsmParser::UnresolvedOperand ptr, val; - if (parseAtomicBinOp(parser, result, "bin_op") || parser.parseOperand(ptr) || - parser.parseComma() || parser.parseOperand(val) || - parseAtomicOrdering(parser, result, "ordering") || - parser.parseOptionalAttrDict(result.attributes) || - parser.parseColonType(type) || - parser.resolveOperand(ptr, LLVM::LLVMPointerType::get(type), - result.operands) || - parser.resolveOperand(val, type, result.operands)) - return failure(); - - result.addTypes(type); - return success(); -} - LogicalResult AtomicRMWOp::verify() { auto ptrType = getPtr().getType().cast(); auto valType = getVal().getType(); if (!ptrType.isOpaque() && valType != ptrType.getElementType()) return emitOpError("expected LLVM IR element type for operand #0 to " "match type for operand #1"); - auto resType = getRes().getType(); - if (resType != valType) - return emitOpError( - "expected LLVM IR result type to match type for operand #1"); if (getBinOp() == AtomicBinOp::fadd || getBinOp() == AtomicBinOp::fsub) { if (!mlir::LLVM::isCompatibleFloatingPointType(valType)) return emitOpError("expected LLVM IR floating point type"); @@ -2384,55 +2296,21 @@ LogicalResult AtomicRMWOp::verify() { } //===----------------------------------------------------------------------===// -// Printer, parser and verifier for LLVM::AtomicCmpXchgOp. +// Verifier for LLVM::AtomicCmpXchgOp. //===----------------------------------------------------------------------===// -void AtomicCmpXchgOp::print(OpAsmPrinter &p) { - p << ' ' << getPtr() << ", " << getCmp() << ", " << getVal() << ' ' - << stringifyAtomicOrdering(getSuccessOrdering()) << ' ' - << stringifyAtomicOrdering(getFailureOrdering()); - p.printOptionalAttrDict((*this)->getAttrs(), - {"success_ordering", "failure_ordering"}); - p << " : " << getVal().getType(); -} - -// ::= `llvm.cmpxchg` ssa-use `,` ssa-use `,` ssa-use -// keyword keyword attribute-dict? `:` type -ParseResult AtomicCmpXchgOp::parse(OpAsmParser &parser, - OperationState &result) { - auto &builder = parser.getBuilder(); - Type type; - OpAsmParser::UnresolvedOperand ptr, cmp, val; - if (parser.parseOperand(ptr) || parser.parseComma() || - parser.parseOperand(cmp) || parser.parseComma() || - parser.parseOperand(val) || - parseAtomicOrdering(parser, result, "success_ordering") || - parseAtomicOrdering(parser, result, "failure_ordering") || - parser.parseOptionalAttrDict(result.attributes) || - parser.parseColonType(type) || - parser.resolveOperand(ptr, LLVM::LLVMPointerType::get(type), - result.operands) || - parser.resolveOperand(cmp, type, result.operands) || - parser.resolveOperand(val, type, result.operands)) - return failure(); - - auto boolType = IntegerType::get(builder.getContext(), 1); - auto resultType = - LLVMStructType::getLiteral(builder.getContext(), {type, boolType}); - result.addTypes(resultType); - - return success(); +/// Returns an LLVM struct type that contains a value type and a boolean type. +static LLVMStructType getValAndBoolStructType(Type valType) { + auto boolType = IntegerType::get(valType.getContext(), 1); + return LLVMStructType::getLiteral(valType.getContext(), {valType, boolType}); } LogicalResult AtomicCmpXchgOp::verify() { auto ptrType = getPtr().getType().cast(); if (!ptrType) return emitOpError("expected LLVM IR pointer type for operand #0"); - auto cmpType = getCmp().getType(); auto valType = getVal().getType(); - if (cmpType != valType) - return emitOpError("expected both value operands to have the same type"); - if (!ptrType.isOpaque() && cmpType != ptrType.getElementType()) + if (!ptrType.isOpaque() && valType != ptrType.getElementType()) return emitOpError("expected LLVM IR element type for operand #0 to " "match type for all other operands"); auto intType = valType.dyn_cast(); diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp index 7f44db50f820..998392ded460 100644 --- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp @@ -348,6 +348,22 @@ convertOperationImpl(Operation &opInst, llvm::IRBuilderBase &builder, #include "mlir/Dialect/LLVMIR/LLVMConversions.inc" #include "mlir/Dialect/LLVMIR/LLVMIntrinsicConversions.inc" + // Helper function to reconstruct the function type for an indirect call given + // the result and argument types. The function cannot reconstruct the type of + // variadic functions since the call operation does not carry enough + // information to distinguish normal and variadic arguments. Supporting + // indirect variadic calls requires an additional type attribute on the call + // operation that stores the LLVM function type of the callee. + // TODO: Support indirect calls to variadic function pointers. + auto getCalleeFunctionType = [&](TypeRange resultTypes, ValueRange args) { + Type resultType = resultTypes.empty() + ? LLVMVoidType::get(opInst.getContext()) + : resultTypes.front(); + return llvm::cast(moduleTranslation.convertType( + LLVMFunctionType::get(opInst.getContext(), resultType, + llvm::to_vector(args.getTypes()), false))); + }; + // Emit function calls. If the "callee" attribute is present, this is a // direct function call and we also need to look up the remapped function // itself. Otherwise, this is an indirect call and the callee is the first @@ -360,12 +376,9 @@ convertOperationImpl(Operation &opInst, llvm::IRBuilderBase &builder, call = builder.CreateCall( moduleTranslation.lookupFunction(attr.getValue()), operandsRef); } else { - auto calleeType = - callOp->getOperands().front().getType().cast(); - auto *calleeFunctionType = cast( - moduleTranslation.convertType(calleeType.getElementType())); - call = builder.CreateCall(calleeFunctionType, operandsRef.front(), - operandsRef.drop_front()); + call = builder.CreateCall(getCalleeFunctionType(callOp.getResultTypes(), + callOp.getArgOperands()), + operandsRef.front(), operandsRef.drop_front()); } llvm::MDNode *branchWeights = convertBranchWeights(callOp.getBranchWeights(), moduleTranslation); @@ -449,12 +462,9 @@ convertOperationImpl(Operation &opInst, llvm::IRBuilderBase &builder, moduleTranslation.lookupBlock(invOp.getSuccessor(0)), moduleTranslation.lookupBlock(invOp.getSuccessor(1)), operandsRef); } else { - auto calleeType = - invOp.getCalleeOperands().front().getType().cast(); - auto *calleeFunctionType = cast( - moduleTranslation.convertType(calleeType.getElementType())); result = builder.CreateInvoke( - calleeFunctionType, operandsRef.front(), + getCalleeFunctionType(invOp.getResultTypes(), invOp.getArgOperands()), + operandsRef.front(), moduleTranslation.lookupBlock(invOp.getSuccessor(0)), moduleTranslation.lookupBlock(invOp.getSuccessor(1)), operandsRef.drop_front()); diff --git a/mlir/test/Conversion/FuncToLLVM/convert-funcs.mlir b/mlir/test/Conversion/FuncToLLVM/convert-funcs.mlir index 9db15b89ccc9..716ef210093d 100644 --- a/mlir/test/Conversion/FuncToLLVM/convert-funcs.mlir +++ b/mlir/test/Conversion/FuncToLLVM/convert-funcs.mlir @@ -52,7 +52,7 @@ func.func private @body(i32) func.func @indirect_const_call(%arg0: i32) { // CHECK-NEXT: %[[ADDR:.*]] = llvm.mlir.addressof @body : !llvm.ptr> %0 = constant @body : (i32) -> () -// CHECK-NEXT: llvm.call %[[ADDR]](%[[ARG0:.*]]) : (i32) -> () +// CHECK-NEXT: llvm.call %[[ADDR]](%[[ARG0:.*]]) : !llvm.ptr>, (i32) -> () call_indirect %0(%arg0) : (i32) -> () // CHECK-NEXT: llvm.return return @@ -60,7 +60,7 @@ func.func @indirect_const_call(%arg0: i32) { // CHECK-LABEL: llvm.func @indirect_call(%arg0: !llvm.ptr>, %arg1: f32) -> i32 { func.func @indirect_call(%arg0: (f32) -> i32, %arg1: f32) -> i32 { -// CHECK-NEXT: %0 = llvm.call %arg0(%arg1) : (f32) -> i32 +// CHECK-NEXT: %0 = llvm.call %arg0(%arg1) : !llvm.ptr>, (f32) -> i32 %0 = call_indirect %arg0(%arg1) : (f32) -> i32 // CHECK-NEXT: llvm.return %0 : i32 return %0 : i32 diff --git a/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir b/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir index 1a8a75d1e2b3..0adda9731548 100644 --- a/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir +++ b/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir @@ -370,7 +370,7 @@ func.func @generic_atomic_rmw(%I : memref<10xi32>, %i : index) { // CHECK-NEXT: llvm.br ^bb1([[init]] : i32) // CHECK-NEXT: ^bb1([[loaded:%.*]]: i32): // CHECK-NEXT: [[pair:%.*]] = llvm.cmpxchg %{{.*}}, [[loaded]], [[loaded]] - // CHECK-SAME: acq_rel monotonic : i32 + // CHECK-SAME: acq_rel monotonic : !llvm.ptr, i32 // CHECK-NEXT: [[new:%.*]] = llvm.extractvalue [[pair]][0] // CHECK-NEXT: [[ok:%.*]] = llvm.extractvalue [[pair]][1] // CHECK-NEXT: llvm.cond_br [[ok]], ^bb2, ^bb1([[new]] : i32) diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir index aa2c43ff42b7..39c89da1ec1f 100644 --- a/mlir/test/Dialect/LLVMIR/invalid.mlir +++ b/mlir/test/Dialect/LLVMIR/invalid.mlir @@ -174,25 +174,51 @@ func.func @store_malformed_elem_type(%foo: !llvm.ptr, %bar: f32) { // ----- -func.func @call_non_function_type(%callee : !llvm.func, %arg : i8) { - // expected-error@+1 {{expected function type}} - llvm.call %callee(%arg) : !llvm.func +func.func @invalid_call() { + // expected-error@+1 {{'llvm.call' op must have either a `callee` attribute or at least an operand}} + "llvm.call"() : () -> () llvm.return } // ----- -func.func @invalid_call() { - // expected-error@+1 {{'llvm.call' op must have either a `callee` attribute or at least an operand}} - "llvm.call"() : () -> () +func.func @call_missing_ptr_type(%callee : !llvm.func, %arg : i8) { + // expected-error@+1 {{expected indirect call to have 2 trailing types}} + llvm.call %callee(%arg) : (i8) -> (i8) + llvm.return +} + +// ----- + +func.func private @standard_func_callee() + +func.func @call_missing_ptr_type(%arg : i8) { + // expected-error@+1 {{expected direct call to have 1 trailing type}} + llvm.call @standard_func_callee(%arg) : !llvm.ptr, (i8) -> (i8) llvm.return } // ----- -func.func @call_non_function_type(%callee : !llvm.func, %arg : i8) { - // expected-error@+1 {{expected function type}} - llvm.call %callee(%arg) : !llvm.func +func.func @call_non_pointer_type(%callee : !llvm.func, %arg : i8) { + // expected-error@+1 {{indirect call expects a pointer as callee: '!llvm.func'}} + llvm.call %callee(%arg) : !llvm.func, (i8) -> (i8) + llvm.return +} + +// ----- + +func.func @call_non_function_type(%callee : !llvm.ptr, %arg : i8) { + // expected-error@+1 {{expected trailing function type}} + llvm.call %callee(%arg) : !llvm.ptr, !llvm.func + llvm.return +} + +// ----- + +func.func @call_void_result_type(%callee : !llvm.ptr, %arg : i8) { + // expected-error@+1 {{expected a non-void result type}} + llvm.call %callee(%arg) : !llvm.ptr, (i8) -> (!llvm.void) llvm.return } @@ -206,6 +232,14 @@ func.func @call_unknown_symbol() { // ----- +func.func @call_variadic(%callee : !llvm.ptr>, %arg : i8) { + // expected-error@+1 {{indirect calls to variadic functions are not supported}} + llvm.call %callee(%arg) : !llvm.ptr>, (i8) -> (i8) + llvm.return +} + +// ----- + func.func private @standard_func_callee() func.func @call_non_llvm() { @@ -216,7 +250,7 @@ func.func @call_non_llvm() { // ----- -func.func @call_non_llvm_indirect(%arg0 : tensor<*xi32>) { +func.func @call_non_llvm_arg(%arg0 : tensor<*xi32>) { // expected-error@+1 {{'llvm.call' op operand #0 must be LLVM dialect-compatible type}} "llvm.call"(%arg0) : (tensor<*xi32>) -> () llvm.return @@ -224,6 +258,14 @@ func.func @call_non_llvm_indirect(%arg0 : tensor<*xi32>) { // ----- +func.func @call_non_llvm_res(%callee : !llvm.ptr) { + // expected-error@+1 {{'llvm.call' op result #0 must be LLVM dialect-compatible type}} + llvm.call %callee() : !llvm.ptr, () -> (tensor<*xi32>) + llvm.return +} + +// ----- + llvm.func @callee_func(i8) -> () func.func @callee_arg_mismatch(%arg0 : i32) { @@ -260,25 +302,9 @@ func.func @indirect_callee_return_mismatch(%callee : !llvm.ptr>) { // ----- -func.func @call_too_many_results(%callee : () -> (i32,i32)) { +func.func @call_too_many_results(%callee : !llvm.ptr) { // expected-error@+1 {{expected function with 0 or 1 result}} - llvm.call %callee() : () -> (i32, i32) - llvm.return -} - -// ----- - -func.func @call_non_llvm_result(%callee : () -> (tensor<*xi32>)) { - // expected-error@+1 {{expected result to have LLVM type}} - llvm.call %callee() : () -> (tensor<*xi32>) - llvm.return -} - -// ----- - -func.func @call_non_llvm_input(%callee : (tensor<*xi32>) -> (), %arg : tensor<*xi32>) { - // expected-error@+1 {{expected LLVM types as inputs}} - llvm.call %callee(%arg) : (tensor<*xi32>) -> () + llvm.call %callee() : !llvm.ptr, () -> (i32, i32) llvm.return } @@ -577,14 +603,14 @@ func.func @atomicrmw_expected_ptr(%f32 : f32) { func.func @atomicrmw_mismatched_operands(%f32_ptr : !llvm.ptr, %i32 : i32) { // expected-error@+1 {{expected LLVM IR element type for operand #0 to match type for operand #1}} - %0 = "llvm.atomicrmw"(%f32_ptr, %i32) {bin_op=11, ordering=1} : (!llvm.ptr, i32) -> f32 + %0 = "llvm.atomicrmw"(%f32_ptr, %i32) {bin_op=11, ordering=1} : (!llvm.ptr, i32) -> i32 llvm.return } // ----- func.func @atomicrmw_mismatched_operands(%f32_ptr : !llvm.ptr, %f32 : f32) { - // expected-error@+1 {{expected LLVM IR result type to match type for operand #1}} + // expected-error@+1 {{op failed to verify that result #0 and operand #1 have the same type}} %0 = "llvm.atomicrmw"(%f32_ptr, %f32) {bin_op=11, ordering=1} : (!llvm.ptr, f32) -> i32 llvm.return } @@ -593,7 +619,7 @@ func.func @atomicrmw_mismatched_operands(%f32_ptr : !llvm.ptr, %f32 : f32) func.func @atomicrmw_expected_float(%i32_ptr : !llvm.ptr, %i32 : i32) { // expected-error@+1 {{expected LLVM IR floating point type}} - %0 = llvm.atomicrmw fadd %i32_ptr, %i32 unordered : i32 + %0 = llvm.atomicrmw fadd %i32_ptr, %i32 unordered : !llvm.ptr, i32 llvm.return } @@ -601,7 +627,7 @@ func.func @atomicrmw_expected_float(%i32_ptr : !llvm.ptr, %i32 : i32) { func.func @atomicrmw_unexpected_xchg_type(%i1_ptr : !llvm.ptr, %i1 : i1) { // expected-error@+1 {{unexpected LLVM IR type for 'xchg' bin_op}} - %0 = llvm.atomicrmw xchg %i1_ptr, %i1 unordered : i1 + %0 = llvm.atomicrmw xchg %i1_ptr, %i1 unordered : !llvm.ptr, i1 llvm.return } @@ -609,7 +635,7 @@ func.func @atomicrmw_unexpected_xchg_type(%i1_ptr : !llvm.ptr, %i1 : i1) { func.func @atomicrmw_expected_int(%f32_ptr : !llvm.ptr, %f32 : f32) { // expected-error@+1 {{expected LLVM IR integer type}} - %0 = llvm.atomicrmw max %f32_ptr, %f32 unordered : f32 + %0 = llvm.atomicrmw max %f32_ptr, %f32 unordered : !llvm.ptr, f32 llvm.return } @@ -632,15 +658,24 @@ func.func @cmpxchg_mismatched_operands(%i64_ptr : !llvm.ptr, %i32 : i32) { // ----- func.func @cmpxchg_mismatched_value_operands(%ptr : !llvm.ptr, %i32 : i32, %i64 : i64) { - // expected-error@+1 {{expected both value operands to have the same type}} + // expected-error@+1 {{op failed to verify that operand #1 and operand #2 have the same type}} %0 = "llvm.cmpxchg"(%ptr, %i32, %i64) {success_ordering=2,failure_ordering=2} : (!llvm.ptr, i32, i64) -> !llvm.struct<(i32, i1)> llvm.return } + +// ----- + +func.func @cmpxchg_mismatched_result(%ptr : !llvm.ptr, %i64 : i64) { + // expected-error@+1 {{op failed to verify that result #0 has an LLVM struct type consisting of the type of operand #2 and a bool}} + %0 = "llvm.cmpxchg"(%ptr, %i64, %i64) {success_ordering=2,failure_ordering=2} : (!llvm.ptr, i64, i64) -> !llvm.struct<(i64, i64)> + llvm.return +} + // ----- func.func @cmpxchg_unexpected_type(%i1_ptr : !llvm.ptr, %i1 : i1) { // expected-error@+1 {{unexpected LLVM IR type}} - %0 = llvm.cmpxchg %i1_ptr, %i1, %i1 monotonic monotonic : i1 + %0 = llvm.cmpxchg %i1_ptr, %i1, %i1 monotonic monotonic : !llvm.ptr, i1 llvm.return } @@ -648,7 +683,7 @@ func.func @cmpxchg_unexpected_type(%i1_ptr : !llvm.ptr, %i1 : i1) { func.func @cmpxchg_at_least_monotonic_success(%i32_ptr : !llvm.ptr, %i32 : i32) { // expected-error@+1 {{ordering must be at least 'monotonic'}} - %0 = llvm.cmpxchg %i32_ptr, %i32, %i32 unordered monotonic : i32 + %0 = llvm.cmpxchg %i32_ptr, %i32, %i32 unordered monotonic : !llvm.ptr, i32 llvm.return } @@ -656,7 +691,7 @@ func.func @cmpxchg_at_least_monotonic_success(%i32_ptr : !llvm.ptr, %i32 : func.func @cmpxchg_at_least_monotonic_failure(%i32_ptr : !llvm.ptr, %i32 : i32) { // expected-error@+1 {{ordering must be at least 'monotonic'}} - %0 = llvm.cmpxchg %i32_ptr, %i32, %i32 monotonic unordered : i32 + %0 = llvm.cmpxchg %i32_ptr, %i32, %i32 monotonic unordered : !llvm.ptr, i32 llvm.return } @@ -664,7 +699,7 @@ func.func @cmpxchg_at_least_monotonic_failure(%i32_ptr : !llvm.ptr, %i32 : func.func @cmpxchg_failure_release(%i32_ptr : !llvm.ptr, %i32 : i32) { // expected-error@+1 {{failure ordering cannot be 'release' or 'acq_rel'}} - %0 = llvm.cmpxchg %i32_ptr, %i32, %i32 acq_rel release : i32 + %0 = llvm.cmpxchg %i32_ptr, %i32, %i32 acq_rel release : !llvm.ptr, i32 llvm.return } @@ -672,7 +707,7 @@ func.func @cmpxchg_failure_release(%i32_ptr : !llvm.ptr, %i32 : i32) { func.func @cmpxchg_failure_acq_rel(%i32_ptr : !llvm.ptr, %i32 : i32) { // expected-error@+1 {{failure ordering cannot be 'release' or 'acq_rel'}} - %0 = llvm.cmpxchg %i32_ptr, %i32, %i32 acq_rel acq_rel : i32 + %0 = llvm.cmpxchg %i32_ptr, %i32, %i32 acq_rel acq_rel : !llvm.ptr, i32 llvm.return } diff --git a/mlir/test/Dialect/LLVMIR/roundtrip.mlir b/mlir/test/Dialect/LLVMIR/roundtrip.mlir index a37b154eca54..3f2097ab7834 100644 --- a/mlir/test/Dialect/LLVMIR/roundtrip.mlir +++ b/mlir/test/Dialect/LLVMIR/roundtrip.mlir @@ -65,14 +65,13 @@ func.func @ops(%arg0: i32, %arg1: f32, // CHECK: %[[STRUCT:.*]] = llvm.call @foo(%[[I32]]) : (i32) -> !llvm.struct<(i32, f64, i32)> // CHECK: %[[VALUE:.*]] = llvm.extractvalue %[[STRUCT]][0] : !llvm.struct<(i32, f64, i32)> // CHECK: %[[NEW_STRUCT:.*]] = llvm.insertvalue %[[VALUE]], %[[STRUCT]][2] : !llvm.struct<(i32, f64, i32)> -// CHECK: %[[FUNC:.*]] = llvm.mlir.addressof @foo : !llvm.ptr (i32)>> -// CHECK: %{{.*}} = llvm.call %[[FUNC]](%[[I32]]) : (i32) -> !llvm.struct<(i32, f64, i32)> +// CHECK: %[[FUNC:.*]] = llvm.mlir.addressof @foo : !llvm.ptr +// CHECK: %{{.*}} = llvm.call %[[FUNC]](%[[I32]]) : !llvm.ptr, (i32) -> !llvm.struct<(i32, f64, i32)> %17 = llvm.call @foo(%arg0) : (i32) -> !llvm.struct<(i32, f64, i32)> %18 = llvm.extractvalue %17[0] : !llvm.struct<(i32, f64, i32)> %19 = llvm.insertvalue %18, %17[2] : !llvm.struct<(i32, f64, i32)> - %20 = llvm.mlir.addressof @foo : !llvm.ptr (i32)>> - %21 = llvm.call %20(%arg0) : (i32) -> !llvm.struct<(i32, f64, i32)> - + %20 = llvm.mlir.addressof @foo : !llvm.ptr + %21 = llvm.call %20(%arg0) : !llvm.ptr, (i32) -> !llvm.struct<(i32, f64, i32)> // Terminator operations and their successors. // @@ -341,16 +340,16 @@ func.func @null() { } // CHECK-LABEL: @atomicrmw -func.func @atomicrmw(%ptr : !llvm.ptr, %val : f32) { - // CHECK: llvm.atomicrmw fadd %{{.*}}, %{{.*}} monotonic : f32 - %0 = llvm.atomicrmw fadd %ptr, %val monotonic : f32 +func.func @atomicrmw(%ptr : !llvm.ptr, %val : f32) { + // CHECK: llvm.atomicrmw fadd %{{.*}}, %{{.*}} monotonic : !llvm.ptr, f32 + %0 = llvm.atomicrmw fadd %ptr, %val monotonic : !llvm.ptr, f32 llvm.return } // CHECK-LABEL: @cmpxchg -func.func @cmpxchg(%ptr : !llvm.ptr, %cmp : i32, %new : i32) { - // CHECK: llvm.cmpxchg %{{.*}}, %{{.*}}, %{{.*}} acq_rel monotonic : i32 - %0 = llvm.cmpxchg %ptr, %cmp, %new acq_rel monotonic : i32 +func.func @cmpxchg(%ptr : !llvm.ptr, %cmp : i32, %new : i32) { + // CHECK: llvm.cmpxchg %{{.*}}, %{{.*}}, %{{.*}} acq_rel monotonic : !llvm.ptr, i32 + %0 = llvm.cmpxchg %ptr, %cmp, %new acq_rel monotonic : !llvm.ptr, i32 llvm.return } @@ -401,8 +400,15 @@ llvm.func @invokeLandingpad() -> i32 attributes { personality = @__gxx_personali llvm.invoke @bar(%8, %6, %4) to ^bb2 unwind ^bb1 : (!llvm.ptr, !llvm.ptr, !llvm.ptr) -> () // CHECK: ^[[BB4:.*]]: -// CHECK: llvm.return %[[a0]] : i32 +// CHECK: %[[FUNC:.*]] = llvm.mlir.addressof @foo : !llvm.ptr +// CHECK: %{{.*}} = llvm.invoke %[[FUNC]]{{.*}}: !llvm.ptr, ^bb4: + %12 = llvm.mlir.addressof @foo : !llvm.ptr + %13 = llvm.invoke %12(%7) to ^bb2 unwind ^bb1 : !llvm.ptr, (i32) -> !llvm.struct<(i32, f64, i32)> + +// CHECK: ^[[BB5:.*]]: +// CHECK: llvm.return %[[a0]] : i32 +^bb5: llvm.return %0 : i32 } diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir index 9d6999db12ab..1bdd5981f656 100644 --- a/mlir/test/Dialect/OpenMP/invalid.mlir +++ b/mlir/test/Dialect/OpenMP/invalid.mlir @@ -524,7 +524,7 @@ combiner { atomic { ^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr): %2 = llvm.load %arg3 : !llvm.ptr - llvm.atomicrmw fadd %arg2, %2 monotonic : f32 + llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32 omp.yield } @@ -1288,7 +1288,7 @@ combiner { atomic { ^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr): %2 = llvm.load %arg3 : !llvm.ptr - llvm.atomicrmw add %arg2, %2 monotonic : i32 + llvm.atomicrmw add %arg2, %2 monotonic : !llvm.ptr, i32 omp.yield } diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir index 56d898e20c3b..ac8fea8ca8f0 100644 --- a/mlir/test/Dialect/OpenMP/ops.mlir +++ b/mlir/test/Dialect/OpenMP/ops.mlir @@ -556,7 +556,7 @@ combiner { atomic { ^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr): %2 = llvm.load %arg3 : !llvm.ptr - llvm.atomicrmw fadd %arg2, %2 monotonic : f32 + llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32 omp.yield } diff --git a/mlir/test/Target/LLVMIR/Import/constant.ll b/mlir/test/Target/LLVMIR/Import/constant.ll index 12175fbc9cb6..2e8be550c45f 100644 --- a/mlir/test/Target/LLVMIR/Import/constant.ll +++ b/mlir/test/Target/LLVMIR/Import/constant.ll @@ -131,7 +131,7 @@ define i32 @function_address_before_def() { store ptr @callee, ptr %1 ; CHECK: %[[INDIR:.*]] = llvm.load %[[PTR]] : !llvm.ptr -> !llvm.ptr %2 = load ptr, ptr %1 - ; CHECK: llvm.call %[[INDIR]]() + ; CHECK: llvm.call %[[INDIR]]() : !llvm.ptr, () -> i32 %3 = call i32 %2() ret i32 %3 } @@ -149,7 +149,7 @@ define i32 @function_address_after_def() { store ptr @callee, ptr %1 ; CHECK: %[[INDIR:.*]] = llvm.load %[[PTR]] : !llvm.ptr -> !llvm.ptr %2 = load ptr, ptr %1 - ; CHECK: llvm.call %[[INDIR]]() + ; CHECK: llvm.call %[[INDIR]]() : !llvm.ptr, () -> i32 %3 = call i32 %2() ret i32 %3 } diff --git a/mlir/test/Target/LLVMIR/Import/instructions.ll b/mlir/test/Target/LLVMIR/Import/instructions.ll index 329605cce790..6b8ce08c1673 100644 --- a/mlir/test/Target/LLVMIR/Import/instructions.ll +++ b/mlir/test/Target/LLVMIR/Import/instructions.ll @@ -367,31 +367,31 @@ define void @load_store(ptr %ptr) { ; CHECK-SAME: %[[PTR2:[a-zA-Z0-9]+]] ; CHECK-SAME: %[[VAL2:[a-zA-Z0-9]+]] define void @atomic_rmw(ptr %ptr1, i32 %val1, ptr %ptr2, float %val2) { - ; CHECK: llvm.atomicrmw xchg %[[PTR1]], %[[VAL1]] acquire : i32 + ; CHECK: llvm.atomicrmw xchg %[[PTR1]], %[[VAL1]] acquire %1 = atomicrmw xchg ptr %ptr1, i32 %val1 acquire - ; CHECK: llvm.atomicrmw add %[[PTR1]], %[[VAL1]] release : i32 + ; CHECK: llvm.atomicrmw add %[[PTR1]], %[[VAL1]] release %2 = atomicrmw add ptr %ptr1, i32 %val1 release - ; CHECK: llvm.atomicrmw sub %[[PTR1]], %[[VAL1]] acq_rel : i32 + ; CHECK: llvm.atomicrmw sub %[[PTR1]], %[[VAL1]] acq_rel %3 = atomicrmw sub ptr %ptr1, i32 %val1 acq_rel - ; CHECK: llvm.atomicrmw _and %[[PTR1]], %[[VAL1]] seq_cst : i32 + ; CHECK: llvm.atomicrmw _and %[[PTR1]], %[[VAL1]] seq_cst %4 = atomicrmw and ptr %ptr1, i32 %val1 seq_cst - ; CHECK: llvm.atomicrmw nand %[[PTR1]], %[[VAL1]] acquire : i32 + ; CHECK: llvm.atomicrmw nand %[[PTR1]], %[[VAL1]] acquire %5 = atomicrmw nand ptr %ptr1, i32 %val1 acquire - ; CHECK: llvm.atomicrmw _or %[[PTR1]], %[[VAL1]] acquire : i32 + ; CHECK: llvm.atomicrmw _or %[[PTR1]], %[[VAL1]] acquire %6 = atomicrmw or ptr %ptr1, i32 %val1 acquire - ; CHECK: llvm.atomicrmw _xor %[[PTR1]], %[[VAL1]] acquire : i32 + ; CHECK: llvm.atomicrmw _xor %[[PTR1]], %[[VAL1]] acquire %7 = atomicrmw xor ptr %ptr1, i32 %val1 acquire - ; CHECK: llvm.atomicrmw max %[[PTR1]], %[[VAL1]] acquire : i32 + ; CHECK: llvm.atomicrmw max %[[PTR1]], %[[VAL1]] acquire %8 = atomicrmw max ptr %ptr1, i32 %val1 acquire - ; CHECK: llvm.atomicrmw min %[[PTR1]], %[[VAL1]] acquire : i32 + ; CHECK: llvm.atomicrmw min %[[PTR1]], %[[VAL1]] acquire %9 = atomicrmw min ptr %ptr1, i32 %val1 acquire - ; CHECK: llvm.atomicrmw umax %[[PTR1]], %[[VAL1]] acquire : i32 + ; CHECK: llvm.atomicrmw umax %[[PTR1]], %[[VAL1]] acquire %10 = atomicrmw umax ptr %ptr1, i32 %val1 acquire - ; CHECK: llvm.atomicrmw umin %[[PTR1]], %[[VAL1]] acquire : i32 + ; CHECK: llvm.atomicrmw umin %[[PTR1]], %[[VAL1]] acquire %11 = atomicrmw umin ptr %ptr1, i32 %val1 acquire - ; CHECK: llvm.atomicrmw fadd %[[PTR2]], %[[VAL2]] acquire : f32 + ; CHECK: llvm.atomicrmw fadd %[[PTR2]], %[[VAL2]] acquire %12 = atomicrmw fadd ptr %ptr2, float %val2 acquire - ; CHECK: llvm.atomicrmw fsub %[[PTR2]], %[[VAL2]] acquire : f32 + ; CHECK: llvm.atomicrmw fsub %[[PTR2]], %[[VAL2]] acquire %13 = atomicrmw fsub ptr %ptr2, float %val2 acquire ret void } @@ -403,9 +403,9 @@ define void @atomic_rmw(ptr %ptr1, i32 %val1, ptr %ptr2, float %val2) { ; CHECK-SAME: %[[VAL1:[a-zA-Z0-9]+]] ; CHECK-SAME: %[[VAL2:[a-zA-Z0-9]+]] define void @atomic_cmpxchg(ptr %ptr1, i32 %val1, i32 %val2) { - ; CHECK: llvm.cmpxchg %[[PTR1]], %[[VAL1]], %[[VAL2]] seq_cst seq_cst : i32 + ; CHECK: llvm.cmpxchg %[[PTR1]], %[[VAL1]], %[[VAL2]] seq_cst seq_cst %1 = cmpxchg ptr %ptr1, i32 %val1, i32 %val2 seq_cst seq_cst - ; CHECK: llvm.cmpxchg %[[PTR1]], %[[VAL1]], %[[VAL2]] monotonic seq_cst : i32 + ; CHECK: llvm.cmpxchg %[[PTR1]], %[[VAL1]], %[[VAL2]] monotonic seq_cst %2 = cmpxchg ptr %ptr1, i32 %val1, i32 %val2 monotonic seq_cst ret void } @@ -415,9 +415,9 @@ define void @atomic_cmpxchg(ptr %ptr1, i32 %val1, i32 %val2) { ; CHECK: llvm.func @fn(i32) -> f32 declare float @fn(i32) -; CHECK-LABEL: @call_fn +; CHECK-LABEL: @direct_call ; CHECK-SAME: %[[ARG1:[a-zA-Z0-9]+]] -define float @call_fn(i32 %arg1) { +define float @direct_call(i32 %arg1) { ; CHECK: llvm.call @fn(%[[ARG1]]) %1 = call float @fn(i32 %arg1) ret float %1 @@ -425,12 +425,12 @@ define float @call_fn(i32 %arg1) { ; // ----- -; CHECK-LABEL: @call_fn_ptr +; CHECK-LABEL: @indirect_call ; CHECK-SAME: %[[PTR:[a-zA-Z0-9]+]] -define void @call_fn_ptr(ptr %fn) { +define void @indirect_call(ptr addrspace(42) %fn) { ; CHECK: %[[C0:[0-9]+]] = llvm.mlir.constant(0 : i16) : i16 - ; CHECK: llvm.call %[[PTR]](%[[C0]]) - call void %fn(i16 0) + ; CHECK: llvm.call %[[PTR]](%[[C0]]) : !llvm.ptr<42>, (i16) -> () + call addrspace(42) void %fn(i16 0) ret void } @@ -447,6 +447,19 @@ define void @gep_static_idx(ptr %ptr) { ; // ----- +; CHECK: @varargs(...) +declare void @varargs(...) + +; CHECK-LABEL: @varargs_call +; CHECK-SAME: %[[ARG1:[a-zA-Z0-9]+]] +define void @varargs_call(i32 %0) { + ; CHECK: llvm.call @varargs(%[[ARG1]]) : (i32) -> () + call void (...) @varargs(i32 %0) + ret void +} + +; // ----- + %sub_struct = type { i32, i8 } %my_struct = type { %sub_struct, [4 x i32] } diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir index 4c4e85ce08ad..589348043fd8 100644 --- a/mlir/test/Target/LLVMIR/llvmir.mlir +++ b/mlir/test/Target/LLVMIR/llvmir.mlir @@ -1008,16 +1008,16 @@ llvm.func @gep(%ptr: !llvm.ptr)>>, %idx: i64, // CHECK-LABEL: define void @indirect_const_call(i64 {{%.*}}) llvm.func @indirect_const_call(%arg0: i64) { // CHECK-NEXT: call void @body(i64 %0) - %0 = llvm.mlir.addressof @body : !llvm.ptr> - llvm.call %0(%arg0) : (i64) -> () + %0 = llvm.mlir.addressof @body : !llvm.ptr + llvm.call %0(%arg0) : !llvm.ptr, (i64) -> () // CHECK-NEXT: ret void llvm.return } -// CHECK-LABEL: define i32 @indirect_call(ptr {{%.*}}, float {{%.*}}) -llvm.func @indirect_call(%arg0: !llvm.ptr>, %arg1: f32) -> i32 { -// CHECK-NEXT: %3 = call i32 %0(float %1) - %0 = llvm.call %arg0(%arg1) : (f32) -> i32 +// CHECK-LABEL: define i32 @indirect_call(ptr addrspace(42) {{%.*}}, float {{%.*}}) +llvm.func @indirect_call(%arg0: !llvm.ptr<42>, %arg1: f32) -> i32 { +// CHECK-NEXT: %3 = call addrspace(42) i32 %0(float %1) + %0 = llvm.call %arg0(%arg1) : !llvm.ptr<42>, (f32) -> i32 // CHECK-NEXT: ret i32 %3 llvm.return %0 : i32 } @@ -1184,8 +1184,15 @@ llvm.func @dereferenceableornullattr_ret_decl() -> (!llvm.ptr {llvm.dereferencea // CHECK-LABEL: declare inreg ptr @inregattr_ret_decl() llvm.func @inregattr_ret_decl() -> (!llvm.ptr {llvm.inreg}) -// CHECK-LABEL: @llvm_varargs(...) -llvm.func @llvm_varargs(...) +// CHECK-LABEL: @varargs(...) +llvm.func @varargs(...) + +// CHECK-LABEL: define void @varargs_call +llvm.func @varargs_call(%arg0 : i32) { +// CHECK: call void (...) @varargs(i32 %{{.*}}) + llvm.call @varargs(%arg0) : (i32) -> () + llvm.return +} llvm.func @intpointerconversion(%arg0 : i32) -> i32 { // CHECK: %2 = inttoptr i32 %0 to ptr @@ -1396,38 +1403,38 @@ llvm.func @atomicrmw( %f32_ptr : !llvm.ptr, %f32 : f32, %i32_ptr : !llvm.ptr, %i32 : i32) { // CHECK: atomicrmw fadd ptr %{{.*}}, float %{{.*}} monotonic - %0 = llvm.atomicrmw fadd %f32_ptr, %f32 monotonic : f32 + %0 = llvm.atomicrmw fadd %f32_ptr, %f32 monotonic : !llvm.ptr, f32 // CHECK: atomicrmw fsub ptr %{{.*}}, float %{{.*}} monotonic - %1 = llvm.atomicrmw fsub %f32_ptr, %f32 monotonic : f32 + %1 = llvm.atomicrmw fsub %f32_ptr, %f32 monotonic : !llvm.ptr, f32 // CHECK: atomicrmw xchg ptr %{{.*}}, float %{{.*}} monotonic - %2 = llvm.atomicrmw xchg %f32_ptr, %f32 monotonic : f32 + %2 = llvm.atomicrmw xchg %f32_ptr, %f32 monotonic : !llvm.ptr, f32 // CHECK: atomicrmw add ptr %{{.*}}, i32 %{{.*}} acquire - %3 = llvm.atomicrmw add %i32_ptr, %i32 acquire : i32 + %3 = llvm.atomicrmw add %i32_ptr, %i32 acquire : !llvm.ptr, i32 // CHECK: atomicrmw sub ptr %{{.*}}, i32 %{{.*}} release - %4 = llvm.atomicrmw sub %i32_ptr, %i32 release : i32 + %4 = llvm.atomicrmw sub %i32_ptr, %i32 release : !llvm.ptr, i32 // CHECK: atomicrmw and ptr %{{.*}}, i32 %{{.*}} acq_rel - %5 = llvm.atomicrmw _and %i32_ptr, %i32 acq_rel : i32 + %5 = llvm.atomicrmw _and %i32_ptr, %i32 acq_rel : !llvm.ptr, i32 // CHECK: atomicrmw nand ptr %{{.*}}, i32 %{{.*}} seq_cst - %6 = llvm.atomicrmw nand %i32_ptr, %i32 seq_cst : i32 + %6 = llvm.atomicrmw nand %i32_ptr, %i32 seq_cst : !llvm.ptr, i32 // CHECK: atomicrmw or ptr %{{.*}}, i32 %{{.*}} monotonic - %7 = llvm.atomicrmw _or %i32_ptr, %i32 monotonic : i32 + %7 = llvm.atomicrmw _or %i32_ptr, %i32 monotonic : !llvm.ptr, i32 // CHECK: atomicrmw xor ptr %{{.*}}, i32 %{{.*}} monotonic - %8 = llvm.atomicrmw _xor %i32_ptr, %i32 monotonic : i32 + %8 = llvm.atomicrmw _xor %i32_ptr, %i32 monotonic : !llvm.ptr, i32 // CHECK: atomicrmw max ptr %{{.*}}, i32 %{{.*}} monotonic - %9 = llvm.atomicrmw max %i32_ptr, %i32 monotonic : i32 + %9 = llvm.atomicrmw max %i32_ptr, %i32 monotonic : !llvm.ptr, i32 // CHECK: atomicrmw min ptr %{{.*}}, i32 %{{.*}} monotonic - %10 = llvm.atomicrmw min %i32_ptr, %i32 monotonic : i32 + %10 = llvm.atomicrmw min %i32_ptr, %i32 monotonic : !llvm.ptr, i32 // CHECK: atomicrmw umax ptr %{{.*}}, i32 %{{.*}} monotonic - %11 = llvm.atomicrmw umax %i32_ptr, %i32 monotonic : i32 + %11 = llvm.atomicrmw umax %i32_ptr, %i32 monotonic : !llvm.ptr, i32 // CHECK: atomicrmw umin ptr %{{.*}}, i32 %{{.*}} monotonic - %12 = llvm.atomicrmw umin %i32_ptr, %i32 monotonic : i32 + %12 = llvm.atomicrmw umin %i32_ptr, %i32 monotonic : !llvm.ptr, i32 llvm.return } // CHECK-LABEL: @cmpxchg llvm.func @cmpxchg(%ptr : !llvm.ptr, %cmp : i32, %val: i32) { // CHECK: cmpxchg ptr %{{.*}}, i32 %{{.*}}, i32 %{{.*}} acq_rel monotonic - %0 = llvm.cmpxchg %ptr, %cmp, %val acq_rel monotonic : i32 + %0 = llvm.cmpxchg %ptr, %cmp, %val acq_rel monotonic : !llvm.ptr, i32 // CHECK: %{{[0-9]+}} = extractvalue { i32, i1 } %{{[0-9]+}}, 0 %1 = llvm.extractvalue %0[0] : !llvm.struct<(i32, i1)> // CHECK: %{{[0-9]+}} = extractvalue { i32, i1 } %{{[0-9]+}}, 1 diff --git a/mlir/test/Target/LLVMIR/openmp-reduction.mlir b/mlir/test/Target/LLVMIR/openmp-reduction.mlir index abeb46b1adc2..d66d65bd2845 100644 --- a/mlir/test/Target/LLVMIR/openmp-reduction.mlir +++ b/mlir/test/Target/LLVMIR/openmp-reduction.mlir @@ -17,7 +17,7 @@ combiner { atomic { ^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr): %2 = llvm.load %arg3 : !llvm.ptr - llvm.atomicrmw fadd %arg2, %2 monotonic : f32 + llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32 omp.yield } @@ -90,7 +90,7 @@ combiner { atomic { ^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr): %2 = llvm.load %arg3 : !llvm.ptr - llvm.atomicrmw fadd %arg2, %2 monotonic : f32 + llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32 omp.yield } @@ -178,7 +178,7 @@ combiner { atomic { ^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr): %2 = llvm.load %arg3 : !llvm.ptr - llvm.atomicrmw fadd %arg2, %2 monotonic : f32 + llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32 omp.yield } @@ -261,7 +261,7 @@ combiner { atomic { ^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr): %2 = llvm.load %arg3 : !llvm.ptr - llvm.atomicrmw fadd %arg2, %2 monotonic : f32 + llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32 omp.yield } @@ -340,7 +340,7 @@ combiner { atomic { ^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr): %2 = llvm.load %arg3 : !llvm.ptr - llvm.atomicrmw fadd %arg2, %2 monotonic : f32 + llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32 omp.yield } From 501094f8c4f81aea1488d5d812499b188e44b4d0 Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Mon, 6 Mar 2023 10:21:05 +0100 Subject: [PATCH 059/147] [mlir][llvm] Add structured loop metadata This commit introduces a structured representation of loop metadata to the LLVM dialect. This attribute explicitly models all known `!llvm.loop` metadata fields and groups them by introducing nested attributes for each namespace. The new attribute replaces the LoopOptionAttr that could only model a limited subset of loop metadata. Reviewed By: gysit Differential Revision: https://reviews.llvm.org/D143064 (cherry picked from commit 889a11783ec0751be6766341936f688f4afd37df) --- .../mlir/Dialect/LLVMIR/LLVMAttrDefs.td | 153 +++++++++-- mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h | 51 ---- mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td | 19 -- .../include/mlir/Dialect/LLVMIR/LLVMOpBase.td | 2 - .../mlir/Target/LLVMIR/ModuleTranslation.h | 26 +- mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp | 170 ------------- mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 59 ++--- mlir/lib/Target/LLVMIR/CMakeLists.txt | 2 + .../LLVMIR/LLVMToLLVMIRTranslation.cpp | 88 +------ .../LLVMIR/LoopAnnotationTranslation.cpp | 221 ++++++++++++++++ .../Target/LLVMIR/LoopAnnotationTranslation.h | 57 +++++ mlir/lib/Target/LLVMIR/ModuleTranslation.cpp | 13 + mlir/test/Dialect/LLVMIR/invalid.mlir | 71 +----- mlir/test/Dialect/LLVMIR/loop-metadata.mlir | 70 +++++ mlir/test/Dialect/LLVMIR/roundtrip.mlir | 16 -- mlir/test/Target/LLVMIR/llvmir.mlir | 38 --- mlir/test/Target/LLVMIR/loop-metadata.mlir | 239 ++++++++++++++++++ 17 files changed, 768 insertions(+), 527 deletions(-) create mode 100644 mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp create mode 100644 mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.h create mode 100644 mlir/test/Dialect/LLVMIR/loop-metadata.mlir create mode 100644 mlir/test/Target/LLVMIR/loop-metadata.mlir diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td index 96936c4dcdc4..570533fae001 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td @@ -50,38 +50,147 @@ def LinkageAttr : LLVM_Attr<"Linkage", "linkage"> { } //===----------------------------------------------------------------------===// -// LoopOptionsAttr +// Loop Attributes //===----------------------------------------------------------------------===// -def LoopOptionsAttr : LLVM_Attr<"LoopOptions", "loopopts"> { +def LoopVectorizeAttr : LLVM_Attr<"LoopVectorize", "loop_vectorize"> { let description = [{ - This attributes encapsulates "loop options". It is means to decorate - branches that are "latches" (loop backedges) and maps to the `!llvm.loop` - metadatas: https://llvm.org/docs/LangRef.html#llvm-loop - It store the options as a pair in a sorted array and expose - APIs to retrieve the value for each option with a stronger type (bool for - example). + This attribute defines vectorization specific loop annotations that map to + the "!llvm.loop.vectorize" metadata. }]; let parameters = (ins - ArrayRefParameter<"std::pair", "">:$options + OptionalParameter<"BoolAttr">:$disable, + OptionalParameter<"BoolAttr">:$predicateEnable, + OptionalParameter<"BoolAttr">:$scalableEnable, + OptionalParameter<"IntegerAttr">:$width, + OptionalParameter<"LoopAnnotationAttr">:$followupVectorized, + OptionalParameter<"LoopAnnotationAttr">:$followupEpilogue, + OptionalParameter<"LoopAnnotationAttr">:$followupAll ); - let extraClassDeclaration = [{ - using OptionValuePair = std::pair; - using OptionsArray = ArrayRef>; - std::optional disableUnroll(); - std::optional disableLICM(); - std::optional interleaveCount(); + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopInterleaveAttr : LLVM_Attr<"LoopInterleave", "loop_interleave"> { + let description = [{ + This attribute defines interleaving specific loop annotations that map to + the "!llvm.loop.interleave" metadata. }]; - let builders = [ - /// Build the LoopOptions Attribute from a sorted array of individual options. - AttrBuilder<(ins "ArrayRef>":$sortedOptions)>, - AttrBuilder<(ins "LoopOptionsAttrBuilder &":$optionBuilders)> - ]; - let hasCustomAssemblyFormat = 1; - let skipDefaultBuilders = 1; + let parameters = (ins + "IntegerAttr":$count + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopUnrollAttr : LLVM_Attr<"LoopUnroll", "loop_unroll"> { + let description = [{ + This attribute defines unrolling specific loop annotations that map to + the "!llvm.loop.unroll" metadata. + }]; + + let parameters = (ins + OptionalParameter<"BoolAttr">:$disable, + OptionalParameter<"IntegerAttr">:$count, + OptionalParameter<"BoolAttr">:$runtimeDisable, + OptionalParameter<"BoolAttr">:$full, + OptionalParameter<"LoopAnnotationAttr">:$followup, + OptionalParameter<"LoopAnnotationAttr">:$followupRemainder + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopUnrollAndJamAttr : LLVM_Attr<"LoopUnrollAndJam", "loop_unroll_and_jam"> { + let description = [{ + This attribute defines "unroll and jam" specific loop annotations that map to + the "!llvm.loop.unroll_and_jam" metadata. + }]; + + let parameters = (ins + OptionalParameter<"BoolAttr">:$disable, + OptionalParameter<"IntegerAttr">:$count, + OptionalParameter<"LoopAnnotationAttr">:$followupOuter, + OptionalParameter<"LoopAnnotationAttr">:$followupInner, + OptionalParameter<"LoopAnnotationAttr">:$followupRemainderOuter, + OptionalParameter<"LoopAnnotationAttr">:$followupRemainderInner, + OptionalParameter<"LoopAnnotationAttr">:$followupAll + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopLICMAttr : LLVM_Attr<"LoopLICM", "loop_licm"> { + let description = [{ + This attribute encapsulates loop invariant code motion (licm) specific loop + annotations. The fields correspond to the "!llvm.licm.disable" and the + "!llvm.loop.licm_versioning.disable" metadata. + }]; + + let parameters = (ins + OptionalParameter<"BoolAttr">:$disable, + OptionalParameter<"BoolAttr">:$versioningDisable + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopDistributeAttr : LLVM_Attr<"LoopDistribute", "loop_distribute"> { + let description = [{ + This attribute defines distribution specific loop annotations that map to + the "!llvm.loop.distribute" metadata. + }]; + + let parameters = (ins + OptionalParameter<"BoolAttr">:$disable, + OptionalParameter<"LoopAnnotationAttr">:$followupCoincident, + OptionalParameter<"LoopAnnotationAttr">:$followupSequential, + OptionalParameter<"LoopAnnotationAttr">:$followupFallback, + OptionalParameter<"LoopAnnotationAttr">:$followupAll + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopPipelineAttr : LLVM_Attr<"LoopPipeline", "loop_pipeline"> { + let description = [{ + This attribute defines pipelining specific loop annotations that map to + the "!llvm.loop.pipeline" metadata. + }]; + + let parameters = (ins + OptionalParameter<"BoolAttr">:$disable, + OptionalParameter<"IntegerAttr">:$initiationinterval + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopAnnotationAttr : LLVM_Attr<"LoopAnnotation", "loop_annotation"> { + let description = [{ + This attributes encapsulates "loop metadata". It is meant to decorate + branches that are "latches" (loop backedges) and maps to the `!llvm.loop` + metadatas: https://llvm.org/docs/LangRef.html#llvm-loop + It stores annotations in attribute parameters and groups related options in + nested attributes to provide structured access. + }]; + + let parameters = (ins + OptionalParameter<"BoolAttr">:$disableNonforced, + OptionalParameter<"LoopVectorizeAttr">:$vectorize, + OptionalParameter<"LoopInterleaveAttr">:$interleave, + OptionalParameter<"LoopUnrollAttr">:$unroll, + OptionalParameter<"LoopUnrollAndJamAttr">:$unrollAndJam, + OptionalParameter<"LoopLICMAttr">:$licm, + OptionalParameter<"LoopDistributeAttr">:$distribute, + OptionalParameter<"LoopPipelineAttr">:$pipeline, + OptionalParameter<"BoolAttr">:$mustProgress, + OptionalArrayRefParameter<"SymbolRefAttr">:$parallelAccesses + ); + + let assemblyFormat = "`<` struct(params) `>`"; } //===----------------------------------------------------------------------===// diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h index d6619ac8c9b9..abbfcf415c54 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h @@ -22,7 +22,6 @@ namespace mlir { namespace LLVM { -class LoopOptionsAttrBuilder; /// This class represents the base attribute for all debug info attributes. class DINodeAttr : public Attribute { @@ -74,54 +73,4 @@ using linkage::Linkage; #define GET_ATTRDEF_CLASSES #include "mlir/Dialect/LLVMIR/LLVMOpsAttrDefs.h.inc" -namespace mlir { -namespace LLVM { - -/// Builder class for LoopOptionsAttr. This helper class allows to progressively -/// build a LoopOptionsAttr one option at a time, and pay the price of attribute -/// creation once all the options are in place. -class LoopOptionsAttrBuilder { -public: - /// Construct a empty builder. - LoopOptionsAttrBuilder() = default; - - /// Construct a builder with an initial list of options from an existing - /// LoopOptionsAttr. - LoopOptionsAttrBuilder(LoopOptionsAttr attr); - - /// Set the `disable_licm` option to the provided value. If no value - /// is provided the option is deleted. - LoopOptionsAttrBuilder &setDisableLICM(std::optional value); - - /// Set the `interleave_count` option to the provided value. If no value - /// is provided the option is deleted. - LoopOptionsAttrBuilder &setInterleaveCount(std::optional count); - - /// Set the `disable_unroll` option to the provided value. If no value - /// is provided the option is deleted. - LoopOptionsAttrBuilder &setDisableUnroll(std::optional value); - - /// Set the `disable_pipeline` option to the provided value. If no value - /// is provided the option is deleted. - LoopOptionsAttrBuilder &setDisablePipeline(std::optional value); - - /// Set the `pipeline_initiation_interval` option to the provided value. - /// If no value is provided the option is deleted. - LoopOptionsAttrBuilder & - setPipelineInitiationInterval(std::optional count); - - /// Returns true if any option has been set. - bool empty() { return options.empty(); } - -private: - template - LoopOptionsAttrBuilder &setOption(LoopOptionCase tag, std::optional value); - - friend class LoopOptionsAttr; - SmallVector options; -}; - -} // namespace LLVM -} // namespace mlir - #endif // MLIR_DIALECT_LLVMIR_LLVMATTRS_H_ diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td index 17b372f3774e..c7ba00e6ec89 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td @@ -489,25 +489,6 @@ def Linkage : DialectAttr< "::mlir::LLVM::LinkageAttr::get($_builder.getContext(), $0)"; } -//===----------------------------------------------------------------------===// -// LoopOptions -//===----------------------------------------------------------------------===// - -def LOptDisableUnroll : I32EnumAttrCase<"disable_unroll", 1>; -def LOptDisableLICM : I32EnumAttrCase<"disable_licm", 2>; -def LOptInterleaveCount : I32EnumAttrCase<"interleave_count", 3>; -def LOptDisablePipeline : I32EnumAttrCase<"disable_pipeline", 4>; -def LOptPipelineInitiationInterval : I32EnumAttrCase<"pipeline_initiation_interval", 5>; - -def LoopOptionCase : I32EnumAttr< - "LoopOptionCase", - "LLVM loop option", - [LOptDisableUnroll, LOptDisableLICM, LOptInterleaveCount, - LOptDisablePipeline, LOptPipelineInitiationInterval - ]> { - let cppNamespace = "::mlir::LLVM"; -} - //===----------------------------------------------------------------------===// // UnnamedAddr //===----------------------------------------------------------------------===// diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td index 3f99dcbcccaa..1753c6d8cde4 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td @@ -39,8 +39,6 @@ def LLVM_Dialect : Dialect { static StringRef getNoAliasScopesAttrName() { return "noalias_scopes"; } static StringRef getAliasScopesAttrName() { return "alias_scopes"; } static StringRef getLoopAttrName() { return "llvm.loop"; } - static StringRef getParallelAccessAttrName() { return "parallel_access"; } - static StringRef getLoopOptionsAttrName() { return "options"; } static StringRef getAccessGroupsAttrName() { return "access_groups"; } static StringRef getStructAttrsAttrName() { return "llvm.struct_attrs"; } static StringRef getTBAAAttrName() { return "llvm.tbaa"; } diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h index 4e85dcf6ab34..59b29f506609 100644 --- a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h +++ b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h @@ -40,6 +40,7 @@ namespace LLVM { namespace detail { class DebugTranslation; +class LoopAnnotationTranslation; } // namespace detail class DINodeAttr; @@ -129,19 +130,6 @@ class ModuleTranslation { llvm::MDNode *getAliasScope(Operation &opInst, SymbolRefAttr aliasScopeRef) const; - /// Returns the LLVM metadata corresponding to a llvm loop's codegen - /// options attribute. - llvm::MDNode *lookupLoopOptionsMetadata(Attribute options) const { - return loopOptionsMetadataMapping.lookup(options); - } - - void mapLoopOptionsMetadata(Attribute options, llvm::MDNode *metadata) { - auto result = loopOptionsMetadataMapping.try_emplace(options, metadata); - (void)result; - assert(result.second && - "attempting to map loop options that was already mapped"); - } - // Sets LLVM metadata for memory operations that are in a parallel loop. void setAccessGroupsMetadata(Operation *op, llvm::Instruction *inst); @@ -152,6 +140,10 @@ class ModuleTranslation { /// TBAA attributes. void setTBAAMetadata(Operation *op, llvm::Instruction *inst); + /// Sets LLVM loop metadata for branch operations that have a loop annotation + /// attribute. + void setLoopMetadata(Operation *op, llvm::Instruction *inst); + /// Converts the type from MLIR LLVM dialect to LLVM. llvm::Type *convertType(Type type); @@ -315,6 +307,9 @@ class ModuleTranslation { /// A converter for translating debug information. std::unique_ptr debugTranslation; + /// A converter for translating loop annotations. + std::unique_ptr loopAnnotationTranslation; + /// Builder for LLVM IR generation of OpenMP constructs. std::unique_ptr ompBuilder; @@ -343,11 +338,6 @@ class ModuleTranslation { /// identified via their branches) and contained memory accesses. DenseMap accessGroupMetadataMapping; - /// Mapping from an attribute describing loop codegen options to its LLVM - /// metadata. The metadata is attached to Latch block branches with this - /// attribute. - DenseMap loopOptionsMetadataMapping; - /// Mapping from an alias scope metadata operation to its LLVM metadata. /// This map is populated on module entry. DenseMap aliasScopeMetadataMapping; diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp index 3b6b67a137a9..8e749259dad9 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp @@ -92,176 +92,6 @@ DISubroutineTypeAttr::verify(function_ref emitError, return success(); } -//===----------------------------------------------------------------------===// -// LoopOptionsAttrBuilder -//===----------------------------------------------------------------------===// - -LoopOptionsAttrBuilder::LoopOptionsAttrBuilder(LoopOptionsAttr attr) - : options(attr.getOptions().begin(), attr.getOptions().end()) {} - -template -LoopOptionsAttrBuilder & -LoopOptionsAttrBuilder::setOption(LoopOptionCase tag, std::optional value) { - auto option = llvm::find_if( - options, [tag](auto option) { return option.first == tag; }); - if (option != options.end()) { - if (value) - option->second = *value; - else - options.erase(option); - } else { - options.push_back(LoopOptionsAttr::OptionValuePair(tag, *value)); - } - return *this; -} - -LoopOptionsAttrBuilder & -LoopOptionsAttrBuilder::setDisableLICM(std::optional value) { - return setOption(LoopOptionCase::disable_licm, value); -} - -/// Set the `interleave_count` option to the provided value. If no value -/// is provided the option is deleted. -LoopOptionsAttrBuilder & -LoopOptionsAttrBuilder::setInterleaveCount(std::optional count) { - return setOption(LoopOptionCase::interleave_count, count); -} - -/// Set the `disable_unroll` option to the provided value. If no value -/// is provided the option is deleted. -LoopOptionsAttrBuilder & -LoopOptionsAttrBuilder::setDisableUnroll(std::optional value) { - return setOption(LoopOptionCase::disable_unroll, value); -} - -/// Set the `disable_pipeline` option to the provided value. If no value -/// is provided the option is deleted. -LoopOptionsAttrBuilder & -LoopOptionsAttrBuilder::setDisablePipeline(std::optional value) { - return setOption(LoopOptionCase::disable_pipeline, value); -} - -/// Set the `pipeline_initiation_interval` option to the provided value. -/// If no value is provided the option is deleted. -LoopOptionsAttrBuilder &LoopOptionsAttrBuilder::setPipelineInitiationInterval( - std::optional count) { - return setOption(LoopOptionCase::pipeline_initiation_interval, count); -} - -//===----------------------------------------------------------------------===// -// LoopOptionsAttr -//===----------------------------------------------------------------------===// - -template -static std::optional -getOption(ArrayRef> options, - LoopOptionCase option) { - auto it = - lower_bound(options, option, [](auto optionPair, LoopOptionCase option) { - return optionPair.first < option; - }); - if (it == options.end()) - return {}; - return static_cast(it->second); -} - -std::optional LoopOptionsAttr::disableUnroll() { - return getOption(getOptions(), LoopOptionCase::disable_unroll); -} - -std::optional LoopOptionsAttr::disableLICM() { - return getOption(getOptions(), LoopOptionCase::disable_licm); -} - -std::optional LoopOptionsAttr::interleaveCount() { - return getOption(getOptions(), LoopOptionCase::interleave_count); -} - -/// Build the LoopOptions Attribute from a sorted array of individual options. -LoopOptionsAttr LoopOptionsAttr::get( - MLIRContext *context, - ArrayRef> sortedOptions) { - assert(llvm::is_sorted(sortedOptions, llvm::less_first()) && - "LoopOptionsAttr ctor expects a sorted options array"); - return Base::get(context, sortedOptions); -} - -/// Build the LoopOptions Attribute from a sorted array of individual options. -LoopOptionsAttr LoopOptionsAttr::get(MLIRContext *context, - LoopOptionsAttrBuilder &optionBuilders) { - llvm::sort(optionBuilders.options, llvm::less_first()); - return Base::get(context, optionBuilders.options); -} - -void LoopOptionsAttr::print(AsmPrinter &printer) const { - printer << "<"; - llvm::interleaveComma(getOptions(), printer, [&](auto option) { - printer << stringifyEnum(option.first) << " = "; - switch (option.first) { - case LoopOptionCase::disable_licm: - case LoopOptionCase::disable_unroll: - case LoopOptionCase::disable_pipeline: - printer << (option.second ? "true" : "false"); - break; - case LoopOptionCase::interleave_count: - case LoopOptionCase::pipeline_initiation_interval: - printer << option.second; - break; - } - }); - printer << ">"; -} - -Attribute LoopOptionsAttr::parse(AsmParser &parser, Type type) { - if (failed(parser.parseLess())) - return {}; - - SmallVector> options; - llvm::SmallDenseSet seenOptions; - auto parseLoopOptions = [&]() -> ParseResult { - StringRef optionName; - if (parser.parseKeyword(&optionName)) - return failure(); - - auto option = symbolizeLoopOptionCase(optionName); - if (!option) - return parser.emitError(parser.getNameLoc(), "unknown loop option: ") - << optionName; - if (!seenOptions.insert(*option).second) - return parser.emitError(parser.getNameLoc(), "loop option present twice"); - if (failed(parser.parseEqual())) - return failure(); - - int64_t value; - switch (*option) { - case LoopOptionCase::disable_licm: - case LoopOptionCase::disable_unroll: - case LoopOptionCase::disable_pipeline: - if (succeeded(parser.parseOptionalKeyword("true"))) - value = 1; - else if (succeeded(parser.parseOptionalKeyword("false"))) - value = 0; - else { - return parser.emitError(parser.getNameLoc(), - "expected boolean value 'true' or 'false'"); - } - break; - case LoopOptionCase::interleave_count: - case LoopOptionCase::pipeline_initiation_interval: - if (failed(parser.parseInteger(value))) - return parser.emitError(parser.getNameLoc(), "expected integer value"); - break; - } - options.push_back(std::make_pair(*option, value)); - return success(); - }; - if (parser.parseCommaSeparatedList(parseLoopOptions) || parser.parseGreater()) - return {}; - - llvm::sort(options, llvm::less_first()); - return get(parser.getContext(), options); -} - //===----------------------------------------------------------------------===// // MemoryEffectsAttr //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp index 04f91755314d..adba1730d6c2 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -2713,7 +2713,10 @@ struct LLVMOpAsmDialectInterface : public OpAsmDialectInterface { .Case([&](auto attr) { + DISubprogramAttr, DISubroutineTypeAttr, LoopAnnotationAttr, + LoopVectorizeAttr, LoopInterleaveAttr, LoopUnrollAttr, + LoopUnrollAndJamAttr, LoopLICMAttr, LoopDistributeAttr, + LoopPipelineAttr>([&](auto attr) { os << decltype(attr)::getMnemonic(); return AliasResult::OverridableAlias; }) @@ -2905,45 +2908,27 @@ LogicalResult LLVMDialect::verifyOperationAttribute(Operation *op, // If the `llvm.loop` attribute is present, enforce the following structure, // which the module translation can assume. if (attr.getName() == LLVMDialect::getLoopAttrName()) { - auto loopAttr = attr.getValue().dyn_cast(); + auto loopAttr = attr.getValue().dyn_cast(); if (!loopAttr) return op->emitOpError() << "expected '" << LLVMDialect::getLoopAttrName() - << "' to be a dictionary attribute"; - std::optional parallelAccessGroup = - loopAttr.getNamed(LLVMDialect::getParallelAccessAttrName()); - if (parallelAccessGroup) { - auto accessGroups = parallelAccessGroup->getValue().dyn_cast(); - if (!accessGroups) - return op->emitOpError() - << "expected '" << LLVMDialect::getParallelAccessAttrName() - << "' to be an array attribute"; - for (Attribute attr : accessGroups) { - auto accessGroupRef = attr.dyn_cast(); - if (!accessGroupRef) - return op->emitOpError() - << "expected '" << attr << "' to be a symbol reference"; - StringAttr metadataName = accessGroupRef.getRootReference(); - auto metadataOp = - SymbolTable::lookupNearestSymbolFrom( - op->getParentOp(), metadataName); - if (!metadataOp) - return op->emitOpError() - << "expected '" << attr << "' to reference a metadata op"; - StringAttr accessGroupName = accessGroupRef.getLeafReference(); - Operation *accessGroupOp = - SymbolTable::lookupNearestSymbolFrom(metadataOp, accessGroupName); - if (!accessGroupOp) - return op->emitOpError() - << "expected '" << attr << "' to reference an access_group op"; - } + << "' to be a loop annotation attribute"; + ArrayRef parallelAccesses = loopAttr.getParallelAccesses(); + if (parallelAccesses.empty()) + return success(); + for (SymbolRefAttr accessGroupRef : parallelAccesses) { + StringAttr metadataName = accessGroupRef.getRootReference(); + auto metadataOp = SymbolTable::lookupNearestSymbolFrom( + op->getParentOp(), metadataName); + if (!metadataOp) + return op->emitOpError() << "expected '" << accessGroupRef + << "' to reference a metadata op"; + StringAttr accessGroupName = accessGroupRef.getLeafReference(); + Operation *accessGroupOp = + SymbolTable::lookupNearestSymbolFrom(metadataOp, accessGroupName); + if (!accessGroupOp) + return op->emitOpError() << "expected '" << accessGroupRef + << "' to reference an access_group op"; } - - std::optional loopOptions = - loopAttr.getNamed(LLVMDialect::getLoopOptionsAttrName()); - if (loopOptions && !loopOptions->getValue().isa()) - return op->emitOpError() - << "expected '" << LLVMDialect::getLoopOptionsAttrName() - << "' to be a `loopopts` attribute"; } if (attr.getName() == LLVMDialect::getStructAttrsAttrName()) { diff --git a/mlir/lib/Target/LLVMIR/CMakeLists.txt b/mlir/lib/Target/LLVMIR/CMakeLists.txt index 97577c036a22..1741176ddac8 100644 --- a/mlir/lib/Target/LLVMIR/CMakeLists.txt +++ b/mlir/lib/Target/LLVMIR/CMakeLists.txt @@ -5,6 +5,7 @@ set(LLVM_OPTIONAL_SOURCES ConvertToLLVMIR.cpp DebugTranslation.cpp DebugImporter.cpp + LoopAnnotationTranslation.cpp ModuleTranslation.cpp ModuleImport.cpp TypeToLLVM.cpp @@ -14,6 +15,7 @@ set(LLVM_OPTIONAL_SOURCES add_mlir_translation_library(MLIRTargetLLVMIRExport DebugTranslation.cpp + LoopAnnotationTranslation.cpp ModuleTranslation.cpp TypeToLLVM.cpp diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp index 998392ded460..af50ae134737 100644 --- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp @@ -167,90 +167,6 @@ static llvm::FastMathFlags getFastmathFlags(FastmathFlagsInterface &op) { return ret; } -/// Returns an LLVM metadata node corresponding to a loop option. This metadata -/// is attached to an llvm.loop node. -static llvm::MDNode *getLoopOptionMetadata(llvm::LLVMContext &ctx, - LoopOptionCase option, - int64_t value) { - StringRef name; - llvm::Constant *cstValue = nullptr; - switch (option) { - case LoopOptionCase::disable_licm: - name = "llvm.licm.disable"; - cstValue = llvm::ConstantInt::getBool(ctx, value); - break; - case LoopOptionCase::disable_unroll: - name = "llvm.loop.unroll.disable"; - cstValue = llvm::ConstantInt::getBool(ctx, value); - break; - case LoopOptionCase::interleave_count: - name = "llvm.loop.interleave.count"; - cstValue = llvm::ConstantInt::get( - llvm::IntegerType::get(ctx, /*NumBits=*/32), value); - break; - case LoopOptionCase::disable_pipeline: - name = "llvm.loop.pipeline.disable"; - cstValue = llvm::ConstantInt::getBool(ctx, value); - break; - case LoopOptionCase::pipeline_initiation_interval: - name = "llvm.loop.pipeline.initiationinterval"; - cstValue = llvm::ConstantInt::get( - llvm::IntegerType::get(ctx, /*NumBits=*/32), value); - break; - } - return llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name), - llvm::ConstantAsMetadata::get(cstValue)}); -} - -static void setLoopMetadata(Operation &opInst, llvm::Instruction &llvmInst, - llvm::IRBuilderBase &builder, - LLVM::ModuleTranslation &moduleTranslation) { - if (Attribute attr = opInst.getAttr(LLVMDialect::getLoopAttrName())) { - llvm::Module *module = builder.GetInsertBlock()->getModule(); - llvm::MDNode *loopMD = moduleTranslation.lookupLoopOptionsMetadata(attr); - if (!loopMD) { - llvm::LLVMContext &ctx = module->getContext(); - - SmallVector loopOptions; - // Reserve operand 0 for loop id self reference. - auto dummy = llvm::MDNode::getTemporary(ctx, std::nullopt); - loopOptions.push_back(dummy.get()); - - auto loopAttr = attr.cast(); - auto parallelAccessGroup = - loopAttr.getNamed(LLVMDialect::getParallelAccessAttrName()); - if (parallelAccessGroup) { - SmallVector parallelAccess; - parallelAccess.push_back( - llvm::MDString::get(ctx, "llvm.loop.parallel_accesses")); - for (SymbolRefAttr accessGroupRef : parallelAccessGroup->getValue() - .cast() - .getAsRange()) - parallelAccess.push_back( - moduleTranslation.getAccessGroup(opInst, accessGroupRef)); - loopOptions.push_back(llvm::MDNode::get(ctx, parallelAccess)); - } - - if (auto loopOptionsAttr = loopAttr.getAs( - LLVMDialect::getLoopOptionsAttrName())) { - for (auto option : loopOptionsAttr.getOptions()) - loopOptions.push_back( - getLoopOptionMetadata(ctx, option.first, option.second)); - } - - // Create loop options and set the first operand to itself. - loopMD = llvm::MDNode::get(ctx, loopOptions); - loopMD->replaceOperandWith(0, loopMD); - - // Store a map from this Attribute to the LLVM metadata in case we - // encounter it again. - moduleTranslation.mapLoopOptionsMetadata(attr, loopMD); - } - - llvmInst.setMetadata(module->getMDKindID("llvm.loop"), loopMD); - } -} - /// Convert the value of a DenseI64ArrayAttr to a vector of unsigned indices. static SmallVector extractPosition(ArrayRef indices) { SmallVector position; @@ -505,7 +421,7 @@ convertOperationImpl(Operation &opInst, llvm::IRBuilderBase &builder, llvm::BranchInst *branch = builder.CreateBr(moduleTranslation.lookupBlock(brOp.getSuccessor())); moduleTranslation.mapBranch(&opInst, branch); - setLoopMetadata(opInst, *branch, builder, moduleTranslation); + moduleTranslation.setLoopMetadata(&opInst, branch); return success(); } if (auto condbrOp = dyn_cast(opInst)) { @@ -516,7 +432,7 @@ convertOperationImpl(Operation &opInst, llvm::IRBuilderBase &builder, moduleTranslation.lookupBlock(condbrOp.getSuccessor(0)), moduleTranslation.lookupBlock(condbrOp.getSuccessor(1)), branchWeights); moduleTranslation.mapBranch(&opInst, branch); - setLoopMetadata(opInst, *branch, builder, moduleTranslation); + moduleTranslation.setLoopMetadata(&opInst, branch); return success(); } if (auto switchOp = dyn_cast(opInst)) { diff --git a/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp new file mode 100644 index 000000000000..273dc1378984 --- /dev/null +++ b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp @@ -0,0 +1,221 @@ +#include "LoopAnnotationTranslation.h" + +using namespace mlir; +using namespace mlir::LLVM; +using namespace mlir::LLVM::detail; + +namespace { +/// Helper class that keeps the state of one attribute to metadata conversion. +struct LoopAnnotationConversion { + LoopAnnotationConversion(LoopAnnotationAttr attr, + ModuleTranslation &moduleTranslation, Operation *op, + LoopAnnotationTranslation &loopAnnotationTranslation) + : attr(attr), moduleTranslation(moduleTranslation), op(op), + loopAnnotationTranslation(loopAnnotationTranslation), + ctx(moduleTranslation.getLLVMContext()) {} + + /// Converts this struct's loop annotation into a corresponding LLVMIR + /// metadata representation. + llvm::MDNode *convert(); + + /// Conversion functions for different payload attribute kinds. + void addUnitNode(StringRef name); + void addUnitNode(StringRef name, BoolAttr attr); + void convertBoolNode(StringRef name, BoolAttr attr, bool negated = false); + void convertI32Node(StringRef name, IntegerAttr attr); + void convertFollowupNode(StringRef name, LoopAnnotationAttr attr); + + /// Conversion functions for each for each loop annotation sub-attribute. + void convertLoopOptions(LoopVectorizeAttr options); + void convertLoopOptions(LoopInterleaveAttr options); + void convertLoopOptions(LoopUnrollAttr options); + void convertLoopOptions(LoopUnrollAndJamAttr options); + void convertLoopOptions(LoopLICMAttr options); + void convertLoopOptions(LoopDistributeAttr options); + void convertLoopOptions(LoopPipelineAttr options); + + LoopAnnotationAttr attr; + ModuleTranslation &moduleTranslation; + Operation *op; + LoopAnnotationTranslation &loopAnnotationTranslation; + llvm::LLVMContext &ctx; + llvm::SmallVector metadataNodes; +}; +} // namespace + +void LoopAnnotationConversion::addUnitNode(StringRef name) { + metadataNodes.push_back( + llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name)})); +} + +void LoopAnnotationConversion::addUnitNode(StringRef name, BoolAttr attr) { + if (attr && attr.getValue()) + addUnitNode(name); +} + +void LoopAnnotationConversion::convertBoolNode(StringRef name, BoolAttr attr, + bool negated) { + if (!attr) + return; + bool val = negated ^ attr.getValue(); + llvm::Constant *cstValue = llvm::ConstantInt::getBool(ctx, val); + metadataNodes.push_back( + llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name), + llvm::ConstantAsMetadata::get(cstValue)})); +} + +void LoopAnnotationConversion::convertI32Node(StringRef name, + IntegerAttr attr) { + if (!attr) + return; + uint32_t val = attr.getInt(); + llvm::Constant *cstValue = llvm::ConstantInt::get( + llvm::IntegerType::get(ctx, /*NumBits=*/32), val, /*isSigned=*/false); + metadataNodes.push_back( + llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name), + llvm::ConstantAsMetadata::get(cstValue)})); +} + +void LoopAnnotationConversion::convertFollowupNode(StringRef name, + LoopAnnotationAttr attr) { + if (!attr) + return; + + llvm::MDNode *node = loopAnnotationTranslation.translate(attr, op); + + metadataNodes.push_back( + llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name), node})); +} + +void LoopAnnotationConversion::convertLoopOptions(LoopVectorizeAttr options) { + convertBoolNode("llvm.loop.vectorize.enable", options.getDisable(), true); + convertBoolNode("llvm.loop.vectorize.predicate.enable", + options.getPredicateEnable()); + convertBoolNode("llvm.loop.vectorize.scalable.enable", + options.getScalableEnable()); + convertI32Node("llvm.loop.vectorize.width", options.getWidth()); + convertFollowupNode("llvm.loop.vectorize.followup_vectorized", + options.getFollowupVectorized()); + convertFollowupNode("llvm.loop.vectorize.followup_epilogue", + options.getFollowupEpilogue()); + convertFollowupNode("llvm.loop.vectorize.followup_all", + options.getFollowupAll()); +} + +void LoopAnnotationConversion::convertLoopOptions(LoopInterleaveAttr options) { + convertI32Node("llvm.loop.interleave.count", options.getCount()); +} + +void LoopAnnotationConversion::convertLoopOptions(LoopUnrollAttr options) { + if (auto disable = options.getDisable()) + addUnitNode(disable.getValue() ? "llvm.loop.unroll.disable" + : "llvm.loop.unroll.enable"); + convertI32Node("llvm.loop.unroll.count", options.getCount()); + convertBoolNode("llvm.loop.unroll.runtime.disable", + options.getRuntimeDisable()); + addUnitNode("llvm.loop.unroll.full", options.getFull()); + convertFollowupNode("llvm.loop.unroll.followup", options.getFollowup()); + convertFollowupNode("llvm.loop.unroll.followup_remainder", + options.getFollowupRemainder()); +} + +void LoopAnnotationConversion::convertLoopOptions( + LoopUnrollAndJamAttr options) { + if (auto disable = options.getDisable()) + addUnitNode(disable.getValue() ? "llvm.loop.unroll_and_jam.disable" + : "llvm.loop.unroll_and_jam.enable"); + convertI32Node("llvm.loop.unroll_and_jam.count", options.getCount()); + convertFollowupNode("llvm.loop.unroll_and_jam.followup_outer", + options.getFollowupOuter()); + convertFollowupNode("llvm.loop.unroll_and_jam.followup_inner", + options.getFollowupInner()); + convertFollowupNode("llvm.loop.unroll_and_jam.followup_remainder_outer", + options.getFollowupRemainderOuter()); + convertFollowupNode("llvm.loop.unroll_and_jam.followup_remainder_inner", + options.getFollowupRemainderInner()); + convertFollowupNode("llvm.loop.unroll_and_jam.followup_all", + options.getFollowupAll()); +} + +void LoopAnnotationConversion::convertLoopOptions(LoopLICMAttr options) { + addUnitNode("llvm.licm.disable", options.getDisable()); + addUnitNode("llvm.loop.licm_versioning.disable", + options.getVersioningDisable()); +} + +void LoopAnnotationConversion::convertLoopOptions(LoopDistributeAttr options) { + convertBoolNode("llvm.loop.distribute.enable", options.getDisable(), true); + convertFollowupNode("llvm.loop.distribute.followup_coincident", + options.getFollowupCoincident()); + convertFollowupNode("llvm.loop.distribute.followup_sequential", + options.getFollowupSequential()); + convertFollowupNode("llvm.loop.distribute.followup_fallback", + options.getFollowupFallback()); + convertFollowupNode("llvm.loop.distribute.followup_all", + options.getFollowupAll()); +} + +void LoopAnnotationConversion::convertLoopOptions(LoopPipelineAttr options) { + convertBoolNode("llvm.loop.pipeline.disable", options.getDisable()); + convertI32Node("llvm.loop.pipeline.initiationinterval", + options.getInitiationinterval()); +} + +llvm::MDNode *LoopAnnotationConversion::convert() { + + // Reserve operand 0 for loop id self reference. + auto dummy = llvm::MDNode::getTemporary(ctx, std::nullopt); + metadataNodes.push_back(dummy.get()); + + addUnitNode("llvm.loop.disable_nonforced", attr.getDisableNonforced()); + addUnitNode("llvm.loop.mustprogress", attr.getMustProgress()); + + if (auto options = attr.getVectorize()) + convertLoopOptions(options); + if (auto options = attr.getInterleave()) + convertLoopOptions(options); + if (auto options = attr.getUnroll()) + convertLoopOptions(options); + if (auto options = attr.getUnrollAndJam()) + convertLoopOptions(options); + if (auto options = attr.getLicm()) + convertLoopOptions(options); + if (auto options = attr.getDistribute()) + convertLoopOptions(options); + if (auto options = attr.getPipeline()) + convertLoopOptions(options); + + ArrayRef parallelAccessGroups = attr.getParallelAccesses(); + if (!parallelAccessGroups.empty()) { + SmallVector parallelAccess; + parallelAccess.push_back( + llvm::MDString::get(ctx, "llvm.loop.parallel_accesses")); + for (SymbolRefAttr accessGroupRef : parallelAccessGroups) + parallelAccess.push_back( + moduleTranslation.getAccessGroup(*op, accessGroupRef)); + metadataNodes.push_back(llvm::MDNode::get(ctx, parallelAccess)); + } + + // Create loop options and set the first operand to itself. + llvm::MDNode *loopMD = llvm::MDNode::get(ctx, metadataNodes); + loopMD->replaceOperandWith(0, loopMD); + + return loopMD; +} + +llvm::MDNode *LoopAnnotationTranslation::translate(LoopAnnotationAttr attr, + Operation *op) { + if (!attr) + return nullptr; + + llvm::MDNode *loopMD = lookupLoopMetadata(attr); + if (loopMD) + return loopMD; + + loopMD = + LoopAnnotationConversion(attr, moduleTranslation, op, *this).convert(); + // Store a map from this Attribute to the LLVM metadata in case we + // encounter it again. + mapLoopMetadata(attr, loopMD); + return loopMD; +} diff --git a/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.h b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.h new file mode 100644 index 000000000000..0bbd5442510f --- /dev/null +++ b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.h @@ -0,0 +1,57 @@ +//===- LoopAnnotationTranslation.h ------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the translation between an MLIR loop annotations and +// the corresponding LLVMIR metadata representation. +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_LIB_TARGET_LLVMIR_LOOPANNOTATIONTRANSLATION_H_ +#define MLIR_LIB_TARGET_LLVMIR_LOOPANNOTATIONTRANSLATION_H_ + +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/Target/LLVMIR/ModuleTranslation.h" + +namespace mlir { +namespace LLVM { +namespace detail { + +/// A helper class that converts a LoopAnnotationAttr into a corresponding +/// llvm::MDNode. +class LoopAnnotationTranslation { +public: + LoopAnnotationTranslation(LLVM::ModuleTranslation &moduleTranslation) + : moduleTranslation(moduleTranslation) {} + + llvm::MDNode *translate(LoopAnnotationAttr attr, Operation *op); + +private: + /// Returns the LLVM metadata corresponding to a llvm loop metadata attribute. + llvm::MDNode *lookupLoopMetadata(Attribute options) const { + return loopMetadataMapping.lookup(options); + } + + void mapLoopMetadata(Attribute options, llvm::MDNode *metadata) { + auto result = loopMetadataMapping.try_emplace(options, metadata); + (void)result; + assert(result.second && + "attempting to map loop options that was already mapped"); + } + + /// Mapping from an attribute describing loop metadata to its LLVM metadata. + /// The metadata is attached to Latch block branches with this attribute. + DenseMap loopMetadataMapping; + + LLVM::ModuleTranslation &moduleTranslation; +}; + +} // namespace detail +} // namespace LLVM +} // namespace mlir + +#endif // MLIR_LIB_TARGET_LLVMIR_LOOPANNOTATIONTRANSLATION_H_ diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp index 5f4833279eb5..aa5627e7a15f 100644 --- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp @@ -15,6 +15,7 @@ #include "AttrKindDetail.h" #include "DebugTranslation.h" +#include "LoopAnnotationTranslation.h" #include "mlir/Dialect/DLTI/DLTI.h" #include "mlir/Dialect/LLVMIR/LLVMDialect.h" #include "mlir/Dialect/LLVMIR/Transforms/LegalizeForExport.h" @@ -411,6 +412,8 @@ ModuleTranslation::ModuleTranslation(Operation *module, : mlirModule(module), llvmModule(std::move(llvmModule)), debugTranslation( std::make_unique(module, *this->llvmModule)), + loopAnnotationTranslation( + std::make_unique(*this)), typeTranslator(this->llvmModule->getContext()), iface(module->getContext()) { assert(satisfiesLLVMModule(mlirModule) && @@ -1215,6 +1218,16 @@ LogicalResult ModuleTranslation::createTBAAMetadata() { return success(); } +void ModuleTranslation::setLoopMetadata(Operation *op, + llvm::Instruction *inst) { + auto attr = + op->getAttrOfType(LLVMDialect::getLoopAttrName()); + if (!attr) + return; + llvm::MDNode *loopMD = loopAnnotationTranslation->translate(attr, op); + inst->setMetadata(llvm::LLVMContext::MD_loop, loopMD); +} + llvm::Type *ModuleTranslation::convertType(Type type) { return typeTranslator.translateType(type); } diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir index 39c89da1ec1f..a425b240ba54 100644 --- a/mlir/test/Dialect/LLVMIR/invalid.mlir +++ b/mlir/test/Dialect/LLVMIR/invalid.mlir @@ -842,7 +842,7 @@ llvm.mlir.global appending @non_array_type_global_appending_linkage() : i32 module { llvm.func @loopOptions() { - // expected-error@below {{expected 'llvm.loop' to be a dictionary attribute}} + // expected-error@below {{expected 'llvm.loop' to be a loop annotation attribute}} llvm.br ^bb4 {llvm.loop = "test"} ^bb4: llvm.return @@ -851,32 +851,10 @@ module { // ----- -module { - llvm.func @loopOptions() { - // expected-error@below {{expected 'parallel_access' to be an array attribute}} - llvm.br ^bb4 {llvm.loop = {parallel_access = "loop"}} - ^bb4: - llvm.return - } -} - -// ----- - -module { - llvm.func @loopOptions() { - // expected-error@below {{expected '"loop"' to be a symbol reference}} - llvm.br ^bb4 {llvm.loop = {parallel_access = ["loop"]}} - ^bb4: - llvm.return - } -} - -// ----- - module { llvm.func @loopOptions() { // expected-error@below {{expected '@func1' to reference a metadata op}} - llvm.br ^bb4 {llvm.loop = {parallel_access = [@func1]}} + llvm.br ^bb4 {llvm.loop = #llvm.loop_annotation} ^bb4: llvm.return } @@ -890,7 +868,7 @@ module { module { llvm.func @loopOptions() { // expected-error@below {{expected '@metadata' to reference an access_group op}} - llvm.br ^bb4 {llvm.loop = {parallel_access = [@metadata]}} + llvm.br ^bb4 {llvm.loop = #llvm.loop_annotation} ^bb4: llvm.return } @@ -900,49 +878,6 @@ module { // ----- -module { - llvm.func @loopOptions() { - // expected-error@below {{expected 'options' to be a `loopopts` attribute}} - llvm.br ^bb4 {llvm.loop = {options = "name"}} - ^bb4: - llvm.return - } -} - -// ----- - -module { - llvm.func @loopOptions() { - // expected-error@below {{unknown loop option: name}} - llvm.br ^bb4 {llvm.loop = {options = #llvm.loopopts}} - ^bb4: - llvm.return - } -} - -// ----- - -module { - llvm.func @loopOptions() { - // expected-error@below {{loop option present twice}} - llvm.br ^bb4 {llvm.loop = {options = #llvm.loopopts}} - ^bb4: - llvm.return - } -} - -// ----- - -module { - llvm.func @accessGroups(%arg0 : !llvm.ptr) { - // expected-error@below {{attribute 'access_groups' failed to satisfy constraint: symbol ref array attribute}} - %0 = llvm.load %arg0 { "access_groups" = "test" } : !llvm.ptr - llvm.return - } -} - -// ----- - module { llvm.func @accessGroups(%arg0 : !llvm.ptr) { // expected-error@below {{expected '@func1' to specify a fully qualified reference}} diff --git a/mlir/test/Dialect/LLVMIR/loop-metadata.mlir b/mlir/test/Dialect/LLVMIR/loop-metadata.mlir new file mode 100644 index 000000000000..edd9592168cb --- /dev/null +++ b/mlir/test/Dialect/LLVMIR/loop-metadata.mlir @@ -0,0 +1,70 @@ +// RUN: mlir-opt %s | mlir-opt | FileCheck %s + +// CHECK-DAG: #[[FOLLOWUP:.*]] = #llvm.loop_annotation +#followup = #llvm.loop_annotation + +// CHECK-DAG: #[[VECTORIZE:.*]] = #llvm.loop_vectorize +#vectorize = #llvm.loop_vectorize< + disable = false, predicateEnable = false, scalableEnable = true, width = 16 : i32, + followupVectorized = #followup, followupEpilogue = #followup, followupAll = #followup +> + +// CHECK-DAG: #[[INTERLEAVE:.*]] = #llvm.loop_interleave +#interleave = #llvm.loop_interleave + +// CHECK-DAG: #[[UNROLL:.*]] = #llvm.loop_unroll +#unroll = #llvm.loop_unroll< + disable = true, count = 32 : i32, runtimeDisable = true, full = false, + followup = #followup, followupRemainder = #followup +> + +// CHECK-DAG: #[[UNROLL_AND_JAM:.*]] = #llvm.loop_unroll_and_jam +#unrollAndJam = #llvm.loop_unroll_and_jam< + disable = false, count = 16 : i32, followupOuter = #followup, followupInner = #followup, + followupRemainderOuter = #followup, followupRemainderInner = #followup, followupAll = #followup +> + +// CHECK-DAG: #[[LICM:.*]] = #llvm.loop_licm +#licm = #llvm.loop_licm + +// CHECK-DAG: #[[DISTRIBUTE:.*]] = #llvm.loop_distribute +#distribute = #llvm.loop_distribute< + disable = true, followupCoincident = #followup, followupSequential = #followup, + followupFallback = #followup, followupAll = #followup +> + +// CHECK-DAG: #[[PIPELINE:.*]] = #llvm.loop_pipeline +#pipeline = #llvm.loop_pipeline + +// CHECK: #[[LOOP_ANNOT:.*]] = #llvm.loop_annotation< +// CHECK-DAG: disableNonforced = false +// CHECK-DAG: mustProgress = true +// CHECK-DAG: unroll = #[[UNROLL]] +// CHECK-DAG: unrollAndJam = #[[UNROLL_AND_JAM]] +// CHECK-DAG: licm = #[[LICM]] +// CHECK-DAG: distribute = #[[DISTRIBUTE]] +// CHECK-DAG: pipeline = #[[PIPELINE]] +// CHECK-DAG: parallelAccesses = @metadata::@group1, @metadata::@group2> +#loopMD = #llvm.loop_annotation + +// CHECK: llvm.func @loop_annotation +llvm.func @loop_annotation() { + // CHECK: llvm.br ^bb1 {llvm.loop = #[[LOOP_ANNOT]] + llvm.br ^bb1 {llvm.loop = #loopMD} +^bb1: + llvm.return +} + +llvm.metadata @metadata { + llvm.access_group @group1 + llvm.access_group @group2 +} diff --git a/mlir/test/Dialect/LLVMIR/roundtrip.mlir b/mlir/test/Dialect/LLVMIR/roundtrip.mlir index 3f2097ab7834..5627d2fee0ab 100644 --- a/mlir/test/Dialect/LLVMIR/roundtrip.mlir +++ b/mlir/test/Dialect/LLVMIR/roundtrip.mlir @@ -497,22 +497,6 @@ func.func @fastmathFlags(%arg0: f32, %arg1: f32, %arg2: i32, %arg3: vector<2 x f return } -module { - // CHECK-LABEL: @loopOptions - llvm.func @loopOptions() { - // CHECK: llvm.br - // CHECK-SAME: llvm.loop = {options = #llvm.loopopts}, parallel_access = [@metadata::@group1]} - llvm.br ^bb1 {llvm.loop = {options = #llvm.loopopts}, parallel_access = [@metadata::@group1]} - ^bb1: - llvm.return - } - // CHECK: llvm.metadata @metadata attributes {test_attribute} { - llvm.metadata @metadata attributes {test_attribute} { - // CHECK: llvm.access_group @group1 - llvm.access_group @group1 - } -} - // CHECK-LABEL: llvm.func @vararg_func llvm.func @vararg_func(%arg0: i32, ...) { // CHECK: %{{.*}} = llvm.mlir.constant(1 : i32) : i32 diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir index 589348043fd8..5a16430b2f20 100644 --- a/mlir/test/Target/LLVMIR/llvmir.mlir +++ b/mlir/test/Target/LLVMIR/llvmir.mlir @@ -1923,44 +1923,6 @@ llvm.func @switch_weights(%arg0: i32) -> i32 { // ----- -module { - llvm.func @loopOptions(%arg1 : i32, %arg2 : i32) { - %0 = llvm.mlir.constant(0 : i32) : i32 - %4 = llvm.alloca %arg1 x i32 : (i32) -> (!llvm.ptr) - llvm.br ^bb3(%0 : i32) - ^bb3(%1: i32): - %2 = llvm.icmp "slt" %1, %arg1 : i32 - // CHECK: br i1 {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] - llvm.cond_br %2, ^bb4, ^bb5 {llvm.loop = {parallel_access = [@metadata::@group1, @metadata::@group2], options = #llvm.loopopts}} - ^bb4: - %3 = llvm.add %1, %arg2 : i32 - // CHECK: = load i32, ptr %{{.*}} !llvm.access.group ![[ACCESS_GROUPS_NODE:[0-9]+]] - %5 = llvm.load %4 { access_groups = [@metadata::@group1, @metadata::@group2] } : !llvm.ptr - // CHECK: br label {{.*}} !llvm.loop ![[LOOP_NODE]] - llvm.br ^bb3(%3 : i32) {llvm.loop = {parallel_access = [@metadata::@group1, @metadata::@group2], options = #llvm.loopopts}} - ^bb5: - llvm.return - } - - llvm.metadata @metadata { - llvm.access_group @group1 - llvm.access_group @group2 - } -} - -// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], ![[PA_NODE:[0-9]+]], ![[UNROLL_DISABLE_NODE:[0-9]+]], ![[LICM_DISABLE_NODE:[0-9]+]], ![[INTERLEAVE_NODE:[0-9]+]], ![[PIPELINE_DISABLE_NODE:[0-9]+]], ![[II_NODE:[0-9]+]]} -// CHECK: ![[PA_NODE]] = !{!"llvm.loop.parallel_accesses", ![[GROUP_NODE1:[0-9]+]], ![[GROUP_NODE2:[0-9]+]]} -// CHECK: ![[GROUP_NODE1]] = distinct !{} -// CHECK: ![[GROUP_NODE2]] = distinct !{} -// CHECK: ![[UNROLL_DISABLE_NODE]] = !{!"llvm.loop.unroll.disable", i1 true} -// CHECK: ![[LICM_DISABLE_NODE]] = !{!"llvm.licm.disable", i1 true} -// CHECK: ![[INTERLEAVE_NODE]] = !{!"llvm.loop.interleave.count", i32 1} -// CHECK: ![[PIPELINE_DISABLE_NODE]] = !{!"llvm.loop.pipeline.disable", i1 true} -// CHECK: ![[II_NODE]] = !{!"llvm.loop.pipeline.initiationinterval", i32 2} -// CHECK: ![[ACCESS_GROUPS_NODE]] = !{![[GROUP_NODE1]], ![[GROUP_NODE2]]} - -// ----- - module { llvm.func @aliasScope(%arg1 : !llvm.ptr, %arg2 : !llvm.ptr, %arg3 : !llvm.ptr) { %0 = llvm.mlir.constant(0 : i32) : i32 diff --git a/mlir/test/Target/LLVMIR/loop-metadata.mlir b/mlir/test/Target/LLVMIR/loop-metadata.mlir new file mode 100644 index 000000000000..f2eaa1cffa33 --- /dev/null +++ b/mlir/test/Target/LLVMIR/loop-metadata.mlir @@ -0,0 +1,239 @@ +// RUN: mlir-translate -mlir-to-llvmir -split-input-file %s | FileCheck %s + +// CHECK-LABEL: @disableNonForced +llvm.func @disableNonForced() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.disable_nonforced"} + +// ----- + +// CHECK-LABEL: @mustprogress +llvm.func @mustprogress() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.mustprogress"} + +// ----- + + +#followup = #llvm.loop_annotation + +// CHECK-LABEL: @vectorizeOptions +llvm.func @vectorizeOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation + >} +^bb1: + llvm.return +} + +// CHECK-DAG: ![[NON_FORCED:[0-9]+]] = !{!"llvm.loop.disable_nonforced"} +// CHECK-DAG: ![[FOLLOWUP:[0-9]+]] = distinct !{![[FOLLOWUP]], ![[NON_FORCED]]} +// CHECK-DAG: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.enable", i1 true} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.predicate.enable", i1 true} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.scalable.enable", i1 false} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.width", i32 16} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.followup_vectorized", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.followup_epilogue", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.followup_all", ![[FOLLOWUP]]} + +// ----- + +// CHECK-LABEL: @interleaveOptions +llvm.func @interleaveOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation>} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], ![[INTERLEAVE_NODE:[0-9]+]]} +// CHECK: ![[INTERLEAVE_NODE]] = !{!"llvm.loop.interleave.count", i32 32} + +// ----- + +#followup = #llvm.loop_annotation + +// CHECK-LABEL: @unrollOptions +llvm.func @unrollOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation + >} +^bb1: + llvm.return +} + +// CHECK-DAG: ![[NON_FORCED:[0-9]+]] = !{!"llvm.loop.disable_nonforced"} +// CHECK-DAG: ![[FOLLOWUP:[0-9]+]] = distinct !{![[FOLLOWUP]], ![[NON_FORCED]]} +// CHECK-DAG: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.disable"} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.count", i32 64} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.runtime.disable", i1 false} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.followup", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.followup_remainder", ![[FOLLOWUP]]} + +// ----- + +// CHECK-LABEL: @unrollOptions2 +llvm.func @unrollOptions2() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation>} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.unroll.enable"} +// CHECK-DAG: ![[VEC_NODE2:[0-9]+]] = !{!"llvm.loop.unroll.full"} + +// ----- + +#followup = #llvm.loop_annotation + +// CHECK-LABEL: @unrollAndJamOptions +llvm.func @unrollAndJamOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation + >} +^bb1: + llvm.return +} + +// CHECK-DAG: ![[NON_FORCED:[0-9]+]] = !{!"llvm.loop.disable_nonforced"} +// CHECK-DAG: ![[FOLLOWUP:[0-9]+]] = distinct !{![[FOLLOWUP]], ![[NON_FORCED]]} +// CHECK-DAG: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.enable"} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.count", i32 8} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.followup_outer", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.followup_inner", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.followup_remainder_outer", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.followup_remainder_inner", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.followup_all", ![[FOLLOWUP]]} + +// ----- + +// CHECK-LABEL: @licmOptions +llvm.func @licmOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation>} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.licm_versioning.disable"} + +// ----- + +// CHECK-LABEL: @licmOptions2 +llvm.func @licmOptions2() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation>} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.licm.disable"} + +// ----- + +#followup = #llvm.loop_annotation + +// CHECK-LABEL: @distributeOptions +llvm.func @distributeOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation + >} +^bb1: + llvm.return +} + +// CHECK-DAG: ![[NON_FORCED:[0-9]+]] = !{!"llvm.loop.disable_nonforced"} +// CHECK-DAG: ![[FOLLOWUP:[0-9]+]] = distinct !{![[FOLLOWUP]], ![[NON_FORCED]]} +// CHECK-DAG: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.distribute.enable", i1 false} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.distribute.followup_coincident", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.distribute.followup_sequential", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.distribute.followup_fallback", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.distribute.followup_all", ![[FOLLOWUP]]} + +// ----- + +// CHECK-LABEL: @pipelineOptions +llvm.func @pipelineOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation>} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.pipeline.disable", i1 false} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.pipeline.initiationinterval", i32 1} + +// ----- + +// CHECK-LABEL: @loopOptions +llvm.func @loopOptions(%arg1 : i32, %arg2 : i32) { + %0 = llvm.mlir.constant(0 : i32) : i32 + %4 = llvm.alloca %arg1 x i32 : (i32) -> (!llvm.ptr) + llvm.br ^bb3(%0 : i32) + ^bb3(%1: i32): + %2 = llvm.icmp "slt" %1, %arg1 : i32 + // CHECK: br i1 {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.cond_br %2, ^bb4, ^bb5 {llvm.loop = #llvm.loop_annotation< + licm = , + interleave = , + unroll = , pipeline = , + parallelAccesses = @metadata::@group1, @metadata::@group2>} + ^bb4: + %3 = llvm.add %1, %arg2 : i32 + // CHECK: = load i32, ptr %{{.*}} !llvm.access.group ![[ACCESS_GROUPS_NODE:[0-9]+]] + %5 = llvm.load %4 { access_groups = [@metadata::@group1, @metadata::@group2] } : !llvm.ptr + // CHECK: br label {{.*}} !llvm.loop ![[LOOP_NODE]] + llvm.br ^bb3(%3 : i32) {llvm.loop = #llvm.loop_annotation< + licm = , + interleave = , + unroll = , pipeline = , + parallelAccesses = @metadata::@group1, @metadata::@group2>} + + ^bb5: + llvm.return +} + +llvm.metadata @metadata { + llvm.access_group @group1 + llvm.access_group @group2 +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: ![[PA_NODE:[0-9]+]] = !{!"llvm.loop.parallel_accesses", ![[GROUP_NODE1:[0-9]+]], ![[GROUP_NODE2:[0-9]+]]} +// CHECK-DAG: ![[GROUP_NODE1:[0-9]+]] = distinct !{} +// CHECK-DAG: ![[GROUP_NODE2:[0-9]+]] = distinct !{} +// CHECK-DAG: ![[UNROLL_DISABLE_NODE:[0-9]+]] = !{!"llvm.loop.unroll.disable"} +// CHECK-DAG: ![[LICM_DISABLE_NODE:[0-9]+]] = !{!"llvm.licm.disable"} +// CHECK-DAG: ![[INTERLEAVE_NODE:[0-9]+]] = !{!"llvm.loop.interleave.count", i32 1} +// CHECK-DAG: ![[PIPELINE_DISABLE_NODE:[0-9]+]] = !{!"llvm.loop.pipeline.disable", i1 true} +// CHECK-DAG: ![[II_NODE:[0-9]+]] = !{!"llvm.loop.pipeline.initiationinterval", i32 2} +// CHECK-DAG: ![[ACCESS_GROUPS_NODE:[0-9]+]] = !{![[GROUP_NODE1]], ![[GROUP_NODE2]]} From 927765d825149c7b344dd4126234254cdf27dd02 Mon Sep 17 00:00:00 2001 From: Tobias Gysi Date: Mon, 6 Mar 2023 10:24:12 +0100 Subject: [PATCH 060/147] [mlir][llvm] Use tablegen for enum conversion. --- mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td | 166 +++++++++++------- .../include/mlir/Dialect/LLVMIR/LLVMOpBase.td | 14 +- mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td | 33 ++-- mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 3 +- .../LLVMIR/LLVMToLLVMIRTranslation.cpp | 117 ------------ mlir/lib/Target/LLVMIR/ModuleImport.cpp | 121 ------------- .../test/Target/LLVMIR/Import/instructions.ll | 8 + mlir/test/Target/LLVMIR/llvmir.mlir | 30 ++-- .../tools/mlir-tblgen/LLVMIRConversionGen.cpp | 18 ++ 9 files changed, 179 insertions(+), 331 deletions(-) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td index c7ba00e6ec89..7fd27b182005 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td @@ -33,42 +33,64 @@ def AsmATTOrIntel : LLVM_EnumAttr< // Atomic Operations //===----------------------------------------------------------------------===// -def AtomicBinOpXchg : I64EnumAttrCase<"xchg", 0>; -def AtomicBinOpAdd : I64EnumAttrCase<"add", 1>; -def AtomicBinOpSub : I64EnumAttrCase<"sub", 2>; -def AtomicBinOpAnd : I64EnumAttrCase<"_and", 3>; -def AtomicBinOpNand : I64EnumAttrCase<"nand", 4>; -def AtomicBinOpOr : I64EnumAttrCase<"_or", 5>; -def AtomicBinOpXor : I64EnumAttrCase<"_xor", 6>; -def AtomicBinOpMax : I64EnumAttrCase<"max", 7>; -def AtomicBinOpMin : I64EnumAttrCase<"min", 8>; -def AtomicBinOpUMax : I64EnumAttrCase<"umax", 9>; -def AtomicBinOpUMin : I64EnumAttrCase<"umin", 10>; -def AtomicBinOpFAdd : I64EnumAttrCase<"fadd", 11>; -def AtomicBinOpFSub : I64EnumAttrCase<"fsub", 12>; -def AtomicBinOp : I64EnumAttr< +def AtomicBinOpXchg : LLVM_EnumAttrCase<"xchg", "xchg", "Xchg", 0>; +def AtomicBinOpAdd : LLVM_EnumAttrCase<"add", "add", "Add", 1>; +def AtomicBinOpSub : LLVM_EnumAttrCase<"sub", "sub", "Sub", 2>; +def AtomicBinOpAnd : LLVM_EnumAttrCase<"_and", "_and", "And", 3>; +def AtomicBinOpNand : LLVM_EnumAttrCase<"nand", "nand", "Nand", 4>; +def AtomicBinOpOr : LLVM_EnumAttrCase<"_or", "_or", "Or", 5>; +def AtomicBinOpXor : LLVM_EnumAttrCase<"_xor", "_xor", "Xor", 6>; +def AtomicBinOpMax : LLVM_EnumAttrCase<"max", "max", "Max", 7>; +def AtomicBinOpMin : LLVM_EnumAttrCase<"min", "min", "Min", 8>; +def AtomicBinOpUMax : LLVM_EnumAttrCase<"umax", "umax", "UMax", 9>; +def AtomicBinOpUMin : LLVM_EnumAttrCase<"umin", "umin", "UMin", 10>; +def AtomicBinOpFAdd : LLVM_EnumAttrCase<"fadd", "fadd", "FAdd", 11>; +def AtomicBinOpFSub : LLVM_EnumAttrCase<"fsub", "fsub", "FSub", 12>; +def AtomicBinOpFMax : LLVM_EnumAttrCase<"fmax", "fmax", "FMax", 13>; +def AtomicBinOpFMin : LLVM_EnumAttrCase<"fmin", "fmin", "FMin", 14>; +def AtomicBinOpUIncWrap : LLVM_EnumAttrCase<"uinc_wrap", + "uinc_wrap", "UIncWrap", 15>; +def AtomicBinOpUDecWrap : LLVM_EnumAttrCase<"udec_wrap", + "udec_wrap", "UDecWrap", 16>; + +// A sentinel value that has no MLIR counterpart. +def AtomicBadBinOp : LLVM_EnumAttrCase<"", "", "BAD_BINOP", 0>; + +def AtomicBinOp : LLVM_EnumAttr< "AtomicBinOp", + "::llvm::AtomicRMWInst::BinOp", "llvm.atomicrmw binary operations", [AtomicBinOpXchg, AtomicBinOpAdd, AtomicBinOpSub, AtomicBinOpAnd, AtomicBinOpNand, AtomicBinOpOr, AtomicBinOpXor, AtomicBinOpMax, AtomicBinOpMin, AtomicBinOpUMax, AtomicBinOpUMin, AtomicBinOpFAdd, - AtomicBinOpFSub]> { + AtomicBinOpFSub, AtomicBinOpFMax, AtomicBinOpFMin, AtomicBinOpUIncWrap, + AtomicBinOpUDecWrap], + [AtomicBadBinOp]> { let cppNamespace = "::mlir::LLVM"; } -def AtomicOrderingNotAtomic : I64EnumAttrCase<"not_atomic", 0>; -def AtomicOrderingUnordered : I64EnumAttrCase<"unordered", 1>; -def AtomicOrderingMonotonic : I64EnumAttrCase<"monotonic", 2>; -def AtomicOrderingAcquire : I64EnumAttrCase<"acquire", 4>; -def AtomicOrderingRelease : I64EnumAttrCase<"release", 5>; -def AtomicOrderingAcquireRelease : I64EnumAttrCase<"acq_rel", 6>; -def AtomicOrderingSequentiallyConsistent : I64EnumAttrCase<"seq_cst", 7>; -def AtomicOrdering : I64EnumAttr< +def AtomicOrderingNotAtomic : LLVM_EnumAttrCase<"not_atomic", + "not_atomic", "NotAtomic", 0>; +def AtomicOrderingUnordered : LLVM_EnumAttrCase<"unordered", + "unordered", "Unordered", 1>; +def AtomicOrderingMonotonic : LLVM_EnumAttrCase<"monotonic", + "monotonic", "Monotonic", 2>; +def AtomicOrderingAcquire : LLVM_EnumAttrCase<"acquire", + "acquire", "Acquire", 4>; +def AtomicOrderingRelease : LLVM_EnumAttrCase<"release", + "release", "Release", 5>; +def AtomicOrderingAcquireRelease : + LLVM_EnumAttrCase<"acq_rel", "acq_rel", "AcquireRelease", 6>; +def AtomicOrderingSequentiallyConsistent : + LLVM_EnumAttrCase<"seq_cst", "seq_cst", "SequentiallyConsistent", 7>; +def AtomicOrdering : LLVM_EnumAttr< "AtomicOrdering", + "::llvm::AtomicOrdering", "Atomic ordering for LLVM's memory model", [AtomicOrderingNotAtomic, AtomicOrderingUnordered, AtomicOrderingMonotonic, AtomicOrderingAcquire, AtomicOrderingRelease, AtomicOrderingAcquireRelease, - AtomicOrderingSequentiallyConsistent]> { + AtomicOrderingSequentiallyConsistent + ]> { let cppNamespace = "::mlir::LLVM"; } @@ -380,59 +402,77 @@ def FastmathFlags : I32BitEnumAttr< } //===----------------------------------------------------------------------===// -// FCmp Predicates +// FCmp and ICmp Predicates //===----------------------------------------------------------------------===// -// Predicate for float comparisons -def FCmpPredicateFALSE : I64EnumAttrCase<"_false", 0>; -def FCmpPredicateOEQ : I64EnumAttrCase<"oeq", 1>; -def FCmpPredicateOGT : I64EnumAttrCase<"ogt", 2>; -def FCmpPredicateOGE : I64EnumAttrCase<"oge", 3>; -def FCmpPredicateOLT : I64EnumAttrCase<"olt", 4>; -def FCmpPredicateOLE : I64EnumAttrCase<"ole", 5>; -def FCmpPredicateONE : I64EnumAttrCase<"one", 6>; -def FCmpPredicateORD : I64EnumAttrCase<"ord", 7>; -def FCmpPredicateUEQ : I64EnumAttrCase<"ueq", 8>; -def FCmpPredicateUGT : I64EnumAttrCase<"ugt", 9>; -def FCmpPredicateUGE : I64EnumAttrCase<"uge", 10>; -def FCmpPredicateULT : I64EnumAttrCase<"ult", 11>; -def FCmpPredicateULE : I64EnumAttrCase<"ule", 12>; -def FCmpPredicateUNE : I64EnumAttrCase<"une", 13>; -def FCmpPredicateUNO : I64EnumAttrCase<"uno", 14>; -def FCmpPredicateTRUE : I64EnumAttrCase<"_true", 15>; - -def FCmpPredicate : I64EnumAttr< +// Predicates for float comparisons +def FCmpPredicateFALSE : LLVM_EnumAttrCase<"_false", "_false", "FCMP_FALSE", 0>; +def FCmpPredicateOEQ : LLVM_EnumAttrCase<"oeq", "oeq", "FCMP_OEQ", 1>; +def FCmpPredicateOGT : LLVM_EnumAttrCase<"ogt", "ogt", "FCMP_OGT", 2>; +def FCmpPredicateOGE : LLVM_EnumAttrCase<"oge", "oge", "FCMP_OGE", 3>; +def FCmpPredicateOLT : LLVM_EnumAttrCase<"olt", "olt", "FCMP_OLT", 4>; +def FCmpPredicateOLE : LLVM_EnumAttrCase<"ole", "ole", "FCMP_OLE", 5>; +def FCmpPredicateONE : LLVM_EnumAttrCase<"one", "one", "FCMP_ONE", 6>; +def FCmpPredicateORD : LLVM_EnumAttrCase<"ord", "ord", "FCMP_ORD", 7>; +def FCmpPredicateUEQ : LLVM_EnumAttrCase<"ueq", "ueq", "FCMP_UEQ", 8>; +def FCmpPredicateUGT : LLVM_EnumAttrCase<"ugt", "ugt", "FCMP_UGT", 9>; +def FCmpPredicateUGE : LLVM_EnumAttrCase<"uge", "uge", "FCMP_UGE", 10>; +def FCmpPredicateULT : LLVM_EnumAttrCase<"ult", "ult", "FCMP_ULT", 11>; +def FCmpPredicateULE : LLVM_EnumAttrCase<"ule", "ule", "FCMP_ULE", 12>; +def FCmpPredicateUNE : LLVM_EnumAttrCase<"une", "une", "FCMP_UNE", 13>; +def FCmpPredicateUNO : LLVM_EnumAttrCase<"uno", "uno", "FCMP_UNO", 14>; +def FCmpPredicateTRUE : LLVM_EnumAttrCase<"_true", "_true", "FCMP_TRUE", 15>; + +// A sentinel value that has no MLIR counterpart. +def ICmpPredicateBad : LLVM_EnumAttrCase<"", "", "BAD_ICMP_PREDICATE", 0>; + +// Predicates for integer comparisons. +def ICmpPredicateEQ : LLVM_EnumAttrCase<"eq", "eq", "ICMP_EQ", 0>; +def ICmpPredicateNE : LLVM_EnumAttrCase<"ne", "ne", "ICMP_NE", 1>; +def ICmpPredicateSLT : LLVM_EnumAttrCase<"slt", "slt", "ICMP_SLT", 2>; +def ICmpPredicateSLE : LLVM_EnumAttrCase<"sle", "sle", "ICMP_SLE", 3>; +def ICmpPredicateSGT : LLVM_EnumAttrCase<"sgt", "sgt", "ICMP_SGT", 4>; +def ICmpPredicateSGE : LLVM_EnumAttrCase<"sge", "sge", "ICMP_SGE", 5>; +def ICmpPredicateULT : LLVM_EnumAttrCase<"ult", "ult", "ICMP_ULT", 6>; +def ICmpPredicateULE : LLVM_EnumAttrCase<"ule", "ule", "ICMP_ULE", 7>; +def ICmpPredicateUGT : LLVM_EnumAttrCase<"ugt", "ugt", "ICMP_UGT", 8>; +def ICmpPredicateUGE : LLVM_EnumAttrCase<"uge", "uge", "ICMP_UGE", 9>; + +// A sentinel value that has no MLIR counterpart. +def FCmpPredicateBad : LLVM_EnumAttrCase<"", "", "BAD_FCMP_PREDICATE", 0>; + +// LLVM's predicate enum contains the floating-point and integer comparison +// cases, while the LLVM dialect uses two separate enums. The floating-point +// predicate enum thus defines all integer predicates as unsupported and +// vice versa. +def FCmpPredicate : LLVM_EnumAttr< "FCmpPredicate", + "::llvm::CmpInst::Predicate", "llvm.fcmp comparison predicate", [FCmpPredicateFALSE, FCmpPredicateOEQ, FCmpPredicateOGT, FCmpPredicateOGE, FCmpPredicateOLT, FCmpPredicateOLE, FCmpPredicateONE, FCmpPredicateORD, FCmpPredicateUEQ, FCmpPredicateUGT, FCmpPredicateUGE, FCmpPredicateULT, - FCmpPredicateULE, FCmpPredicateUNE, FCmpPredicateUNO, FCmpPredicateTRUE + FCmpPredicateULE, FCmpPredicateUNE, FCmpPredicateUNO, FCmpPredicateTRUE], + [ICmpPredicateEQ, ICmpPredicateNE, ICmpPredicateSLT, ICmpPredicateSLE, + ICmpPredicateSGT, ICmpPredicateSGE, ICmpPredicateULT, ICmpPredicateULE, + ICmpPredicateUGT, ICmpPredicateUGE, FCmpPredicateBad, ICmpPredicateBad ]> { let cppNamespace = "::mlir::LLVM"; } -//===----------------------------------------------------------------------===// -// ICmp Predicates -//===----------------------------------------------------------------------===// - -// Predicate for integer comparisons. -def ICmpPredicateEQ : I64EnumAttrCase<"eq", 0>; -def ICmpPredicateNE : I64EnumAttrCase<"ne", 1>; -def ICmpPredicateSLT : I64EnumAttrCase<"slt", 2>; -def ICmpPredicateSLE : I64EnumAttrCase<"sle", 3>; -def ICmpPredicateSGT : I64EnumAttrCase<"sgt", 4>; -def ICmpPredicateSGE : I64EnumAttrCase<"sge", 5>; -def ICmpPredicateULT : I64EnumAttrCase<"ult", 6>; -def ICmpPredicateULE : I64EnumAttrCase<"ule", 7>; -def ICmpPredicateUGT : I64EnumAttrCase<"ugt", 8>; -def ICmpPredicateUGE : I64EnumAttrCase<"uge", 9>; -def ICmpPredicate : I64EnumAttr< +def ICmpPredicate : LLVM_EnumAttr< "ICmpPredicate", - "llvm.icmp comparison predicate", + "::llvm::CmpInst::Predicate", + "lvm.icmp comparison predicate", [ICmpPredicateEQ, ICmpPredicateNE, ICmpPredicateSLT, ICmpPredicateSLE, ICmpPredicateSGT, ICmpPredicateSGE, ICmpPredicateULT, ICmpPredicateULE, - ICmpPredicateUGT, ICmpPredicateUGE]> { + ICmpPredicateUGT, ICmpPredicateUGE], + [FCmpPredicateFALSE, FCmpPredicateOEQ, FCmpPredicateOGT, FCmpPredicateOGE, + FCmpPredicateOLT, FCmpPredicateOLE, FCmpPredicateONE, FCmpPredicateORD, + FCmpPredicateUEQ, FCmpPredicateUGT, FCmpPredicateUGE, FCmpPredicateULT, + FCmpPredicateULE, FCmpPredicateUNE, FCmpPredicateUNO, FCmpPredicateTRUE, + FCmpPredicateBad, ICmpPredicateBad + ]> { let cppNamespace = "::mlir::LLVM"; } diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td index 1753c6d8cde4..b51f974cdc9c 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td @@ -310,14 +310,24 @@ class LLVM_EnumAttrCase : // - `name`: name of the C++ enum class in MLIR API; // - `llvmName`: name of the C++ enum in LLVM API; // - `description`: textual description for documentation purposes; -// - `cases`: list of enum cases. +// - `cases`: list of enum cases; +// - `unsupportedCases`: optional list of unsupported enum cases. // For example, `LLVM_EnumAttr cases> : + list cases, + list unsupportedCases = []> : I64EnumAttr { + + // List of unsupported cases that have no conversion to an MLIR value. + list unsupported = unsupportedCases; + // The equivalent enum class name in LLVM. string llvmClassName = llvmName; } diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td index f318173549aa..5ff36907f1c6 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td @@ -122,12 +122,13 @@ def LLVM_ICmpOp : LLVM_ArithmeticCmpOp<"icmp", [Pure]> { let hasCustomAssemblyFormat = 1; string llvmInstName = "ICmp"; string llvmBuilder = [{ - $res = builder.CreateICmp(getLLVMCmpPredicate($predicate), $lhs, $rhs); + $res = builder.CreateICmp( + convertICmpPredicateToLLVM($predicate), $lhs, $rhs); }]; string mlirBuilder = [{ auto *iCmpInst = cast(inst); - $res = $_builder.create<$_qualCppClassName>( - $_location, getICmpPredicate(iCmpInst->getPredicate()), $lhs, $rhs); + $res = $_builder.create<$_qualCppClassName>($_location, + convertICmpPredicateFromLLVM(iCmpInst->getPredicate()), $lhs, $rhs); }]; // Set the $predicate index to -1 to indicate there is no matching operand // and decrement the following indices. @@ -145,12 +146,12 @@ def LLVM_FCmpOp : LLVM_ArithmeticCmpOp<"fcmp", [ let hasCustomAssemblyFormat = 1; string llvmInstName = "FCmp"; string llvmBuilder = [{ - $res = builder.CreateFCmp(getLLVMCmpPredicate($predicate), $lhs, $rhs); + $res = builder.CreateFCmp(convertFCmpPredicateToLLVM($predicate), $lhs, $rhs); }]; string mlirBuilder = [{ auto *fCmpInst = cast(inst); auto op = $_builder.create<$_qualCppClassName>( - $_location, getFCmpPredicate(fCmpInst->getPredicate()), $lhs, $rhs); + $_location, convertFCmpPredicateFromLLVM(fCmpInst->getPredicate()), $lhs, $rhs); moduleImport.setFastmathFlagsAttr(inst, op); $res = op; }]; @@ -1670,15 +1671,15 @@ def LLVM_AtomicRMWOp : LLVM_Op<"atomicrmw", [ }]; string llvmInstName = "AtomicRMW"; string llvmBuilder = [{ - $res = builder.CreateAtomicRMW(getLLVMAtomicBinOp($bin_op), $ptr, $val, - llvm::MaybeAlign(), - getLLVMAtomicOrdering($ordering)); + $res = builder.CreateAtomicRMW( + convertAtomicBinOpToLLVM($bin_op), $ptr, $val, llvm::MaybeAlign(), + convertAtomicOrderingToLLVM($ordering)); }]; string mlirBuilder = [{ auto *atomicInst = cast(inst); $res = $_builder.create($_location, $_resultType, - getLLVMAtomicBinOp(atomicInst->getOperation()), $ptr, $val, - getLLVMAtomicOrdering(atomicInst->getOrdering())); + convertAtomicBinOpFromLLVM(atomicInst->getOperation()), $ptr, $val, + convertAtomicOrderingFromLLVM(atomicInst->getOrdering())); }]; // Only $ptr and $val are llvm instruction operands. list llvmArgIndices = [-1, 0, 1, -1]; @@ -1706,15 +1707,15 @@ def LLVM_AtomicCmpXchgOp : LLVM_Op<"cmpxchg", [ string llvmInstName = "AtomicCmpXchg"; string llvmBuilder = [{ $res = builder.CreateAtomicCmpXchg($ptr, $cmp, $val, llvm::MaybeAlign(), - getLLVMAtomicOrdering($success_ordering), - getLLVMAtomicOrdering($failure_ordering)); + convertAtomicOrderingToLLVM($success_ordering), + convertAtomicOrderingToLLVM($failure_ordering)); }]; string mlirBuilder = [{ auto *cmpXchgInst = cast(inst); $res = $_builder.create( $_location, $_resultType, $ptr, $cmp, $val, - getLLVMAtomicOrdering(cmpXchgInst->getSuccessOrdering()), - getLLVMAtomicOrdering(cmpXchgInst->getFailureOrdering())); + convertAtomicOrderingFromLLVM(cmpXchgInst->getSuccessOrdering()), + convertAtomicOrderingFromLLVM(cmpXchgInst->getFailureOrdering())); }]; } @@ -1724,14 +1725,14 @@ def LLVM_FenceOp : LLVM_Op<"fence"> { string llvmInstName = "Fence"; let llvmBuilder = [{ llvm::LLVMContext &llvmContext = builder.getContext(); - builder.CreateFence(getLLVMAtomicOrdering($ordering), + builder.CreateFence(convertAtomicOrderingToLLVM($ordering), llvmContext.getOrInsertSyncScopeID($syncscope)); }]; string mlirBuilder = [{ llvm::FenceInst *fenceInst = cast(inst); $_op = $_builder.create( $_location, - getLLVMAtomicOrdering(fenceInst->getOrdering()), + convertAtomicOrderingFromLLVM(fenceInst->getOrdering()), getLLVMSyncScope(fenceInst)); }]; let hasCustomAssemblyFormat = 1; diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp index adba1730d6c2..9c271833f890 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -2267,7 +2267,8 @@ LogicalResult AtomicRMWOp::verify() { if (!ptrType.isOpaque() && valType != ptrType.getElementType()) return emitOpError("expected LLVM IR element type for operand #0 to " "match type for operand #1"); - if (getBinOp() == AtomicBinOp::fadd || getBinOp() == AtomicBinOp::fsub) { + if (getBinOp() == AtomicBinOp::fadd || getBinOp() == AtomicBinOp::fsub || + getBinOp() == AtomicBinOp::fmin || getBinOp() == AtomicBinOp::fmax) { if (!mlir::LLVM::isCompatibleFloatingPointType(valType)) return emitOpError("expected LLVM IR floating point type"); } else if (getBinOp() == AtomicBinOp::xchg) { diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp index af50ae134737..5c98f922b22e 100644 --- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp @@ -28,123 +28,6 @@ using mlir::LLVM::detail::getLLVMConstant; #include "mlir/Dialect/LLVMIR/LLVMConversionEnumsToLLVM.inc" -/// Convert MLIR integer comparison predicate to LLVM IR comparison predicate. -static llvm::CmpInst::Predicate getLLVMCmpPredicate(ICmpPredicate p) { - switch (p) { - case LLVM::ICmpPredicate::eq: - return llvm::CmpInst::Predicate::ICMP_EQ; - case LLVM::ICmpPredicate::ne: - return llvm::CmpInst::Predicate::ICMP_NE; - case LLVM::ICmpPredicate::slt: - return llvm::CmpInst::Predicate::ICMP_SLT; - case LLVM::ICmpPredicate::sle: - return llvm::CmpInst::Predicate::ICMP_SLE; - case LLVM::ICmpPredicate::sgt: - return llvm::CmpInst::Predicate::ICMP_SGT; - case LLVM::ICmpPredicate::sge: - return llvm::CmpInst::Predicate::ICMP_SGE; - case LLVM::ICmpPredicate::ult: - return llvm::CmpInst::Predicate::ICMP_ULT; - case LLVM::ICmpPredicate::ule: - return llvm::CmpInst::Predicate::ICMP_ULE; - case LLVM::ICmpPredicate::ugt: - return llvm::CmpInst::Predicate::ICMP_UGT; - case LLVM::ICmpPredicate::uge: - return llvm::CmpInst::Predicate::ICMP_UGE; - } - llvm_unreachable("incorrect comparison predicate"); -} - -static llvm::CmpInst::Predicate getLLVMCmpPredicate(FCmpPredicate p) { - switch (p) { - case LLVM::FCmpPredicate::_false: - return llvm::CmpInst::Predicate::FCMP_FALSE; - case LLVM::FCmpPredicate::oeq: - return llvm::CmpInst::Predicate::FCMP_OEQ; - case LLVM::FCmpPredicate::ogt: - return llvm::CmpInst::Predicate::FCMP_OGT; - case LLVM::FCmpPredicate::oge: - return llvm::CmpInst::Predicate::FCMP_OGE; - case LLVM::FCmpPredicate::olt: - return llvm::CmpInst::Predicate::FCMP_OLT; - case LLVM::FCmpPredicate::ole: - return llvm::CmpInst::Predicate::FCMP_OLE; - case LLVM::FCmpPredicate::one: - return llvm::CmpInst::Predicate::FCMP_ONE; - case LLVM::FCmpPredicate::ord: - return llvm::CmpInst::Predicate::FCMP_ORD; - case LLVM::FCmpPredicate::ueq: - return llvm::CmpInst::Predicate::FCMP_UEQ; - case LLVM::FCmpPredicate::ugt: - return llvm::CmpInst::Predicate::FCMP_UGT; - case LLVM::FCmpPredicate::uge: - return llvm::CmpInst::Predicate::FCMP_UGE; - case LLVM::FCmpPredicate::ult: - return llvm::CmpInst::Predicate::FCMP_ULT; - case LLVM::FCmpPredicate::ule: - return llvm::CmpInst::Predicate::FCMP_ULE; - case LLVM::FCmpPredicate::une: - return llvm::CmpInst::Predicate::FCMP_UNE; - case LLVM::FCmpPredicate::uno: - return llvm::CmpInst::Predicate::FCMP_UNO; - case LLVM::FCmpPredicate::_true: - return llvm::CmpInst::Predicate::FCMP_TRUE; - } - llvm_unreachable("incorrect comparison predicate"); -} - -static llvm::AtomicRMWInst::BinOp getLLVMAtomicBinOp(AtomicBinOp op) { - switch (op) { - case LLVM::AtomicBinOp::xchg: - return llvm::AtomicRMWInst::BinOp::Xchg; - case LLVM::AtomicBinOp::add: - return llvm::AtomicRMWInst::BinOp::Add; - case LLVM::AtomicBinOp::sub: - return llvm::AtomicRMWInst::BinOp::Sub; - case LLVM::AtomicBinOp::_and: - return llvm::AtomicRMWInst::BinOp::And; - case LLVM::AtomicBinOp::nand: - return llvm::AtomicRMWInst::BinOp::Nand; - case LLVM::AtomicBinOp::_or: - return llvm::AtomicRMWInst::BinOp::Or; - case LLVM::AtomicBinOp::_xor: - return llvm::AtomicRMWInst::BinOp::Xor; - case LLVM::AtomicBinOp::max: - return llvm::AtomicRMWInst::BinOp::Max; - case LLVM::AtomicBinOp::min: - return llvm::AtomicRMWInst::BinOp::Min; - case LLVM::AtomicBinOp::umax: - return llvm::AtomicRMWInst::BinOp::UMax; - case LLVM::AtomicBinOp::umin: - return llvm::AtomicRMWInst::BinOp::UMin; - case LLVM::AtomicBinOp::fadd: - return llvm::AtomicRMWInst::BinOp::FAdd; - case LLVM::AtomicBinOp::fsub: - return llvm::AtomicRMWInst::BinOp::FSub; - } - llvm_unreachable("incorrect atomic binary operator"); -} - -static llvm::AtomicOrdering getLLVMAtomicOrdering(AtomicOrdering ordering) { - switch (ordering) { - case LLVM::AtomicOrdering::not_atomic: - return llvm::AtomicOrdering::NotAtomic; - case LLVM::AtomicOrdering::unordered: - return llvm::AtomicOrdering::Unordered; - case LLVM::AtomicOrdering::monotonic: - return llvm::AtomicOrdering::Monotonic; - case LLVM::AtomicOrdering::acquire: - return llvm::AtomicOrdering::Acquire; - case LLVM::AtomicOrdering::release: - return llvm::AtomicOrdering::Release; - case LLVM::AtomicOrdering::acq_rel: - return llvm::AtomicOrdering::AcquireRelease; - case LLVM::AtomicOrdering::seq_cst: - return llvm::AtomicOrdering::SequentiallyConsistent; - } - llvm_unreachable("incorrect atomic ordering"); -} - static llvm::FastMathFlags getFastmathFlags(FastmathFlagsInterface &op) { using llvmFMF = llvm::FastMathFlags; using FuncT = void (llvmFMF::*)(bool); diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp index 697ccb80971b..befb077cc82e 100644 --- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp @@ -97,127 +97,6 @@ static FloatType getDLFloatType(MLIRContext &ctx, int32_t bitwidth) { } } -static ICmpPredicate getICmpPredicate(llvm::CmpInst::Predicate pred) { - switch (pred) { - default: - llvm_unreachable("incorrect comparison predicate"); - case llvm::CmpInst::Predicate::ICMP_EQ: - return LLVM::ICmpPredicate::eq; - case llvm::CmpInst::Predicate::ICMP_NE: - return LLVM::ICmpPredicate::ne; - case llvm::CmpInst::Predicate::ICMP_SLT: - return LLVM::ICmpPredicate::slt; - case llvm::CmpInst::Predicate::ICMP_SLE: - return LLVM::ICmpPredicate::sle; - case llvm::CmpInst::Predicate::ICMP_SGT: - return LLVM::ICmpPredicate::sgt; - case llvm::CmpInst::Predicate::ICMP_SGE: - return LLVM::ICmpPredicate::sge; - case llvm::CmpInst::Predicate::ICMP_ULT: - return LLVM::ICmpPredicate::ult; - case llvm::CmpInst::Predicate::ICMP_ULE: - return LLVM::ICmpPredicate::ule; - case llvm::CmpInst::Predicate::ICMP_UGT: - return LLVM::ICmpPredicate::ugt; - case llvm::CmpInst::Predicate::ICMP_UGE: - return LLVM::ICmpPredicate::uge; - } - llvm_unreachable("incorrect integer comparison predicate"); -} - -static FCmpPredicate getFCmpPredicate(llvm::CmpInst::Predicate pred) { - switch (pred) { - default: - llvm_unreachable("incorrect comparison predicate"); - case llvm::CmpInst::Predicate::FCMP_FALSE: - return LLVM::FCmpPredicate::_false; - case llvm::CmpInst::Predicate::FCMP_TRUE: - return LLVM::FCmpPredicate::_true; - case llvm::CmpInst::Predicate::FCMP_OEQ: - return LLVM::FCmpPredicate::oeq; - case llvm::CmpInst::Predicate::FCMP_ONE: - return LLVM::FCmpPredicate::one; - case llvm::CmpInst::Predicate::FCMP_OLT: - return LLVM::FCmpPredicate::olt; - case llvm::CmpInst::Predicate::FCMP_OLE: - return LLVM::FCmpPredicate::ole; - case llvm::CmpInst::Predicate::FCMP_OGT: - return LLVM::FCmpPredicate::ogt; - case llvm::CmpInst::Predicate::FCMP_OGE: - return LLVM::FCmpPredicate::oge; - case llvm::CmpInst::Predicate::FCMP_ORD: - return LLVM::FCmpPredicate::ord; - case llvm::CmpInst::Predicate::FCMP_ULT: - return LLVM::FCmpPredicate::ult; - case llvm::CmpInst::Predicate::FCMP_ULE: - return LLVM::FCmpPredicate::ule; - case llvm::CmpInst::Predicate::FCMP_UGT: - return LLVM::FCmpPredicate::ugt; - case llvm::CmpInst::Predicate::FCMP_UGE: - return LLVM::FCmpPredicate::uge; - case llvm::CmpInst::Predicate::FCMP_UNO: - return LLVM::FCmpPredicate::uno; - case llvm::CmpInst::Predicate::FCMP_UEQ: - return LLVM::FCmpPredicate::ueq; - case llvm::CmpInst::Predicate::FCMP_UNE: - return LLVM::FCmpPredicate::une; - } - llvm_unreachable("incorrect floating point comparison predicate"); -} - -static AtomicOrdering getLLVMAtomicOrdering(llvm::AtomicOrdering ordering) { - switch (ordering) { - case llvm::AtomicOrdering::NotAtomic: - return LLVM::AtomicOrdering::not_atomic; - case llvm::AtomicOrdering::Unordered: - return LLVM::AtomicOrdering::unordered; - case llvm::AtomicOrdering::Monotonic: - return LLVM::AtomicOrdering::monotonic; - case llvm::AtomicOrdering::Acquire: - return LLVM::AtomicOrdering::acquire; - case llvm::AtomicOrdering::Release: - return LLVM::AtomicOrdering::release; - case llvm::AtomicOrdering::AcquireRelease: - return LLVM::AtomicOrdering::acq_rel; - case llvm::AtomicOrdering::SequentiallyConsistent: - return LLVM::AtomicOrdering::seq_cst; - } - llvm_unreachable("incorrect atomic ordering"); -} - -static AtomicBinOp getLLVMAtomicBinOp(llvm::AtomicRMWInst::BinOp binOp) { - switch (binOp) { - case llvm::AtomicRMWInst::Xchg: - return LLVM::AtomicBinOp::xchg; - case llvm::AtomicRMWInst::Add: - return LLVM::AtomicBinOp::add; - case llvm::AtomicRMWInst::Sub: - return LLVM::AtomicBinOp::sub; - case llvm::AtomicRMWInst::And: - return LLVM::AtomicBinOp::_and; - case llvm::AtomicRMWInst::Nand: - return LLVM::AtomicBinOp::nand; - case llvm::AtomicRMWInst::Or: - return LLVM::AtomicBinOp::_or; - case llvm::AtomicRMWInst::Xor: - return LLVM::AtomicBinOp::_xor; - case llvm::AtomicRMWInst::Max: - return LLVM::AtomicBinOp::max; - case llvm::AtomicRMWInst::Min: - return LLVM::AtomicBinOp::min; - case llvm::AtomicRMWInst::UMax: - return LLVM::AtomicBinOp::umax; - case llvm::AtomicRMWInst::UMin: - return LLVM::AtomicBinOp::umin; - case llvm::AtomicRMWInst::FAdd: - return LLVM::AtomicBinOp::fadd; - case llvm::AtomicRMWInst::FSub: - return LLVM::AtomicBinOp::fsub; - default: - llvm_unreachable("unsupported atomic binary operation"); - } -} - /// Converts the sync scope identifier of `fenceInst` to the string /// representation necessary to build the LLVM dialect fence operation. static StringRef getLLVMSyncScope(llvm::FenceInst *fenceInst) { diff --git a/mlir/test/Target/LLVMIR/Import/instructions.ll b/mlir/test/Target/LLVMIR/Import/instructions.ll index 6b8ce08c1673..438b4f0c3584 100644 --- a/mlir/test/Target/LLVMIR/Import/instructions.ll +++ b/mlir/test/Target/LLVMIR/Import/instructions.ll @@ -393,6 +393,14 @@ define void @atomic_rmw(ptr %ptr1, i32 %val1, ptr %ptr2, float %val2) { %12 = atomicrmw fadd ptr %ptr2, float %val2 acquire ; CHECK: llvm.atomicrmw fsub %[[PTR2]], %[[VAL2]] acquire %13 = atomicrmw fsub ptr %ptr2, float %val2 acquire + ; CHECK: llvm.atomicrmw fmax %[[PTR2]], %[[VAL2]] acquire + %14 = atomicrmw fmax ptr %ptr2, float %val2 acquire + ; CHECK: llvm.atomicrmw fmin %[[PTR2]], %[[VAL2]] acquire + %15 = atomicrmw fmin ptr %ptr2, float %val2 acquire + ; CHECK: llvm.atomicrmw uinc_wrap %[[PTR1]], %[[VAL1]] acquire + %16 = atomicrmw uinc_wrap ptr %ptr1, i32 %val1 acquire + ; CHECK: llvm.atomicrmw udec_wrap %[[PTR1]], %[[VAL1]] acquire + %17 = atomicrmw udec_wrap ptr %ptr1, i32 %val1 acquire ret void } diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir index 5a16430b2f20..bbaa8570d0e4 100644 --- a/mlir/test/Target/LLVMIR/llvmir.mlir +++ b/mlir/test/Target/LLVMIR/llvmir.mlir @@ -1406,28 +1406,36 @@ llvm.func @atomicrmw( %0 = llvm.atomicrmw fadd %f32_ptr, %f32 monotonic : !llvm.ptr, f32 // CHECK: atomicrmw fsub ptr %{{.*}}, float %{{.*}} monotonic %1 = llvm.atomicrmw fsub %f32_ptr, %f32 monotonic : !llvm.ptr, f32 + // CHECK: atomicrmw fmax ptr %{{.*}}, float %{{.*}} monotonic + %2 = llvm.atomicrmw fmax %f32_ptr, %f32 monotonic : !llvm.ptr, f32 + // CHECK: atomicrmw fmin ptr %{{.*}}, float %{{.*}} monotonic + %3 = llvm.atomicrmw fmin %f32_ptr, %f32 monotonic : !llvm.ptr, f32 // CHECK: atomicrmw xchg ptr %{{.*}}, float %{{.*}} monotonic - %2 = llvm.atomicrmw xchg %f32_ptr, %f32 monotonic : !llvm.ptr, f32 + %4 = llvm.atomicrmw xchg %f32_ptr, %f32 monotonic : !llvm.ptr, f32 // CHECK: atomicrmw add ptr %{{.*}}, i32 %{{.*}} acquire - %3 = llvm.atomicrmw add %i32_ptr, %i32 acquire : !llvm.ptr, i32 + %5 = llvm.atomicrmw add %i32_ptr, %i32 acquire : !llvm.ptr, i32 // CHECK: atomicrmw sub ptr %{{.*}}, i32 %{{.*}} release - %4 = llvm.atomicrmw sub %i32_ptr, %i32 release : !llvm.ptr, i32 + %6 = llvm.atomicrmw sub %i32_ptr, %i32 release : !llvm.ptr, i32 // CHECK: atomicrmw and ptr %{{.*}}, i32 %{{.*}} acq_rel - %5 = llvm.atomicrmw _and %i32_ptr, %i32 acq_rel : !llvm.ptr, i32 + %7 = llvm.atomicrmw _and %i32_ptr, %i32 acq_rel : !llvm.ptr, i32 // CHECK: atomicrmw nand ptr %{{.*}}, i32 %{{.*}} seq_cst - %6 = llvm.atomicrmw nand %i32_ptr, %i32 seq_cst : !llvm.ptr, i32 + %8 = llvm.atomicrmw nand %i32_ptr, %i32 seq_cst : !llvm.ptr, i32 // CHECK: atomicrmw or ptr %{{.*}}, i32 %{{.*}} monotonic - %7 = llvm.atomicrmw _or %i32_ptr, %i32 monotonic : !llvm.ptr, i32 + %9 = llvm.atomicrmw _or %i32_ptr, %i32 monotonic : !llvm.ptr, i32 // CHECK: atomicrmw xor ptr %{{.*}}, i32 %{{.*}} monotonic - %8 = llvm.atomicrmw _xor %i32_ptr, %i32 monotonic : !llvm.ptr, i32 + %10 = llvm.atomicrmw _xor %i32_ptr, %i32 monotonic : !llvm.ptr, i32 // CHECK: atomicrmw max ptr %{{.*}}, i32 %{{.*}} monotonic - %9 = llvm.atomicrmw max %i32_ptr, %i32 monotonic : !llvm.ptr, i32 + %11 = llvm.atomicrmw max %i32_ptr, %i32 monotonic : !llvm.ptr, i32 // CHECK: atomicrmw min ptr %{{.*}}, i32 %{{.*}} monotonic - %10 = llvm.atomicrmw min %i32_ptr, %i32 monotonic : !llvm.ptr, i32 + %12 = llvm.atomicrmw min %i32_ptr, %i32 monotonic : !llvm.ptr, i32 // CHECK: atomicrmw umax ptr %{{.*}}, i32 %{{.*}} monotonic - %11 = llvm.atomicrmw umax %i32_ptr, %i32 monotonic : !llvm.ptr, i32 + %13 = llvm.atomicrmw umax %i32_ptr, %i32 monotonic : !llvm.ptr, i32 // CHECK: atomicrmw umin ptr %{{.*}}, i32 %{{.*}} monotonic - %12 = llvm.atomicrmw umin %i32_ptr, %i32 monotonic : !llvm.ptr, i32 + %14 = llvm.atomicrmw umin %i32_ptr, %i32 monotonic : !llvm.ptr, i32 + // CHECK: atomicrmw uinc_wrap ptr %{{.*}}, i32 %{{.*}} monotonic + %15 = llvm.atomicrmw uinc_wrap %i32_ptr, %i32 monotonic : !llvm.ptr, i32 + // CHECK: atomicrmw udec_wrap ptr %{{.*}}, i32 %{{.*}} monotonic + %16 = llvm.atomicrmw udec_wrap %i32_ptr, %i32 monotonic : !llvm.ptr, i32 llvm.return } diff --git a/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp b/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp index 1aa5232f9671..926ebdf20d78 100644 --- a/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp +++ b/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp @@ -365,6 +365,18 @@ class LLVMEnumAttr : public tblgen::EnumAttr { return cases; } + + std::vector getAllUnsupportedCases() const { + const auto *inits = def->getValueAsListInit("unsupported"); + + std::vector cases; + cases.reserve(inits->size()); + + for (const llvm::Init *init : *inits) + cases.emplace_back(cast(init)); + + return cases; + } }; // Wraper class around a Tablegen definition of a C-style LLVM enum attribute. @@ -473,6 +485,12 @@ static void emitOneEnumFromConversion(const llvm::Record *record, os << formatv(" return {0}::{1}::{2};\n", cppNamespace, cppClassName, cppEnumerant); } + for (const auto &enumerant : enumAttr.getAllUnsupportedCases()) { + StringRef llvmEnumerant = enumerant.getLLVMEnumerant(); + os << formatv(" case {0}::{1}:\n", llvmClass, llvmEnumerant); + os << formatv(" llvm_unreachable(\"unsupported case {0}::{1}\");\n", + enumAttr.getLLVMClassName(), llvmEnumerant); + } os << " }\n"; os << formatv(" llvm_unreachable(\"unknown {0} type\");", From bef18b5d75abaf7e53fed90e873f54c489819527 Mon Sep 17 00:00:00 2001 From: Tobias Gysi Date: Mon, 6 Mar 2023 10:25:54 +0100 Subject: [PATCH 061/147] [mlir][llvm] Fix bug in constant import from LLVM IR. The revision addresses a bug during constant expression traversal when importing LLVM IR. A constant expression may have cyclic dependencies, for example, when a constant is initialized with its address. This revision extends the constant expression traversal to detect cyclic dependencies and adds a test to verify this case is handled properly. Reviewed By: Dinistro Differential Revision: https://reviews.llvm.org/D143152 (cherry picked from commit b6a2e33500cd29c8cce40acd08ff1dbd7559656f) --- mlir/lib/Target/LLVMIR/ModuleImport.cpp | 72 ++++++---- mlir/test/Target/LLVMIR/Import/constant.ll | 124 ++++++++++-------- .../Target/LLVMIR/Import/global-variables.ll | 9 +- .../Import/incorrect-constant-caching.ll | 34 ++--- .../incorrect-constexpr-inst-caching.ll | 40 +++--- 5 files changed, 155 insertions(+), 124 deletions(-) diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp index befb077cc82e..4d70e8536f40 100644 --- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp @@ -859,37 +859,57 @@ ModuleImport::convertGlobalCtorsAndDtors(llvm::GlobalVariable *globalVar) { SetVector ModuleImport::getConstantsToConvert(llvm::Constant *constant) { - // Traverse the constant dependencies in post order. - SmallVector workList; - SmallVector orderedList; - workList.push_back(constant); + // Return the empty set if the constant has been translated before. + if (valueMapping.count(constant)) + return {}; + + // Traverse the constants in post-order and stop the traversal if a constant + // already has a `valueMapping` from an earlier constant translation or if the + // constant is traversed a second time. + SetVector orderedSet; + SetVector workList; + DenseMap> adjacencyLists; + workList.insert(constant); while (!workList.empty()) { - llvm::Constant *current = workList.pop_back_val(); - // Skip constants that have been converted before and store all other ones. - if (valueMapping.count(current)) + llvm::Constant *current = workList.back(); + // Collect all dependencies of the current constant and add them to the + // adjacency list if none has been computed before. + auto adjacencyIt = adjacencyLists.find(current); + if (adjacencyIt == adjacencyLists.end()) { + adjacencyIt = adjacencyLists.try_emplace(current).first; + // Add all constant operands to the adjacency list and skip any other + // values such as basic block addresses. + for (llvm::Value *operand : current->operands()) + if (auto *constDependency = dyn_cast(operand)) + adjacencyIt->getSecond().push_back(constDependency); + // Use the getElementValue method to add the dependencies of zero + // initialized aggregate constants since they do not take any operands. + if (auto *constAgg = dyn_cast(current)) { + unsigned numElements = constAgg->getElementCount().getFixedValue(); + for (unsigned i = 0, e = numElements; i != e; ++i) + adjacencyIt->getSecond().push_back(constAgg->getElementValue(i)); + } + } + // Add the current constant to the `orderedSet` of the traversed nodes if + // all its dependencies have been traversed before. Additionally, remove the + // constant from the `workList` and continue the traversal. + if (adjacencyIt->getSecond().empty()) { + orderedSet.insert(current); + workList.pop_back(); continue; - orderedList.push_back(current); - // Add the current constant's dependencies to the work list. Only add - // constant dependencies and skip any other values such as basic block - // addresses. - for (llvm::Value *operand : current->operands()) - if (auto *constDependency = dyn_cast(operand)) - workList.push_back(constDependency); - // Use the `getElementValue` method to add the dependencies of zero - // initialized aggregate constants since they do not take any operands. - if (auto *constAgg = dyn_cast(current)) { - unsigned numElements = constAgg->getElementCount().getFixedValue(); - for (unsigned i = 0, e = numElements; i != e; ++i) - workList.push_back(constAgg->getElementValue(i)); } + // Add the next dependency from the adjacency list to the `workList` and + // continue the traversal. Remove the dependency from the adjacency list to + // mark that it has been processed. Only enqueue the dependency if it has no + // `valueMapping` from an earlier translation and if it has not been + // enqueued before. + llvm::Constant *dependency = adjacencyIt->getSecond().pop_back_val(); + if (valueMapping.count(dependency) || workList.count(dependency) || + orderedSet.count(dependency)) + continue; + workList.insert(dependency); } - // Add the constants in reverse post order to the result set to ensure all - // dependencies are satisfied. Avoid storing duplicates since LLVM constants - // are uniqued and only one `valueMapping` entry per constant is possible. - SetVector orderedSet; - for (llvm::Constant *orderedConst : llvm::reverse(orderedList)) - orderedSet.insert(orderedConst); return orderedSet; } diff --git a/mlir/test/Target/LLVMIR/Import/constant.ll b/mlir/test/Target/LLVMIR/Import/constant.ll index 2e8be550c45f..85cde7b1bcfe 100644 --- a/mlir/test/Target/LLVMIR/Import/constant.ll +++ b/mlir/test/Target/LLVMIR/Import/constant.ll @@ -60,10 +60,10 @@ define ptr @null_constant() { ; CHECK-LABEL: @gep_const_expr define ptr @gep_const_expr() { - ; CHECK: %[[ADDR:[0-9]+]] = llvm.mlir.addressof @global : !llvm.ptr - ; CHECK: %[[IDX:[0-9]+]] = llvm.mlir.constant(2 : i32) : i32 - ; CHECK: %[[GEP:[0-9]+]] = llvm.getelementptr %[[ADDR]][%[[IDX]]] : (!llvm.ptr, i32) -> !llvm.ptr - ; CHECK: llvm.return %[[GEP]] : !llvm.ptr + ; CHECK-DAG: %[[ADDR:[0-9]+]] = llvm.mlir.addressof @global : !llvm.ptr + ; CHECK-DAG: %[[IDX:[0-9]+]] = llvm.mlir.constant(2 : i32) : i32 + ; CHECK-DAG: %[[GEP:[0-9]+]] = llvm.getelementptr %[[ADDR]][%[[IDX]]] : (!llvm.ptr, i32) -> !llvm.ptr + ; CHECK-DAG: llvm.return %[[GEP]] : !llvm.ptr ret ptr getelementptr (i32, ptr @global, i32 2) } @@ -73,14 +73,14 @@ define ptr @gep_const_expr() { ; CHECK-LABEL: @const_expr_with_duplicate define i64 @const_expr_with_duplicate() { - ; CHECK: %[[ADDR:[0-9]+]] = llvm.mlir.addressof @global : !llvm.ptr - ; CHECK: %[[IDX:[0-9]+]] = llvm.mlir.constant(7 : i32) : i32 - ; CHECK: %[[GEP:[0-9]+]] = llvm.getelementptr %[[ADDR]][%[[IDX]]] : (!llvm.ptr, i32) -> !llvm.ptr - ; CHECK: %[[DUP:[0-9]+]] = llvm.ptrtoint %[[GEP]] : !llvm.ptr to i64 + ; CHECK-DAG: %[[ADDR:[0-9]+]] = llvm.mlir.addressof @global : !llvm.ptr + ; CHECK-DAG: %[[IDX:[0-9]+]] = llvm.mlir.constant(7 : i32) : i32 + ; CHECK-DAG: %[[GEP:[0-9]+]] = llvm.getelementptr %[[ADDR]][%[[IDX]]] : (!llvm.ptr, i32) -> !llvm.ptr + ; CHECK-DAG: %[[DUP:[0-9]+]] = llvm.ptrtoint %[[GEP]] : !llvm.ptr to i64 ; Verify the duplicate sub expression is converted only once. - ; CHECK: %[[SUM:[0-9]+]] = llvm.add %[[DUP]], %[[DUP]] : i64 - ; CHECK: llvm.return %[[SUM]] : i64 + ; CHECK-DAG: %[[SUM:[0-9]+]] = llvm.add %[[DUP]], %[[DUP]] : i64 + ; CHECK-DAG: llvm.return %[[SUM]] : i64 ret i64 add (i64 ptrtoint (ptr getelementptr (i32, ptr @global, i32 7) to i64), i64 ptrtoint (ptr getelementptr (i32, ptr @global, i32 7) to i64)) } @@ -92,27 +92,27 @@ define i64 @const_expr_with_duplicate() { ; CHECK-LABEL: @const_expr_with_aggregate() define i64 @const_expr_with_aggregate() { ; Compute the vector elements. - ; CHECK: %[[VAL1:[0-9]+]] = llvm.mlir.constant(33 : i64) : i64 - ; CHECK: %[[ADDR:[0-9]+]] = llvm.mlir.addressof @global : !llvm.ptr - ; CHECK: %[[IDX1:[0-9]+]] = llvm.mlir.constant(7 : i32) : i32 - ; CHECK: %[[GEP1:[0-9]+]] = llvm.getelementptr %[[ADDR]][%[[IDX1]]] : (!llvm.ptr, i32) -> !llvm.ptr - ; CHECK: %[[VAL2:[0-9]+]] = llvm.ptrtoint %[[GEP1]] : !llvm.ptr to i64 + ; CHECK-DAG: %[[VAL1:[0-9]+]] = llvm.mlir.constant(33 : i64) : i64 + ; CHECK-DAG: %[[ADDR:[0-9]+]] = llvm.mlir.addressof @global : !llvm.ptr + ; CHECK-DAG: %[[IDX1:[0-9]+]] = llvm.mlir.constant(7 : i32) : i32 + ; CHECK-DAG: %[[GEP1:[0-9]+]] = llvm.getelementptr %[[ADDR]][%[[IDX1]]] : (!llvm.ptr, i32) -> !llvm.ptr + ; CHECK-DAG: %[[VAL2:[0-9]+]] = llvm.ptrtoint %[[GEP1]] : !llvm.ptr to i64 ; Fill the vector. - ; CHECK: %[[VEC1:[0-9]+]] = llvm.mlir.undef : vector<2xi64> - ; CHECK: %[[IDX2:[0-9]+]] = llvm.mlir.constant(0 : i32) : i32 - ; CHECK: %[[VEC2:[0-9]+]] = llvm.insertelement %[[VAL1]], %[[VEC1]][%[[IDX2]] : i32] : vector<2xi64> - ; CHECK: %[[IDX3:[0-9]+]] = llvm.mlir.constant(1 : i32) : i32 - ; CHECK: %[[VEC3:[0-9]+]] = llvm.insertelement %[[VAL2]], %[[VEC2]][%[[IDX3]] : i32] : vector<2xi64> - ; CHECK: %[[IDX4:[0-9]+]] = llvm.mlir.constant(42 : i32) : i32 + ; CHECK-DAG: %[[VEC1:[0-9]+]] = llvm.mlir.undef : vector<2xi64> + ; CHECK-DAG: %[[IDX2:[0-9]+]] = llvm.mlir.constant(0 : i32) : i32 + ; CHECK-DAG: %[[VEC2:[0-9]+]] = llvm.insertelement %[[VAL1]], %[[VEC1]][%[[IDX2]] : i32] : vector<2xi64> + ; CHECK-DAG: %[[IDX3:[0-9]+]] = llvm.mlir.constant(1 : i32) : i32 + ; CHECK-DAG: %[[VEC3:[0-9]+]] = llvm.insertelement %[[VAL2]], %[[VEC2]][%[[IDX3]] : i32] : vector<2xi64> + ; CHECK-DAG: %[[IDX4:[0-9]+]] = llvm.mlir.constant(42 : i32) : i32 ; Compute the extract index. - ; CHECK: %[[GEP2:[0-9]+]] = llvm.getelementptr %[[ADDR]][%[[IDX4]]] : (!llvm.ptr, i32) -> !llvm.ptr - ; CHECK: %[[IDX5:[0-9]+]] = llvm.ptrtoint %[[GEP2]] : !llvm.ptr to i64 + ; CHECK-DAG: %[[GEP2:[0-9]+]] = llvm.getelementptr %[[ADDR]][%[[IDX4]]] : (!llvm.ptr, i32) -> !llvm.ptr + ; CHECK-DAG: %[[IDX5:[0-9]+]] = llvm.ptrtoint %[[GEP2]] : !llvm.ptr to i64 ; Extract the vector element. - ; CHECK: %[[ELEM:[0-9]+]] = llvm.extractelement %[[VEC3]][%[[IDX5]] : i64] : vector<2xi64> - ; CHECK: llvm.return %[[ELEM]] : i64 + ; CHECK-DAG: %[[ELEM:[0-9]+]] = llvm.extractelement %[[VEC3]][%[[IDX5]] : i64] : vector<2xi64> + ; CHECK-DAG: llvm.return %[[ELEM]] : i64 ret i64 extractelement ( <2 x i64> , i64 ptrtoint (ptr getelementptr (i32, ptr @global, i32 42) to i64)) @@ -158,43 +158,43 @@ define i32 @function_address_after_def() { ; Verify the aggregate constant import. -; CHECK: %[[C0:.+]] = llvm.mlir.constant(9 : i32) : i32 -; CHECK: %[[C1:.+]] = llvm.mlir.constant(4 : i8) : i8 -; CHECK: %[[C2:.+]] = llvm.mlir.constant(8 : i16) : i16 -; CHECK: %[[C3:.+]] = llvm.mlir.constant(7 : i32) : i32 -; CHECK: %[[ROOT:.+]] = llvm.mlir.undef : !llvm.struct<"simple_agg_type", (i32, i8, i16, i32)> -; CHECK: %[[CHAIN0:.+]] = llvm.insertvalue %[[C0]], %[[ROOT]][0] -; CHECK: %[[CHAIN1:.+]] = llvm.insertvalue %[[C1]], %[[CHAIN0]][1] -; CHECK: %[[CHAIN2:.+]] = llvm.insertvalue %[[C2]], %[[CHAIN1]][2] -; CHECK: %[[CHAIN3:.+]] = llvm.insertvalue %[[C3]], %[[CHAIN2]][3] -; CHECK: llvm.return %[[CHAIN3]] +; CHECK-DAG: %[[C0:.+]] = llvm.mlir.constant(9 : i32) : i32 +; CHECK-DAG: %[[C1:.+]] = llvm.mlir.constant(4 : i8) : i8 +; CHECK-DAG: %[[C2:.+]] = llvm.mlir.constant(8 : i16) : i16 +; CHECK-DAG: %[[C3:.+]] = llvm.mlir.constant(7 : i32) : i32 +; CHECK-DAG: %[[ROOT:.+]] = llvm.mlir.undef : !llvm.struct<"simple_agg_type", (i32, i8, i16, i32)> +; CHECK-DAG: %[[CHAIN0:.+]] = llvm.insertvalue %[[C0]], %[[ROOT]][0] +; CHECK-DAG: %[[CHAIN1:.+]] = llvm.insertvalue %[[C1]], %[[CHAIN0]][1] +; CHECK-DAG: %[[CHAIN2:.+]] = llvm.insertvalue %[[C2]], %[[CHAIN1]][2] +; CHECK-DAG: %[[CHAIN3:.+]] = llvm.insertvalue %[[C3]], %[[CHAIN2]][3] +; CHECK-DAG: llvm.return %[[CHAIN3]] %simple_agg_type = type {i32, i8, i16, i32} @simple_agg = global %simple_agg_type {i32 9, i8 4, i16 8, i32 7} -; CHECK: %[[C1:.+]] = llvm.mlir.constant(1 : i32) : i32 -; CHECK: %[[C2:.+]] = llvm.mlir.constant(2 : i8) : i8 -; CHECK: %[[C3:.+]] = llvm.mlir.constant(3 : i16) : i16 -; CHECK: %[[C4:.+]] = llvm.mlir.constant(4 : i32) : i32 -; CHECK: %[[NESTED:.+]] = llvm.mlir.undef : !llvm.struct<"simple_agg_type", (i32, i8, i16, i32)> -; CHECK: %[[CHAIN0:.+]] = llvm.insertvalue %[[C1]], %[[NESTED]][0] -; CHECK: %[[CHAIN1:.+]] = llvm.insertvalue %[[C2]], %[[CHAIN0]][1] -; CHECK: %[[CHAIN2:.+]] = llvm.insertvalue %[[C3]], %[[CHAIN1]][2] -; CHECK: %[[CHAIN3:.+]] = llvm.insertvalue %[[C4]], %[[CHAIN2]][3] -; CHECK: %[[NULL:.+]] = llvm.mlir.null : !llvm.ptr -; CHECK: %[[ROOT:.+]] = llvm.mlir.undef : !llvm.struct<"nested_agg_type", (struct<"simple_agg_type", (i32, i8, i16, i32)>, ptr)> -; CHECK: %[[CHAIN4:.+]] = llvm.insertvalue %[[CHAIN3]], %[[ROOT]][0] -; CHECK: %[[CHAIN5:.+]] = llvm.insertvalue %[[NULL]], %[[CHAIN4]][1] -; CHECK: llvm.return %[[CHAIN5]] +; CHECK-DAG: %[[C1:.+]] = llvm.mlir.constant(1 : i32) : i32 +; CHECK-DAG: %[[C2:.+]] = llvm.mlir.constant(2 : i8) : i8 +; CHECK-DAG: %[[C3:.+]] = llvm.mlir.constant(3 : i16) : i16 +; CHECK-DAG: %[[C4:.+]] = llvm.mlir.constant(4 : i32) : i32 +; CHECK-DAG: %[[NESTED:.+]] = llvm.mlir.undef : !llvm.struct<"simple_agg_type", (i32, i8, i16, i32)> +; CHECK-DAG: %[[CHAIN0:.+]] = llvm.insertvalue %[[C1]], %[[NESTED]][0] +; CHECK-DAG: %[[CHAIN1:.+]] = llvm.insertvalue %[[C2]], %[[CHAIN0]][1] +; CHECK-DAG: %[[CHAIN2:.+]] = llvm.insertvalue %[[C3]], %[[CHAIN1]][2] +; CHECK-DAG: %[[CHAIN3:.+]] = llvm.insertvalue %[[C4]], %[[CHAIN2]][3] +; CHECK-DAG: %[[NULL:.+]] = llvm.mlir.null : !llvm.ptr +; CHECK-DAG: %[[ROOT:.+]] = llvm.mlir.undef : !llvm.struct<"nested_agg_type", (struct<"simple_agg_type", (i32, i8, i16, i32)>, ptr)> +; CHECK-DAG: %[[CHAIN4:.+]] = llvm.insertvalue %[[CHAIN3]], %[[ROOT]][0] +; CHECK-DAG: %[[CHAIN5:.+]] = llvm.insertvalue %[[NULL]], %[[CHAIN4]][1] +; CHECK-DAG: llvm.return %[[CHAIN5]] %nested_agg_type = type {%simple_agg_type, ptr} @nested_agg = global %nested_agg_type { %simple_agg_type{i32 1, i8 2, i16 3, i32 4}, ptr null } -; CHECK: %[[NULL:.+]] = llvm.mlir.null : !llvm.ptr -; CHECK: %[[ROOT:.+]] = llvm.mlir.undef : !llvm.vec<2 x ptr> -; CHECK: %[[P0:.+]] = llvm.mlir.constant(0 : i32) : i32 -; CHECK: %[[CHAIN0:.+]] = llvm.insertelement %[[NULL]], %[[ROOT]][%[[P0]] : i32] : !llvm.vec<2 x ptr> -; CHECK: %[[P1:.+]] = llvm.mlir.constant(1 : i32) : i32 -; CHECK: %[[CHAIN1:.+]] = llvm.insertelement %[[NULL]], %[[CHAIN0]][%[[P1]] : i32] : !llvm.vec<2 x ptr> -; CHECK: llvm.return %[[CHAIN1]] : !llvm.vec<2 x ptr> +; CHECK-DAG: %[[NULL:.+]] = llvm.mlir.null : !llvm.ptr +; CHECK-DAG: %[[ROOT:.+]] = llvm.mlir.undef : !llvm.vec<2 x ptr> +; CHECK-DAG: %[[P0:.+]] = llvm.mlir.constant(0 : i32) : i32 +; CHECK-DAG: %[[CHAIN0:.+]] = llvm.insertelement %[[NULL]], %[[ROOT]][%[[P0]] : i32] : !llvm.vec<2 x ptr> +; CHECK-DAG: %[[P1:.+]] = llvm.mlir.constant(1 : i32) : i32 +; CHECK-DAG: %[[CHAIN1:.+]] = llvm.insertelement %[[NULL]], %[[CHAIN0]][%[[P1]] : i32] : !llvm.vec<2 x ptr> +; CHECK-DAG: llvm.return %[[CHAIN1]] : !llvm.vec<2 x ptr> @vector_agg = global <2 x ptr> ; // ----- @@ -214,3 +214,15 @@ define i64 @const_exprs_with_duplicate() { %2 = add i64 %1, ptrtoint (ptr getelementptr (i32, ptr @global, i32 42) to i64) ret i64 %2 } + +; // ----- + +; Verify the import of constant expressions with cyclic dependencies. + +@cyclic = internal constant i64 mul (i64 ptrtoint (ptr @cyclic to i64), i64 ptrtoint (ptr @cyclic to i64)) + +; CHECK-LABEL: @cyclic +; CHECK: %[[ADDR:.+]] = llvm.mlir.addressof @cyclic +; CHECK: %[[VAL0:.+]] = llvm.ptrtoint %[[ADDR]] +; CHECK: %[[VAL1:.+]] = llvm.mul %[[VAL0]], %[[VAL0]] +; CHECK: llvm.return %[[VAL1]] diff --git a/mlir/test/Target/LLVMIR/Import/global-variables.ll b/mlir/test/Target/LLVMIR/Import/global-variables.ll index 5e68de9b2c01..3b8be65eacd4 100644 --- a/mlir/test/Target/LLVMIR/Import/global-variables.ll +++ b/mlir/test/Target/LLVMIR/Import/global-variables.ll @@ -33,11 +33,10 @@ ; CHECK: llvm.mlir.global internal constant @global_gep_const_expr ; CHECK-SAME: {addr_space = 0 : i32, dso_local} : !llvm.ptr { -; CHECK: %[[ADDR:[0-9]+]] = llvm.mlir.addressof @global_int : !llvm.ptr -; CHECK: %[[IDX:[0-9]+]] = llvm.mlir.constant(2 : i32) : i32 -; CHECK: %[[GEP:[0-9]+]] = llvm.getelementptr %[[ADDR]][%[[IDX]]] : (!llvm.ptr, i32) -> !llvm.ptr -; CHECK: llvm.return %[[GEP]] : !llvm.ptr -; CHECK: } +; CHECK-DAG: %[[ADDR:[0-9]+]] = llvm.mlir.addressof @global_int : !llvm.ptr +; CHECK-DAG: %[[IDX:[0-9]+]] = llvm.mlir.constant(2 : i32) : i32 +; CHECK-DAG: %[[GEP:[0-9]+]] = llvm.getelementptr %[[ADDR]][%[[IDX]]] : (!llvm.ptr, i32) -> !llvm.ptr +; CHECK-DAG llvm.return %[[GEP]] : !llvm.ptr @global_gep_const_expr = internal constant ptr getelementptr (i32, ptr @global_int, i32 2) ; // ----- diff --git a/mlir/test/Target/LLVMIR/Import/incorrect-constant-caching.ll b/mlir/test/Target/LLVMIR/Import/incorrect-constant-caching.ll index 1953ecd68372..386709b3b09b 100644 --- a/mlir/test/Target/LLVMIR/Import/incorrect-constant-caching.ll +++ b/mlir/test/Target/LLVMIR/Import/incorrect-constant-caching.ll @@ -8,23 +8,23 @@ ; only wrote minimum level of checks. %my_struct = type {i32, ptr} -; CHECK: llvm.mlir.constant(8 : i32) : i32 -; CHECK: llvm.mlir.addressof @str0 : !llvm.ptr -; CHECK: llvm.mlir.constant(0 : i32) : i32 -; CHECK: llvm.getelementptr -; CHECK: llvm.mlir.undef : !llvm.struct<"my_struct", (i32, ptr)> -; CHECK: llvm.insertvalue -; CHECK: llvm.insertvalue -; CHECK: llvm.mlir.constant(7 : i32) : i32 -; CHECK: llvm.mlir.addressof @str1 : !llvm.ptr -; CHECK: llvm.getelementptr -; CHECK: llvm.mlir.undef : !llvm.struct<"my_struct", (i32, ptr)> -; CHECK: llvm.insertvalue -; CHECK: llvm.insertvalue -; CHECK: llvm.mlir.undef : !llvm.array<2 x struct<"my_struct", (i32, ptr)>> -; CHECK: llvm.insertvalue -; CHECK: llvm.insertvalue -; CHECK: llvm.return +; CHECK-DAG: llvm.mlir.constant(8 : i32) : i32 +; CHECK-DAG: llvm.mlir.addressof @str0 : !llvm.ptr +; CHECK-DAG: llvm.mlir.constant(0 : i32) : i32 +; CHECK-DAG: llvm.getelementptr +; CHECK-DAG: llvm.mlir.undef : !llvm.struct<"my_struct", (i32, ptr)> +; CHECK-DAG: llvm.insertvalue +; CHECK-DAG: llvm.insertvalue +; CHECK-DAG: llvm.mlir.constant(7 : i32) : i32 +; CHECK-DAG: llvm.mlir.addressof @str1 : !llvm.ptr +; CHECK-DAG: llvm.getelementptr +; CHECK-DAG: llvm.mlir.undef : !llvm.struct<"my_struct", (i32, ptr)> +; CHECK-DAG: llvm.insertvalue +; CHECK-DAG: llvm.insertvalue +; CHECK-DAG: llvm.mlir.undef : !llvm.array<2 x struct<"my_struct", (i32, ptr)>> +; CHECK-DAG: llvm.insertvalue +; CHECK-DAG: llvm.insertvalue +; CHECK-DAG: llvm.return @str0 = private unnamed_addr constant [5 x i8] c"aaaa\00" @str1 = private unnamed_addr constant [5 x i8] c"bbbb\00" @g = global [2 x %my_struct] [%my_struct {i32 8, ptr getelementptr ([5 x i8], ptr @str0, i32 0, i32 1)}, %my_struct {i32 7, ptr getelementptr ([5 x i8], ptr @str1, i32 0, i32 1)}] diff --git a/mlir/test/Target/LLVMIR/Import/incorrect-constexpr-inst-caching.ll b/mlir/test/Target/LLVMIR/Import/incorrect-constexpr-inst-caching.ll index 15957050e079..75ba51550b1e 100644 --- a/mlir/test/Target/LLVMIR/Import/incorrect-constexpr-inst-caching.ll +++ b/mlir/test/Target/LLVMIR/Import/incorrect-constexpr-inst-caching.ll @@ -5,26 +5,26 @@ ; Thus, we only wrote minimum level of checks. %my_struct = type {i32, ptr} -; CHECK: llvm.mlir.constant(8 : i32) : i32 -; CHECK: llvm.mlir.addressof @str0 : !llvm.ptr -; CHECK: llvm.mlir.constant(0 : i32) : i32 -; CHECK: llvm.mlir.constant(1 : i32) : i32 -; CHECK: llvm.getelementptr -; CHECK: llvm.mlir.undef : !llvm.struct<"my_struct", (i32, ptr)> -; CHECK: llvm.insertvalue -; CHECK: llvm.insertvalue -; CHECK: llvm.mlir.constant(7 : i32) : i32 -; CHECK: llvm.mlir.addressof @str1 : !llvm.ptr -; CHECK: llvm.mlir.constant(2 : i32) : i32 -; CHECK: llvm.mlir.constant(3 : i32) : i32 -; CHECK: llvm.getelementptr -; CHECK: llvm.mlir.undef : !llvm.struct<"my_struct", (i32, ptr)> -; CHECK: llvm.insertvalue -; CHECK: llvm.insertvalue -; CHECK: llvm.mlir.undef : !llvm.array<2 x struct<"my_struct", (i32, ptr)>> -; CHECK: llvm.insertvalue -; CHECK: llvm.insertvalue -; CHECK: llvm.return +; CHECK-DAG: llvm.mlir.constant(8 : i32) : i32 +; CHECK-DAG: llvm.mlir.addressof @str0 : !llvm.ptr +; CHECK-DAG: llvm.mlir.constant(0 : i32) : i32 +; CHECK-DAG: llvm.mlir.constant(1 : i32) : i32 +; CHECK-DAG: llvm.getelementptr +; CHECK-DAG: llvm.mlir.undef : !llvm.struct<"my_struct", (i32, ptr)> +; CHECK-DAG: llvm.insertvalue +; CHECK-DAG: llvm.insertvalue +; CHECK-DAG: llvm.mlir.constant(7 : i32) : i32 +; CHECK-DAG: llvm.mlir.addressof @str1 : !llvm.ptr +; CHECK-DAG: llvm.mlir.constant(2 : i32) : i32 +; CHECK-DAG: llvm.mlir.constant(3 : i32) : i32 +; CHECK-DAG: llvm.getelementptr +; CHECK-DAG: llvm.mlir.undef : !llvm.struct<"my_struct", (i32, ptr)> +; CHECK-DAG: llvm.insertvalue +; CHECK-DAG: llvm.insertvalue +; CHECK-DAG: llvm.mlir.undef : !llvm.array<2 x struct<"my_struct", (i32, ptr)>> +; CHECK-DAG: llvm.insertvalue +; CHECK-DAG: llvm.insertvalue +; CHECK-DAG: llvm.return @str0 = private unnamed_addr constant [5 x i8] c"aaaa\00" @str1 = private unnamed_addr constant [5 x i8] c"bbbb\00" @g = global [2 x %my_struct] [%my_struct {i32 8, ptr getelementptr ([5 x i8], ptr @str0, i32 0, i32 1)}, %my_struct {i32 7, ptr getelementptr ([5 x i8], ptr @str1, i32 2, i32 3)}] From 752d21c965706e2423e5efc385ec1a1898cd48f5 Mon Sep 17 00:00:00 2001 From: Tobias Gysi Date: Mon, 6 Mar 2023 10:27:35 +0100 Subject: [PATCH 062/147] [mlir][llvm] Drop opaque ptr test in LLVM IR import. After switching all LLVM IR import tests to opaque pointers the specialized opaque pointer test file is redundant. Reviewed By: Dinistro Differential Revision: https://reviews.llvm.org/D143370 (cherry picked from commit 7dd974a11532c7495f54168f69d6ab3942308fbe) --- .../test/Target/LLVMIR/Import/instructions.ll | 4 +- mlir/test/Target/LLVMIR/Import/opaque.ll | 53 ------------------- 2 files changed, 2 insertions(+), 55 deletions(-) delete mode 100644 mlir/test/Target/LLVMIR/Import/opaque.ll diff --git a/mlir/test/Target/LLVMIR/Import/instructions.ll b/mlir/test/Target/LLVMIR/Import/instructions.ll index 438b4f0c3584..15abc3dc96a7 100644 --- a/mlir/test/Target/LLVMIR/Import/instructions.ll +++ b/mlir/test/Target/LLVMIR/Import/instructions.ll @@ -448,7 +448,7 @@ define void @indirect_call(ptr addrspace(42) %fn) { ; CHECK-SAME: %[[PTR:[a-zA-Z0-9]+]] define void @gep_static_idx(ptr %ptr) { ; CHECK: %[[IDX:.+]] = llvm.mlir.constant(7 : i32) - ; CHECK: llvm.getelementptr inbounds %[[PTR]][%[[IDX]]] : (!llvm.ptr, i32) -> !llvm.ptr + ; CHECK: llvm.getelementptr inbounds %[[PTR]][%[[IDX]]] : (!llvm.ptr, i32) -> !llvm.ptr, f32 %1 = getelementptr inbounds float, ptr %ptr, i32 7 ret void } @@ -476,7 +476,7 @@ define void @varargs_call(i32 %0) { ; CHECK-SAME: %[[IDX:[a-zA-Z0-9]+]] define void @gep_dynamic_idx(ptr %ptr, i32 %idx) { ; CHECK: %[[C0:.+]] = llvm.mlir.constant(0 : i32) - ; CHECK: llvm.getelementptr %[[PTR]][%[[C0]], 1, %[[IDX]]] + ; CHECK: llvm.getelementptr %[[PTR]][%[[C0]], 1, %[[IDX]]]{{.*}}"my_struct" %1 = getelementptr %my_struct, ptr %ptr, i32 0, i32 1, i32 %idx ret void } diff --git a/mlir/test/Target/LLVMIR/Import/opaque.ll b/mlir/test/Target/LLVMIR/Import/opaque.ll deleted file mode 100644 index 2d20b2c0b771..000000000000 --- a/mlir/test/Target/LLVMIR/Import/opaque.ll +++ /dev/null @@ -1,53 +0,0 @@ -; RUN: mlir-translate -import-llvm -split-input-file -opaque-pointers %s | FileCheck %s - -; CHECK-LABEL: @opaque_ptr_load -define i32 @opaque_ptr_load(ptr %0) { - ; CHECK: = llvm.load %{{.*}} : !llvm.ptr -> i32 - %2 = load i32, ptr %0, align 4 - ret i32 %2 -} - -; // ----- - -; CHECK-LABEL: @opaque_ptr_store -define void @opaque_ptr_store(i32 %0, ptr %1) { - ; CHECK: llvm.store %{{.*}}, %{{.*}} : i32, !llvm.ptr - store i32 %0, ptr %1, align 4 - ret void -} - -; // ----- - -; CHECK-LABEL: @opaque_ptr_ptr_store -define void @opaque_ptr_ptr_store(ptr %0, ptr %1) { - ; CHECK: llvm.store %{{.*}}, %{{.*}} : !llvm.ptr, !llvm.ptr - store ptr %0, ptr %1, align 8 - ret void -} - -; // ----- - -; CHECK-LABEL: @opaque_ptr_alloca -define ptr @opaque_ptr_alloca(i32 %0) { - ; CHECK: = llvm.alloca %{{.*}} x f32 {alignment = 4 : i64} : (i32) -> !llvm.ptr - %2 = alloca float, i32 %0, align 4 - ret ptr %2 -} - -; // ----- - -; CHECK-LABEL: @opaque_ptr_gep -define ptr @opaque_ptr_gep(ptr %0, i32 %1) { - ; CHECK: = llvm.getelementptr inbounds %{{.*}}[%{{.*}}] : (!llvm.ptr, i32) -> !llvm.ptr, f32 - %3 = getelementptr inbounds float, ptr %0, i32 %1 - ret ptr %3 -} - -; // ----- - -; CHECK-LABEL: @opaque_ptr_gep -define ptr @opaque_ptr_gep_struct(ptr %0, i32 %1){ - ; CHECK: = llvm.getelementptr %{{.*}}[%{{.*}}, 0, 1] : (!llvm.ptr, i32) -> !llvm.ptr, !llvm.struct<(struct<(f32, f64)>, struct<(i32, i64)>)> - %3 = getelementptr { { float, double }, { i32, i64 } }, ptr %0, i32 %1, i32 0, i32 1 - ret ptr %3 -} From 321eb61ad91c95ae66a2bfd8b37575a1bcb1b8c3 Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Mon, 6 Mar 2023 10:29:04 +0100 Subject: [PATCH 063/147] [mlir][llvm] Add missing license header (NFC) This commit adds a missing license header that was forgotten in https://reviews.llvm.org/D143064. (cherry picked from commit a9787577ee572965ee31ef2b7699ce66ea949a51) --- mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp index 273dc1378984..b8b6e3c8ffdd 100644 --- a/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp @@ -1,3 +1,11 @@ +//===- LoopAnnotationTranslation.cpp - Loop annotation export -------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + #include "LoopAnnotationTranslation.h" using namespace mlir; From ed4f94e5e1f964ff0c52f268b95ea23bbe4fc299 Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Mon, 6 Mar 2023 10:31:24 +0100 Subject: [PATCH 064/147] [mlir][llvm] Add support for loop metadata import This commit introduces functionality to import loop metadata. Loop metadata nodes are transformed into LoopAnnotationAttrs and attached to the corresponding branch operations. Reviewed By: gysit Differential Revision: https://reviews.llvm.org/D143376 (cherry picked from commit b83caa32dc86cfc4d7f7b160284c2f7d002dea75) --- .../include/mlir/Target/LLVMIR/ModuleImport.h | 18 +- mlir/lib/Target/LLVMIR/CMakeLists.txt | 2 + .../LLVMIR/LLVMIRToLLVMTranslation.cpp | 40 +- .../Target/LLVMIR/LoopAnnotationImporter.cpp | 417 ++++++++++++++++++ .../Target/LLVMIR/LoopAnnotationImporter.h | 53 +++ mlir/lib/Target/LLVMIR/ModuleImport.cpp | 27 +- .../Target/LLVMIR/Import/import-failure.ll | 177 ++++++++ .../Target/LLVMIR/Import/metadata-loop.ll | 237 ++++++++++ 8 files changed, 951 insertions(+), 20 deletions(-) create mode 100644 mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp create mode 100644 mlir/lib/Target/LLVMIR/LoopAnnotationImporter.h diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h index a19123e9ee2f..23b1fbc29dd7 100644 --- a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h +++ b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h @@ -33,6 +33,7 @@ namespace LLVM { namespace detail { class DebugImporter; +class LoopAnnotationImporter; } // namespace detail /// Module import implementation class that provides methods to import globals @@ -164,11 +165,16 @@ class ModuleImport { return tbaaMapping.lookup(node); } - /// Returns the MLIR symbol reference mapped to the given LLVM access - /// group metadata `node`. - SymbolRefAttr lookupAccessGroupAttr(const llvm::MDNode *node) const { - return accessGroupMapping.lookup(node); - } + /// Returns the symbol references pointing to the access group operations that + /// map to the access group nodes starting from the access group metadata + /// `node`. Returns failure, if any of the symbol references cannot be found. + FailureOr> + lookupAccessGroupAttrs(const llvm::MDNode *node) const; + + /// Returns the loop annotation attribute that corresponds to the given LLVM + /// loop metadata `node`. + LoopAnnotationAttr translateLoopAnnotationAttr(const llvm::MDNode *node, + Location loc) const; private: /// Clears the block and value mapping before processing a new region. @@ -303,6 +309,8 @@ class ModuleImport { LLVM::TypeFromLLVMIRTranslator typeTranslator; /// Stateful debug information importer. std::unique_ptr debugImporter; + /// Loop annotation importer. + std::unique_ptr loopAnnotationImporter; }; } // namespace LLVM diff --git a/mlir/lib/Target/LLVMIR/CMakeLists.txt b/mlir/lib/Target/LLVMIR/CMakeLists.txt index 1741176ddac8..c8e7797c7612 100644 --- a/mlir/lib/Target/LLVMIR/CMakeLists.txt +++ b/mlir/lib/Target/LLVMIR/CMakeLists.txt @@ -5,6 +5,7 @@ set(LLVM_OPTIONAL_SOURCES ConvertToLLVMIR.cpp DebugTranslation.cpp DebugImporter.cpp + LoopAnnotationImporter.cpp LoopAnnotationTranslation.cpp ModuleTranslation.cpp ModuleImport.cpp @@ -55,6 +56,7 @@ add_mlir_translation_library(MLIRToLLVMIRTranslationRegistration add_mlir_translation_library(MLIRTargetLLVMIRImport DebugImporter.cpp + LoopAnnotationImporter.cpp ModuleImport.cpp TypeFromLLVM.cpp diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp index da15cd9b6624..3d438326b893 100644 --- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp @@ -72,7 +72,7 @@ static LogicalResult convertIntrinsicImpl(OpBuilder &odsBuilder, static ArrayRef getSupportedMetadataImpl() { static const SmallVector convertibleMetadata = { llvm::LLVMContext::MD_prof, llvm::LLVMContext::MD_tbaa, - llvm::LLVMContext::MD_access_group}; + llvm::LLVMContext::MD_access_group, llvm::LLVMContext::MD_loop}; return convertibleMetadata; } @@ -141,28 +141,38 @@ static LogicalResult setTBAAAttr(const llvm::MDNode *node, Operation *op, return success(); } -/// Searches the symbol references pointing to the access group operations that -/// map to the access group nodes starting from the access group metadata +/// Looks up all the symbol references pointing to the access group operations +/// that map to the access group nodes starting from the access group metadata /// `node`, and attaches all of them to the imported operation if the lookups /// succeed. Returns failure otherwise. static LogicalResult setAccessGroupAttr(const llvm::MDNode *node, Operation *op, LLVM::ModuleImport &moduleImport) { - // An access group node is either access group or an access group list. - SmallVector accessGroups; - if (!node->getNumOperands()) - accessGroups.push_back(moduleImport.lookupAccessGroupAttr(node)); - for (const llvm::MDOperand &operand : node->operands()) { - auto *node = cast(operand.get()); - accessGroups.push_back(moduleImport.lookupAccessGroupAttr(node)); - } - // Exit if one of the access group node lookups failed. - if (llvm::is_contained(accessGroups, nullptr)) + FailureOr> accessGroups = + moduleImport.lookupAccessGroupAttrs(node); + if (failed(accessGroups)) return failure(); + SmallVector accessGroupAttrs(accessGroups->begin(), + accessGroups->end()); op->setAttr(LLVMDialect::getAccessGroupsAttrName(), - ArrayAttr::get(op->getContext(), accessGroups)); + ArrayAttr::get(op->getContext(), accessGroupAttrs)); return success(); } + +/// Converts the given loop metadata node to an MLIR loop annotation attribute +/// and attaches it to the imported operation if the translation succeeds. +/// Returns failure otherwise. +static LogicalResult setLoopAttr(const llvm::MDNode *node, Operation *op, + LLVM::ModuleImport &moduleImport) { + LoopAnnotationAttr attr = + moduleImport.translateLoopAnnotationAttr(node, op->getLoc()); + if (!attr) + return failure(); + + op->setAttr(LLVMDialect::getLoopAttrName(), attr); + return success(); +} + namespace { /// Implementation of the dialect interface that converts operations belonging @@ -191,6 +201,8 @@ class LLVMDialectLLVMIRImportInterface : public LLVMImportDialectInterface { return setTBAAAttr(node, op, moduleImport); if (kind == llvm::LLVMContext::MD_access_group) return setAccessGroupAttr(node, op, moduleImport); + if (kind == llvm::LLVMContext::MD_loop) + return setLoopAttr(node, op, moduleImport); // A handler for a supported metadata kind is missing. llvm_unreachable("unknown metadata type"); diff --git a/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp b/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp new file mode 100644 index 000000000000..a3218e13307e --- /dev/null +++ b/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp @@ -0,0 +1,417 @@ +//===- LoopAnnotationImporter.cpp - Loop annotation import ----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "LoopAnnotationImporter.h" +#include "llvm/IR/Constants.h" + +using namespace mlir; +using namespace mlir::LLVM; +using namespace mlir::LLVM::detail; + +namespace { +/// Helper class that keeps the state of one metadata to attribute conversion. +struct LoopMetadataConversion { + LoopMetadataConversion(const llvm::MDNode *node, ModuleImport &moduleImport, + Location loc, + LoopAnnotationImporter &loopAnnotationImporter) + : node(node), moduleImport(moduleImport), loc(loc), + loopAnnotationImporter(loopAnnotationImporter), + ctx(loc->getContext()){}; + /// Converts this structs loop metadata node into a LoopAnnotationAttr. + LoopAnnotationAttr convert(); + + LogicalResult initPropertyMap(); + + /// Helper function to get and erase a property. + const llvm::MDNode *lookupAndEraseProperty(StringRef name); + + /// Helper functions to lookup and convert MDNodes into a specifc attribute + /// kind. These functions return null-attributes if there is no node with the + /// specified name, or failure, if the node is ill-formatted. + FailureOr lookupUnitNode(StringRef name); + FailureOr lookupBoolNode(StringRef name, bool negated = false); + FailureOr lookupIntNode(StringRef name); + FailureOr lookupMDNode(StringRef name); + FailureOr> lookupMDNodes(StringRef name); + FailureOr lookupFollowupNode(StringRef name); + FailureOr lookupBooleanUnitNode(StringRef enableName, + StringRef disableName, + bool negated = false); + + /// Conversion functions for sub-attributes. + FailureOr convertVectorizeAttr(); + FailureOr convertInterleaveAttr(); + FailureOr convertUnrollAttr(); + FailureOr convertUnrollAndJamAttr(); + FailureOr convertLICMAttr(); + FailureOr convertDistributeAttr(); + FailureOr convertPipelineAttr(); + FailureOr> convertParallelAccesses(); + + llvm::StringMap propertyMap; + const llvm::MDNode *node; + ModuleImport &moduleImport; + Location loc; + LoopAnnotationImporter &loopAnnotationImporter; + MLIRContext *ctx; +}; +} // namespace + +LogicalResult LoopMetadataConversion::initPropertyMap() { + // Check if it's a valid node. + if (node->getNumOperands() == 0 || + dyn_cast(node->getOperand(0)) != node) + return emitWarning(loc) << "invalid loop node"; + + for (const llvm::MDOperand &operand : llvm::drop_begin(node->operands())) { + // Skip over DILocations. + if (isa(operand)) + continue; + + auto *property = dyn_cast(operand); + if (!property) + return emitWarning(loc) << "expected all loop properties to be either " + "debug locations or metadata nodes"; + + if (property->getNumOperands() == 0) + return emitWarning(loc) << "cannot import empty loop property"; + + auto *nameNode = dyn_cast(property->getOperand(0)); + if (!nameNode) + return emitWarning(loc) << "cannot import loop property without a name"; + StringRef name = nameNode->getString(); + + bool succ = propertyMap.try_emplace(name, property).second; + if (!succ) + return emitWarning(loc) + << "cannot import loop properties with duplicated names " << name; + } + + return success(); +} + +const llvm::MDNode * +LoopMetadataConversion::lookupAndEraseProperty(StringRef name) { + auto it = propertyMap.find(name); + if (it == propertyMap.end()) + return nullptr; + const llvm::MDNode *property = it->getValue(); + propertyMap.erase(it); + return property; +} + +FailureOr LoopMetadataConversion::lookupUnitNode(StringRef name) { + const llvm::MDNode *property = lookupAndEraseProperty(name); + if (!property) + return BoolAttr(nullptr); + + if (property->getNumOperands() != 1) + return emitWarning(loc) + << "expected metadata node " << name << " to hold no value"; + + return BoolAttr::get(ctx, true); +} + +FailureOr LoopMetadataConversion::lookupBooleanUnitNode( + StringRef enableName, StringRef disableName, bool negated) { + auto enable = lookupUnitNode(enableName); + auto disable = lookupUnitNode(disableName); + if (failed(enable) || failed(disable)) + return failure(); + + if (*enable && *disable) + return emitWarning(loc) + << "expected metadata nodes " << enableName << " and " << disableName + << " to be mutually exclusive."; + + if (*enable) + return BoolAttr::get(ctx, !negated); + + if (*disable) + return BoolAttr::get(ctx, negated); + return BoolAttr(nullptr); +} + +FailureOr LoopMetadataConversion::lookupBoolNode(StringRef name, + bool negated) { + const llvm::MDNode *property = lookupAndEraseProperty(name); + if (!property) + return BoolAttr(nullptr); + + auto emitNodeWarning = [&]() { + return emitWarning(loc) + << "expected metadata node " << name << " to hold a boolean value"; + }; + + if (property->getNumOperands() != 2) + return emitNodeWarning(); + llvm::ConstantInt *val = + llvm::mdconst::dyn_extract(property->getOperand(1)); + if (!val || val->getBitWidth() != 1) + return emitNodeWarning(); + + return BoolAttr::get(ctx, val->getValue().getLimitedValue(1) ^ negated); +} + +FailureOr LoopMetadataConversion::lookupIntNode(StringRef name) { + const llvm::MDNode *property = lookupAndEraseProperty(name); + if (!property) + return IntegerAttr(nullptr); + + auto emitNodeWarning = [&]() { + return emitWarning(loc) + << "expected metadata node " << name << " to hold an i32 value"; + }; + + if (property->getNumOperands() != 2) + return emitNodeWarning(); + + llvm::ConstantInt *val = + llvm::mdconst::dyn_extract(property->getOperand(1)); + if (!val || val->getBitWidth() != 32) + return emitNodeWarning(); + + return IntegerAttr::get(IntegerType::get(ctx, 32), + val->getValue().getLimitedValue()); +} + +FailureOr LoopMetadataConversion::lookupMDNode(StringRef name) { + const llvm::MDNode *property = lookupAndEraseProperty(name); + if (!property) + return nullptr; + + auto emitNodeWarning = [&]() { + return emitWarning(loc) + << "expected metadata node " << name << " to hold an MDNode"; + }; + + if (property->getNumOperands() != 2) + return emitNodeWarning(); + + auto *node = dyn_cast(property->getOperand(1)); + if (!node) + return emitNodeWarning(); + + return node; +} + +FailureOr> +LoopMetadataConversion::lookupMDNodes(StringRef name) { + const llvm::MDNode *property = lookupAndEraseProperty(name); + SmallVector res; + if (!property) + return res; + + auto emitNodeWarning = [&]() { + return emitWarning(loc) << "expected metadata node " << name + << " to hold one or multiple MDNodes"; + }; + + if (property->getNumOperands() < 2) + return emitNodeWarning(); + + for (unsigned i = 1, e = property->getNumOperands(); i < e; ++i) { + auto *node = dyn_cast(property->getOperand(i)); + if (!node) + return emitNodeWarning(); + res.push_back(node); + } + + return res; +} + +FailureOr +LoopMetadataConversion::lookupFollowupNode(StringRef name) { + auto node = lookupMDNode(name); + if (failed(node)) + return failure(); + if (*node == nullptr) + return LoopAnnotationAttr(nullptr); + + return loopAnnotationImporter.translate(*node, loc); +} + +static bool isEmptyOrNull(const Attribute attr) { return !attr; } + +template +static bool isEmptyOrNull(const SmallVectorImpl &vec) { + return vec.empty(); +} + +/// Helper function that only creates and attribute of type T if all argument +/// conversion were successfull and at least one of them holds a non-null value. +template +static T createIfNonNull(MLIRContext *ctx, const P &...args) { + bool anyFailed = (failed(args) || ...); + if (anyFailed) + return {}; + + bool allEmpty = (isEmptyOrNull(*args) && ...); + if (allEmpty) + return {}; + + return T::get(ctx, *args...); +} + +FailureOr LoopMetadataConversion::convertVectorizeAttr() { + FailureOr enable = + lookupBoolNode("llvm.loop.vectorize.enable", true); + FailureOr predicateEnable = + lookupBoolNode("llvm.loop.vectorize.predicate.enable"); + FailureOr scalableEnable = + lookupBoolNode("llvm.loop.vectorize.scalable.enable"); + FailureOr width = lookupIntNode("llvm.loop.vectorize.width"); + FailureOr followupVec = + lookupFollowupNode("llvm.loop.vectorize.followup_vectorized"); + FailureOr followupEpi = + lookupFollowupNode("llvm.loop.vectorize.followup_epilogue"); + FailureOr followupAll = + lookupFollowupNode("llvm.loop.vectorize.followup_all"); + + return createIfNonNull(ctx, enable, predicateEnable, + scalableEnable, width, followupVec, + followupEpi, followupAll); +} + +FailureOr LoopMetadataConversion::convertInterleaveAttr() { + FailureOr count = lookupIntNode("llvm.loop.interleave.count"); + return createIfNonNull(ctx, count); +} + +FailureOr LoopMetadataConversion::convertUnrollAttr() { + FailureOr disable = lookupBooleanUnitNode( + "llvm.loop.unroll.enable", "llvm.loop.unroll.disable", /*negated=*/true); + FailureOr count = lookupIntNode("llvm.loop.unroll.count"); + FailureOr runtimeDisable = + lookupUnitNode("llvm.loop.unroll.runtime.disable"); + FailureOr full = lookupUnitNode("llvm.loop.unroll.full"); + FailureOr followup = + lookupFollowupNode("llvm.loop.unroll.followup"); + FailureOr followupRemainder = + lookupFollowupNode("llvm.loop.unroll.followup_remainder"); + + return createIfNonNull(ctx, disable, count, runtimeDisable, + full, followup, followupRemainder); +} + +FailureOr +LoopMetadataConversion::convertUnrollAndJamAttr() { + FailureOr disable = lookupBooleanUnitNode( + "llvm.loop.unroll_and_jam.enable", "llvm.loop.unroll_and_jam.disable", + /*negated=*/true); + FailureOr count = + lookupIntNode("llvm.loop.unroll_and_jam.count"); + FailureOr followupOuter = + lookupFollowupNode("llvm.loop.unroll_and_jam.followup_outer"); + FailureOr followupInner = + lookupFollowupNode("llvm.loop.unroll_and_jam.followup_inner"); + FailureOr followupRemainderOuter = + lookupFollowupNode("llvm.loop.unroll_and_jam.followup_remainder_outer"); + FailureOr followupRemainderInner = + lookupFollowupNode("llvm.loop.unroll_and_jam.followup_remainder_inner"); + FailureOr followupAll = + lookupFollowupNode("llvm.loop.unroll_and_jam.followup_all"); + return createIfNonNull( + ctx, disable, count, followupOuter, followupInner, followupRemainderOuter, + followupRemainderInner, followupAll); +} + +FailureOr LoopMetadataConversion::convertLICMAttr() { + FailureOr disable = lookupUnitNode("llvm.licm.disable"); + FailureOr versioningDisable = + lookupUnitNode("llvm.loop.licm_versioning.disable"); + return createIfNonNull(ctx, disable, versioningDisable); +} + +FailureOr LoopMetadataConversion::convertDistributeAttr() { + FailureOr disable = + lookupBoolNode("llvm.loop.distribute.enable", true); + FailureOr followupCoincident = + lookupFollowupNode("llvm.loop.distribute.followup_coincident"); + FailureOr followupSequential = + lookupFollowupNode("llvm.loop.distribute.followup_sequential"); + FailureOr followupFallback = + lookupFollowupNode("llvm.loop.distribute.followup_fallback"); + FailureOr followupAll = + lookupFollowupNode("llvm.loop.distribute.followup_all"); + return createIfNonNull(ctx, disable, followupCoincident, + followupSequential, + followupFallback, followupAll); +} + +FailureOr LoopMetadataConversion::convertPipelineAttr() { + FailureOr disable = lookupBoolNode("llvm.loop.pipeline.disable"); + FailureOr initiationinterval = + lookupIntNode("llvm.loop.pipeline.initiationinterval"); + return createIfNonNull(ctx, disable, initiationinterval); +} + +FailureOr> +LoopMetadataConversion::convertParallelAccesses() { + FailureOr> nodes = + lookupMDNodes("llvm.loop.parallel_accesses"); + if (failed(nodes)) + return failure(); + SmallVector refs; + for (llvm::MDNode *node : *nodes) { + FailureOr> accessGroups = + moduleImport.lookupAccessGroupAttrs(node); + if (failed(accessGroups)) + return emitWarning(loc) << "could not lookup access group"; + llvm::append_range(refs, *accessGroups); + } + return refs; +} + +LoopAnnotationAttr LoopMetadataConversion::convert() { + if (failed(initPropertyMap())) + return {}; + + FailureOr disableNonForced = + lookupUnitNode("llvm.loop.disable_nonforced"); + FailureOr vecAttr = convertVectorizeAttr(); + FailureOr interleaveAttr = convertInterleaveAttr(); + FailureOr unrollAttr = convertUnrollAttr(); + FailureOr unrollAndJamAttr = convertUnrollAndJamAttr(); + FailureOr licmAttr = convertLICMAttr(); + FailureOr distributeAttr = convertDistributeAttr(); + FailureOr pipelineAttr = convertPipelineAttr(); + FailureOr mustProgress = lookupUnitNode("llvm.loop.mustprogress"); + FailureOr> parallelAccesses = + convertParallelAccesses(); + + // Drop the metadata if there are parts that cannot be imported. + if (!propertyMap.empty()) { + for (auto name : propertyMap.keys()) + emitWarning(loc) << "unknown loop annotation " << name; + return {}; + } + + return createIfNonNull( + ctx, disableNonForced, vecAttr, interleaveAttr, unrollAttr, + unrollAndJamAttr, licmAttr, distributeAttr, pipelineAttr, mustProgress, + parallelAccesses); +} + +LoopAnnotationAttr LoopAnnotationImporter::translate(const llvm::MDNode *node, + Location loc) { + if (!node) + return {}; + + // Note: This check is necessary to distinguish between failed translations + // and not yet attempted translations. + auto it = loopMetadataMapping.find(node); + if (it != loopMetadataMapping.end()) + return it->getSecond(); + + LoopAnnotationAttr attr = + LoopMetadataConversion(node, moduleImport, loc, *this).convert(); + + mapLoopMetadata(node, attr); + return attr; +} diff --git a/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.h b/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.h new file mode 100644 index 000000000000..bd6f5ef350e6 --- /dev/null +++ b/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.h @@ -0,0 +1,53 @@ +//===- LoopAnnotationImporter.h ---------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the translation between LLVMIR loop metadata and the +// corresponding MLIR representation. +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_LIB_TARGET_LLVMIR_LOOPANNOTATIONIMPORTER_H_ +#define MLIR_LIB_TARGET_LLVMIR_LOOPANNOTATIONIMPORTER_H_ + +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/Target/LLVMIR/ModuleImport.h" + +namespace mlir { +namespace LLVM { +namespace detail { + +/// A helper class that converts a `llvm.loop` metadata node into a +/// corresponding LoopAnnotationAttr. +class LoopAnnotationImporter { +public: + explicit LoopAnnotationImporter(ModuleImport &moduleImport) + : moduleImport(moduleImport) {} + LoopAnnotationAttr translate(const llvm::MDNode *node, Location loc); + +private: + /// Returns the LLVM metadata corresponding to a llvm loop metadata attribute. + LoopAnnotationAttr lookupLoopMetadata(const llvm::MDNode *node) const { + return loopMetadataMapping.lookup(node); + } + + void mapLoopMetadata(const llvm::MDNode *metadata, LoopAnnotationAttr attr) { + auto result = loopMetadataMapping.try_emplace(metadata, attr); + (void)result; + assert(result.second && + "attempting to map loop options that was already mapped"); + } + + ModuleImport &moduleImport; + DenseMap loopMetadataMapping; +}; + +} // namespace detail +} // namespace LLVM +} // namespace mlir + +#endif // MLIR_LIB_TARGET_LLVMIR_LOOPANNOTATIONIMPORTER_H_ diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp index 4d70e8536f40..341a8473da4c 100644 --- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp @@ -16,6 +16,7 @@ #include "AttrKindDetail.h" #include "DebugImporter.h" +#include "LoopAnnotationImporter.h" #include "mlir/Dialect/DLTI/DLTI.h" #include "mlir/Dialect/LLVMIR/LLVMDialect.h" @@ -241,7 +242,8 @@ ModuleImport::ModuleImport(ModuleOp mlirModule, mlirModule(mlirModule), llvmModule(std::move(llvmModule)), iface(mlirModule->getContext()), typeTranslator(*mlirModule->getContext()), - debugImporter(std::make_unique(mlirModule)) { + debugImporter(std::make_unique(mlirModule)), + loopAnnotationImporter(std::make_unique(*this)) { builder.setInsertionPointToStart(mlirModule.getBody()); } @@ -1566,6 +1568,29 @@ LogicalResult ModuleImport::processBasicBlock(llvm::BasicBlock *bb, return success(); } +FailureOr> +ModuleImport::lookupAccessGroupAttrs(const llvm::MDNode *node) const { + // An access group node is either a single access group or an access group + // list. + SmallVector accessGroups; + if (!node->getNumOperands()) + accessGroups.push_back(accessGroupMapping.lookup(node)); + for (const llvm::MDOperand &operand : node->operands()) { + auto *node = cast(operand.get()); + accessGroups.push_back(accessGroupMapping.lookup(node)); + } + // Exit if one of the access group node lookups failed. + if (llvm::is_contained(accessGroups, nullptr)) + return failure(); + return accessGroups; +} + +LoopAnnotationAttr +ModuleImport::translateLoopAnnotationAttr(const llvm::MDNode *node, + Location loc) const { + return loopAnnotationImporter->translate(node, loc); +} + OwningOpRef mlir::translateLLVMIRToModule(std::unique_ptr llvmModule, MLIRContext *context) { diff --git a/mlir/test/Target/LLVMIR/Import/import-failure.ll b/mlir/test/Target/LLVMIR/Import/import-failure.ll index 107d886d697b..acce75b4fcb0 100644 --- a/mlir/test/Target/LLVMIR/Import/import-failure.ll +++ b/mlir/test/Target/LLVMIR/Import/import-failure.ll @@ -262,3 +262,180 @@ define void @access_group(ptr %arg1) { !0 = !{!1} !1 = distinct !{!"unsupported access group"} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: warning: expected all loop properties to be either debug locations or metadata nodes +; CHECK: import-failure.ll +; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, i32 42} +define void @invalid_loop_node(i64 %n, ptr %A) { +entry: + br label %end, !llvm.loop !0 +end: + ret void +} + +!0 = distinct !{!0, i32 42} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: warning: cannot import empty loop property +; CHECK: import-failure.ll +; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1} +define void @invalid_loop_node(i64 %n, ptr %A) { +entry: + br label %end, !llvm.loop !0 +end: + ret void +} + +!0 = distinct !{!0, !1} +!1 = distinct !{} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: warning: cannot import loop property without a name +; CHECK: import-failure.ll +; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1} +define void @invalid_loop_node(i64 %n, ptr %A) { +entry: + br label %end, !llvm.loop !0 +end: + ret void +} + +!0 = distinct !{!0, !1} +!1 = distinct !{i1 0} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: warning: cannot import loop properties with duplicated names llvm.loop.disable_nonforced +; CHECK: import-failure.ll +; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1, !1} +define void @unsupported_loop_annotation(i64 %n, ptr %A) { +entry: + br label %end, !llvm.loop !0 +end: + ret void +} + +!0 = distinct !{!0, !1, !1} +!1 = !{!"llvm.loop.disable_nonforced"} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: warning: expected metadata node llvm.loop.disable_nonforced to hold no value +; CHECK: import-failure.ll +; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1} +define void @unsupported_loop_annotation(i64 %n, ptr %A) { +entry: + br label %end, !llvm.loop !0 +end: + ret void +} + +!0 = distinct !{!0, !1} +!1 = !{!"llvm.loop.disable_nonforced", i1 0} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: warning: expected metadata nodes llvm.loop.unroll.enable and llvm.loop.unroll.disable to be mutually exclusive +; CHECK: import-failure.ll +; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1, !2} +define void @unsupported_loop_annotation(i64 %n, ptr %A) { +entry: + br label %end, !llvm.loop !0 +end: + ret void +} + +!0 = distinct !{!0, !1, !2} +!1 = !{!"llvm.loop.unroll.enable"} +!2 = !{!"llvm.loop.unroll.disable"} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: warning: expected metadata node llvm.loop.vectorize.enable to hold a boolean value +; CHECK: import-failure.ll +; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1} +define void @unsupported_loop_annotation(i64 %n, ptr %A) { +entry: + br label %end, !llvm.loop !0 +end: + ret void +} + +!0 = distinct !{!0, !1} +!1 = !{!"llvm.loop.vectorize.enable"} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: warning: expected metadata node llvm.loop.vectorize.width to hold an i32 value +; CHECK: import-failure.ll +; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1} +define void @unsupported_loop_annotation(i64 %n, ptr %A) { +entry: + br label %end, !llvm.loop !0 +end: + ret void +} + +!0 = distinct !{!0, !1} +!1 = !{!"llvm.loop.vectorize.width", !0} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: warning: expected metadata node llvm.loop.vectorize.followup_all to hold an MDNode +; CHECK: import-failure.ll +; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1} +define void @unsupported_loop_annotation(i64 %n, ptr %A) { +entry: + br label %end, !llvm.loop !0 +end: + ret void +} + +!0 = distinct !{!0, !1} +!1 = !{!"llvm.loop.vectorize.followup_all", i32 42} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: warning: expected metadata node llvm.loop.parallel_accesses to hold one or multiple MDNodes +; CHECK: import-failure.ll +; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1} +define void @unsupported_loop_annotation(i64 %n, ptr %A) { +entry: + br label %end, !llvm.loop !0 +end: + ret void +} + +!0 = distinct !{!0, !1} +!1 = !{!"llvm.loop.parallel_accesses", i32 42} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: warning: unknown loop annotation llvm.loop.typo +; CHECK: import-failure.ll +; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1, !2} +define void @unsupported_loop_annotation(i64 %n, ptr %A) { +entry: + br label %end, !llvm.loop !0 +end: + ret void +} + +!0 = distinct !{!0, !1, !2} +!1 = !{!"llvm.loop.disable_nonforced"} +!2 = !{!"llvm.loop.typo"} diff --git a/mlir/test/Target/LLVMIR/Import/metadata-loop.ll b/mlir/test/Target/LLVMIR/Import/metadata-loop.ll index 87ac08ecd7e8..93d29b0edea5 100644 --- a/mlir/test/Target/LLVMIR/Import/metadata-loop.ll +++ b/mlir/test/Target/LLVMIR/Import/metadata-loop.ll @@ -25,3 +25,240 @@ define void @access_group(ptr %arg1) { !3 = distinct !{} !4 = distinct !{} !5 = distinct !{} + +; // ----- + +; CHECK: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation + +; CHECK-LABEL: @simple +define void @simple(i64 %n, ptr %A) { +entry: +; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]} + br label %end, !llvm.loop !1 +end: + ret void +} + +!1 = distinct !{!1, !2, !3} +!2 = !{!"llvm.loop.disable_nonforced"} +!3 = !{!"llvm.loop.mustprogress"} + +; // ----- + +; CHECK-DAG: #[[FOLLOWUP:.*]] = #llvm.loop_annotation +; CHECK-DAG: #[[VECTORIZE_ATTR:.*]] = #llvm.loop_vectorize +; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation + +; CHECK-LABEL: @vectorize +define void @vectorize(i64 %n, ptr %A) { +entry: +; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]} + br label %end, !llvm.loop !1 +end: + ret void +} + +!1 = distinct !{!1, !2, !3, !4, !5, !6, !7, !8} +!2 = !{!"llvm.loop.vectorize.enable", i1 1} +!3 = !{!"llvm.loop.vectorize.predicate.enable", i1 1} +!4 = !{!"llvm.loop.vectorize.scalable.enable", i1 0} +!5 = !{!"llvm.loop.vectorize.width", i32 16} +!6 = !{!"llvm.loop.vectorize.followup_vectorized", !9} +!7 = !{!"llvm.loop.vectorize.followup_epilogue", !9} +!8 = !{!"llvm.loop.vectorize.followup_all", !9} + +!9 = distinct !{!9, !10} +!10 = !{!"llvm.loop.disable_nonforced"} + +; // ----- + +; CHECK-DAG: #[[INTERLEAVE_ATTR:.*]] = #llvm.loop_interleave +; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation + +; CHECK-LABEL: @interleave +define void @interleave(i64 %n, ptr %A) { +entry: +; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]} + br label %end, !llvm.loop !1 +end: + ret void +} + +!1 = distinct !{!1, !2} +!2 = !{!"llvm.loop.interleave.count", i32 8} + +; // ----- + +; CHECK-DAG: #[[FOLLOWUP:.*]] = #llvm.loop_annotation +; CHECK-DAG: #[[UNROLL_ATTR:.*]] = #llvm.loop_unroll +; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation + +; CHECK-LABEL: @unroll +define void @unroll(i64 %n, ptr %A) { +entry: +; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]} + br label %end, !llvm.loop !1 +end: + ret void +} + +!1 = distinct !{!1, !2, !3, !4, !5, !6, !7} +!2 = !{!"llvm.loop.unroll.enable"} +!3 = !{!"llvm.loop.unroll.count", i32 16} +!4 = !{!"llvm.loop.unroll.runtime.disable"} +!5 = !{!"llvm.loop.unroll.full"} +!6 = !{!"llvm.loop.unroll.followup", !8} +!7 = !{!"llvm.loop.unroll.followup_remainder", !8} + +!8 = distinct !{!8, !9} +!9 = !{!"llvm.loop.disable_nonforced"} + +; // ----- + +; CHECK-DAG: #[[UNROLL_ATTR:.*]] = #llvm.loop_unroll +; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation + +; CHECK-LABEL: @unroll_disable +define void @unroll_disable(i64 %n, ptr %A) { +entry: +; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]} + br label %end, !llvm.loop !1 +end: + ret void +} + +!1 = distinct !{!1, !2} +!2 = !{!"llvm.loop.unroll.disable"} + +; // ----- + +; CHECK-DAG: #[[FOLLOWUP:.*]] = #llvm.loop_annotation +; CHECK-DAG: #[[UNROLL_AND_JAM_ATTR:.*]] = #llvm.loop_unroll_and_jam +; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation + +; CHECK-LABEL: @unroll_and_jam +define void @unroll_and_jam(i64 %n, ptr %A) { +entry: +; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]} + br label %end, !llvm.loop !1 +end: + ret void +} + +!1 = distinct !{!1, !2, !3, !4, !5, !6, !7, !8} +!2 = !{!"llvm.loop.unroll_and_jam.enable"} +!3 = !{!"llvm.loop.unroll_and_jam.count", i32 32} +!4 = !{!"llvm.loop.unroll_and_jam.followup_outer", !9} +!5 = !{!"llvm.loop.unroll_and_jam.followup_inner", !9} +!6 = !{!"llvm.loop.unroll_and_jam.followup_remainder_outer", !9} +!7 = !{!"llvm.loop.unroll_and_jam.followup_remainder_inner", !9} +!8 = !{!"llvm.loop.unroll_and_jam.followup_all", !9} + +!9 = distinct !{!9, !10} +!10 = !{!"llvm.loop.disable_nonforced"} + +; // ----- + +; CHECK-DAG: #[[LICM_ATTR:.*]] = #llvm.loop_licm +; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation + +; CHECK-LABEL: @licm +define void @licm(i64 %n, ptr %A) { +entry: +; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]} + br label %end, !llvm.loop !1 +end: + ret void +} + +!1 = distinct !{!1, !2, !3} +!2 = !{!"llvm.licm.disable"} +!3 = !{!"llvm.loop.licm_versioning.disable"} + +; // ----- + +; CHECK-DAG: #[[FOLLOWUP:.*]] = #llvm.loop_annotation +; CHECK-DAG: #[[DISTRIBUTE_ATTR:.*]] = #llvm.loop_distribute +; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation + +; CHECK-LABEL: @distribute +define void @distribute(i64 %n, ptr %A) { +entry: +; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]} + br label %end, !llvm.loop !1 +end: + ret void +} + +!1 = distinct !{!1, !2, !3, !4, !5, !6} +!2 = !{!"llvm.loop.distribute.enable", i1 0} +!3 = !{!"llvm.loop.distribute.followup_coincident", !9} +!4 = !{!"llvm.loop.distribute.followup_sequential", !9} +!5 = !{!"llvm.loop.distribute.followup_fallback", !9} +!6 = !{!"llvm.loop.distribute.followup_all", !9} + +!9 = distinct !{!9, !10} +!10 = !{!"llvm.loop.disable_nonforced"} + +; // ----- + +; CHECK-DAG: #[[PIPELINE_ATTR:.*]] = #llvm.loop_pipeline +; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation + +; CHECK-LABEL: @pipeline +define void @pipeline(i64 %n, ptr %A) { +entry: +; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]} + br label %end, !llvm.loop !1 +end: + ret void +} + +!1 = distinct !{!1, !2, !3} +!2 = !{!"llvm.loop.pipeline.disable", i1 0} +!3 = !{!"llvm.loop.pipeline.initiationinterval", i32 2} + +; // ----- + +; CHECK: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation + +; CHECK: llvm.metadata @__llvm_global_metadata { +; CHECK: llvm.access_group @[[GROUP0]] + +; CHECK-LABEL: @parallel_accesses +define void @parallel_accesses(ptr %arg) { +entry: + %0 = load i32, ptr %arg, !llvm.access.group !0 +; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]} + br label %end, !llvm.loop !1 +end: + ret void +} + +!0 = distinct !{} +!1 = distinct !{!1, !2} +!2 = !{!"llvm.loop.parallel_accesses", !0} + +; // ----- + +; CHECK: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation + +; CHECK: llvm.metadata @__llvm_global_metadata { +; CHECK: llvm.access_group @[[GROUP0]] +; CHECK: llvm.access_group @[[GROUP1]] + +; CHECK-LABEL: @multiple_parallel_accesses +define void @multiple_parallel_accesses(ptr %arg) { +entry: + %0 = load i32, ptr %arg, !llvm.access.group !0 + %1 = load i32, ptr %arg, !llvm.access.group !3 +; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]} + br label %end, !llvm.loop !1 +end: + ret void +} + +!0 = distinct !{} +!1 = distinct !{!1, !2} +!2 = !{!"llvm.loop.parallel_accesses", !0, !3} +!3 = distinct !{} From f9c30f6886b94a18b20a1b83c4f4b56872a416ca Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Mon, 6 Mar 2023 10:33:39 +0100 Subject: [PATCH 065/147] [mlir][llvm] Add MD_prof import error handling This commit adds additional checks and warning messages to the MD_prof import. As LLVM does not verify most metadata, the import has the be resilient towards ill-formatted inputs. Reviewed By: gysit Differential Revision: https://reviews.llvm.org/D143492 (cherry picked from commit c6ac7e9dd5999aec825a1eb540dc3e0fd35243ae) --- .../LLVMIR/LLVMIRToLLVMTranslation.cpp | 31 +++-- .../Target/LLVMIR/Import/import-failure.ll | 129 +++++++++++++++++- 2 files changed, 148 insertions(+), 12 deletions(-) diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp index 3d438326b893..207ef8b42809 100644 --- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp @@ -84,36 +84,48 @@ static LogicalResult setProfilingAttr(OpBuilder &builder, llvm::MDNode *node, LLVM::ModuleImport &moduleImport) { // Return success for empty metadata nodes since there is nothing to import. if (!node->getNumOperands()) - return success(); + return op->emitWarning() << "expected non-empty profiling metadata node"; auto *name = dyn_cast(node->getOperand(0)); if (!name) - return failure(); + return op->emitWarning() + << "expected profiling metadata node to have a string identifier"; // Handle function entry count metadata. if (name->getString().equals("function_entry_count")) { + auto emitNodeWarning = [&]() { + return op->emitWarning() + << "expected function_entry_count to hold a single i64 value"; + }; + // TODO support function entry count metadata with GUID fields. if (node->getNumOperands() != 2) - return failure(); + return emitNodeWarning(); llvm::ConstantInt *entryCount = - llvm::mdconst::extract(node->getOperand(1)); + llvm::mdconst::dyn_extract(node->getOperand(1)); + if (!entryCount) + return emitNodeWarning(); if (auto funcOp = dyn_cast(op)) { funcOp.setFunctionEntryCount(entryCount->getZExtValue()); return success(); } - return failure(); + return op->emitWarning() + << "expected function_entry_count to be attached to a function"; } if (!name->getString().equals("branch_weights")) - return failure(); + return op->emitWarning() + << "unknown profiling metadata node " << name->getString(); // Handle branch weights metadata. SmallVector branchWeights; branchWeights.reserve(node->getNumOperands() - 1); for (unsigned i = 1, e = node->getNumOperands(); i != e; ++i) { llvm::ConstantInt *branchWeight = - llvm::mdconst::extract(node->getOperand(i)); + llvm::mdconst::dyn_extract(node->getOperand(i)); + if (!branchWeight) + return op->emitWarning() << "expected branch weights to be integers"; branchWeights.push_back(branchWeight->getZExtValue()); } @@ -124,7 +136,10 @@ static LogicalResult setProfilingAttr(OpBuilder &builder, llvm::MDNode *node, builder.getI32VectorAttr(branchWeights)); return success(); }) - .Default([](auto) { return failure(); }); + .Default([op](auto) { + return op->emitWarning() + << op->getName() << " does not support branch weights"; + }); } /// Searches the symbol reference pointing to the metadata operation that diff --git a/mlir/test/Target/LLVMIR/Import/import-failure.ll b/mlir/test/Target/LLVMIR/Import/import-failure.ll index acce75b4fcb0..1db992736d99 100644 --- a/mlir/test/Target/LLVMIR/Import/import-failure.ll +++ b/mlir/test/Target/LLVMIR/Import/import-failure.ll @@ -56,8 +56,7 @@ define void @unhandled_intrinsic(ptr %arg1, ptr %arg2) { ; // ----- -; CHECK: import-failure.ll -; CHECK-SAME: warning: unhandled metadata: !0 = !{!"unknown metadata"} on br i1 %arg1, label %bb1, label %bb2, !prof !0 +; CHECK: warning: unhandled metadata: !0 = !{!"unknown metadata"} on br i1 %arg1, label %bb1, label %bb2, !prof !0 define i64 @unhandled_metadata(i1 %arg1, i64 %arg2) { entry: br i1 %arg1, label %bb1, label %bb2, !prof !0 @@ -71,8 +70,7 @@ bb2: ; // ----- -; CHECK: import-failure.ll -; CHECK-SAME: warning: unhandled function metadata: !0 = !{!"unknown metadata"} on define void @unhandled_func_metadata(i1 %arg1, i64 %arg2) !prof !0 +; CHECK: warning: unhandled function metadata: !0 = !{!"unknown metadata"} on define void @unhandled_func_metadata(i1 %arg1, i64 %arg2) !prof !0 define void @unhandled_func_metadata(i1 %arg1, i64 %arg2) !prof !0 { ret void } @@ -439,3 +437,126 @@ end: !0 = distinct !{!0, !1, !2} !1 = !{!"llvm.loop.disable_nonforced"} !2 = !{!"llvm.loop.typo"} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: warning: expected non-empty profiling metadata node +; CHECK: warning: unhandled metadata: !0 = !{} +define void @cond_br(i1 %arg) { +entry: + br i1 %arg, label %bb1, label %bb2, !prof !0 +bb1: + ret void +bb2: + ret void +} + +!0 = !{} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: warning: expected profiling metadata node to have a string identifier +; CHECK: import-failure.ll:{{.*}} warning: unhandled metadata: !0 = !{i32 64} +define void @cond_br(i1 %arg) { +entry: + br i1 %arg, label %bb1, label %bb2, !prof !0 +bb1: + ret void +bb2: + ret void +} + +!0 = !{i32 64} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: warning: expected function_entry_count to hold a single i64 value +; CHECK: warning: unhandled function metadata: !0 = !{!"function_entry_count"} +define void @cond_br(i1 %arg) !prof !0 { +entry: + br i1 %arg, label %bb1, label %bb2 +bb1: + ret void +bb2: + ret void +} + +!0 = !{!"function_entry_count"} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: warning: expected function_entry_count to hold a single i64 value +; CHECK: warning: unhandled function metadata: !0 = !{!"function_entry_count", !"string"} +define void @cond_br(i1 %arg) !prof !0 { +entry: + br i1 %arg, label %bb1, label %bb2 +bb1: + ret void +bb2: + ret void +} + +!0 = !{!"function_entry_count", !"string"} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: warning: expected function_entry_count to be attached to a function +; CHECK: warning: unhandled metadata: !0 = !{!"function_entry_count", i64 42} +define void @cond_br(i1 %arg) { +entry: + br i1 %arg, label %bb1, label %bb2, !prof !0 +bb1: + ret void +bb2: + ret void +} + +!0 = !{!"function_entry_count", i64 42} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: warning: unknown profiling metadata node unknown_prof_type +; CHECK: warning: unhandled metadata: !0 = !{!"unknown_prof_type"} +define void @cond_br(i1 %arg) { +entry: + br i1 %arg, label %bb1, label %bb2, !prof !0 +bb1: + ret void +bb2: + ret void +} + +!0 = !{!"unknown_prof_type"} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: warning: expected branch weights to be integers +; CHECK: warning: unhandled metadata: !0 = !{!"branch_weights", !"foo"} +define void @cond_br(i1 %arg) { +entry: + br i1 %arg, label %bb1, label %bb2, !prof !0 +bb1: + ret void +bb2: + ret void +} + +!0 = !{!"branch_weights", !"foo"} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: warning: llvm.func does not support branch weights +; CHECK: import-failure.ll:{{.*}} warning: unhandled function metadata: !0 = !{!"branch_weights", i32 64} +define void @cond_br(i1 %arg) !prof !0 { + ret void +} + +!0 = !{!"branch_weights", i32 64} From 53d342ea3f6bc26d5719f92404b99959eca9c4ec Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Mon, 6 Mar 2023 10:34:39 +0100 Subject: [PATCH 066/147] [mlir][llvm] Purge struct_attr This commit removes the `llvm.struct_attr` which was used to bundle result attributes that were previously attached to multiple results. This extension isn't part of LLVM as result attribute semantics cannot be supported on a struct field granularity. Furthermore, many usages promoted result attributes to argument attributes but this does not necessary preserve the semantics. Reviewed By: gysit Differential Revision: https://reviews.llvm.org/D143473 (cherry picked from commit 851a89715cadf411bd66a7cbb9d7567580f08f4c) --- .../include/mlir/Dialect/LLVMIR/LLVMOpBase.td | 5 -- mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp | 86 +++++++------------ mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 50 ----------- .../emit-c-wrappers-for-external-callers.mlir | 43 +++++++--- ...mit-c-wrappers-for-external-functions.mlir | 42 ++++++--- mlir/test/Dialect/LLVMIR/invalid.mlir | 63 -------------- 6 files changed, 93 insertions(+), 196 deletions(-) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td index b51f974cdc9c..b2ca396ab72a 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td @@ -40,7 +40,6 @@ def LLVM_Dialect : Dialect { static StringRef getAliasScopesAttrName() { return "alias_scopes"; } static StringRef getLoopAttrName() { return "llvm.loop"; } static StringRef getAccessGroupsAttrName() { return "access_groups"; } - static StringRef getStructAttrsAttrName() { return "llvm.struct_attrs"; } static StringRef getTBAAAttrName() { return "llvm.tbaa"; } /// Names of llvm parameter attributes. @@ -73,10 +72,6 @@ def LLVM_Dialect : Dialect { /// effect when lowering to the LLVMDialect. static StringRef getReadnoneAttrName() { return "llvm.readnone"; } - /// Verifies if the attribute is a well-formed value for "llvm.struct_attrs" - static LogicalResult verifyStructAttr( - Operation *op, Attribute attr, Type annotatedType); - /// Verifies if the given string is a well-formed data layout descriptor. /// Uses `reportError` to report errors. static LogicalResult verifyDataLayoutString( diff --git a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp index 6ff52566dee3..f70cb469f3f3 100644 --- a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp +++ b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp @@ -78,54 +78,26 @@ static void filterFuncAttributes(func::FuncOp func, bool filterArgAndResAttrs, } } -/// Helper function for wrapping all attributes into a single DictionaryAttr -static auto wrapAsStructAttrs(OpBuilder &b, ArrayAttr attrs) { - return DictionaryAttr::get( - b.getContext(), - b.getNamedAttr(LLVM::LLVMDialect::getStructAttrsAttrName(), attrs)); -} +/// Adds a an empty set of argument attributes for the newly added argument in +/// front of the existing ones. +static void prependEmptyArgAttr(OpBuilder &builder, + SmallVectorImpl &newFuncAttrs, + func::FuncOp func) { + auto argAttrs = func.getArgAttrs(); + // Nothing to do when there were no arg attrs beforehand. + if (!argAttrs) + return; -/// Combines all result attributes into a single DictionaryAttr -/// and prepends to argument attrs. -/// This is intended to be used to format the attributes for a C wrapper -/// function when the result(s) is converted to the first function argument -/// (in the multiple return case, all returns get wrapped into a single -/// argument). The total number of argument attributes should be equal to -/// (number of function arguments) + 1. -static void -prependResAttrsToArgAttrs(OpBuilder &builder, - SmallVectorImpl &attributes, - func::FuncOp func) { size_t numArguments = func.getNumArguments(); - auto allAttrs = SmallVector( - numArguments + 1, DictionaryAttr::get(builder.getContext())); - NamedAttribute *argAttrs = nullptr; - for (auto *it = attributes.begin(); it != attributes.end();) { - if (it->getName() == func.getArgAttrsAttrName()) { - auto arrayAttrs = it->getValue().cast(); - assert(arrayAttrs.size() == numArguments && - "Number of arg attrs and args should match"); - std::copy(arrayAttrs.begin(), arrayAttrs.end(), allAttrs.begin() + 1); - argAttrs = it; - } else if (it->getName() == func.getResAttrsAttrName()) { - auto arrayAttrs = it->getValue().cast(); - assert(!arrayAttrs.empty() && "expected array to be non-empty"); - allAttrs[0] = (arrayAttrs.size() == 1) - ? arrayAttrs[0] - : wrapAsStructAttrs(builder, arrayAttrs); - it = attributes.erase(it); - continue; - } - it++; - } - - auto newArgAttrs = builder.getNamedAttr(func.getArgAttrsAttrName(), - builder.getArrayAttr(allAttrs)); - if (!argAttrs) { - attributes.emplace_back(newArgAttrs); - return; - } - *argAttrs = newArgAttrs; + SmallVector newArgAttrs; + newArgAttrs.reserve(numArguments + 1); + // Insert empty dictionary for the new argument. + newArgAttrs.push_back(builder.getDictionaryAttr({})); + + llvm::append_range(newArgAttrs, *argAttrs); + auto newNamedAttr = builder.getNamedAttr(func.getArgAttrsAttrName(), + builder.getArrayAttr(newArgAttrs)); + newFuncAttrs.push_back(newNamedAttr); } /// Creates an auxiliary function with pointer-to-memref-descriptor-struct @@ -141,12 +113,16 @@ static void wrapForExternalCallers(OpBuilder &rewriter, Location loc, func::FuncOp funcOp, LLVM::LLVMFuncOp newFuncOp) { auto type = funcOp.getFunctionType(); - SmallVector attributes; - filterFuncAttributes(funcOp, /*filterArgAndResAttrs=*/false, attributes); auto [wrapperFuncType, resultIsNowArg] = typeConverter.convertFunctionTypeCWrapper(type); + + SmallVector attributes; + // Only modify the argument and result attributes when the result is now an + // argument. if (resultIsNowArg) - prependResAttrsToArgAttrs(rewriter, attributes, funcOp); + prependEmptyArgAttr(rewriter, attributes, funcOp); + filterFuncAttributes(funcOp, /*filterArgAndResAttrs=*/resultIsNowArg, + attributes); auto wrapperFuncOp = rewriter.create( loc, llvm::formatv("_mlir_ciface_{0}", funcOp.getName()).str(), wrapperFuncType, LLVM::Linkage::External, /*dsoLocal*/ false, @@ -207,10 +183,13 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc, assert(wrapperType && "unexpected type conversion failure"); SmallVector attributes; - filterFuncAttributes(funcOp, /*filterArgAndResAttrs=*/false, attributes); - + // Only modify the argument and result attributes when the result is now an + // argument. if (resultIsNowArg) - prependResAttrsToArgAttrs(builder, attributes, funcOp); + prependEmptyArgAttr(builder, attributes, funcOp); + filterFuncAttributes(funcOp, /*filterArgAndResAttrs=*/resultIsNowArg, + attributes); + // Create the auxiliary function. auto wrapperFunc = builder.create( loc, llvm::formatv("_mlir_ciface_{0}", funcOp.getName()).str(), @@ -312,8 +291,7 @@ struct FuncOpConversionBase : public ConvertOpToLLVMPattern { auto newResAttrDicts = (funcOp.getNumResults() == 1) ? resAttrDicts - : rewriter.getArrayAttr( - {wrapAsStructAttrs(rewriter, resAttrDicts)}); + : rewriter.getArrayAttr(rewriter.getDictionaryAttr({})); attributes.push_back( rewriter.getNamedAttr(funcOp.getResAttrsAttrName(), newResAttrDicts)); } diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp index 9c271833f890..e8ada51c7aef 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -2932,12 +2932,6 @@ LogicalResult LLVMDialect::verifyOperationAttribute(Operation *op, } } - if (attr.getName() == LLVMDialect::getStructAttrsAttrName()) { - return op->emitOpError() - << "'" << LLVM::LLVMDialect::getStructAttrsAttrName() - << "' is permitted only in argument or result attributes"; - } - // If the data layout attribute is present, it must use the LLVM data layout // syntax. Try parsing it and report errors in case of failure. Users of this // attribute may assume it is well-formed and can pass it to the (asserting) @@ -2954,46 +2948,6 @@ LogicalResult LLVMDialect::verifyOperationAttribute(Operation *op, << "' to be a string attributes"; } -LogicalResult LLVMDialect::verifyStructAttr(Operation *op, Attribute attr, - Type annotatedType) { - auto structType = annotatedType.dyn_cast(); - if (!structType) { - const auto emitIncorrectAnnotatedType = [&op]() { - return op->emitError() - << "expected '" << LLVMDialect::getStructAttrsAttrName() - << "' to annotate '!llvm.struct' or '!llvm.ptr>'"; - }; - const auto ptrType = annotatedType.dyn_cast(); - if (!ptrType) - return emitIncorrectAnnotatedType(); - structType = ptrType.getElementType().dyn_cast(); - if (!structType) - return emitIncorrectAnnotatedType(); - } - - const auto arrAttrs = attr.dyn_cast(); - if (!arrAttrs) - return op->emitError() << "expected '" - << LLVMDialect::getStructAttrsAttrName() - << "' to be an array attribute"; - - if (structType.getBody().size() != arrAttrs.size()) - return op->emitError() - << "size of '" << LLVMDialect::getStructAttrsAttrName() - << "' must match the size of the annotated '!llvm.struct'"; - return success(); -} - -static LogicalResult verifyFuncOpInterfaceStructAttr(Operation *op, - Attribute attr, - Type annotatedType) { - if (isa(op)) - return LLVMDialect::verifyStructAttr(op, attr, annotatedType); - return op->emitError() << "expected '" - << LLVMDialect::getStructAttrsAttrName() - << "' to be used on function-like operations"; -} - LogicalResult LLVMDialect::verifyParameterAttribute(Operation *op, Type paramType, NamedAttribute paramAttr) { @@ -3045,10 +2999,6 @@ LogicalResult LLVMDialect::verifyParameterAttribute(Operation *op, return success(); }; - // Note: The struct parameter attributes are not lowered to LLVM IR. - if (name == LLVMDialect::getStructAttrsAttrName()) - return verifyFuncOpInterfaceStructAttr(op, paramAttr.getValue(), paramType); - // Check a unit attribute that is attached to a pointer value. if (name == LLVMDialect::getNoAliasAttrName() || name == LLVMDialect::getReadonlyAttrName() || diff --git a/mlir/test/Conversion/FuncToLLVM/emit-c-wrappers-for-external-callers.mlir b/mlir/test/Conversion/FuncToLLVM/emit-c-wrappers-for-external-callers.mlir index 8b9360a09900..826ca9540ae5 100644 --- a/mlir/test/Conversion/FuncToLLVM/emit-c-wrappers-for-external-callers.mlir +++ b/mlir/test/Conversion/FuncToLLVM/emit-c-wrappers-for-external-callers.mlir @@ -2,7 +2,7 @@ // CHECK: llvm.func @res_attrs_with_memref_return() -> (!llvm.struct{{.*}} {test.returnOne}) // CHECK-LABEL: llvm.func @_mlir_ciface_res_attrs_with_memref_return -// CHECK: %{{.*}}: !llvm.ptr{{.*}} {test.returnOne} +// CHECK-NOT: test.returnOne func.func @res_attrs_with_memref_return() -> (memref {test.returnOne}) { %0 = memref.alloc() : memref return %0 : memref @@ -10,24 +10,30 @@ func.func @res_attrs_with_memref_return() -> (memref {test.returnOne}) { // CHECK: llvm.func @res_attrs_with_value_return() -> (f32 {test.returnOne = 1 : i64}) // CHECK-LABEL: llvm.func @_mlir_ciface_res_attrs_with_value_return -// CHECK: -> (f32 {test.returnOne = 1 : i64}) +// CHECK-SAME: -> (f32 {test.returnOne = 1 : i64}) func.func @res_attrs_with_value_return() -> (f32 {test.returnOne = 1}) { %0 = arith.constant 1.00 : f32 return %0 : f32 } -// CHECK: llvm.func @multiple_return() -> (!llvm.struct<{{.*}}> {llvm.struct_attrs = [{test.returnOne = 1 : i64}, {test.returnThree = 3 : i64, test.returnTwo = 2 : i64}]}) +// CHECK: llvm.func @multiple_return() -> !llvm.struct<{{.*}}> // CHECK-LABEL: llvm.func @_mlir_ciface_multiple_return -// CHECK: (%{{.*}}: !llvm.ptr<{{.*}}> {llvm.struct_attrs = [{test.returnOne = 1 : i64}, {test.returnThree = 3 : i64, test.returnTwo = 2 : i64}]}) +// CHECK-SAME: !llvm.ptr +// CHECK-NOT: test.returnOne +// CHECK-NOT: test.returnTwo +// CHECK-NOT: test.returnThree func.func @multiple_return() -> (memref {test.returnOne = 1}, f32 {test.returnTwo = 2, test.returnThree = 3}) { %0 = memref.alloc() : memref %1 = arith.constant 1.00 : f32 return %0, %1 : memref, f32 } -// CHECK: llvm.func @multiple_return_missing_res_attr() -> (!llvm.struct<{{.*}}> {llvm.struct_attrs = [{test.returnOne = 1 : i64}, {}, {test.returnThree = 3 : i64, test.returnTwo = 2 : i64}]}) +// CHECK: llvm.func @multiple_return_missing_res_attr() -> !llvm.struct<{{.*}}> // CHECK-LABEL: llvm.func @_mlir_ciface_multiple_return_missing_res_attr -// CHECK: (%{{.*}}: !llvm.ptr<{{.*}}> {llvm.struct_attrs = [{test.returnOne = 1 : i64}, {}, {test.returnThree = 3 : i64, test.returnTwo = 2 : i64}]}) +// CHECK-SAME: !llvm.ptr +// CHECK-NOT: test.returnOne +// CHECK-NOT: test.returnTwo +// CHECK-NOT: test.returnThree func.func @multiple_return_missing_res_attr() -> (memref {test.returnOne = 1}, i64, f32 {test.returnTwo = 2, test.returnThree = 3}) { %0 = memref.alloc() : memref %1 = arith.constant 2 : i64 @@ -37,7 +43,9 @@ func.func @multiple_return_missing_res_attr() -> (memref {test.returnOne = // CHECK: llvm.func @one_arg_attr_no_res_attrs_with_memref_return({{.*}}) -> !llvm.struct{{.*}} // CHECK-LABEL: llvm.func @_mlir_ciface_one_arg_attr_no_res_attrs_with_memref_return -// CHECK: %{{.*}}: !llvm.ptr<{{.*}}>, %{{.*}}: !llvm.ptr<{{.*}}> {test.argOne = 1 : i64} +// CHECK-SAME: !llvm.ptr +// CHECK-SAME: !llvm.ptr +// CHECK-SAME: {test.argOne = 1 : i64}) func.func @one_arg_attr_no_res_attrs_with_memref_return(%arg0: memref {test.argOne = 1}) -> memref { %0 = memref.alloc() : memref return %0 : memref @@ -45,7 +53,10 @@ func.func @one_arg_attr_no_res_attrs_with_memref_return(%arg0: memref {test // CHECK: llvm.func @one_arg_attr_one_res_attr_with_memref_return({{.*}}) -> (!llvm.struct<{{.*}}> {test.returnOne = 1 : i64}) // CHECK-LABEL: llvm.func @_mlir_ciface_one_arg_attr_one_res_attr_with_memref_return -// CHECK: (%{{.*}}: !llvm.ptr<{{.*}}> {test.returnOne = 1 : i64}, %{{.*}}: !llvm.ptr<{{.*}}> {test.argOne = 1 : i64} +// CHECK-SAME: !llvm.ptr +// CHECK-NOT: test.returnOne +// CHECK-SAME: !llvm.ptr +// CHECK-SAME: {test.argOne = 1 : i64}) func.func @one_arg_attr_one_res_attr_with_memref_return(%arg0: memref {test.argOne = 1}) -> (memref {test.returnOne = 1}) { %0 = memref.alloc() : memref return %0 : memref @@ -53,15 +64,23 @@ func.func @one_arg_attr_one_res_attr_with_memref_return(%arg0: memref {test // CHECK: llvm.func @one_arg_attr_one_res_attr_with_value_return({{.*}}) -> (f32 {test.returnOne = 1 : i64}) // CHECK-LABEL: llvm.func @_mlir_ciface_one_arg_attr_one_res_attr_with_value_return -// CHECK: (%{{.*}}: !llvm.ptr<{{.*}}> {test.argOne = 1 : i64}) -> (f32 {test.returnOne = 1 : i64}) +// CHECK-SAME: !llvm.ptr +// CHECK-SAME: {test.argOne = 1 : i64}) +// CHECK-SAME: -> (f32 {test.returnOne = 1 : i64}) func.func @one_arg_attr_one_res_attr_with_value_return(%arg0: memref {test.argOne = 1}) -> (f32 {test.returnOne = 1}) { %0 = arith.constant 1.00 : f32 return %0 : f32 } -// CHECK: llvm.func @multiple_arg_attr_multiple_res_attr({{.*}}) -> (!llvm.struct<{{.*}}> {llvm.struct_attrs = [{}, {test.returnOne = 1 : i64}, {test.returnTwo = 2 : i64}]}) +// CHECK: llvm.func @multiple_arg_attr_multiple_res_attr({{.*}}) -> !llvm.struct<{{.*}}> // CHECK-LABEL: llvm.func @_mlir_ciface_multiple_arg_attr_multiple_res_attr -// CHECK: (%{{.*}}: !llvm.ptr<{{.*}}> {llvm.struct_attrs = [{}, {test.returnOne = 1 : i64}, {test.returnTwo = 2 : i64}]}, %{{.*}}: !llvm.ptr<{{.*}}> {test.argZero = 0 : i64}, %{{.*}}: f32, %{{.*}}: i32 {test.argTwo = 2 : i64} +// CHECK-SAME: !llvm.ptr +// CHECK-NOT: test.returnOne +// CHECK-NOT: test.returnTwo +// CHECK-SAME: !llvm.ptr +// CHECK-SAME: {test.argZero = 0 : i64} +// CHECK-SAME: f32 +// CHECK-SAME: i32 {test.argTwo = 2 : i64}) func.func @multiple_arg_attr_multiple_res_attr(%arg0: memref {test.argZero = 0}, %arg1: f32, %arg2: i32 {test.argTwo = 2}) -> (f32, memref {test.returnOne = 1}, i32 {test.returnTwo = 2}) { %0 = arith.constant 1.00 : f32 %1 = memref.alloc() : memref @@ -71,8 +90,8 @@ func.func @multiple_arg_attr_multiple_res_attr(%arg0: memref {test.argZero // CHECK: llvm.func @drop_linkage_attr() -> (!llvm.struct{{.*}} {test.returnOne}) // CHECK-LABEL: llvm.func @_mlir_ciface_drop_linkage_attr +// CHECK-SAME: !llvm.ptr // CHECK-NOT: llvm.linkage -// CHECK: %{{.*}}: !llvm.ptr{{.*}} {test.returnOne} func.func @drop_linkage_attr() -> (memref {test.returnOne}) attributes { llvm.linkage = #llvm.linkage } { %0 = memref.alloc() : memref return %0 : memref diff --git a/mlir/test/Conversion/FuncToLLVM/emit-c-wrappers-for-external-functions.mlir b/mlir/test/Conversion/FuncToLLVM/emit-c-wrappers-for-external-functions.mlir index 9d73072aab9d..f43c23773d9d 100644 --- a/mlir/test/Conversion/FuncToLLVM/emit-c-wrappers-for-external-functions.mlir +++ b/mlir/test/Conversion/FuncToLLVM/emit-c-wrappers-for-external-functions.mlir @@ -2,46 +2,64 @@ // CHECK: llvm.func @res_attrs_with_memref_return() -> (!llvm.struct{{.*}} {test.returnOne}) // CHECK-LABEL: llvm.func @_mlir_ciface_res_attrs_with_memref_return -// CHECK: !llvm.ptr{{.*}} {test.returnOne} +// CHECK-SAME: !llvm.ptr +// CHECK-NOT: test.returnOne func.func private @res_attrs_with_memref_return() -> (memref {test.returnOne}) // CHECK: llvm.func @res_attrs_with_value_return() -> (f32 {test.returnOne = 1 : i64}) // CHECK-LABEL: llvm.func @_mlir_ciface_res_attrs_with_value_return -// CHECK: -> (f32 {test.returnOne = 1 : i64}) +// CHECK-SAME: -> (f32 {test.returnOne = 1 : i64}) func.func private @res_attrs_with_value_return() -> (f32 {test.returnOne = 1}) -// CHECK: llvm.func @multiple_return() -> (!llvm.struct<{{.*}}> {llvm.struct_attrs = [{test.returnOne = 1 : i64}, {test.returnThree = 3 : i64, test.returnTwo = 2 : i64}]}) +// CHECK: llvm.func @multiple_return() -> !llvm.struct<{{.*}}> // CHECK-LABEL: llvm.func @_mlir_ciface_multiple_return -// CHECK: (!llvm.ptr<{{.*}}> {llvm.struct_attrs = [{test.returnOne = 1 : i64}, {test.returnThree = 3 : i64, test.returnTwo = 2 : i64}]}) +// CHECK-NOT: test.returnOne +// CHECK-NOT: test.returnTwo +// CHECK-NOT: test.returnThree func.func private @multiple_return() -> (memref {test.returnOne = 1}, f32 {test.returnTwo = 2, test.returnThree = 3}) -// CHECK: llvm.func @multiple_return_missing_res_attr() -> (!llvm.struct<{{.*}}> {llvm.struct_attrs = [{test.returnOne = 1 : i64}, {}, {test.returnThree = 3 : i64, test.returnTwo = 2 : i64}]}) +// CHECK: llvm.func @multiple_return_missing_res_attr() -> !llvm.struct<{{.*}}> // CHECK-LABEL: llvm.func @_mlir_ciface_multiple_return_missing_res_attr -// CHECK: (!llvm.ptr<{{.*}}> {llvm.struct_attrs = [{test.returnOne = 1 : i64}, {}, {test.returnThree = 3 : i64, test.returnTwo = 2 : i64}]}) +// CHECK-NOT: test.returnOne +// CHECK-NOT: test.returnTwo +// CHECK-NOT: test.returnThree func.func private @multiple_return_missing_res_attr() -> (memref {test.returnOne = 1}, i64, f32 {test.returnTwo = 2, test.returnThree = 3}) // CHECK: llvm.func @one_arg_attr_no_res_attrs_with_memref_return({{.*}}) -> !llvm.struct{{.*}} // CHECK-LABEL: llvm.func @_mlir_ciface_one_arg_attr_no_res_attrs_with_memref_return -// CHECK: !llvm.ptr<{{.*}}>, !llvm.ptr<{{.*}}> {test.argOne = 1 : i64} +// CHECK-SAME: !llvm.ptr +// CHECK-SAME: !llvm.ptr +// CHECK-SAME: {test.argOne = 1 : i64}) func.func private @one_arg_attr_no_res_attrs_with_memref_return(%arg0: memref {test.argOne = 1}) -> memref // CHECK: llvm.func @one_arg_attr_one_res_attr_with_memref_return({{.*}}) -> (!llvm.struct<{{.*}}> {test.returnOne = 1 : i64}) // CHECK-LABEL: llvm.func @_mlir_ciface_one_arg_attr_one_res_attr_with_memref_return -// CHECK: (!llvm.ptr<{{.*}}> {test.returnOne = 1 : i64}, !llvm.ptr<{{.*}}> {test.argOne = 1 : i64} +// CHECK-SAME: !llvm.ptr +// CHECK-NOT: test.returnOne +// CHECK-SAME: !llvm.ptr +// CHECK-SAME: {test.argOne = 1 : i64}) func.func private @one_arg_attr_one_res_attr_with_memref_return(%arg0: memref {test.argOne = 1}) -> (memref {test.returnOne = 1}) // CHECK: llvm.func @one_arg_attr_one_res_attr_with_value_return({{.*}}) -> (f32 {test.returnOne = 1 : i64}) // CHECK-LABEL: llvm.func @_mlir_ciface_one_arg_attr_one_res_attr_with_value_return -// CHECK: (!llvm.ptr<{{.*}}> {test.argOne = 1 : i64}) -> (f32 {test.returnOne = 1 : i64}) +// CHECK-SAME: !llvm.ptr +// CHECK-SAME: {test.argOne = 1 : i64} +// CHECK-SAME: -> (f32 {test.returnOne = 1 : i64}) func.func private @one_arg_attr_one_res_attr_with_value_return(%arg0: memref {test.argOne = 1}) -> (f32 {test.returnOne = 1}) -// CHECK: llvm.func @multiple_arg_attr_multiple_res_attr({{.*}}) -> (!llvm.struct<{{.*}}> {llvm.struct_attrs = [{}, {test.returnOne = 1 : i64}, {test.returnTwo = 2 : i64}]}) +// CHECK: llvm.func @multiple_arg_attr_multiple_res_attr({{.*}}) -> !llvm.struct<{{.*}}> // CHECK-LABEL: llvm.func @_mlir_ciface_multiple_arg_attr_multiple_res_attr -// CHECK: (!llvm.ptr<{{.*}}> {llvm.struct_attrs = [{}, {test.returnOne = 1 : i64}, {test.returnTwo = 2 : i64}]}, !llvm.ptr<{{.*}}> {test.argZero = 0 : i64}, f32, i32 {test.argTwo = 2 : i64} +// CHECK-SAME: !llvm.ptr +// CHECK-NOT: test.returnOne +// CHECK-NOT: test.returnTwo +// CHECK-SAME: !llvm.ptr +// CHECK-SAME: {test.argZero = 0 : i64} +// CHECK-SAME: f32 +// CHECK-SAME: i32 {test.argTwo = 2 : i64}) func.func private @multiple_arg_attr_multiple_res_attr(%arg0: memref {test.argZero = 0}, %arg1: f32, %arg2: i32 {test.argTwo = 2}) -> (f32, memref {test.returnOne = 1}, i32 {test.returnTwo = 2}) // CHECK: llvm.func weak @drop_linkage_attr() -> (!llvm.struct{{.*}} {test.returnOne}) // CHECK-LABEL: llvm.func @_mlir_ciface_drop_linkage_attr +// CHECK-SAME: !llvm.ptr // CHECK-NOT: llvm.linkage -// CHECK: !llvm.ptr{{.*}} {test.returnOne} func.func private @drop_linkage_attr() -> (memref {test.returnOne}) attributes { llvm.linkage = #llvm.linkage } diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir index a425b240ba54..4547d13dc38b 100644 --- a/mlir/test/Dialect/LLVMIR/invalid.mlir +++ b/mlir/test/Dialect/LLVMIR/invalid.mlir @@ -1247,69 +1247,6 @@ llvm.mlir.global internal @side_effecting_global() : !llvm.struct<(i8)> { // ----- -// expected-error@+1 {{'llvm.struct_attrs' is permitted only in argument or result attributes}} -func.func @struct_attrs_in_op() attributes {llvm.struct_attrs = []} { - return -} - -// ----- - -// expected-error@+1 {{expected 'llvm.struct_attrs' to annotate '!llvm.struct' or '!llvm.ptr>'}} -func.func @invalid_struct_attr_arg_type(%arg0 : i32 {llvm.struct_attrs = []}) { - return -} - -// ----- - -// expected-error@+1 {{expected 'llvm.struct_attrs' to annotate '!llvm.struct' or '!llvm.ptr>'}} -func.func @invalid_struct_attr_pointer_arg_type(%arg0 : !llvm.ptr {llvm.struct_attrs = []}) { - return -} - -// ----- - -// expected-error@+1 {{expected 'llvm.struct_attrs' to be an array attribute}} -func.func @invalid_arg_struct_attr_value(%arg0 : !llvm.struct<(i32)> {llvm.struct_attrs = {}}) { - return -} - -// ----- - -// expected-error@+1 {{size of 'llvm.struct_attrs' must match the size of the annotated '!llvm.struct'}} -func.func @invalid_arg_struct_attr_size(%arg0 : !llvm.struct<(i32)> {llvm.struct_attrs = []}) { - return -} - -// ----- - -// expected-error@+1 {{expected 'llvm.struct_attrs' to annotate '!llvm.struct' or '!llvm.ptr>'}} -func.func @invalid_struct_attr_res_type(%arg0 : i32) -> (i32 {llvm.struct_attrs = []}) { - return %arg0 : i32 -} - -// ----- - -// expected-error@+1 {{expected 'llvm.struct_attrs' to annotate '!llvm.struct' or '!llvm.ptr>'}} -func.func @invalid_struct_attr_pointer_res_type(%arg0 : !llvm.ptr) -> (!llvm.ptr {llvm.struct_attrs = []}) { - return %arg0 : !llvm.ptr -} - -// ----- - -// expected-error@+1 {{expected 'llvm.struct_attrs' to be an array attribute}} -func.func @invalid_res_struct_attr_value(%arg0 : !llvm.struct<(i32)>) -> (!llvm.struct<(i32)> {llvm.struct_attrs = {}}) { - return %arg0 : !llvm.struct<(i32)> -} - -// ----- - -// expected-error@+1 {{size of 'llvm.struct_attrs' must match the size of the annotated '!llvm.struct'}} -func.func @invalid_res_struct_attr_size(%arg0 : !llvm.struct<(i32)>) -> (!llvm.struct<(i32)> {llvm.struct_attrs = []}) { - return %arg0 : !llvm.struct<(i32)> -} - -// ----- - func.func @insert_vector_invalid_source_vector_size(%arg0 : vector<16385xi8>, %arg1 : vector<[16]xi8>) { // expected-error@+1 {{op failed to verify that vectors are not bigger than 2^17 bits.}} %0 = llvm.intr.vector.insert %arg0, %arg1[0] : vector<16385xi8> into vector<[16]xi8> From 69748c77ac5a874a347652d0928533b066179d86 Mon Sep 17 00:00:00 2001 From: Tobias Gysi Date: Mon, 6 Mar 2023 10:35:31 +0100 Subject: [PATCH 067/147] [mlir][llvm] Add extra attributes to the atomic ops. The revision adds a number of extra arguments to the atomic read modify write and compare and exchange operations. The extra arguments include the volatile, weak, syncscope, and alignment attributes. The implementation also adapts the fence operation to use a assembly format and generalizes the helper used to obtain the syncscope name. Reviewed By: Dinistro Differential Revision: https://reviews.llvm.org/D143554 (cherry picked from commit 7f97895f5bf7a9b6300ad1fbf1239f87a619be83) --- mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td | 130 +++++++++++------- .../Conversion/MemRefToLLVM/MemRefToLLVM.cpp | 9 +- .../Conversion/SCFToOpenMP/SCFToOpenMP.cpp | 2 +- mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 86 ++++-------- mlir/lib/Target/LLVMIR/ModuleImport.cpp | 30 ++-- mlir/test/Dialect/LLVMIR/roundtrip.mlir | 4 + .../test/Target/LLVMIR/Import/instructions.ll | 10 ++ mlir/test/Target/LLVMIR/llvmir.mlir | 10 ++ 8 files changed, 159 insertions(+), 122 deletions(-) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td index 5ff36907f1c6..39b79f1ed228 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td @@ -169,9 +169,8 @@ def LLVM_FRemOp : LLVM_FloatArithmeticOp<"frem", "FRem">; def LLVM_FNegOp : LLVM_UnaryFloatArithmeticOp< LLVM_ScalarOrVectorOf, "fneg", "FNeg">; -// Common code definition that is used to verify and set the alignment attribute -// of LLVM ops that accept such an attribute. -class MemoryOpWithAlignmentBase { +// Common code definitions used to set memory operation attributes and flags. +class MemoryOpBase { code setAlignmentCode = [{ if ($alignment.has_value()) { auto align = *$alignment; @@ -179,10 +178,15 @@ class MemoryOpWithAlignmentBase { inst->setAlignment(llvm::Align(align)); } }]; -} - -// Code definition that is used for nontemporal metadata creation. -class MemoryOpWithAlignmentAndAttributes : MemoryOpWithAlignmentBase { + code setVolatileCode = [{ + inst->setVolatile($volatile_); + }]; + code setSyncScopeCode = [{ + if ($syncscope.has_value()) { + llvm::LLVMContext &llvmContext = builder.getContext(); + inst->setSyncScopeID(llvmContext.getOrInsertSyncScopeID(*$syncscope)); + } + }]; code setNonTemporalMetadataCode = [{ if ($nontemporal) { llvm::Module *module = builder.GetInsertBlock()->getModule(); @@ -192,22 +196,19 @@ class MemoryOpWithAlignmentAndAttributes : MemoryOpWithAlignmentBase { inst->setMetadata(module->getMDKindID("nontemporal"), metadata); } }]; - code setAccessGroupsMetadataCode = [{ moduleTranslation.setAccessGroupsMetadata(op, inst); }]; - code setAliasScopeMetadataCode = [{ moduleTranslation.setAliasScopeMetadata(op, inst); }]; - code setTBAAMetadataCode = [{ moduleTranslation.setTBAAMetadata(op, inst); }]; } // Memory-related operations. -def LLVM_AllocaOp : LLVM_Op<"alloca">, MemoryOpWithAlignmentBase { +def LLVM_AllocaOp : LLVM_Op<"alloca">, MemoryOpBase { let arguments = (ins AnyInteger:$arraySize, OptionalAttr:$alignment, OptionalAttr:$elem_type); @@ -344,7 +345,7 @@ def LLVM_GEPOp : LLVM_Op<"getelementptr", [Pure]> { let hasVerifier = 1; } -def LLVM_LoadOp : LLVM_Op<"load">, MemoryOpWithAlignmentAndAttributes { +def LLVM_LoadOp : LLVM_Op<"load">, MemoryOpBase { let arguments = (ins Arg, "", [MemRead]>:$addr, OptionalAttr:$access_groups, OptionalAttr:$alias_scopes, @@ -383,7 +384,7 @@ def LLVM_LoadOp : LLVM_Op<"load">, MemoryOpWithAlignmentAndAttributes { let hasVerifier = 1; } -def LLVM_StoreOp : LLVM_Op<"store">, MemoryOpWithAlignmentAndAttributes { +def LLVM_StoreOp : LLVM_Op<"store">, MemoryOpBase { let arguments = (ins LLVM_LoadableType:$value, Arg,"",[MemWrite]>:$addr, OptionalAttr:$access_groups, @@ -1655,79 +1656,111 @@ def LLVM_ConstantOp def LLVM_AtomicRMWType : AnyTypeOf<[LLVM_AnyFloat, AnyInteger]>; -// FIXME: Need to add alignment and syncscope attribute to MLIR atomicrmw -// operation. def LLVM_AtomicRMWOp : LLVM_Op<"atomicrmw", [ TypesMatchWith<"result #0 and operand #1 have the same type", - "val", "res", "$_self">]> { + "val", "res", "$_self">]>, MemoryOpBase { let arguments = (ins AtomicBinOp:$bin_op, LLVM_PointerTo:$ptr, - LLVM_AtomicRMWType:$val, AtomicOrdering:$ordering); + LLVM_AtomicRMWType:$val, AtomicOrdering:$ordering, + OptionalAttr:$syncscope, + OptionalAttr:$alignment, + UnitAttr:$volatile_); let results = (outs LLVM_AtomicRMWType:$res); - let hasVerifier = 1; let assemblyFormat = [{ - $bin_op $ptr `,` $val $ordering - attr-dict `:` qualified(type($ptr)) `,` type($val) + (`volatile` $volatile_^)? $bin_op $ptr `,` $val + (`syncscope` `(` $syncscope^ `)`)? $ordering attr-dict `:` + qualified(type($ptr)) `,` type($val) }]; string llvmInstName = "AtomicRMW"; string llvmBuilder = [{ - $res = builder.CreateAtomicRMW( + auto *inst = builder.CreateAtomicRMW( convertAtomicBinOpToLLVM($bin_op), $ptr, $val, llvm::MaybeAlign(), convertAtomicOrderingToLLVM($ordering)); - }]; + $res = inst; + }] # setVolatileCode + # setSyncScopeCode + # setAlignmentCode; string mlirBuilder = [{ auto *atomicInst = cast(inst); - $res = $_builder.create($_location, $_resultType, + unsigned alignment = atomicInst->getAlign().value(); + $res = $_builder.create($_location, convertAtomicBinOpFromLLVM(atomicInst->getOperation()), $ptr, $val, - convertAtomicOrderingFromLLVM(atomicInst->getOrdering())); + convertAtomicOrderingFromLLVM(atomicInst->getOrdering()), + getLLVMSyncScope(atomicInst), alignment, atomicInst->isVolatile()); }]; - // Only $ptr and $val are llvm instruction operands. - list llvmArgIndices = [-1, 0, 1, -1]; + list llvmArgIndices = [-1, 0, 1, -1, -1, -1, -1]; + let builders = [ + OpBuilder<(ins "LLVM::AtomicBinOp":$binOp, "Value":$ptr, "Value":$val, + "LLVM::AtomicOrdering":$ordering, + CArg<"StringRef", "StringRef()">:$syncscope, + CArg<"unsigned", "0">:$alignment, CArg<"bool", "false">:$isVolatile + )> + ]; + let hasVerifier = 1; } def LLVM_AtomicCmpXchgType : AnyTypeOf<[AnyInteger, LLVM_AnyPointer]>; -// FIXME: Need to add alignment attribute to MLIR cmpxchg operation. def LLVM_AtomicCmpXchgOp : LLVM_Op<"cmpxchg", [ TypesMatchWith<"operand #1 and operand #2 have the same type", "val", "cmp", "$_self">, TypesMatchWith<"result #0 has an LLVM struct type consisting of " - "the type of operand #2 and a bool", - "val", "res", "getValAndBoolStructType($_self)">]> { + "the type of operand #2 and a bool", "val", "res", + "getValAndBoolStructType($_self)">]>, MemoryOpBase { let arguments = (ins LLVM_PointerTo:$ptr, LLVM_AtomicCmpXchgType:$cmp, LLVM_AtomicCmpXchgType:$val, AtomicOrdering:$success_ordering, - AtomicOrdering:$failure_ordering); + AtomicOrdering:$failure_ordering, + OptionalAttr:$syncscope, + OptionalAttr:$alignment, + UnitAttr:$weak, + UnitAttr:$volatile_); let results = (outs LLVM_AnyStruct:$res); - let hasVerifier = 1; let assemblyFormat = [{ - $ptr `,` $cmp `,` $val $success_ordering $failure_ordering + (`weak` $weak^)? (`volatile` $volatile_^)? $ptr `,` $cmp `,` $val + (`syncscope` `(` $syncscope^ `)`)? $success_ordering $failure_ordering attr-dict `:` qualified(type($ptr)) `,` type($val) }]; string llvmInstName = "AtomicCmpXchg"; string llvmBuilder = [{ - $res = builder.CreateAtomicCmpXchg($ptr, $cmp, $val, llvm::MaybeAlign(), - convertAtomicOrderingToLLVM($success_ordering), - convertAtomicOrderingToLLVM($failure_ordering)); - }]; + auto *inst = builder.CreateAtomicCmpXchg($ptr, $cmp, $val, + llvm::MaybeAlign(), convertAtomicOrderingToLLVM($success_ordering), + convertAtomicOrderingToLLVM($failure_ordering)); + $res = inst; + inst->setWeak($weak); + }] # setVolatileCode + # setSyncScopeCode + # setAlignmentCode; string mlirBuilder = [{ auto *cmpXchgInst = cast(inst); + unsigned alignment = cmpXchgInst->getAlign().value(); $res = $_builder.create( - $_location, $_resultType, $ptr, $cmp, $val, + $_location, $ptr, $cmp, $val, convertAtomicOrderingFromLLVM(cmpXchgInst->getSuccessOrdering()), - convertAtomicOrderingFromLLVM(cmpXchgInst->getFailureOrdering())); + convertAtomicOrderingFromLLVM(cmpXchgInst->getFailureOrdering()), + getLLVMSyncScope(cmpXchgInst), alignment, cmpXchgInst->isWeak(), + cmpXchgInst->isVolatile()); }]; + let builders = [ + OpBuilder<(ins "Value":$ptr, "Value":$cmp, "Value":$val, + "LLVM::AtomicOrdering":$successOrdering, + "LLVM::AtomicOrdering":$failureOrdering, + CArg<"StringRef", "StringRef()">:$syncscope, + CArg<"unsigned", "0">:$alignment, CArg<"bool", "false">:$isWeak, + CArg<"bool", "false">:$isVolatile + )> + ]; + let hasVerifier = 1; } -def LLVM_FenceOp : LLVM_Op<"fence"> { - let arguments = (ins AtomicOrdering:$ordering, StrAttr:$syncscope); - let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder]; +def LLVM_FenceOp : LLVM_Op<"fence">, MemoryOpBase { + let arguments = (ins AtomicOrdering:$ordering, + OptionalAttr:$syncscope); + let assemblyFormat = "(`syncscope` `(` $syncscope^ `)`)? $ordering attr-dict"; string llvmInstName = "Fence"; let llvmBuilder = [{ - llvm::LLVMContext &llvmContext = builder.getContext(); - builder.CreateFence(convertAtomicOrderingToLLVM($ordering), - llvmContext.getOrInsertSyncScopeID($syncscope)); - }]; + auto *inst = builder.CreateFence(convertAtomicOrderingToLLVM($ordering)); + }] # setSyncScopeCode; string mlirBuilder = [{ llvm::FenceInst *fenceInst = cast(inst); $_op = $_builder.create( @@ -1735,7 +1768,12 @@ def LLVM_FenceOp : LLVM_Op<"fence"> { convertAtomicOrderingFromLLVM(fenceInst->getOrdering()), getLLVMSyncScope(fenceInst)); }]; - let hasCustomAssemblyFormat = 1; + let builders = [ + LLVM_VoidResultTypeOpBuilder, + LLVM_ZeroResultOpBuilder, + OpBuilder<(ins "LLVM::AtomicOrdering":$ordering, + CArg<"StringRef", "StringRef()">:$syncscope)> + ]; let hasVerifier = 1; } diff --git a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp index 7132560ad8b1..f5da371f550d 100644 --- a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp +++ b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp @@ -570,12 +570,8 @@ struct GenericAtomicRMWOpLowering // Append the cmpxchg op to the end of the loop block. auto successOrdering = LLVM::AtomicOrdering::acq_rel; auto failureOrdering = LLVM::AtomicOrdering::monotonic; - auto boolType = IntegerType::get(rewriter.getContext(), 1); - auto pairType = LLVM::LLVMStructType::getLiteral(rewriter.getContext(), - {valueType, boolType}); auto cmpxchg = rewriter.create( - loc, pairType, dataPtr, loopArgument, result, successOrdering, - failureOrdering); + loc, dataPtr, loopArgument, result, successOrdering, failureOrdering); // Extract the %new_loaded and %ok values from the pair. Value newLoaded = rewriter.create(loc, cmpxchg, 0); Value ok = rewriter.create(loc, cmpxchg, 1); @@ -1620,13 +1616,12 @@ struct AtomicRMWOpLowering : public LoadStoreOpLowering { auto maybeKind = matchSimpleAtomicOp(atomicOp); if (!maybeKind) return failure(); - auto resultType = adaptor.getValue().getType(); auto memRefType = atomicOp.getMemRefType(); auto dataPtr = getStridedElementPtr(atomicOp.getLoc(), memRefType, adaptor.getMemref(), adaptor.getIndices(), rewriter); rewriter.replaceOpWithNewOp( - atomicOp, resultType, *maybeKind, dataPtr, adaptor.getValue(), + atomicOp, *maybeKind, dataPtr, adaptor.getValue(), LLVM::AtomicOrdering::acq_rel); return success(); } diff --git a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp index 1c3e35d4a91d..a4acf0431b72 100644 --- a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp +++ b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp @@ -229,7 +229,7 @@ static omp::ReductionDeclareOp addAtomicRMW(OpBuilder &builder, builder.setInsertionPointToEnd(atomicBlock); Value loaded = builder.create(reduce.getLoc(), atomicBlock->getArgument(1)); - builder.create(reduce.getLoc(), type, atomicKind, + builder.create(reduce.getLoc(), atomicKind, atomicBlock->getArgument(0), loaded, LLVM::AtomicOrdering::monotonic); builder.create(reduce.getLoc(), ArrayRef()); diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp index e8ada51c7aef..73a2dc9dea63 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -2227,40 +2227,18 @@ LogicalResult LLVM::ConstantOp::verify() { OpFoldResult LLVM::ConstantOp::fold(FoldAdaptor) { return getValue(); } //===----------------------------------------------------------------------===// -// Utility functions for parsing atomic ops -//===----------------------------------------------------------------------===// - -// Helper function to parse a keyword into the specified attribute named by -// `attrName`. The keyword must match one of the string values defined by the -// AtomicOrdering enum. The resulting I64 attribute is added to the `result` -// state. -static ParseResult parseAtomicOrdering(OpAsmParser &parser, - OperationState &result, - StringRef attrName) { - SMLoc loc; - StringRef ordering; - if (parser.getCurrentLocation(&loc) || parser.parseKeyword(&ordering)) - return failure(); - - // Replace the keyword `ordering` with an integer attribute. - auto kind = symbolizeAtomicOrdering(ordering); - if (!kind) { - return parser.emitError(loc) - << "'" << ordering << "' is an incorrect value of the '" << attrName - << "' attribute"; - } - - auto value = static_cast(*kind); - auto attr = parser.getBuilder().getI64IntegerAttr(value); - result.addAttribute(attrName, attr); +// AtomicRMWOp +//===----------------------------------------------------------------------===// - return success(); +void AtomicRMWOp::build(OpBuilder &builder, OperationState &state, + AtomicBinOp binOp, Value ptr, Value val, + AtomicOrdering ordering, StringRef syncscope, + unsigned alignment, bool isVolatile) { + build(builder, state, val.getType(), binOp, ptr, val, ordering, + !syncscope.empty() ? builder.getStringAttr(syncscope) : nullptr, + alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile); } -//===----------------------------------------------------------------------===// -// Verifier for LLVM::AtomicRMWOp. -//===----------------------------------------------------------------------===// - LogicalResult AtomicRMWOp::verify() { auto ptrType = getPtr().getType().cast(); auto valType = getVal().getType(); @@ -2297,7 +2275,7 @@ LogicalResult AtomicRMWOp::verify() { } //===----------------------------------------------------------------------===// -// Verifier for LLVM::AtomicCmpXchgOp. +// AtomicCmpXchgOp //===----------------------------------------------------------------------===// /// Returns an LLVM struct type that contains a value type and a boolean type. @@ -2306,6 +2284,18 @@ static LLVMStructType getValAndBoolStructType(Type valType) { return LLVMStructType::getLiteral(valType.getContext(), {valType, boolType}); } +void AtomicCmpXchgOp::build(OpBuilder &builder, OperationState &state, + Value ptr, Value cmp, Value val, + AtomicOrdering successOrdering, + AtomicOrdering failureOrdering, StringRef syncscope, + unsigned alignment, bool isWeak, bool isVolatile) { + build(builder, state, getValAndBoolStructType(val.getType()), ptr, cmp, val, + successOrdering, failureOrdering, + !syncscope.empty() ? builder.getStringAttr(syncscope) : nullptr, + alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isWeak, + isVolatile); +} + LogicalResult AtomicCmpXchgOp::verify() { auto ptrType = getPtr().getType().cast(); if (!ptrType) @@ -2331,35 +2321,13 @@ LogicalResult AtomicCmpXchgOp::verify() { } //===----------------------------------------------------------------------===// -// Printer, parser and verifier for LLVM::FenceOp. +// FenceOp //===----------------------------------------------------------------------===// -// ::= `llvm.fence` (`syncscope(`strAttr`)`)? keyword -// attribute-dict? -ParseResult FenceOp::parse(OpAsmParser &parser, OperationState &result) { - StringAttr sScope; - StringRef syncscopeKeyword = "syncscope"; - if (!failed(parser.parseOptionalKeyword(syncscopeKeyword))) { - if (parser.parseLParen() || - parser.parseAttribute(sScope, syncscopeKeyword, result.attributes) || - parser.parseRParen()) - return failure(); - } else { - result.addAttribute(syncscopeKeyword, - parser.getBuilder().getStringAttr("")); - } - if (parseAtomicOrdering(parser, result, "ordering") || - parser.parseOptionalAttrDict(result.attributes)) - return failure(); - return success(); -} - -void FenceOp::print(OpAsmPrinter &p) { - StringRef syncscopeKeyword = "syncscope"; - p << ' '; - if (!(*this)->getAttr(syncscopeKeyword).cast().getValue().empty()) - p << "syncscope(" << (*this)->getAttr(syncscopeKeyword) << ") "; - p << stringifyAtomicOrdering(getOrdering()); +void FenceOp::build(OpBuilder &builder, OperationState &state, + AtomicOrdering ordering, StringRef syncscope) { + build(builder, state, ordering, + syncscope.empty() ? nullptr : builder.getStringAttr(syncscope)); } LogicalResult FenceOp::verify() { diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp index 341a8473da4c..b075c75d950b 100644 --- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp @@ -98,15 +98,27 @@ static FloatType getDLFloatType(MLIRContext &ctx, int32_t bitwidth) { } } -/// Converts the sync scope identifier of `fenceInst` to the string -/// representation necessary to build the LLVM dialect fence operation. -static StringRef getLLVMSyncScope(llvm::FenceInst *fenceInst) { - llvm::LLVMContext &llvmContext = fenceInst->getContext(); - SmallVector syncScopeNames; - llvmContext.getSyncScopeNames(syncScopeNames); - for (StringRef name : syncScopeNames) - if (fenceInst->getSyncScopeID() == llvmContext.getOrInsertSyncScopeID(name)) - return name; +/// Converts the sync scope identifier of `inst` to the string representation +/// necessary to build an atomic LLVM dialect operation. Returns the empty +/// string if the operation has either no sync scope or the default system-level +/// sync scope attached. The atomic operations only set their sync scope +/// attribute if they have a non-default sync scope attached. +static StringRef getLLVMSyncScope(llvm::Instruction *inst) { + std::optional syncScopeID = + llvm::getAtomicSyncScopeID(inst); + if (!syncScopeID) + return ""; + + // Search the sync scope name for the given identifier. The default + // system-level sync scope thereby maps to the empty string. + SmallVector syncScopeName; + llvm::LLVMContext &llvmContext = inst->getContext(); + llvmContext.getSyncScopeNames(syncScopeName); + auto *it = llvm::find_if(syncScopeName, [&](StringRef name) { + return *syncScopeID == llvmContext.getOrInsertSyncScopeID(name); + }); + if (it != syncScopeName.end()) + return *it; llvm_unreachable("incorrect sync scope identifier"); } diff --git a/mlir/test/Dialect/LLVMIR/roundtrip.mlir b/mlir/test/Dialect/LLVMIR/roundtrip.mlir index 5627d2fee0ab..e789978d713b 100644 --- a/mlir/test/Dialect/LLVMIR/roundtrip.mlir +++ b/mlir/test/Dialect/LLVMIR/roundtrip.mlir @@ -343,6 +343,8 @@ func.func @null() { func.func @atomicrmw(%ptr : !llvm.ptr, %val : f32) { // CHECK: llvm.atomicrmw fadd %{{.*}}, %{{.*}} monotonic : !llvm.ptr, f32 %0 = llvm.atomicrmw fadd %ptr, %val monotonic : !llvm.ptr, f32 + // CHECK: llvm.atomicrmw volatile fsub %{{.*}}, %{{.*}} syncscope("singlethread") monotonic {alignment = 16 : i64} : !llvm.ptr, f32 + %1 = llvm.atomicrmw volatile fsub %ptr, %val syncscope("singlethread") monotonic {alignment = 16 : i64} : !llvm.ptr, f32 llvm.return } @@ -350,6 +352,8 @@ func.func @atomicrmw(%ptr : !llvm.ptr, %val : f32) { func.func @cmpxchg(%ptr : !llvm.ptr, %cmp : i32, %new : i32) { // CHECK: llvm.cmpxchg %{{.*}}, %{{.*}}, %{{.*}} acq_rel monotonic : !llvm.ptr, i32 %0 = llvm.cmpxchg %ptr, %cmp, %new acq_rel monotonic : !llvm.ptr, i32 + // CHECK: llvm.cmpxchg weak volatile %{{.*}}, %{{.*}}, %{{.*}} syncscope("singlethread") acq_rel monotonic {alignment = 16 : i64} : !llvm.ptr, i32 + %1 = llvm.cmpxchg weak volatile %ptr, %cmp, %new syncscope("singlethread") acq_rel monotonic {alignment = 16 : i64} : !llvm.ptr, i32 llvm.return } diff --git a/mlir/test/Target/LLVMIR/Import/instructions.ll b/mlir/test/Target/LLVMIR/Import/instructions.ll index 15abc3dc96a7..14dbf07371bf 100644 --- a/mlir/test/Target/LLVMIR/Import/instructions.ll +++ b/mlir/test/Target/LLVMIR/Import/instructions.ll @@ -401,6 +401,11 @@ define void @atomic_rmw(ptr %ptr1, i32 %val1, ptr %ptr2, float %val2) { %16 = atomicrmw uinc_wrap ptr %ptr1, i32 %val1 acquire ; CHECK: llvm.atomicrmw udec_wrap %[[PTR1]], %[[VAL1]] acquire %17 = atomicrmw udec_wrap ptr %ptr1, i32 %val1 acquire + + ; CHECK: llvm.atomicrmw volatile + ; CHECK-SAME: syncscope("singlethread") + ; CHECK-SAME: {alignment = 8 : i64} + %18 = atomicrmw volatile udec_wrap ptr %ptr1, i32 %val1 syncscope("singlethread") acquire, align 8 ret void } @@ -415,6 +420,11 @@ define void @atomic_cmpxchg(ptr %ptr1, i32 %val1, i32 %val2) { %1 = cmpxchg ptr %ptr1, i32 %val1, i32 %val2 seq_cst seq_cst ; CHECK: llvm.cmpxchg %[[PTR1]], %[[VAL1]], %[[VAL2]] monotonic seq_cst %2 = cmpxchg ptr %ptr1, i32 %val1, i32 %val2 monotonic seq_cst + + ; CHECK: llvm.cmpxchg weak volatile + ; CHECK-SAME: syncscope("singlethread") + ; CHECK-SAME: {alignment = 8 : i64} + %3 = cmpxchg weak volatile ptr %ptr1, i32 %val1, i32 %val2 syncscope("singlethread") monotonic seq_cst, align 8 ret void } diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir index bbaa8570d0e4..840b1f2caf04 100644 --- a/mlir/test/Target/LLVMIR/llvmir.mlir +++ b/mlir/test/Target/LLVMIR/llvmir.mlir @@ -1436,6 +1436,11 @@ llvm.func @atomicrmw( %15 = llvm.atomicrmw uinc_wrap %i32_ptr, %i32 monotonic : !llvm.ptr, i32 // CHECK: atomicrmw udec_wrap ptr %{{.*}}, i32 %{{.*}} monotonic %16 = llvm.atomicrmw udec_wrap %i32_ptr, %i32 monotonic : !llvm.ptr, i32 + + // CHECK: atomicrmw volatile + // CHECK-SAME: syncscope("singlethread") + // CHECK-SAME: align 8 + %17 = llvm.atomicrmw volatile udec_wrap %i32_ptr, %i32 syncscope("singlethread") monotonic {alignment = 8 : i64} : !llvm.ptr, i32 llvm.return } @@ -1447,6 +1452,11 @@ llvm.func @cmpxchg(%ptr : !llvm.ptr, %cmp : i32, %val: i32) { %1 = llvm.extractvalue %0[0] : !llvm.struct<(i32, i1)> // CHECK: %{{[0-9]+}} = extractvalue { i32, i1 } %{{[0-9]+}}, 1 %2 = llvm.extractvalue %0[1] : !llvm.struct<(i32, i1)> + + // CHECK: cmpxchg weak volatile + // CHECK-SAME: syncscope("singlethread") + // CHECK-SAME: align 8 + %3 = llvm.cmpxchg weak volatile %ptr, %cmp, %val syncscope("singlethread") acq_rel monotonic {alignment = 8 : i64} : !llvm.ptr, i32 llvm.return } From 9ba5616a268e5509f3d5ebcbfe62d4904a829661 Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Mon, 6 Mar 2023 10:36:32 +0100 Subject: [PATCH 068/147] [mlir][llvm] Fuse MD_access_group & MD_loop import This commit moves the importing logic of access group metadata into the loop annotation importer. These two metadata imports can be grouped because access groups are only used in combination with `llvm.loop.parallel_accesses`. As a nice side effect, this commit decouples the LoopAnnotationImporter from the ModuleImport class. Differential Revision: https://reviews.llvm.org/D143577 (cherry picked from commit e630a502230f8779bddd214094d28fef61fde866) --- .../include/mlir/Target/LLVMIR/ModuleImport.h | 3 - .../Target/LLVMIR/LoopAnnotationImporter.cpp | 71 ++++++++++++++++--- .../Target/LLVMIR/LoopAnnotationImporter.h | 30 ++++++-- mlir/lib/Target/LLVMIR/ModuleImport.cpp | 53 +++----------- .../Target/LLVMIR/Import/import-failure.ll | 18 ++++- 5 files changed, 109 insertions(+), 66 deletions(-) diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h index 23b1fbc29dd7..3265c3237241 100644 --- a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h +++ b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h @@ -302,9 +302,6 @@ class ModuleImport { /// to the LLVMIR dialect TBAA operations corresponding to these /// nodes. DenseMap tbaaMapping; - /// Mapping between original LLVM access group metadata nodes and the symbol - /// references pointing to the imported MLIR access group operations. - DenseMap accessGroupMapping; /// The stateful type translator (contains named structs). LLVM::TypeFromLLVMIRTranslator typeTranslator; /// Stateful debug information importer. diff --git a/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp b/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp index a3218e13307e..a3cbf2bcd47c 100644 --- a/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp +++ b/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp @@ -16,11 +16,9 @@ using namespace mlir::LLVM::detail; namespace { /// Helper class that keeps the state of one metadata to attribute conversion. struct LoopMetadataConversion { - LoopMetadataConversion(const llvm::MDNode *node, ModuleImport &moduleImport, - Location loc, + LoopMetadataConversion(const llvm::MDNode *node, Location loc, LoopAnnotationImporter &loopAnnotationImporter) - : node(node), moduleImport(moduleImport), loc(loc), - loopAnnotationImporter(loopAnnotationImporter), + : node(node), loc(loc), loopAnnotationImporter(loopAnnotationImporter), ctx(loc->getContext()){}; /// Converts this structs loop metadata node into a LoopAnnotationAttr. LoopAnnotationAttr convert(); @@ -55,7 +53,6 @@ struct LoopMetadataConversion { llvm::StringMap propertyMap; const llvm::MDNode *node; - ModuleImport &moduleImport; Location loc; LoopAnnotationImporter &loopAnnotationImporter; MLIRContext *ctx; @@ -233,7 +230,7 @@ LoopMetadataConversion::lookupFollowupNode(StringRef name) { if (*node == nullptr) return LoopAnnotationAttr(nullptr); - return loopAnnotationImporter.translate(*node, loc); + return loopAnnotationImporter.translateLoopAnnotation(*node, loc); } static bool isEmptyOrNull(const Attribute attr) { return !attr; } @@ -360,7 +357,7 @@ LoopMetadataConversion::convertParallelAccesses() { SmallVector refs; for (llvm::MDNode *node : *nodes) { FailureOr> accessGroups = - moduleImport.lookupAccessGroupAttrs(node); + loopAnnotationImporter.lookupAccessGroupAttrs(node); if (failed(accessGroups)) return emitWarning(loc) << "could not lookup access group"; llvm::append_range(refs, *accessGroups); @@ -398,8 +395,9 @@ LoopAnnotationAttr LoopMetadataConversion::convert() { parallelAccesses); } -LoopAnnotationAttr LoopAnnotationImporter::translate(const llvm::MDNode *node, - Location loc) { +LoopAnnotationAttr +LoopAnnotationImporter::translateLoopAnnotation(const llvm::MDNode *node, + Location loc) { if (!node) return {}; @@ -409,9 +407,60 @@ LoopAnnotationAttr LoopAnnotationImporter::translate(const llvm::MDNode *node, if (it != loopMetadataMapping.end()) return it->getSecond(); - LoopAnnotationAttr attr = - LoopMetadataConversion(node, moduleImport, loc, *this).convert(); + LoopAnnotationAttr attr = LoopMetadataConversion(node, loc, *this).convert(); mapLoopMetadata(node, attr); return attr; } + +LogicalResult LoopAnnotationImporter::translateAccessGroup( + const llvm::MDNode *node, Location loc, MetadataOp metadataOp) { + SmallVector accessGroups; + if (!node->getNumOperands()) + accessGroups.push_back(node); + for (const llvm::MDOperand &operand : node->operands()) { + auto *childNode = dyn_cast(operand); + if (!childNode) + return emitWarning(loc) + << "expected access group operands to be metadata nodes"; + accessGroups.push_back(cast(operand.get())); + } + + // Convert all entries of the access group list to access group operations. + for (const llvm::MDNode *accessGroup : accessGroups) { + if (accessGroupMapping.count(accessGroup)) + continue; + // Verify the access group node is distinct and empty. + if (accessGroup->getNumOperands() != 0 || !accessGroup->isDistinct()) + return emitWarning(loc) + << "expected an access group node to be empty and distinct"; + + OpBuilder::InsertionGuard guard(builder); + builder.setInsertionPointToEnd(&metadataOp.getBody().back()); + auto groupOp = builder.create( + loc, llvm::formatv("group_{0}", accessGroupMapping.size()).str()); + // Add a mapping from the access group node to the symbol reference pointing + // to the newly created operation. + accessGroupMapping[accessGroup] = SymbolRefAttr::get( + builder.getContext(), metadataOp.getSymName(), + FlatSymbolRefAttr::get(builder.getContext(), groupOp.getSymName())); + } + return success(); +} + +FailureOr> +LoopAnnotationImporter::lookupAccessGroupAttrs(const llvm::MDNode *node) const { + // An access group node is either a single access group or an access group + // list. + SmallVector accessGroups; + if (!node->getNumOperands()) + accessGroups.push_back(accessGroupMapping.lookup(node)); + for (const llvm::MDOperand &operand : node->operands()) { + auto *node = cast(operand.get()); + accessGroups.push_back(accessGroupMapping.lookup(node)); + } + // Exit if one of the access group node lookups failed. + if (llvm::is_contained(accessGroups, nullptr)) + return failure(); + return accessGroups; +} diff --git a/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.h b/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.h index bd6f5ef350e6..5d69a63a2150 100644 --- a/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.h +++ b/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.h @@ -21,13 +21,28 @@ namespace mlir { namespace LLVM { namespace detail { -/// A helper class that converts a `llvm.loop` metadata node into a -/// corresponding LoopAnnotationAttr. +/// A helper class that converts llvm.loop metadata nodes into corresponding +/// LoopAnnotationAttrs and llvm.access.group nodes into +/// AccessGroupMetadataOps. class LoopAnnotationImporter { public: - explicit LoopAnnotationImporter(ModuleImport &moduleImport) - : moduleImport(moduleImport) {} - LoopAnnotationAttr translate(const llvm::MDNode *node, Location loc); + explicit LoopAnnotationImporter(OpBuilder &builder) : builder(builder) {} + LoopAnnotationAttr translateLoopAnnotation(const llvm::MDNode *node, + Location loc); + + /// Converts all LLVM access groups starting from node to MLIR access group + /// operations mested in the region of metadataOp. It stores a mapping from + /// every nested access group nod to the symbol pointing to the translated + /// operation. Returns success if all conversions succeed and failure + /// otherwise. + LogicalResult translateAccessGroup(const llvm::MDNode *node, Location loc, + MetadataOp metadataOp); + + /// Returns the symbol references pointing to the access group operations that + /// map to the access group nodes starting from the access group metadata + /// node. Returns failure, if any of the symbol references cannot be found. + FailureOr> + lookupAccessGroupAttrs(const llvm::MDNode *node) const; private: /// Returns the LLVM metadata corresponding to a llvm loop metadata attribute. @@ -42,8 +57,11 @@ class LoopAnnotationImporter { "attempting to map loop options that was already mapped"); } - ModuleImport &moduleImport; + OpBuilder &builder; DenseMap loopMetadataMapping; + /// Mapping between original LLVM access group metadata nodes and the symbol + /// references pointing to the imported MLIR access group operations. + DenseMap accessGroupMapping; }; } // namespace detail diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp index b075c75d950b..7bcef5d85e62 100644 --- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp @@ -255,7 +255,8 @@ ModuleImport::ModuleImport(ModuleOp mlirModule, iface(mlirModule->getContext()), typeTranslator(*mlirModule->getContext()), debugImporter(std::make_unique(mlirModule)), - loopAnnotationImporter(std::make_unique(*this)) { + loopAnnotationImporter( + std::make_unique(builder)) { builder.setInsertionPointToStart(mlirModule.getBody()); } @@ -512,35 +513,11 @@ LogicalResult ModuleImport::processTBAAMetadata(const llvm::MDNode *node) { LogicalResult ModuleImport::processAccessGroupMetadata(const llvm::MDNode *node) { - // An access group node is either access group or an access group list. Start - // by collecting all access groups to translate. - SmallVector accessGroups; - if (!node->getNumOperands()) - accessGroups.push_back(node); - for (const llvm::MDOperand &operand : node->operands()) - accessGroups.push_back(cast(operand.get())); - - // Convert all entries of the access group list to access group operations. - for (const llvm::MDNode *accessGroup : accessGroups) { - if (accessGroupMapping.count(accessGroup)) - continue; - // Verify the access group node is distinct and empty. - Location loc = mlirModule.getLoc(); - if (accessGroup->getNumOperands() != 0 || !accessGroup->isDistinct()) - return emitError(loc) << "unsupported access group node: " - << diagMD(accessGroup, llvmModule.get()); - - MetadataOp metadataOp = getGlobalMetadataOp(); - OpBuilder::InsertionGuard guard(builder); - builder.setInsertionPointToEnd(&metadataOp.getBody().back()); - auto groupOp = builder.create( - loc, (Twine("group_") + Twine(accessGroupMapping.size())).str()); - // Add a mapping from the access group node to the symbol reference pointing - // to the newly created operation. - accessGroupMapping[accessGroup] = SymbolRefAttr::get( - builder.getContext(), metadataOp.getSymName(), - FlatSymbolRefAttr::get(builder.getContext(), groupOp.getSymName())); - } + Location loc = mlirModule.getLoc(); + if (failed(loopAnnotationImporter->translateAccessGroup( + node, loc, getGlobalMetadataOp()))) + return emitError(loc) << "unsupported access group node: " + << diagMD(node, llvmModule.get()); return success(); } @@ -1582,25 +1559,13 @@ LogicalResult ModuleImport::processBasicBlock(llvm::BasicBlock *bb, FailureOr> ModuleImport::lookupAccessGroupAttrs(const llvm::MDNode *node) const { - // An access group node is either a single access group or an access group - // list. - SmallVector accessGroups; - if (!node->getNumOperands()) - accessGroups.push_back(accessGroupMapping.lookup(node)); - for (const llvm::MDOperand &operand : node->operands()) { - auto *node = cast(operand.get()); - accessGroups.push_back(accessGroupMapping.lookup(node)); - } - // Exit if one of the access group node lookups failed. - if (llvm::is_contained(accessGroups, nullptr)) - return failure(); - return accessGroups; + return loopAnnotationImporter->lookupAccessGroupAttrs(node); } LoopAnnotationAttr ModuleImport::translateLoopAnnotationAttr(const llvm::MDNode *node, Location loc) const { - return loopAnnotationImporter->translate(node, loc); + return loopAnnotationImporter->translateLoopAnnotation(node, loc); } OwningOpRef diff --git a/mlir/test/Target/LLVMIR/Import/import-failure.ll b/mlir/test/Target/LLVMIR/Import/import-failure.ll index 1db992736d99..d09d4eb9e90e 100644 --- a/mlir/test/Target/LLVMIR/Import/import-failure.ll +++ b/mlir/test/Target/LLVMIR/Import/import-failure.ll @@ -241,7 +241,8 @@ define dso_local void @tbaa(ptr %0) { ; // ----- ; CHECK: import-failure.ll -; CHECK-SAME: error: unsupported access group node: !0 = !{} +; CHECK-SAME: warning: expected an access group node to be empty and distinct +; CHECK: error: unsupported access group node: !0 = !{} define void @access_group(ptr %arg1) { %1 = load i32, ptr %arg1, !llvm.access.group !0 ret void @@ -252,7 +253,8 @@ define void @access_group(ptr %arg1) { ; // ----- ; CHECK: import-failure.ll -; CHECK-SAME: error: unsupported access group node: !1 = distinct !{!"unsupported access group"} +; CHECK-SAME: warning: expected an access group node to be empty and distinct +; CHECK: error: unsupported access group node: !0 = !{!1} define void @access_group(ptr %arg1) { %1 = load i32, ptr %arg1, !llvm.access.group !0 ret void @@ -263,6 +265,18 @@ define void @access_group(ptr %arg1) { ; // ----- +; CHECK: import-failure.ll +; CHECK-SAME: warning: expected access group operands to be metadata nodes +; CHECK: error: unsupported access group node: !0 = !{i1 false} +define void @access_group(ptr %arg1) { + %1 = load i32, ptr %arg1, !llvm.access.group !0 + ret void +} + +!0 = !{i1 false} + +; // ----- + ; CHECK: import-failure.ll ; CHECK-SAME: warning: expected all loop properties to be either debug locations or metadata nodes ; CHECK: import-failure.ll From 2314bf2a3bd3dce5bbb9c2678df2c7f5e4ae79b4 Mon Sep 17 00:00:00 2001 From: Tobias Gysi Date: Mon, 6 Mar 2023 10:43:59 +0100 Subject: [PATCH 069/147] [mlir][llvm] Store memory op metadata using op attributes. The revision introduces operation attributes to store tbaa metadata on load and store operations rather than relying using dialect attributes. At the same time, the change also ensures the provided getters and setters instead are used instead of a string based lookup. The latter is done for the tbaa, access groups, and alias scope attributes. The goal of this change is to ensure the metadata attributes are only placed on operations that have the corresponding operation attributes. This is imported since only these operations later on translate these attributes to LLVM IR. Dialect attributes placed on other operations are lost during the translation. Reviewed By: vzakhari, Dinistro Differential Revision: https://reviews.llvm.org/D143654 (cherry picked from commit fef08da4b75fc751c6117df2a0213a0b075d05f5) --- flang/lib/Optimizer/CodeGen/TBAABuilder.cpp | 11 +- flang/test/Fir/tbaa.fir | 58 ++++----- .../include/mlir/Dialect/LLVMIR/LLVMOpBase.td | 4 - mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td | 2 + .../mlir/Target/LLVMIR/ModuleTranslation.h | 19 ++- mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 117 +++++++++--------- .../LLVMIR/LLVMIRToLLVMTranslation.cpp | 47 ++++--- .../LLVMIR/LoopAnnotationTranslation.cpp | 2 +- mlir/lib/Target/LLVMIR/ModuleTranslation.cpp | 112 ++++++++++------- mlir/test/Dialect/LLVMIR/tbaa-invalid.mlir | 12 +- .../Target/LLVMIR/Import/import-failure.ll | 2 - mlir/test/Target/LLVMIR/tbaa.mlir | 8 +- 12 files changed, 218 insertions(+), 176 deletions(-) diff --git a/flang/lib/Optimizer/CodeGen/TBAABuilder.cpp b/flang/lib/Optimizer/CodeGen/TBAABuilder.cpp index 2d206ed2dcbf..c42081869c7a 100644 --- a/flang/lib/Optimizer/CodeGen/TBAABuilder.cpp +++ b/flang/lib/Optimizer/CodeGen/TBAABuilder.cpp @@ -12,6 +12,7 @@ #include "TBAABuilder.h" #include "flang/Optimizer/Dialect/FIRType.h" +#include "llvm/ADT/TypeSwitch.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" @@ -159,9 +160,13 @@ void TBAABuilder::attachTBAATag(Operation *op, Type baseFIRType, else tbaaTagSym = getDataAccessTag(baseFIRType, accessFIRType, gep); - if (tbaaTagSym) - op->setAttr(LLVMDialect::getTBAAAttrName(), - ArrayAttr::get(op->getContext(), tbaaTagSym)); + if (!tbaaTagSym) + return; + + auto tbaaAttr = ArrayAttr::get(op->getContext(), tbaaTagSym); + llvm::TypeSwitch(op) + .Case([&](auto memOp) { memOp.setTbaaAttr(tbaaAttr); }) + .Default([](auto) { llvm_unreachable("expected LoadOp or StoreOp"); }); } } // namespace fir diff --git a/flang/test/Fir/tbaa.fir b/flang/test/Fir/tbaa.fir index ac94ea94cb3c..66261e6ee002 100644 --- a/flang/test/Fir/tbaa.fir +++ b/flang/test/Fir/tbaa.fir @@ -28,10 +28,10 @@ module { // CHECK: %[[VAL_4:.*]] = llvm.mlir.constant(0 : i64) : i64 // CHECK: %[[VAL_5:.*]] = llvm.mlir.constant(10 : i32) : i32 // CHECK: %[[VAL_6:.*]] = llvm.getelementptr %[[VAL_0]][0, 0] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>) -> !llvm.ptr>> -// CHECK: %[[VAL_7:.*]] = llvm.load %[[VAL_6]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT:tag_[0-9]*]]]} : !llvm.ptr>> +// CHECK: %[[VAL_7:.*]] = llvm.load %[[VAL_6]] {tbaa = [@__flang_tbaa::@[[BOXT:tag_[0-9]*]]]} : !llvm.ptr>> // CHECK: %[[VAL_8:.*]] = llvm.mlir.constant(0 : i64) : i64 // CHECK: %[[VAL_9:.*]] = llvm.getelementptr %[[VAL_0]][0, 7, 0, 2] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>) -> !llvm.ptr -// CHECK: %[[VAL_10:.*]] = llvm.load %[[VAL_9]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr +// CHECK: %[[VAL_10:.*]] = llvm.load %[[VAL_9]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr // CHECK: %[[VAL_11:.*]] = llvm.mul %[[VAL_4]], %[[VAL_10]] : i64 // CHECK: %[[VAL_12:.*]] = llvm.add %[[VAL_11]], %[[VAL_8]] : i64 // CHECK: %[[VAL_13:.*]] = llvm.bitcast %[[VAL_7]] : !llvm.ptr> to !llvm.ptr @@ -40,11 +40,11 @@ module { // CHECK: %[[VAL_16:.*]] = llvm.mlir.constant(0 : i64) : i64 // CHECK: %[[VAL_17:.*]] = llvm.mlir.constant(-1 : i32) : i32 // CHECK: %[[VAL_18:.*]] = llvm.getelementptr %[[VAL_0]][0, 8] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>) -> !llvm.ptr> -// CHECK: %[[VAL_19:.*]] = llvm.load %[[VAL_18]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr> +// CHECK: %[[VAL_19:.*]] = llvm.load %[[VAL_18]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr> // CHECK: %[[VAL_20:.*]] = llvm.getelementptr %[[VAL_0]][0, 1] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>) -> !llvm.ptr -// CHECK: %[[VAL_21:.*]] = llvm.load %[[VAL_20]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr +// CHECK: %[[VAL_21:.*]] = llvm.load %[[VAL_20]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr // CHECK: %[[VAL_22:.*]] = llvm.getelementptr %[[VAL_0]][0, 4] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>) -> !llvm.ptr -// CHECK: %[[VAL_23:.*]] = llvm.load %[[VAL_22]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr +// CHECK: %[[VAL_23:.*]] = llvm.load %[[VAL_22]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr // CHECK: %[[VAL_24:.*]] = llvm.mlir.undef : !llvm.struct<(ptr>, i64, i32, i8, i8, i8, i8, ptr, array<1 x i64>)> // CHECK: %[[VAL_25:.*]] = llvm.insertvalue %[[VAL_21]], %[[VAL_24]][1] : !llvm.struct<(ptr>, i64, i32, i8, i8, i8, i8, ptr, array<1 x i64>)> // CHECK: %[[VAL_26:.*]] = llvm.mlir.constant(20180515 : i32) : i32 @@ -64,15 +64,15 @@ module { // CHECK: %[[VAL_40:.*]] = llvm.insertvalue %[[VAL_39]], %[[VAL_38]][7] : !llvm.struct<(ptr>, i64, i32, i8, i8, i8, i8, ptr, array<1 x i64>)> // CHECK: %[[VAL_41:.*]] = llvm.bitcast %[[VAL_15]] : !llvm.ptr> to !llvm.ptr> // CHECK: %[[VAL_42:.*]] = llvm.insertvalue %[[VAL_41]], %[[VAL_40]][0] : !llvm.struct<(ptr>, i64, i32, i8, i8, i8, i8, ptr, array<1 x i64>)> -// CHECK: llvm.store %[[VAL_42]], %[[VAL_2]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr>, i64, i32, i8, i8, i8, i8, ptr, array<1 x i64>)>> +// CHECK: llvm.store %[[VAL_42]], %[[VAL_2]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr>, i64, i32, i8, i8, i8, i8, ptr, array<1 x i64>)>> // CHECK: %[[VAL_43:.*]] = llvm.getelementptr %[[VAL_2]][0, 4] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, ptr, array<1 x i64>)>>) -> !llvm.ptr -// CHECK: %[[VAL_44:.*]] = llvm.load %[[VAL_43]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr +// CHECK: %[[VAL_44:.*]] = llvm.load %[[VAL_43]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr // CHECK: %[[VAL_45:.*]] = llvm.icmp "eq" %[[VAL_44]], %[[VAL_3]] : i8 // CHECK: llvm.cond_br %[[VAL_45]], ^bb1, ^bb2 // CHECK: ^bb1: // CHECK: %[[VAL_46:.*]] = llvm.getelementptr %[[VAL_2]][0, 0] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, ptr, array<1 x i64>)>>) -> !llvm.ptr> -// CHECK: %[[VAL_47:.*]] = llvm.load %[[VAL_46]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr> -// CHECK: llvm.store %[[VAL_5]], %[[VAL_47]] {llvm.tbaa = [@__flang_tbaa::@[[DATAT:tag_[0-9]*]]]} : !llvm.ptr +// CHECK: %[[VAL_47:.*]] = llvm.load %[[VAL_46]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr> +// CHECK: llvm.store %[[VAL_5]], %[[VAL_47]] {tbaa = [@__flang_tbaa::@[[DATAT:tag_[0-9]*]]]} : !llvm.ptr // CHECK: llvm.br ^bb2 // CHECK: ^bb2: // CHECK: llvm.return @@ -133,24 +133,24 @@ module { // CHECK: %[[VAL_8:.*]] = llvm.mlir.addressof @_QQcl.2E2F64756D6D792E66393000 : !llvm.ptr> // CHECK: %[[VAL_9:.*]] = llvm.bitcast %[[VAL_8]] : !llvm.ptr> to !llvm.ptr // CHECK: %[[VAL_10:.*]] = llvm.call @_FortranAioBeginExternalListOutput(%[[VAL_6]], %[[VAL_9]], %[[VAL_5]]) {fastmathFlags = #llvm.fastmath} : (i32, !llvm.ptr, i32) -> !llvm.ptr -// CHECK: %[[VAL_11:.*]] = llvm.load %[[VAL_7]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT:tag_[0-9]*]]]} : !llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>> -// CHECK: llvm.store %[[VAL_11]], %[[VAL_3]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>> +// CHECK: %[[VAL_11:.*]] = llvm.load %[[VAL_7]] {tbaa = [@__flang_tbaa::@[[BOXT:tag_[0-9]*]]]} : !llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>> +// CHECK: llvm.store %[[VAL_11]], %[[VAL_3]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>> // CHECK: %[[VAL_12:.*]] = llvm.getelementptr %[[VAL_3]][0, 7, %[[VAL_4]], 0] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>, i64) -> !llvm.ptr -// CHECK: %[[VAL_13:.*]] = llvm.load %[[VAL_12]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr +// CHECK: %[[VAL_13:.*]] = llvm.load %[[VAL_12]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr // CHECK: %[[VAL_14:.*]] = llvm.getelementptr %[[VAL_3]][0, 7, %[[VAL_4]], 1] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>, i64) -> !llvm.ptr -// CHECK: %[[VAL_15:.*]] = llvm.load %[[VAL_14]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr +// CHECK: %[[VAL_15:.*]] = llvm.load %[[VAL_14]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr // CHECK: %[[VAL_16:.*]] = llvm.getelementptr %[[VAL_3]][0, 7, %[[VAL_4]], 2] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>, i64) -> !llvm.ptr -// CHECK: %[[VAL_17:.*]] = llvm.load %[[VAL_16]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr +// CHECK: %[[VAL_17:.*]] = llvm.load %[[VAL_16]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr // CHECK: %[[VAL_18:.*]] = llvm.getelementptr %[[VAL_3]][0, 8] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>) -> !llvm.ptr> -// CHECK: %[[VAL_19:.*]] = llvm.load %[[VAL_18]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr> +// CHECK: %[[VAL_19:.*]] = llvm.load %[[VAL_18]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr> // CHECK: %[[VAL_20:.*]] = llvm.mlir.constant(0 : i64) : i64 // CHECK: %[[VAL_21:.*]] = llvm.mlir.constant(-1 : i32) : i32 // CHECK: %[[VAL_22:.*]] = llvm.getelementptr %[[VAL_3]][0, 1] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>) -> !llvm.ptr -// CHECK: %[[VAL_23:.*]] = llvm.load %[[VAL_22]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr +// CHECK: %[[VAL_23:.*]] = llvm.load %[[VAL_22]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr // CHECK: %[[VAL_24:.*]] = llvm.getelementptr %[[VAL_3]][0, 4] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>) -> !llvm.ptr -// CHECK: %[[VAL_25:.*]] = llvm.load %[[VAL_24]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr +// CHECK: %[[VAL_25:.*]] = llvm.load %[[VAL_24]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr // CHECK: %[[VAL_26:.*]] = llvm.getelementptr %[[VAL_3]][0, 8] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>) -> !llvm.ptr> -// CHECK: %[[VAL_27:.*]] = llvm.load %[[VAL_26]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr> +// CHECK: %[[VAL_27:.*]] = llvm.load %[[VAL_26]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr> // CHECK: %[[VAL_28:.*]] = llvm.mlir.undef : !llvm.struct<(ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)> // CHECK: %[[VAL_29:.*]] = llvm.insertvalue %[[VAL_23]], %[[VAL_28]][1] : !llvm.struct<(ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)> // CHECK: %[[VAL_30:.*]] = llvm.mlir.constant(20180515 : i32) : i32 @@ -169,13 +169,13 @@ module { // CHECK: %[[VAL_43:.*]] = llvm.bitcast %[[VAL_27]] : !llvm.ptr to !llvm.ptr // CHECK: %[[VAL_44:.*]] = llvm.insertvalue %[[VAL_43]], %[[VAL_42]][8] : !llvm.struct<(ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)> // CHECK: %[[VAL_45:.*]] = llvm.getelementptr %[[VAL_3]][0, 7, 0, 0] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>) -> !llvm.ptr -// CHECK: %[[VAL_46:.*]] = llvm.load %[[VAL_45]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr +// CHECK: %[[VAL_46:.*]] = llvm.load %[[VAL_45]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr // CHECK: %[[VAL_47:.*]] = llvm.getelementptr %[[VAL_3]][0, 7, 0, 1] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>) -> !llvm.ptr -// CHECK: %[[VAL_48:.*]] = llvm.load %[[VAL_47]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr +// CHECK: %[[VAL_48:.*]] = llvm.load %[[VAL_47]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr // CHECK: %[[VAL_49:.*]] = llvm.getelementptr %[[VAL_3]][0, 7, 0, 2] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>) -> !llvm.ptr -// CHECK: %[[VAL_50:.*]] = llvm.load %[[VAL_49]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr +// CHECK: %[[VAL_50:.*]] = llvm.load %[[VAL_49]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr // CHECK: %[[VAL_51:.*]] = llvm.getelementptr %[[VAL_3]][0, 0] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>) -> !llvm.ptr>> -// CHECK: %[[VAL_52:.*]] = llvm.load %[[VAL_51]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr>> +// CHECK: %[[VAL_52:.*]] = llvm.load %[[VAL_51]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr>> // CHECK: %[[VAL_53:.*]] = llvm.mlir.constant(0 : i64) : i64 // CHECK: %[[VAL_54:.*]] = llvm.mlir.constant(1 : i64) : i64 // CHECK: %[[VAL_55:.*]] = llvm.icmp "eq" %[[VAL_48]], %[[VAL_53]] : i64 @@ -185,7 +185,7 @@ module { // CHECK: %[[VAL_59:.*]] = llvm.insertvalue %[[VAL_50]], %[[VAL_58]][7, 0, 2] : !llvm.struct<(ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)> // CHECK: %[[VAL_60:.*]] = llvm.bitcast %[[VAL_52]] : !llvm.ptr> to !llvm.ptr> // CHECK: %[[VAL_61:.*]] = llvm.insertvalue %[[VAL_60]], %[[VAL_59]][0] : !llvm.struct<(ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)> -// CHECK: llvm.store %[[VAL_61]], %[[VAL_1]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>> +// CHECK: llvm.store %[[VAL_61]], %[[VAL_1]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>> // CHECK: %[[VAL_62:.*]] = llvm.bitcast %[[VAL_1]] : !llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>> to !llvm.ptr>, i64, i32, i8, i8, i8, i8, ptr, array<1 x i64>)>> // CHECK: %[[VAL_63:.*]] = llvm.call @_FortranAioOutputDescriptor(%[[VAL_10]], %[[VAL_62]]) {fastmathFlags = #llvm.fastmath} : (!llvm.ptr, !llvm.ptr>, i64, i32, i8, i8, i8, i8, ptr, array<1 x i64>)>>) -> i1 // CHECK: %[[VAL_64:.*]] = llvm.call @_FortranAioEndIoStatement(%[[VAL_10]]) {fastmathFlags = #llvm.fastmath} : (!llvm.ptr) -> i32 @@ -253,7 +253,7 @@ func.func @tbaa(%arg0: !fir.box>) -> i32 { // CHECK-LABEL: llvm.func @tbaa( // CHECK-SAME: %[[VAL_0:.*]]: !llvm.ptr, i64, i32, i8, i8, i8, i8)>>) -> i32 { // CHECK: %[[VAL_1:.*]] = llvm.getelementptr %[[VAL_0]][0, 3] : (!llvm.ptr, i64, i32, i8, i8, i8, i8)>>) -> !llvm.ptr -// CHECK: %[[VAL_2:.*]] = llvm.load %[[VAL_1]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT:tag_[0-9]*]]]} : !llvm.ptr +// CHECK: %[[VAL_2:.*]] = llvm.load %[[VAL_1]] {tbaa = [@__flang_tbaa::@[[BOXT:tag_[0-9]*]]]} : !llvm.ptr // CHECK: llvm.return %[[VAL_2]] : i32 // CHECK: } @@ -275,7 +275,7 @@ func.func @tbaa(%arg0: !fir.box>) -> i1 { // CHECK-LABEL: llvm.func @tbaa( // CHECK-SAME: %[[VAL_0:.*]]: !llvm.ptr, i64, i32, i8, i8, i8, i8)>>) -> i1 { // CHECK: %[[VAL_1:.*]] = llvm.getelementptr %[[VAL_0]][0, 3] : (!llvm.ptr, i64, i32, i8, i8, i8, i8)>>) -> !llvm.ptr -// CHECK: %[[VAL_2:.*]] = llvm.load %[[VAL_1]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT:tag_[0-9]*]]]} : !llvm.ptr +// CHECK: %[[VAL_2:.*]] = llvm.load %[[VAL_1]] {tbaa = [@__flang_tbaa::@[[BOXT:tag_[0-9]*]]]} : !llvm.ptr // CHECK: %[[VAL_3:.*]] = llvm.mlir.constant(0 : i32) : i32 // CHECK: %[[VAL_4:.*]] = llvm.icmp "ne" %[[VAL_2]], %[[VAL_3]] : i32 // CHECK: llvm.return %[[VAL_4]] : i1 @@ -299,7 +299,7 @@ func.func @tbaa(%arg0: !fir.box) -> i32 { // CHECK-LABEL: llvm.func @tbaa( // CHECK-SAME: %[[VAL_0:.*]]: !llvm.ptr, i64, i32, i8, i8, i8, i8)>>) -> i32 { // CHECK: %[[VAL_1:.*]] = llvm.getelementptr %[[VAL_0]][0, 1] : (!llvm.ptr, i64, i32, i8, i8, i8, i8)>>) -> !llvm.ptr -// CHECK: %[[VAL_2:.*]] = llvm.load %[[VAL_1]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT:tag_[0-9]*]]]} : !llvm.ptr +// CHECK: %[[VAL_2:.*]] = llvm.load %[[VAL_1]] {tbaa = [@__flang_tbaa::@[[BOXT:tag_[0-9]*]]]} : !llvm.ptr // CHECK: llvm.return %[[VAL_2]] : i32 // CHECK: } @@ -321,7 +321,7 @@ func.func @tbaa(%arg0: !fir.box>) -> i1 { // CHECK-LABEL: llvm.func @tbaa( // CHECK-SAME: %[[VAL_0:.*]]: !llvm.ptr, i64, i32, i8, i8, i8, i8)>>) -> i1 { // CHECK: %[[VAL_1:.*]] = llvm.getelementptr %[[VAL_0]][0, 5] : (!llvm.ptr, i64, i32, i8, i8, i8, i8)>>) -> !llvm.ptr -// CHECK: %[[VAL_2:.*]] = llvm.load %[[VAL_1]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT:tag_[0-9]*]]]} : !llvm.ptr +// CHECK: %[[VAL_2:.*]] = llvm.load %[[VAL_1]] {tbaa = [@__flang_tbaa::@[[BOXT:tag_[0-9]*]]]} : !llvm.ptr // CHECK: %[[VAL_3:.*]] = llvm.mlir.constant(2 : i32) : i32 // CHECK: %[[VAL_4:.*]] = llvm.and %[[VAL_2]], %[[VAL_3]] : i32 // CHECK: %[[VAL_5:.*]] = llvm.mlir.constant(0 : i32) : i32 @@ -353,11 +353,11 @@ func.func @tbaa(%arg0: !fir.box>) { // CHECK: %[[VAL_4:.*]] = llvm.sub %[[VAL_1]], %[[VAL_2]] : i64 // CHECK: %[[VAL_5:.*]] = llvm.mul %[[VAL_4]], %[[VAL_2]] : i64 // CHECK: %[[VAL_6:.*]] = llvm.getelementptr %[[VAL_0]][0, 7, 0, 2] : (!llvm.ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)>>) -> !llvm.ptr -// CHECK: %[[VAL_7:.*]] = llvm.load %[[VAL_6]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT:tag_[0-9]*]]]} : !llvm.ptr +// CHECK: %[[VAL_7:.*]] = llvm.load %[[VAL_6]] {tbaa = [@__flang_tbaa::@[[BOXT:tag_[0-9]*]]]} : !llvm.ptr // CHECK: %[[VAL_8:.*]] = llvm.mul %[[VAL_5]], %[[VAL_7]] : i64 // CHECK: %[[VAL_9:.*]] = llvm.add %[[VAL_8]], %[[VAL_3]] : i64 // CHECK: %[[VAL_10:.*]] = llvm.getelementptr %[[VAL_0]][0, 0] : (!llvm.ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)>>) -> !llvm.ptr> -// CHECK: %[[VAL_11:.*]] = llvm.load %[[VAL_10]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr> +// CHECK: %[[VAL_11:.*]] = llvm.load %[[VAL_10]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr> // CHECK: %[[VAL_12:.*]] = llvm.bitcast %[[VAL_11]] : !llvm.ptr to !llvm.ptr // CHECK: %[[VAL_13:.*]] = llvm.getelementptr %[[VAL_12]]{{\[}}%[[VAL_9]]] : (!llvm.ptr, i64) -> !llvm.ptr // CHECK: %[[VAL_14:.*]] = llvm.bitcast %[[VAL_13]] : !llvm.ptr to !llvm.ptr diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td index b2ca396ab72a..fbc0e6d0a596 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td @@ -36,11 +36,7 @@ def LLVM_Dialect : Dialect { let extraClassDeclaration = [{ /// Name of the data layout attributes. static StringRef getDataLayoutAttrName() { return "llvm.data_layout"; } - static StringRef getNoAliasScopesAttrName() { return "noalias_scopes"; } - static StringRef getAliasScopesAttrName() { return "alias_scopes"; } static StringRef getLoopAttrName() { return "llvm.loop"; } - static StringRef getAccessGroupsAttrName() { return "access_groups"; } - static StringRef getTBAAAttrName() { return "llvm.tbaa"; } /// Names of llvm parameter attributes. static StringRef getAlignAttrName() { return "llvm.align"; } diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td index 39b79f1ed228..8e4b8340a52b 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td @@ -350,6 +350,7 @@ def LLVM_LoadOp : LLVM_Op<"load">, MemoryOpBase { OptionalAttr:$access_groups, OptionalAttr:$alias_scopes, OptionalAttr:$noalias_scopes, + OptionalAttr:$tbaa, OptionalAttr:$alignment, UnitAttr:$volatile_, UnitAttr:$nontemporal); let results = (outs LLVM_LoadableType:$res); @@ -390,6 +391,7 @@ def LLVM_StoreOp : LLVM_Op<"store">, MemoryOpBase { OptionalAttr:$access_groups, OptionalAttr:$alias_scopes, OptionalAttr:$noalias_scopes, + OptionalAttr:$tbaa, OptionalAttr:$alignment, UnitAttr:$volatile_, UnitAttr:$nontemporal); string llvmInstName = "Store"; diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h index 59b29f506609..2b08d96b680b 100644 --- a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h +++ b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h @@ -120,15 +120,14 @@ class ModuleTranslation { /// in these blocks. void forgetMapping(Region ®ion); - /// Returns the LLVM metadata corresponding to a reference to an mlir LLVM - /// dialect access group operation. - llvm::MDNode *getAccessGroup(Operation &opInst, + /// Returns the LLVM metadata corresponding to a symbol reference to an mlir + /// LLVM dialect access group operation. + llvm::MDNode *getAccessGroup(Operation *op, SymbolRefAttr accessGroupRef) const; - /// Returns the LLVM metadata corresponding to a reference to an mlir LLVM - /// dialect alias scope operation - llvm::MDNode *getAliasScope(Operation &opInst, - SymbolRefAttr aliasScopeRef) const; + /// Returns the LLVM metadata corresponding to a symbol reference to an mlir + /// LLVM dialect alias scope operation + llvm::MDNode *getAliasScope(Operation *op, SymbolRefAttr aliasScopeRef) const; // Sets LLVM metadata for memory operations that are in a parallel loop. void setAccessGroupsMetadata(Operation *op, llvm::Instruction *inst); @@ -287,9 +286,9 @@ class ModuleTranslation { /// metadata nodes for them and their domains. LogicalResult createAliasScopeMetadata(); - /// Returns the LLVM metadata corresponding to a reference to an mlir LLVM - /// dialect TBAATagOp operation. - llvm::MDNode *getTBAANode(Operation &memOp, SymbolRefAttr tagRef) const; + /// Returns the LLVM metadata corresponding to a symbol reference to an mlir + /// LLVM dialect TBAATagOp operation. + llvm::MDNode *getTBAANode(Operation *op, SymbolRefAttr tagRef) const; /// Process tbaa LLVM Metadata operations and create LLVM /// metadata nodes for them. diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp index 73a2dc9dea63..72526228b4d8 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -668,53 +668,60 @@ Type LLVM::GEPOp::getSourceElementType() { // Builder, printer and parser for for LLVM::LoadOp. //===----------------------------------------------------------------------===// -LogicalResult verifySymbolAttribute( - Operation *op, StringRef attributeName, +/// Verifies the given array attribute contains symbol references and checks the +/// referenced symbol types using the provided verification function. +LogicalResult verifyMemOpSymbolRefs( + Operation *op, StringRef name, ArrayAttr symbolRefs, llvm::function_ref verifySymbolType) { - if (Attribute attribute = op->getAttr(attributeName)) { - // Verify that the attribute is a symbol ref array attribute, - // because this constraint is not verified for all attribute - // names processed here (e.g. 'tbaa'). This verification - // is redundant in some cases. - if (!(attribute.isa() && - llvm::all_of(attribute.cast(), [&](Attribute attr) { - return attr && attr.isa(); - }))) - return op->emitOpError("attribute '") - << attributeName - << "' failed to satisfy constraint: symbol ref array attribute"; - - for (SymbolRefAttr symbolRef : - attribute.cast().getAsRange()) { - StringAttr metadataName = symbolRef.getRootReference(); - StringAttr symbolName = symbolRef.getLeafReference(); - // We want @metadata::@symbol, not just @symbol - if (metadataName == symbolName) { - return op->emitOpError() << "expected '" << symbolRef - << "' to specify a fully qualified reference"; - } - auto metadataOp = SymbolTable::lookupNearestSymbolFrom( - op->getParentOp(), metadataName); - if (!metadataOp) - return op->emitOpError() - << "expected '" << symbolRef << "' to reference a metadata op"; - Operation *symbolOp = - SymbolTable::lookupNearestSymbolFrom(metadataOp, symbolName); - if (!symbolOp) - return op->emitOpError() - << "expected '" << symbolRef << "' to be a valid reference"; - if (failed(verifySymbolType(symbolOp, symbolRef))) { - return failure(); - } + assert(symbolRefs && "expected a non-null attribute"); + + // Verify that the attribute is a symbol ref array attribute, + // because this constraint is not verified for all attribute + // names processed here (e.g. 'tbaa'). This verification + // is redundant in some cases. + if (!llvm::all_of(symbolRefs, [](Attribute attr) { + return attr && attr.isa(); + })) + return op->emitOpError("attribute '") + << name + << "' failed to satisfy constraint: symbol ref array attribute"; + + for (SymbolRefAttr symbolRef : symbolRefs.getAsRange()) { + StringAttr metadataName = symbolRef.getRootReference(); + StringAttr symbolName = symbolRef.getLeafReference(); + // We want @metadata::@symbol, not just @symbol + if (metadataName == symbolName) { + return op->emitOpError() << "expected '" << symbolRef + << "' to specify a fully qualified reference"; + } + auto metadataOp = SymbolTable::lookupNearestSymbolFrom( + op->getParentOp(), metadataName); + if (!metadataOp) + return op->emitOpError() + << "expected '" << symbolRef << "' to reference a metadata op"; + Operation *symbolOp = + SymbolTable::lookupNearestSymbolFrom(metadataOp, symbolName); + if (!symbolOp) + return op->emitOpError() + << "expected '" << symbolRef << "' to be a valid reference"; + if (failed(verifySymbolType(symbolOp, symbolRef))) { + return failure(); } } + return success(); } -// Verifies that metadata ops are wired up properly. +/// Verifies the given array attribute contains symbol references that point to +/// metadata operations of the given type. template -static LogicalResult verifyOpMetadata(Operation *op, StringRef attributeName) { +static LogicalResult +verifyMemOpSymbolRefsPointTo(Operation *op, StringRef name, + std::optional symbolRefs) { + if (!symbolRefs) + return success(); + auto verifySymbolType = [op](Operation *symbolOp, SymbolRefAttr symbolRef) -> LogicalResult { if (!isa(symbolOp)) { @@ -724,35 +731,33 @@ static LogicalResult verifyOpMetadata(Operation *op, StringRef attributeName) { } return success(); }; - - return verifySymbolAttribute(op, attributeName, verifySymbolType); + return verifyMemOpSymbolRefs(op, name, *symbolRefs, verifySymbolType); } -static LogicalResult verifyMemoryOpMetadata(Operation *op) { - // access_groups - if (failed(verifyOpMetadata( - op, LLVMDialect::getAccessGroupsAttrName()))) +/// Verifies the types of the metadata operations referenced by aliasing and +/// access group metadata. +template +LogicalResult verifyMemOpMetadata(OpTy memOp) { + if (failed(verifyMemOpSymbolRefsPointTo( + memOp, memOp.getAccessGroupsAttrName(), memOp.getAccessGroups()))) return failure(); - // alias_scopes - if (failed(verifyOpMetadata( - op, LLVMDialect::getAliasScopesAttrName()))) + if (failed(verifyMemOpSymbolRefsPointTo( + memOp, memOp.getAliasScopesAttrName(), memOp.getAliasScopes()))) return failure(); - // noalias_scopes - if (failed(verifyOpMetadata( - op, LLVMDialect::getNoAliasScopesAttrName()))) + if (failed(verifyMemOpSymbolRefsPointTo( + memOp, memOp.getNoaliasScopesAttrName(), memOp.getNoaliasScopes()))) return failure(); - // tbaa - if (failed(verifyOpMetadata(op, - LLVMDialect::getTBAAAttrName()))) + if (failed(verifyMemOpSymbolRefsPointTo( + memOp, memOp.getTbaaAttrName(), memOp.getTbaa()))) return failure(); return success(); } -LogicalResult LoadOp::verify() { return verifyMemoryOpMetadata(*this); } +LogicalResult LoadOp::verify() { return verifyMemOpMetadata(*this); } void LoadOp::build(OpBuilder &builder, OperationState &result, Type t, Value addr, unsigned alignment, bool isVolatile, @@ -828,7 +833,7 @@ ParseResult LoadOp::parse(OpAsmParser &parser, OperationState &result) { // Builder, printer and parser for LLVM::StoreOp. //===----------------------------------------------------------------------===// -LogicalResult StoreOp::verify() { return verifyMemoryOpMetadata(*this); } +LogicalResult StoreOp::verify() { return verifyMemOpMetadata(*this); } void StoreOp::build(OpBuilder &builder, OperationState &result, Value value, Value addr, unsigned alignment, bool isVolatile, diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp index 207ef8b42809..7d36d40c7049 100644 --- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp @@ -76,6 +76,30 @@ static ArrayRef getSupportedMetadataImpl() { return convertibleMetadata; } +namespace { +/// Helper class to attach metadata attributes to specific operation types. It +/// specializes TypeSwitch to take an Operation and return a LogicalResult. +template +struct AttributeSetter { + AttributeSetter(Operation *op) : op(op) {} + + /// Calls `attachFn` on the provided Operation if it has one of + /// the given operation types. Returns failure otherwise. + template + LogicalResult apply(CallableT &&attachFn) { + return llvm::TypeSwitch(op) + .Case([&attachFn](auto concreteOp) { + attachFn(concreteOp); + return success(); + }) + .Default([&](auto) { return failure(); }); + } + +private: + Operation *op; +}; +} // namespace + /// Converts the given profiling metadata `node` to an MLIR profiling attribute /// and attaches it to the imported operation if the translation succeeds. /// Returns failure otherwise. @@ -129,16 +153,10 @@ static LogicalResult setProfilingAttr(OpBuilder &builder, llvm::MDNode *node, branchWeights.push_back(branchWeight->getZExtValue()); } - // Attach the branch weights to the operations that support it. - return llvm::TypeSwitch(op) - .Case([&](auto branchWeightOp) { + return AttributeSetter(op).apply( + [&](auto branchWeightOp) { branchWeightOp.setBranchWeightsAttr( builder.getI32VectorAttr(branchWeights)); - return success(); - }) - .Default([op](auto) { - return op->emitWarning() - << op->getName() << " does not support branch weights"; }); } @@ -151,9 +169,9 @@ static LogicalResult setTBAAAttr(const llvm::MDNode *node, Operation *op, if (!tbaaTagSym) return failure(); - op->setAttr(LLVMDialect::getTBAAAttrName(), - ArrayAttr::get(op->getContext(), tbaaTagSym)); - return success(); + return AttributeSetter(op).apply([&](auto memOp) { + memOp.setTbaaAttr(ArrayAttr::get(memOp.getContext(), tbaaTagSym)); + }); } /// Looks up all the symbol references pointing to the access group operations @@ -169,9 +187,10 @@ static LogicalResult setAccessGroupAttr(const llvm::MDNode *node, Operation *op, SmallVector accessGroupAttrs(accessGroups->begin(), accessGroups->end()); - op->setAttr(LLVMDialect::getAccessGroupsAttrName(), - ArrayAttr::get(op->getContext(), accessGroupAttrs)); - return success(); + return AttributeSetter(op).apply([&](auto memOp) { + memOp.setAccessGroupsAttr( + ArrayAttr::get(memOp.getContext(), accessGroupAttrs)); + }); } /// Converts the given loop metadata node to an MLIR loop annotation attribute diff --git a/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp index b8b6e3c8ffdd..e406849af67e 100644 --- a/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp @@ -200,7 +200,7 @@ llvm::MDNode *LoopAnnotationConversion::convert() { llvm::MDString::get(ctx, "llvm.loop.parallel_accesses")); for (SymbolRefAttr accessGroupRef : parallelAccessGroups) parallelAccess.push_back( - moduleTranslation.getAccessGroup(*op, accessGroupRef)); + moduleTranslation.getAccessGroup(op, accessGroupRef)); metadataNodes.push_back(llvm::MDNode::get(ctx, parallelAccess)); } diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp index aa5627e7a15f..3834bf02dcbc 100644 --- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp @@ -986,12 +986,12 @@ LogicalResult ModuleTranslation::convertFunctions() { } llvm::MDNode * -ModuleTranslation::getAccessGroup(Operation &opInst, +ModuleTranslation::getAccessGroup(Operation *op, SymbolRefAttr accessGroupRef) const { auto metadataName = accessGroupRef.getRootReference(); auto accessGroupName = accessGroupRef.getLeafReference(); auto metadataOp = SymbolTable::lookupNearestSymbolFrom( - opInst.getParentOp(), metadataName); + op->getParentOp(), metadataName); auto *accessGroupOp = SymbolTable::lookupNearestSymbolFrom(metadataOp, accessGroupName); return accessGroupMetadataMapping.lookup(accessGroupOp); @@ -1010,23 +1010,28 @@ LogicalResult ModuleTranslation::createAccessGroupMetadata() { void ModuleTranslation::setAccessGroupsMetadata(Operation *op, llvm::Instruction *inst) { - auto accessGroups = - op->getAttrOfType(LLVMDialect::getAccessGroupsAttrName()); - if (accessGroups && !accessGroups.empty()) { + auto populateGroupsMetadata = [&](std::optional groupRefs) { + if (!groupRefs || groupRefs->empty()) + return; + llvm::Module *module = inst->getModule(); - SmallVector metadatas; - for (SymbolRefAttr accessGroupRef : - accessGroups.getAsRange()) - metadatas.push_back(getAccessGroup(*op, accessGroupRef)); - - llvm::MDNode *unionMD = nullptr; - if (metadatas.size() == 1) - unionMD = llvm::cast(metadatas.front()); - else if (metadatas.size() >= 2) - unionMD = llvm::MDNode::get(module->getContext(), metadatas); - - inst->setMetadata(module->getMDKindID("llvm.access.group"), unionMD); - } + SmallVector groupMDs; + for (SymbolRefAttr groupRef : groupRefs->getAsRange()) + groupMDs.push_back(getAccessGroup(op, groupRef)); + + llvm::MDNode *node = nullptr; + if (groupMDs.size() == 1) + node = llvm::cast(groupMDs.front()); + else if (groupMDs.size() >= 2) + node = llvm::MDNode::get(module->getContext(), groupMDs); + + inst->setMetadata(llvm::LLVMContext::MD_access_group, node); + }; + + llvm::TypeSwitch(op) + .Case( + [&](auto memOp) { populateGroupsMetadata(memOp.getAccessGroups()); }) + .Default([](auto) { llvm_unreachable("expected LoadOp or StoreOp"); }); } LogicalResult ModuleTranslation::createAliasScopeMetadata() { @@ -1067,12 +1072,12 @@ LogicalResult ModuleTranslation::createAliasScopeMetadata() { } llvm::MDNode * -ModuleTranslation::getAliasScope(Operation &opInst, +ModuleTranslation::getAliasScope(Operation *op, SymbolRefAttr aliasScopeRef) const { StringAttr metadataName = aliasScopeRef.getRootReference(); StringAttr scopeName = aliasScopeRef.getLeafReference(); auto metadataOp = SymbolTable::lookupNearestSymbolFrom( - opInst.getParentOp(), metadataName); + op->getParentOp(), metadataName); Operation *aliasScopeOp = SymbolTable::lookupNearestSymbolFrom(metadataOp, scopeName); return aliasScopeMetadataMapping.lookup(aliasScopeOp); @@ -1080,50 +1085,63 @@ ModuleTranslation::getAliasScope(Operation &opInst, void ModuleTranslation::setAliasScopeMetadata(Operation *op, llvm::Instruction *inst) { - auto populateScopeMetadata = [this, op, inst](StringRef attrName, - StringRef llvmMetadataName) { - auto scopes = op->getAttrOfType(attrName); - if (!scopes || scopes.empty()) + auto populateScopeMetadata = [&](std::optional scopeRefs, + unsigned kind) { + if (!scopeRefs || scopeRefs->empty()) return; llvm::Module *module = inst->getModule(); SmallVector scopeMDs; - for (SymbolRefAttr scopeRef : scopes.getAsRange()) - scopeMDs.push_back(getAliasScope(*op, scopeRef)); - llvm::MDNode *unionMD = llvm::MDNode::get(module->getContext(), scopeMDs); - inst->setMetadata(module->getMDKindID(llvmMetadataName), unionMD); + for (SymbolRefAttr scopeRef : scopeRefs->getAsRange()) + scopeMDs.push_back(getAliasScope(op, scopeRef)); + llvm::MDNode *node = llvm::MDNode::get(module->getContext(), scopeMDs); + inst->setMetadata(kind, node); }; - populateScopeMetadata(LLVMDialect::getAliasScopesAttrName(), "alias.scope"); - populateScopeMetadata(LLVMDialect::getNoAliasScopesAttrName(), "noalias"); + llvm::TypeSwitch(op) + .Case([&](auto memOp) { + populateScopeMetadata(memOp.getAliasScopes(), + llvm::LLVMContext::MD_alias_scope); + populateScopeMetadata(memOp.getNoaliasScopes(), + llvm::LLVMContext::MD_noalias); + }) + .Default([](auto) { llvm_unreachable("expected LoadOp or StoreOp"); }); } -llvm::MDNode *ModuleTranslation::getTBAANode(Operation &memOp, +llvm::MDNode *ModuleTranslation::getTBAANode(Operation *op, SymbolRefAttr tagRef) const { StringAttr metadataName = tagRef.getRootReference(); StringAttr tagName = tagRef.getLeafReference(); auto metadataOp = SymbolTable::lookupNearestSymbolFrom( - memOp.getParentOp(), metadataName); + op->getParentOp(), metadataName); Operation *tagOp = SymbolTable::lookupNearestSymbolFrom(metadataOp, tagName); return tbaaMetadataMapping.lookup(tagOp); } void ModuleTranslation::setTBAAMetadata(Operation *op, llvm::Instruction *inst) { - auto tbaa = op->getAttrOfType(LLVMDialect::getTBAAAttrName()); - if (!tbaa || tbaa.empty()) - return; - // LLVM IR currently does not support attaching more than one - // TBAA access tag to a memory accessing instruction. - // It may be useful to support this in future, but for the time being - // just ignore the metadata if MLIR operation has multiple access tags. - if (tbaa.size() > 1) { - op->emitWarning() << "TBAA access tags were not translated, because LLVM " - "IR only supports a single tag per instruction"; - return; - } - SymbolRefAttr tagRef = tbaa[0].cast(); - llvm::MDNode *tagNode = getTBAANode(*op, tagRef); - inst->setMetadata(llvm::LLVMContext::MD_tbaa, tagNode); + auto populateTBAAMetadata = [&](std::optional tagRefs) { + if (!tagRefs || tagRefs->empty()) + return; + + // LLVM IR currently does not support attaching more than one + // TBAA access tag to a memory accessing instruction. + // It may be useful to support this in future, but for the time being + // just ignore the metadata if MLIR operation has multiple access tags. + if (tagRefs->size() > 1) { + op->emitWarning() << "TBAA access tags were not translated, because LLVM " + "IR only supports a single tag per instruction"; + return; + } + + SymbolRefAttr tagRef = (*tagRefs)[0].cast(); + llvm::MDNode *node = getTBAANode(op, tagRef); + inst->setMetadata(llvm::LLVMContext::MD_tbaa, node); + }; + + llvm::TypeSwitch(op) + .Case( + [&](auto memOp) { populateTBAAMetadata(memOp.getTbaa()); }) + .Default([](auto) { llvm_unreachable("expected LoadOp or StoreOp"); }); } LogicalResult ModuleTranslation::createTBAAMetadata() { diff --git a/mlir/test/Dialect/LLVMIR/tbaa-invalid.mlir b/mlir/test/Dialect/LLVMIR/tbaa-invalid.mlir index 0513dc77ff86..a747d596bd96 100644 --- a/mlir/test/Dialect/LLVMIR/tbaa-invalid.mlir +++ b/mlir/test/Dialect/LLVMIR/tbaa-invalid.mlir @@ -8,7 +8,7 @@ module { llvm.func @tbaa(%arg0: !llvm.ptr) { %0 = llvm.mlir.constant(1 : i8) : i8 // expected-error@below {{expected '@tbaa_tag_1' to specify a fully qualified reference}} - llvm.store %0, %arg0 {llvm.tbaa = [@tbaa_tag_1]} : i8, !llvm.ptr + llvm.store %0, %arg0 {tbaa = [@tbaa_tag_1]} : i8, !llvm.ptr llvm.return } } @@ -17,8 +17,8 @@ module { llvm.func @tbaa(%arg0: !llvm.ptr) { %0 = llvm.mlir.constant(1 : i8) : i8 - // expected-error@below {{attribute 'llvm.tbaa' failed to satisfy constraint: symbol ref array attribute}} - llvm.store %0, %arg0 {llvm.tbaa = ["sym"]} : i8, !llvm.ptr + // expected-error@below {{attribute 'tbaa' failed to satisfy constraint: symbol ref array attribute}} + llvm.store %0, %arg0 {tbaa = ["sym"]} : i8, !llvm.ptr llvm.return } @@ -28,7 +28,7 @@ module { llvm.func @tbaa(%arg0: !llvm.ptr) { %0 = llvm.mlir.constant(1 : i8) : i8 // expected-error@below {{expected '@metadata::@group1' to resolve to a llvm.tbaa_tag}} - llvm.store %0, %arg0 {llvm.tbaa = [@metadata::@group1]} : i8, !llvm.ptr + llvm.store %0, %arg0 {tbaa = [@metadata::@group1]} : i8, !llvm.ptr llvm.return } llvm.metadata @metadata { @@ -42,7 +42,7 @@ module { llvm.func @tbaa(%arg0: !llvm.ptr) { %0 = llvm.mlir.constant(1 : i8) : i8 // expected-error@below {{expected '@metadata::@sym' to be a valid reference}} - llvm.store %0, %arg0 {llvm.tbaa = [@metadata::@sym]} : i8, !llvm.ptr + llvm.store %0, %arg0 {tbaa = [@metadata::@sym]} : i8, !llvm.ptr llvm.return } llvm.metadata @metadata { @@ -54,7 +54,7 @@ module { llvm.func @tbaa(%arg0: !llvm.ptr) { %0 = llvm.mlir.constant(1 : i8) : i8 // expected-error@below {{expected '@tbaa::@sym' to reference a metadata op}} - llvm.store %0, %arg0 {llvm.tbaa = [@tbaa::@sym]} : i8, !llvm.ptr + llvm.store %0, %arg0 {tbaa = [@tbaa::@sym]} : i8, !llvm.ptr llvm.return } diff --git a/mlir/test/Target/LLVMIR/Import/import-failure.ll b/mlir/test/Target/LLVMIR/Import/import-failure.ll index d09d4eb9e90e..1163d703256f 100644 --- a/mlir/test/Target/LLVMIR/Import/import-failure.ll +++ b/mlir/test/Target/LLVMIR/Import/import-failure.ll @@ -566,8 +566,6 @@ bb2: ; // ----- -; CHECK: import-failure.ll -; CHECK-SAME: warning: llvm.func does not support branch weights ; CHECK: import-failure.ll:{{.*}} warning: unhandled function metadata: !0 = !{!"branch_weights", i32 64} define void @cond_br(i1 %arg) !prof !0 { ret void diff --git a/mlir/test/Target/LLVMIR/tbaa.mlir b/mlir/test/Target/LLVMIR/tbaa.mlir index 26a96e47aee6..84f27be92326 100644 --- a/mlir/test/Target/LLVMIR/tbaa.mlir +++ b/mlir/test/Target/LLVMIR/tbaa.mlir @@ -16,11 +16,11 @@ module { %1 = llvm.mlir.constant(1 : i32) : i32 %2 = llvm.getelementptr inbounds %arg1[%0, 1] : (!llvm.ptr, i32) -> !llvm.ptr, !llvm.struct<"struct.agg2_t", (i64, i64)> // CHECK: load i64, ptr %{{.*}},{{.*}}!tbaa ![[LTAG:[0-9]*]] - %3 = llvm.load %2 {llvm.tbaa = [@__tbaa::@tbaa_tag_4]} : !llvm.ptr -> i64 + %3 = llvm.load %2 {tbaa = [@__tbaa::@tbaa_tag_4]} : !llvm.ptr -> i64 %4 = llvm.trunc %3 : i64 to i32 %5 = llvm.getelementptr inbounds %arg0[%0, 0] : (!llvm.ptr, i32) -> !llvm.ptr, !llvm.struct<"struct.agg1_t", (i32, i32)> // CHECK: store i32 %{{.*}}, ptr %{{.*}},{{.*}}!tbaa ![[STAG:[0-9]*]] - llvm.store %4, %5 {llvm.tbaa = [@__tbaa::@tbaa_tag_7]} : i32, !llvm.ptr + llvm.store %4, %5 {tbaa = [@__tbaa::@tbaa_tag_7]} : i32, !llvm.ptr llvm.return } } @@ -60,11 +60,11 @@ module { %1 = llvm.mlir.constant(1 : i32) : i32 %2 = llvm.getelementptr inbounds %arg1[%0, 0] : (!llvm.ptr, i32) -> !llvm.ptr, !llvm.struct<"struct.agg2_t", (f32, f32)> // CHECK: load float, ptr %{{.*}},{{.*}}!tbaa ![[LTAG:[0-9]*]] - %3 = llvm.load %2 {llvm.tbaa = [@__tbaa::@tbaa_tag_4]} : !llvm.ptr -> f32 + %3 = llvm.load %2 {tbaa = [@__tbaa::@tbaa_tag_4]} : !llvm.ptr -> f32 %4 = llvm.fptosi %3 : f32 to i32 %5 = llvm.getelementptr inbounds %arg0[%0, 0] : (!llvm.ptr, i32) -> !llvm.ptr, !llvm.struct<"struct.agg1_t", (i32, i32)> // CHECK: store i32 %{{.*}}, ptr %{{.*}},{{.*}}!tbaa ![[STAG:[0-9]*]] - llvm.store %4, %5 {llvm.tbaa = [@__tbaa::@tbaa_tag_7]} : i32, !llvm.ptr + llvm.store %4, %5 {tbaa = [@__tbaa::@tbaa_tag_7]} : i32, !llvm.ptr llvm.return } } From 8a5f516962b8415c38f4e80ca1f8d7c415f712bf Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Mon, 6 Mar 2023 10:44:43 +0100 Subject: [PATCH 070/147] [mlir][llvm] Fix TBAA verfication crash This commit fixes a crash of the TBAA verification that happened due to accessing memory through invalid pointers. A DenseMap does not guarantee that pointers to its elements remain valid after additional elements are inserted. A testcase that caused this crash had more than 100 TBAA metadata operations and thus no test is added. Instead, there is now an assertion that ensures that the graph class is used correctly. Reviewed By: vzakhari Differential Revision: https://reviews.llvm.org/D143653 (cherry picked from commit fc2c791e89cd10ab9225421f215c2267e832977f) --- mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 60 +++++++++++++--------- 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp index 72526228b4d8..1be548f8a025 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -2478,29 +2478,28 @@ struct TBAAGraphNode { // TBAA graph. class TBAAGraph { - // Mapping between symbol names defined by TBAA - // operations and corresponding TBAAGraphNode's. - DenseMap nodeMap; - // Synthetic root node that has all graph nodes - // in its operands list. - TBAAGraphNode root; - public: using iterator = SmallVectorImpl::iterator; + // Creates a new graph with nodes corresponding to `symbolNames` defined by a + // set of TBAA operations. + TBAAGraph(ArrayRef symbolNames) { + for (auto symbol : symbolNames) { + TBAAGraphNode &node = nodeMap[symbol]; + assert(node.symbol.empty() && "node is already in the graph"); + node.symbol = symbol; + } + + // Fill the graph operands once all nodes were added. Otherwise, + // reallocation can lead to pointer invalidation. + for (auto symbol : symbolNames) + root.operands.push_back(&nodeMap[symbol]); + } + iterator begin() { return root.operands.begin(); } iterator end() { return root.operands.end(); } TBAAGraphNode *getEntryNode() { return &root; } - // Add new graph node corresponding to `symbol` - // defined by a TBAA operation. - void addNodeDefinition(StringAttr symbol) { - TBAAGraphNode &node = nodeMap[symbol]; - assert(node.symbol.empty() && "node is already in the graph"); - node.symbol = symbol; - root.operands.push_back(&node); - } - // Get a pointer to TBAAGraphNode corresponding // to `symbol`. The node must be already in the graph. TBAAGraphNode *operator[](StringAttr symbol) { @@ -2508,8 +2507,17 @@ class TBAAGraph { assert(it != nodeMap.end() && "node must be in the graph"); return &it->second; } + +private: + // Mapping between symbol names defined by TBAA + // operations and corresponding TBAAGraphNode's. + DenseMap nodeMap; + // Synthetic root node that has all graph nodes + // in its operands list. + TBAAGraphNode root; }; } // end anonymous namespace + namespace llvm { // GraphTraits definitions for using TBAAGraph with // scc_iterator. @@ -2541,20 +2549,26 @@ LogicalResult MetadataOp::verifyRegions() { Region &body = getBody(); // Symbol names defined by TBAARootMetadataOp and TBAATypeDescriptorOp. llvm::SmallDenseSet definedGraphSymbols; - // Complete TBAA graph consisting of TBAARootMetadataOp, - // TBAATypeDescriptorOp, and TBAATagOp symbols. It is used - // for detecting cycles in the TBAA graph, which is illegal. - TBAAGraph tbaaGraph; - for (Operation &op : body.getOps()) + // Collection of symbol names to ensure a stable ordering of the pointers. + // Otherwise, error messages might not be deterministic. + SmallVector symbolNames; + + for (Operation &op : body.getOps()) { if (isa(op) || isa(op)) { StringAttr symbolDef = cast(op).getNameAttr(); definedGraphSymbols.insert(symbolDef); - tbaaGraph.addNodeDefinition(symbolDef); + symbolNames.push_back(symbolDef); } else if (auto tagOp = dyn_cast(op)) { - tbaaGraph.addNodeDefinition(tagOp.getSymNameAttr()); + symbolNames.push_back(tagOp.getSymNameAttr()); } + } + + // Complete TBAA graph consisting of TBAARootMetadataOp, + // TBAATypeDescriptorOp, and TBAATagOp symbols. It is used + // for detecting cycles in the TBAA graph, which is illegal. + TBAAGraph tbaaGraph(symbolNames); // Verify that TBAA metadata operations refer symbols // from definedGraphSymbols only. Note that TBAATagOp From a65660a167abc011fc92a80d0973a1e3a8506981 Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Mon, 6 Mar 2023 10:45:29 +0100 Subject: [PATCH 071/147] [mlir][llvm] Adapt loop metadata to match llvm This commit adds support for the "llvm.loop.isvectorized" metadata and ensures that the unroll followups match llvm's naming. Reviewed By: gysit Differential Revision: https://reviews.llvm.org/D143730 (cherry picked from commit 4d7c879d33a22a125923449150e493df3dc9fd05) --- .../mlir/Dialect/LLVMIR/LLVMAttrDefs.td | 6 ++-- .../Target/LLVMIR/LoopAnnotationImporter.cpp | 35 ++++++++++++++++--- .../LLVMIR/LoopAnnotationTranslation.cpp | 24 +++++++++---- mlir/test/Dialect/LLVMIR/loop-metadata.mlir | 6 ++-- .../Target/LLVMIR/Import/metadata-loop.ll | 18 +++++----- mlir/test/Target/LLVMIR/loop-metadata.mlir | 19 ++++++++-- 6 files changed, 82 insertions(+), 26 deletions(-) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td index 570533fae001..3f7d6f38b865 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td @@ -96,8 +96,9 @@ def LoopUnrollAttr : LLVM_Attr<"LoopUnroll", "loop_unroll"> { OptionalParameter<"IntegerAttr">:$count, OptionalParameter<"BoolAttr">:$runtimeDisable, OptionalParameter<"BoolAttr">:$full, - OptionalParameter<"LoopAnnotationAttr">:$followup, - OptionalParameter<"LoopAnnotationAttr">:$followupRemainder + OptionalParameter<"LoopAnnotationAttr">:$followupUnrolled, + OptionalParameter<"LoopAnnotationAttr">:$followupRemainder, + OptionalParameter<"LoopAnnotationAttr">:$followupAll ); let assemblyFormat = "`<` struct(params) `>`"; @@ -187,6 +188,7 @@ def LoopAnnotationAttr : LLVM_Attr<"LoopAnnotation", "loop_annotation"> { OptionalParameter<"LoopDistributeAttr">:$distribute, OptionalParameter<"LoopPipelineAttr">:$pipeline, OptionalParameter<"BoolAttr">:$mustProgress, + OptionalParameter<"BoolAttr">:$isVectorized, OptionalArrayRefParameter<"SymbolRefAttr">:$parallelAccesses ); diff --git a/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp b/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp index a3cbf2bcd47c..53e6f9c50e89 100644 --- a/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp +++ b/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp @@ -33,6 +33,7 @@ struct LoopMetadataConversion { /// specified name, or failure, if the node is ill-formatted. FailureOr lookupUnitNode(StringRef name); FailureOr lookupBoolNode(StringRef name, bool negated = false); + FailureOr lookupIntNodeAsBoolAttr(StringRef name); FailureOr lookupIntNode(StringRef name); FailureOr lookupMDNode(StringRef name); FailureOr> lookupMDNodes(StringRef name); @@ -155,6 +156,27 @@ FailureOr LoopMetadataConversion::lookupBoolNode(StringRef name, return BoolAttr::get(ctx, val->getValue().getLimitedValue(1) ^ negated); } +FailureOr +LoopMetadataConversion::lookupIntNodeAsBoolAttr(StringRef name) { + const llvm::MDNode *property = lookupAndEraseProperty(name); + if (!property) + return BoolAttr(nullptr); + + auto emitNodeWarning = [&]() { + return emitWarning(loc) + << "expected metadata node " << name << " to hold an integer value"; + }; + + if (property->getNumOperands() != 2) + return emitNodeWarning(); + llvm::ConstantInt *val = + llvm::mdconst::dyn_extract(property->getOperand(1)); + if (!val || val->getBitWidth() != 32) + return emitNodeWarning(); + + return BoolAttr::get(ctx, val->getValue().getLimitedValue(1)); +} + FailureOr LoopMetadataConversion::lookupIntNode(StringRef name) { const llvm::MDNode *property = lookupAndEraseProperty(name); if (!property) @@ -287,13 +309,16 @@ FailureOr LoopMetadataConversion::convertUnrollAttr() { FailureOr runtimeDisable = lookupUnitNode("llvm.loop.unroll.runtime.disable"); FailureOr full = lookupUnitNode("llvm.loop.unroll.full"); - FailureOr followup = - lookupFollowupNode("llvm.loop.unroll.followup"); + FailureOr followupUnrolled = + lookupFollowupNode("llvm.loop.unroll.followup_unrolled"); FailureOr followupRemainder = lookupFollowupNode("llvm.loop.unroll.followup_remainder"); + FailureOr followupAll = + lookupFollowupNode("llvm.loop.unroll.followup_all"); return createIfNonNull(ctx, disable, count, runtimeDisable, - full, followup, followupRemainder); + full, followupUnrolled, + followupRemainder, followupAll); } FailureOr @@ -379,6 +404,8 @@ LoopAnnotationAttr LoopMetadataConversion::convert() { FailureOr distributeAttr = convertDistributeAttr(); FailureOr pipelineAttr = convertPipelineAttr(); FailureOr mustProgress = lookupUnitNode("llvm.loop.mustprogress"); + FailureOr isVectorized = + lookupIntNodeAsBoolAttr("llvm.loop.isvectorized"); FailureOr> parallelAccesses = convertParallelAccesses(); @@ -392,7 +419,7 @@ LoopAnnotationAttr LoopMetadataConversion::convert() { return createIfNonNull( ctx, disableNonForced, vecAttr, interleaveAttr, unrollAttr, unrollAndJamAttr, licmAttr, distributeAttr, pipelineAttr, mustProgress, - parallelAccesses); + isVectorized, parallelAccesses); } LoopAnnotationAttr diff --git a/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp index e406849af67e..b02433b33edf 100644 --- a/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp @@ -29,6 +29,7 @@ struct LoopAnnotationConversion { /// Conversion functions for different payload attribute kinds. void addUnitNode(StringRef name); void addUnitNode(StringRef name, BoolAttr attr); + void addI32NodeWithVal(StringRef name, uint32_t val); void convertBoolNode(StringRef name, BoolAttr attr, bool negated = false); void convertI32Node(StringRef name, IntegerAttr attr); void convertFollowupNode(StringRef name, LoopAnnotationAttr attr); @@ -61,6 +62,14 @@ void LoopAnnotationConversion::addUnitNode(StringRef name, BoolAttr attr) { addUnitNode(name); } +void LoopAnnotationConversion::addI32NodeWithVal(StringRef name, uint32_t val) { + llvm::Constant *cstValue = llvm::ConstantInt::get( + llvm::IntegerType::get(ctx, /*NumBits=*/32), val, /*isSigned=*/false); + metadataNodes.push_back( + llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name), + llvm::ConstantAsMetadata::get(cstValue)})); +} + void LoopAnnotationConversion::convertBoolNode(StringRef name, BoolAttr attr, bool negated) { if (!attr) @@ -76,12 +85,7 @@ void LoopAnnotationConversion::convertI32Node(StringRef name, IntegerAttr attr) { if (!attr) return; - uint32_t val = attr.getInt(); - llvm::Constant *cstValue = llvm::ConstantInt::get( - llvm::IntegerType::get(ctx, /*NumBits=*/32), val, /*isSigned=*/false); - metadataNodes.push_back( - llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name), - llvm::ConstantAsMetadata::get(cstValue)})); + addI32NodeWithVal(name, attr.getInt()); } void LoopAnnotationConversion::convertFollowupNode(StringRef name, @@ -122,9 +126,12 @@ void LoopAnnotationConversion::convertLoopOptions(LoopUnrollAttr options) { convertBoolNode("llvm.loop.unroll.runtime.disable", options.getRuntimeDisable()); addUnitNode("llvm.loop.unroll.full", options.getFull()); - convertFollowupNode("llvm.loop.unroll.followup", options.getFollowup()); + convertFollowupNode("llvm.loop.unroll.followup_unrolled", + options.getFollowupUnrolled()); convertFollowupNode("llvm.loop.unroll.followup_remainder", options.getFollowupRemainder()); + convertFollowupNode("llvm.loop.unroll.followup_all", + options.getFollowupAll()); } void LoopAnnotationConversion::convertLoopOptions( @@ -177,6 +184,9 @@ llvm::MDNode *LoopAnnotationConversion::convert() { addUnitNode("llvm.loop.disable_nonforced", attr.getDisableNonforced()); addUnitNode("llvm.loop.mustprogress", attr.getMustProgress()); + // "isvectorized" is encoded as an i32 value. + if (BoolAttr isVectorized = attr.getIsVectorized()) + addI32NodeWithVal("llvm.loop.isvectorized", isVectorized.getValue()); if (auto options = attr.getVectorize()) convertLoopOptions(options); diff --git a/mlir/test/Dialect/LLVMIR/loop-metadata.mlir b/mlir/test/Dialect/LLVMIR/loop-metadata.mlir index edd9592168cb..0d434323f870 100644 --- a/mlir/test/Dialect/LLVMIR/loop-metadata.mlir +++ b/mlir/test/Dialect/LLVMIR/loop-metadata.mlir @@ -12,10 +12,10 @@ // CHECK-DAG: #[[INTERLEAVE:.*]] = #llvm.loop_interleave #interleave = #llvm.loop_interleave -// CHECK-DAG: #[[UNROLL:.*]] = #llvm.loop_unroll +// CHECK-DAG: #[[UNROLL:.*]] = #llvm.loop_unroll #unroll = #llvm.loop_unroll< disable = true, count = 32 : i32, runtimeDisable = true, full = false, - followup = #followup, followupRemainder = #followup + followupUnrolled = #followup, followupRemainder = #followup, followupAll = #followup > // CHECK-DAG: #[[UNROLL_AND_JAM:.*]] = #llvm.loop_unroll_and_jam @@ -44,6 +44,7 @@ // CHECK-DAG: licm = #[[LICM]] // CHECK-DAG: distribute = #[[DISTRIBUTE]] // CHECK-DAG: pipeline = #[[PIPELINE]] +// CHECK-DAG: isVectorized = false // CHECK-DAG: parallelAccesses = @metadata::@group1, @metadata::@group2> #loopMD = #llvm.loop_annotation // CHECK: llvm.func @loop_annotation diff --git a/mlir/test/Target/LLVMIR/Import/metadata-loop.ll b/mlir/test/Target/LLVMIR/Import/metadata-loop.ll index 93d29b0edea5..1ddd5e2b8d20 100644 --- a/mlir/test/Target/LLVMIR/Import/metadata-loop.ll +++ b/mlir/test/Target/LLVMIR/Import/metadata-loop.ll @@ -28,7 +28,7 @@ define void @access_group(ptr %arg1) { ; // ----- -; CHECK: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation +; CHECK: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation ; CHECK-LABEL: @simple define void @simple(i64 %n, ptr %A) { @@ -39,9 +39,10 @@ end: ret void } -!1 = distinct !{!1, !2, !3} +!1 = distinct !{!1, !2, !3, !4} !2 = !{!"llvm.loop.disable_nonforced"} !3 = !{!"llvm.loop.mustprogress"} +!4 = !{!"llvm.loop.isvectorized", i32 1} ; // ----- @@ -90,7 +91,7 @@ end: ; // ----- ; CHECK-DAG: #[[FOLLOWUP:.*]] = #llvm.loop_annotation -; CHECK-DAG: #[[UNROLL_ATTR:.*]] = #llvm.loop_unroll +; CHECK-DAG: #[[UNROLL_ATTR:.*]] = #llvm.loop_unroll ; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation ; CHECK-LABEL: @unroll @@ -102,16 +103,17 @@ end: ret void } -!1 = distinct !{!1, !2, !3, !4, !5, !6, !7} +!1 = distinct !{!1, !2, !3, !4, !5, !6, !7, !8} !2 = !{!"llvm.loop.unroll.enable"} !3 = !{!"llvm.loop.unroll.count", i32 16} !4 = !{!"llvm.loop.unroll.runtime.disable"} !5 = !{!"llvm.loop.unroll.full"} -!6 = !{!"llvm.loop.unroll.followup", !8} -!7 = !{!"llvm.loop.unroll.followup_remainder", !8} +!6 = !{!"llvm.loop.unroll.followup_unrolled", !9} +!7 = !{!"llvm.loop.unroll.followup_remainder", !9} +!8 = !{!"llvm.loop.unroll.followup_all", !9} -!8 = distinct !{!8, !9} -!9 = !{!"llvm.loop.disable_nonforced"} +!9 = distinct !{!9, !10} +!10 = !{!"llvm.loop.disable_nonforced"} ; // ----- diff --git a/mlir/test/Target/LLVMIR/loop-metadata.mlir b/mlir/test/Target/LLVMIR/loop-metadata.mlir index f2eaa1cffa33..9bed3ae39162 100644 --- a/mlir/test/Target/LLVMIR/loop-metadata.mlir +++ b/mlir/test/Target/LLVMIR/loop-metadata.mlir @@ -26,6 +26,18 @@ llvm.func @mustprogress() { // ----- +// CHECK-LABEL: @isvectorized +llvm.func @isvectorized() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.isvectorized", i32 1} + +// ----- #followup = #llvm.loop_annotation @@ -73,7 +85,7 @@ llvm.func @unrollOptions() { // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation + followupUnrolled = #followup, followupRemainder = #followup, followupAll = #followup> >} ^bb1: llvm.return @@ -81,12 +93,13 @@ llvm.func @unrollOptions() { // CHECK-DAG: ![[NON_FORCED:[0-9]+]] = !{!"llvm.loop.disable_nonforced"} // CHECK-DAG: ![[FOLLOWUP:[0-9]+]] = distinct !{![[FOLLOWUP]], ![[NON_FORCED]]} -// CHECK-DAG: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} // CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.disable"} // CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.count", i32 64} // CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.runtime.disable", i1 false} -// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.followup", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.followup_unrolled", ![[FOLLOWUP]]} // CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.followup_remainder", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.followup_all", ![[FOLLOWUP]]} // ----- From 920aa72fd44e34698ac1db60380c016aaed7fbc1 Mon Sep 17 00:00:00 2001 From: Tobias Gysi Date: Mon, 6 Mar 2023 10:46:08 +0100 Subject: [PATCH 072/147] Reland [mlir][func] Use the generated pass options in func to llvm. Update the FuncToLLVM pass to use the generated constructors and the generated pass option struct. The hand written constructor got out of sync after some refactorings. Using a generated constructor and options struct ensures the everything remains in sync. Reviewed By: zero9178 This reverts commit 39da46826da82c24ca4407c13ad7feb8e5dc32a1 and relands commit 771d9c05afc2515b474fb53db857716dfdfc1dcf which was originally reverted due to https://lab.llvm.org/buildbot#builders/61/builds/39694 Differential Revision: https://reviews.llvm.org/D143733 (cherry picked from commit 5dc34eb379f40614d5c2381d46e5c8d6d6af63fd) --- .../FuncToLLVM/ConvertFuncToLLVMPass.h | 11 +------ mlir/include/mlir/Conversion/Passes.td | 3 +- mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp | 31 ++----------------- mlir/test/CAPI/execution_engine.c | 2 +- .../mlir-vulkan-runner/mlir-vulkan-runner.cpp | 6 ++-- 5 files changed, 10 insertions(+), 43 deletions(-) diff --git a/mlir/include/mlir/Conversion/FuncToLLVM/ConvertFuncToLLVMPass.h b/mlir/include/mlir/Conversion/FuncToLLVM/ConvertFuncToLLVMPass.h index 701c6acac62f..f6be8a0fcb2b 100644 --- a/mlir/include/mlir/Conversion/FuncToLLVM/ConvertFuncToLLVMPass.h +++ b/mlir/include/mlir/Conversion/FuncToLLVM/ConvertFuncToLLVMPass.h @@ -13,20 +13,11 @@ #include namespace mlir { -class LowerToLLVMOptions; -class ModuleOp; -template -class OperationPass; class Pass; -#define GEN_PASS_DECL_CONVERTFUNCTOLLVM +#define GEN_PASS_DECL_CONVERTFUNCTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" -/// Creates a pass to convert the Func dialect into the LLVMIR dialect. -std::unique_ptr> createConvertFuncToLLVMPass(); -std::unique_ptr> -createConvertFuncToLLVMPass(const LowerToLLVMOptions &options); - } // namespace mlir #endif // MLIR_CONVERSION_FUNCTOLLVM_CONVERTFUNCTOLLVMPASS_H_ diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td index fc03d0037f1c..a2865922e7a0 100644 --- a/mlir/include/mlir/Conversion/Passes.td +++ b/mlir/include/mlir/Conversion/Passes.td @@ -271,7 +271,7 @@ def ConvertControlFlowToSPIRV : Pass<"convert-cf-to-spirv"> { // FuncToLLVM //===----------------------------------------------------------------------===// -def ConvertFuncToLLVM : Pass<"convert-func-to-llvm", "ModuleOp"> { +def ConvertFuncToLLVMPass : Pass<"convert-func-to-llvm", "ModuleOp"> { let summary = "Convert from the Func dialect to the LLVM dialect"; let description = [{ Convert Func dialect operations into the LLVM IR dialect operations. @@ -295,7 +295,6 @@ def ConvertFuncToLLVM : Pass<"convert-func-to-llvm", "ModuleOp"> { returns are updated accordingly. Block argument types are updated to use LLVM IR types. }]; - let constructor = "mlir::createConvertFuncToLLVMPass()"; let dependentDialects = ["LLVM::LLVMDialect"]; let options = [ Option<"useBarePtrCallConv", "use-bare-ptr-memref-call-conv", "bool", diff --git a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp index f70cb469f3f3..178b5f03e9a2 100644 --- a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp +++ b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp @@ -48,7 +48,7 @@ #include namespace mlir { -#define GEN_PASS_DEF_CONVERTFUNCTOLLVM +#define GEN_PASS_DEF_CONVERTFUNCTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" } // namespace mlir @@ -705,15 +705,8 @@ void mlir::populateFuncToLLVMConversionPatterns(LLVMTypeConverter &converter, namespace { /// A pass converting Func operations into the LLVM IR dialect. struct ConvertFuncToLLVMPass - : public impl::ConvertFuncToLLVMBase { - ConvertFuncToLLVMPass() = default; - ConvertFuncToLLVMPass(bool useBarePtrCallConv, unsigned indexBitwidth, - bool useAlignedAlloc, - const llvm::DataLayout &dataLayout) { - this->useBarePtrCallConv = useBarePtrCallConv; - this->indexBitwidth = indexBitwidth; - this->dataLayout = dataLayout.getStringRepresentation(); - } + : public impl::ConvertFuncToLLVMPassBase { + using Base::Base; /// Run the dialect converter on the module. void runOnOperation() override { @@ -754,21 +747,3 @@ struct ConvertFuncToLLVMPass } }; } // namespace - -std::unique_ptr> mlir::createConvertFuncToLLVMPass() { - return std::make_unique(); -} - -std::unique_ptr> -mlir::createConvertFuncToLLVMPass(const LowerToLLVMOptions &options) { - auto allocLowering = options.allocLowering; - // There is no way to provide additional patterns for pass, so - // AllocLowering::None will always fail. - assert(allocLowering != LowerToLLVMOptions::AllocLowering::None && - "ConvertFuncToLLVMPass doesn't support AllocLowering::None"); - bool useAlignedAlloc = - (allocLowering == LowerToLLVMOptions::AllocLowering::AlignedAlloc); - return std::make_unique( - options.useBarePtrCallConv, options.getIndexBitwidth(), useAlignedAlloc, - options.dataLayout); -} diff --git a/mlir/test/CAPI/execution_engine.c b/mlir/test/CAPI/execution_engine.c index 96ad8eb1a402..582b2f8fe222 100644 --- a/mlir/test/CAPI/execution_engine.c +++ b/mlir/test/CAPI/execution_engine.c @@ -34,7 +34,7 @@ void lowerModuleToLLVM(MlirContext ctx, MlirModule module) { MlirPassManager pm = mlirPassManagerCreate(ctx); MlirOpPassManager opm = mlirPassManagerGetNestedUnder( pm, mlirStringRefCreateFromCString("func.func")); - mlirPassManagerAddOwnedPass(pm, mlirCreateConversionConvertFuncToLLVM()); + mlirPassManagerAddOwnedPass(pm, mlirCreateConversionConvertFuncToLLVMPass()); mlirOpPassManagerAddOwnedPass( opm, mlirCreateConversionArithToLLVMConversionPass()); MlirLogicalResult status = mlirPassManagerRun(pm, module); diff --git a/mlir/tools/mlir-vulkan-runner/mlir-vulkan-runner.cpp b/mlir/tools/mlir-vulkan-runner/mlir-vulkan-runner.cpp index d52621a45075..d939b1aa65f3 100644 --- a/mlir/tools/mlir-vulkan-runner/mlir-vulkan-runner.cpp +++ b/mlir/tools/mlir-vulkan-runner/mlir-vulkan-runner.cpp @@ -69,11 +69,13 @@ static LogicalResult runMLIRPasses(Operation *op, modulePM.addPass(spirv::createSPIRVWebGPUPreparePass()); passManager.addPass(createConvertGpuLaunchFuncToVulkanLaunchFuncPass()); - LowerToLLVMOptions llvmOptions(module.getContext(), DataLayout(module)); passManager.addPass(createMemRefToLLVMConversionPass()); passManager.addPass(createConvertVectorToLLVMPass()); passManager.nest().addPass(LLVM::createRequestCWrappersPass()); - passManager.addPass(createConvertFuncToLLVMPass(llvmOptions)); + ConvertFuncToLLVMPassOptions funcToLLVMOptions{}; + funcToLLVMOptions.indexBitwidth = + DataLayout(module).getTypeSizeInBits(IndexType::get(module.getContext())); + passManager.addPass(createConvertFuncToLLVMPass(funcToLLVMOptions)); passManager.addPass(createReconcileUnrealizedCastsPass()); passManager.addPass(createConvertVulkanLaunchFuncToVulkanCallsPass()); From 3ec762bb256b5ebfe72cf00f790c9b3902a5dfd0 Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Mon, 6 Mar 2023 10:47:07 +0100 Subject: [PATCH 073/147] [milr][llvm] Add remaining loop metadata support This commit adds support for the last two loop metadata nodes produced anywhere in the llvm-project. Reviewed By: gysit Differential Revision: https://reviews.llvm.org/D143746 (cherry picked from commit 7f249e45eca41dd6030d4600c40a40033afce603) --- .../mlir/Dialect/LLVMIR/LLVMAttrDefs.td | 28 +++++++++++++++ mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 9 ++--- .../Target/LLVMIR/LoopAnnotationImporter.cpp | 19 +++++++++-- .../LLVMIR/LoopAnnotationTranslation.cpp | 15 ++++++++ mlir/test/Dialect/LLVMIR/loop-metadata.mlir | 10 ++++++ .../Target/LLVMIR/Import/metadata-loop.ll | 34 +++++++++++++++++++ mlir/test/Target/LLVMIR/loop-metadata.mlir | 26 ++++++++++++++ 7 files changed, 135 insertions(+), 6 deletions(-) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td index 3f7d6f38b865..b150c71b37d4 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td @@ -169,6 +169,32 @@ def LoopPipelineAttr : LLVM_Attr<"LoopPipeline", "loop_pipeline"> { let assemblyFormat = "`<` struct(params) `>`"; } +def LoopPeeledAttr : LLVM_Attr<"LoopPeeled", "loop_peeled"> { + let description = [{ + This attribute defines pipelining specific loop annotations that map to + the "!llvm.loop.peeled" metadata. + }]; + + let parameters = (ins + OptionalParameter<"IntegerAttr">:$count + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopUnswitchAttr : LLVM_Attr<"LoopUnswitch", "loop_unswitch"> { + let description = [{ + This attribute defines pipelining specific loop annotations that map to + the "!llvm.loop.unswitch" metadata. + }]; + + let parameters = (ins + OptionalParameter<"BoolAttr">:$partialDisable + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + def LoopAnnotationAttr : LLVM_Attr<"LoopAnnotation", "loop_annotation"> { let description = [{ This attributes encapsulates "loop metadata". It is meant to decorate @@ -187,6 +213,8 @@ def LoopAnnotationAttr : LLVM_Attr<"LoopAnnotation", "loop_annotation"> { OptionalParameter<"LoopLICMAttr">:$licm, OptionalParameter<"LoopDistributeAttr">:$distribute, OptionalParameter<"LoopPipelineAttr">:$pipeline, + OptionalParameter<"LoopPeeledAttr">:$peeled, + OptionalParameter<"LoopUnswitchAttr">:$unswitch, OptionalParameter<"BoolAttr">:$mustProgress, OptionalParameter<"BoolAttr">:$isVectorized, OptionalArrayRefParameter<"SymbolRefAttr">:$parallelAccesses diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp index 1be548f8a025..bdd78e427db2 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -2704,10 +2704,11 @@ struct LLVMOpAsmDialectInterface : public OpAsmDialectInterface { DISubprogramAttr, DISubroutineTypeAttr, LoopAnnotationAttr, LoopVectorizeAttr, LoopInterleaveAttr, LoopUnrollAttr, LoopUnrollAndJamAttr, LoopLICMAttr, LoopDistributeAttr, - LoopPipelineAttr>([&](auto attr) { - os << decltype(attr)::getMnemonic(); - return AliasResult::OverridableAlias; - }) + LoopPipelineAttr, LoopPeeledAttr, LoopUnswitchAttr>( + [&](auto attr) { + os << decltype(attr)::getMnemonic(); + return AliasResult::OverridableAlias; + }) .Default([](Attribute) { return AliasResult::NoAlias; }); } }; diff --git a/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp b/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp index 53e6f9c50e89..2986b64538c2 100644 --- a/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp +++ b/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp @@ -50,6 +50,8 @@ struct LoopMetadataConversion { FailureOr convertLICMAttr(); FailureOr convertDistributeAttr(); FailureOr convertPipelineAttr(); + FailureOr convertPeeledAttr(); + FailureOr convertUnswitchAttr(); FailureOr> convertParallelAccesses(); llvm::StringMap propertyMap; @@ -373,6 +375,17 @@ FailureOr LoopMetadataConversion::convertPipelineAttr() { return createIfNonNull(ctx, disable, initiationinterval); } +FailureOr LoopMetadataConversion::convertPeeledAttr() { + FailureOr count = lookupIntNode("llvm.loop.peeled.count"); + return createIfNonNull(ctx, count); +} + +FailureOr LoopMetadataConversion::convertUnswitchAttr() { + FailureOr partialDisable = + lookupUnitNode("llvm.loop.unswitch.partial.disable"); + return createIfNonNull(ctx, partialDisable); +} + FailureOr> LoopMetadataConversion::convertParallelAccesses() { FailureOr> nodes = @@ -403,6 +416,8 @@ LoopAnnotationAttr LoopMetadataConversion::convert() { FailureOr licmAttr = convertLICMAttr(); FailureOr distributeAttr = convertDistributeAttr(); FailureOr pipelineAttr = convertPipelineAttr(); + FailureOr peeledAttr = convertPeeledAttr(); + FailureOr unswitchAttr = convertUnswitchAttr(); FailureOr mustProgress = lookupUnitNode("llvm.loop.mustprogress"); FailureOr isVectorized = lookupIntNodeAsBoolAttr("llvm.loop.isvectorized"); @@ -418,8 +433,8 @@ LoopAnnotationAttr LoopMetadataConversion::convert() { return createIfNonNull( ctx, disableNonForced, vecAttr, interleaveAttr, unrollAttr, - unrollAndJamAttr, licmAttr, distributeAttr, pipelineAttr, mustProgress, - isVectorized, parallelAccesses); + unrollAndJamAttr, licmAttr, distributeAttr, pipelineAttr, peeledAttr, + unswitchAttr, mustProgress, isVectorized, parallelAccesses); } LoopAnnotationAttr diff --git a/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp index b02433b33edf..5f27f97f2fd4 100644 --- a/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp @@ -42,6 +42,8 @@ struct LoopAnnotationConversion { void convertLoopOptions(LoopLICMAttr options); void convertLoopOptions(LoopDistributeAttr options); void convertLoopOptions(LoopPipelineAttr options); + void convertLoopOptions(LoopPeeledAttr options); + void convertLoopOptions(LoopUnswitchAttr options); LoopAnnotationAttr attr; ModuleTranslation &moduleTranslation; @@ -176,6 +178,15 @@ void LoopAnnotationConversion::convertLoopOptions(LoopPipelineAttr options) { options.getInitiationinterval()); } +void LoopAnnotationConversion::convertLoopOptions(LoopPeeledAttr options) { + convertI32Node("llvm.loop.peeled.count", options.getCount()); +} + +void LoopAnnotationConversion::convertLoopOptions(LoopUnswitchAttr options) { + addUnitNode("llvm.loop.unswitch.partial.disable", + options.getPartialDisable()); +} + llvm::MDNode *LoopAnnotationConversion::convert() { // Reserve operand 0 for loop id self reference. @@ -202,6 +213,10 @@ llvm::MDNode *LoopAnnotationConversion::convert() { convertLoopOptions(options); if (auto options = attr.getPipeline()) convertLoopOptions(options); + if (auto options = attr.getPeeled()) + convertLoopOptions(options); + if (auto options = attr.getUnswitch()) + convertLoopOptions(options); ArrayRef parallelAccessGroups = attr.getParallelAccesses(); if (!parallelAccessGroups.empty()) { diff --git a/mlir/test/Dialect/LLVMIR/loop-metadata.mlir b/mlir/test/Dialect/LLVMIR/loop-metadata.mlir index 0d434323f870..8841a1fad918 100644 --- a/mlir/test/Dialect/LLVMIR/loop-metadata.mlir +++ b/mlir/test/Dialect/LLVMIR/loop-metadata.mlir @@ -36,6 +36,12 @@ // CHECK-DAG: #[[PIPELINE:.*]] = #llvm.loop_pipeline #pipeline = #llvm.loop_pipeline +// CHECK-DAG: #[[PEELED:.*]] = #llvm.loop_peeled +#peeled = #llvm.loop_peeled + +// CHECK-DAG: #[[UNSWITCH:.*]] = #llvm.loop_unswitch +#unswitch = #llvm.loop_unswitch + // CHECK: #[[LOOP_ANNOT:.*]] = #llvm.loop_annotation< // CHECK-DAG: disableNonforced = false // CHECK-DAG: mustProgress = true @@ -44,6 +50,8 @@ // CHECK-DAG: licm = #[[LICM]] // CHECK-DAG: distribute = #[[DISTRIBUTE]] // CHECK-DAG: pipeline = #[[PIPELINE]] +// CHECK-DAG: peeled = #[[PEELED]] +// CHECK-DAG: unswitch = #[[UNSWITCH]] // CHECK-DAG: isVectorized = false // CHECK-DAG: parallelAccesses = @metadata::@group1, @metadata::@group2> #loopMD = #llvm.loop_annotation diff --git a/mlir/test/Target/LLVMIR/Import/metadata-loop.ll b/mlir/test/Target/LLVMIR/Import/metadata-loop.ll index 1ddd5e2b8d20..75ef607d9f18 100644 --- a/mlir/test/Target/LLVMIR/Import/metadata-loop.ll +++ b/mlir/test/Target/LLVMIR/Import/metadata-loop.ll @@ -222,6 +222,40 @@ end: ; // ----- +; CHECK-DAG: #[[PEELED_ATTR:.*]] = #llvm.loop_peeled +; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation + +; CHECK-LABEL: @peeled +define void @peeled(i64 %n, ptr %A) { +entry: +; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]} + br label %end, !llvm.loop !1 +end: + ret void +} + +!1 = distinct !{!1, !2} +!2 = !{!"llvm.loop.peeled.count", i32 5} + +; // ----- + +; CHECK-DAG: #[[UNSWITCH_ATTR:.*]] = #llvm.loop_unswitch +; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation + +; CHECK-LABEL: @unswitched +define void @unswitched(i64 %n, ptr %A) { +entry: +; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]} + br label %end, !llvm.loop !1 +end: + ret void +} + +!1 = distinct !{!1, !2} +!2 = !{!"llvm.loop.unswitch.partial.disable"} + +; // ----- + ; CHECK: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation ; CHECK: llvm.metadata @__llvm_global_metadata { diff --git a/mlir/test/Target/LLVMIR/loop-metadata.mlir b/mlir/test/Target/LLVMIR/loop-metadata.mlir index 9bed3ae39162..e7d06775bf9f 100644 --- a/mlir/test/Target/LLVMIR/loop-metadata.mlir +++ b/mlir/test/Target/LLVMIR/loop-metadata.mlir @@ -207,6 +207,32 @@ llvm.func @pipelineOptions() { // ----- +// CHECK-LABEL: @peeledOptions +llvm.func @peeledOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation>} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.peeled.count", i32 3} + +// ----- + +// CHECK-LABEL: @unswitchOptions +llvm.func @unswitchOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation>} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.unswitch.partial.disable"} + +// ----- + // CHECK-LABEL: @loopOptions llvm.func @loopOptions(%arg1 : i32, %arg2 : i32) { %0 = llvm.mlir.constant(0 : i32) : i32 From 3c10b8e28699ce9ae2d060cf95daa342ce99fec7 Mon Sep 17 00:00:00 2001 From: Tobias Gysi Date: Mon, 6 Mar 2023 10:47:51 +0100 Subject: [PATCH 074/147] [mlir][llvm] Improve LoadOp and StoreOp import. The revision supports importing the volatile keyword and nontemporal metadata for the LoadOp and StoreOp. Additionally, it updates the builders and uses an assembly format for printing and parsing. The operation type still requires custom parse and print methods due to the current handling of typed and opaque pointers. Reviewed By: Dinistro Differential Revision: https://reviews.llvm.org/D143714 (cherry picked from commit 240c6f2643991b1fe69c03ddcc8be0104e699327) --- mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td | 33 ++-- mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 150 +++++++----------- .../test/Target/LLVMIR/Import/instructions.ll | 15 +- .../Target/LLVMIR/Import/metadata-loop.ll | 7 +- 4 files changed, 93 insertions(+), 112 deletions(-) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td index 8e4b8340a52b..d79e511b210d 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td @@ -189,11 +189,10 @@ class MemoryOpBase { }]; code setNonTemporalMetadataCode = [{ if ($nontemporal) { - llvm::Module *module = builder.GetInsertBlock()->getModule(); llvm::MDNode *metadata = llvm::MDNode::get( inst->getContext(), llvm::ConstantAsMetadata::get( builder.getInt32(1))); - inst->setMetadata(module->getMDKindID("nontemporal"), metadata); + inst->setMetadata(llvm::LLVMContext::MD_nontemporal, metadata); } }]; code setAccessGroupsMetadataCode = [{ @@ -355,6 +354,10 @@ def LLVM_LoadOp : LLVM_Op<"load">, MemoryOpBase { UnitAttr:$nontemporal); let results = (outs LLVM_LoadableType:$res); string llvmInstName = "Load"; + let assemblyFormat = [{ + (`volatile` $volatile_^)? $addr attr-dict `:` + custom(type($addr), type($res)) + }]; string llvmBuilder = [{ auto *inst = builder.CreateLoad($_resultType, $addr, $volatile_); }] # setAlignmentCode @@ -365,9 +368,12 @@ def LLVM_LoadOp : LLVM_Op<"load">, MemoryOpBase { # [{ $res = inst; }]; - // FIXME: Import attributes. string mlirBuilder = [{ - $res = $_builder.create($_location, $_resultType, $addr); + auto *loadInst = cast(inst); + unsigned alignment = loadInst->getAlign().value(); + $res = $_builder.create($_location, $_resultType, $addr, + alignment, loadInst->isVolatile(), + loadInst->hasMetadata(llvm::LLVMContext::MD_nontemporal)); }]; let builders = [ OpBuilder<(ins "Value":$addr, CArg<"unsigned", "0">:$alignment, @@ -378,9 +384,10 @@ def LLVM_LoadOp : LLVM_Op<"load">, MemoryOpBase { "when the pointer type is opaque"); build($_builder, $_state, type, addr, alignment, isVolatile, isNonTemporal); }]>, - OpBuilder<(ins "Type":$t, "Value":$addr, + OpBuilder<(ins "Type":$type, "Value":$addr, CArg<"unsigned", "0">:$alignment, CArg<"bool", "false">:$isVolatile, - CArg<"bool", "false">:$isNonTemporal)>,]; + CArg<"bool", "false">:$isNonTemporal)> + ]; let hasCustomAssemblyFormat = 1; let hasVerifier = 1; } @@ -395,6 +402,10 @@ def LLVM_StoreOp : LLVM_Op<"store">, MemoryOpBase { OptionalAttr:$alignment, UnitAttr:$volatile_, UnitAttr:$nontemporal); string llvmInstName = "Store"; + let assemblyFormat = [{ + (`volatile` $volatile_^)? $value `,` $addr attr-dict `:` + custom(type($value), type($addr)) + }]; string llvmBuilder = [{ auto *inst = builder.CreateStore($value, $addr, $volatile_); }] # setAlignmentCode @@ -402,16 +413,18 @@ def LLVM_StoreOp : LLVM_Op<"store">, MemoryOpBase { # setAccessGroupsMetadataCode # setAliasScopeMetadataCode # setTBAAMetadataCode; - // FIXME: Import attributes. string mlirBuilder = [{ - $_op = $_builder.create($_location, $value, $addr); + auto *storeInst = cast(inst); + unsigned alignment = storeInst->getAlign().value(); + $_op = $_builder.create($_location, $value, $addr, + alignment, storeInst->isVolatile(), + storeInst->hasMetadata(llvm::LLVMContext::MD_nontemporal)); }]; let builders = [ OpBuilder<(ins "Value":$value, "Value":$addr, CArg<"unsigned", "0">:$alignment, CArg<"bool", "false">:$isVolatile, CArg<"bool", "false">:$isNonTemporal)> - ]; - let hasCustomAssemblyFormat = 1; + ]; let hasVerifier = 1; } diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp index bdd78e427db2..1b890a4a49e7 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -44,8 +44,6 @@ using mlir::LLVM::linkage::getMaxEnumValForLinkage; #include "mlir/Dialect/LLVMIR/LLVMOpsDialect.cpp.inc" -static constexpr const char kVolatileAttrName[] = "volatile_"; -static constexpr const char kNonTemporalAttrName[] = "nontemporal"; static constexpr const char kElemTypeAttrName[] = "elem_type"; #include "mlir/Dialect/LLVMIR/LLVMOpsInterfaces.cpp.inc" @@ -665,7 +663,7 @@ Type LLVM::GEPOp::getSourceElementType() { } //===----------------------------------------------------------------------===// -// Builder, printer and parser for for LLVM::LoadOp. +// LoadOp //===----------------------------------------------------------------------===// /// Verifies the given array attribute contains symbol references and checks the @@ -759,29 +757,13 @@ LogicalResult verifyMemOpMetadata(OpTy memOp) { LogicalResult LoadOp::verify() { return verifyMemOpMetadata(*this); } -void LoadOp::build(OpBuilder &builder, OperationState &result, Type t, +void LoadOp::build(OpBuilder &builder, OperationState &state, Type type, Value addr, unsigned alignment, bool isVolatile, bool isNonTemporal) { - result.addOperands(addr); - result.addTypes(t); - if (isVolatile) - result.addAttribute(kVolatileAttrName, builder.getUnitAttr()); - if (isNonTemporal) - result.addAttribute(kNonTemporalAttrName, builder.getUnitAttr()); - if (alignment != 0) - result.addAttribute("alignment", builder.getI64IntegerAttr(alignment)); -} - -void LoadOp::print(OpAsmPrinter &p) { - p << ' '; - if (getVolatile_()) - p << "volatile "; - p << getAddr(); - p.printOptionalAttrDict((*this)->getAttrs(), - {kVolatileAttrName, kElemTypeAttrName}); - p << " : " << getAddr().getType(); - if (getAddr().getType().cast().isOpaque()) - p << " -> " << getType(); + build(builder, state, type, addr, /*access_groups=*/nullptr, + /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr, + alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile, + isNonTemporal); } // Extract the pointee type from the LLVM pointer type wrapped in MLIR. Return @@ -797,105 +779,85 @@ getLoadStoreElementType(OpAsmParser &parser, Type type, SMLoc trailingTypeLoc) { return llvmTy.getElementType(); } -// ::= `llvm.load` `volatile` ssa-use attribute-dict? `:` type -// (`->` type)? -ParseResult LoadOp::parse(OpAsmParser &parser, OperationState &result) { - OpAsmParser::UnresolvedOperand addr; - Type type; +/// Parses the LoadOp type either using the typed or opaque pointer format. +// TODO: Drop once the typed pointer assembly format is not needed anymore. +static ParseResult parseLoadType(OpAsmParser &parser, Type &type, + Type &elementType) { SMLoc trailingTypeLoc; - - if (succeeded(parser.parseOptionalKeyword("volatile"))) - result.addAttribute(kVolatileAttrName, parser.getBuilder().getUnitAttr()); - - if (parser.parseOperand(addr) || - parser.parseOptionalAttrDict(result.attributes) || parser.parseColon() || - parser.getCurrentLocation(&trailingTypeLoc) || parser.parseType(type) || - parser.resolveOperand(addr, type, result.operands)) + if (parser.getCurrentLocation(&trailingTypeLoc) || parser.parseType(type)) return failure(); - std::optional elemTy = + std::optional pointerElementType = getLoadStoreElementType(parser, type, trailingTypeLoc); - if (!elemTy) + if (!pointerElementType) return failure(); - if (*elemTy) { - result.addTypes(*elemTy); + if (*pointerElementType) { + elementType = *pointerElementType; return success(); } - Type trailingType; - if (parser.parseArrow() || parser.parseType(trailingType)) + if (parser.parseArrow() || parser.parseType(elementType)) return failure(); - result.addTypes(trailingType); return success(); } +/// Prints the LoadOp type either using the typed or opaque pointer format. +// TODO: Drop once the typed pointer assembly format is not needed anymore. +static void printLoadType(OpAsmPrinter &printer, Operation *op, Type type, + Type elementType) { + printer << type; + auto pointerType = cast(type); + if (pointerType.isOpaque()) + printer << " -> " << elementType; +} + //===----------------------------------------------------------------------===// -// Builder, printer and parser for LLVM::StoreOp. +// StoreOp //===----------------------------------------------------------------------===// LogicalResult StoreOp::verify() { return verifyMemOpMetadata(*this); } -void StoreOp::build(OpBuilder &builder, OperationState &result, Value value, +void StoreOp::build(OpBuilder &builder, OperationState &state, Value value, Value addr, unsigned alignment, bool isVolatile, bool isNonTemporal) { - result.addOperands({value, addr}); - result.addTypes({}); - if (isVolatile) - result.addAttribute(kVolatileAttrName, builder.getUnitAttr()); - if (isNonTemporal) - result.addAttribute(kNonTemporalAttrName, builder.getUnitAttr()); - if (alignment != 0) - result.addAttribute("alignment", builder.getI64IntegerAttr(alignment)); + build(builder, state, value, addr, /*access_groups=*/nullptr, + /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr, + alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile, + isNonTemporal); } -void StoreOp::print(OpAsmPrinter &p) { - p << ' '; - if (getVolatile_()) - p << "volatile "; - p << getValue() << ", " << getAddr(); - p.printOptionalAttrDict((*this)->getAttrs(), {kVolatileAttrName}); - p << " : "; - if (getAddr().getType().cast().isOpaque()) - p << getValue().getType() << ", "; - p << getAddr().getType(); -} - -// ::= `llvm.store` `volatile` ssa-use `,` ssa-use -// attribute-dict? `:` type (`,` type)? -ParseResult StoreOp::parse(OpAsmParser &parser, OperationState &result) { - OpAsmParser::UnresolvedOperand addr, value; - Type type; +/// Parses the StoreOp type either using the typed or opaque pointer format. +// TODO: Drop once the typed pointer assembly format is not needed anymore. +static ParseResult parseStoreType(OpAsmParser &parser, Type &elementType, + Type &type) { SMLoc trailingTypeLoc; - - if (succeeded(parser.parseOptionalKeyword("volatile"))) - result.addAttribute(kVolatileAttrName, parser.getBuilder().getUnitAttr()); - - if (parser.parseOperand(value) || parser.parseComma() || - parser.parseOperand(addr) || - parser.parseOptionalAttrDict(result.attributes) || parser.parseColon() || - parser.getCurrentLocation(&trailingTypeLoc) || parser.parseType(type)) + if (parser.getCurrentLocation(&trailingTypeLoc) || + parser.parseType(elementType)) return failure(); - Type operandType; - if (succeeded(parser.parseOptionalComma())) { - operandType = type; - if (parser.parseType(type)) - return failure(); - } else { - std::optional maybeOperandType = - getLoadStoreElementType(parser, type, trailingTypeLoc); - if (!maybeOperandType) - return failure(); - operandType = *maybeOperandType; - } + if (succeeded(parser.parseOptionalComma())) + return parser.parseType(type); - if (parser.resolveOperand(value, operandType, result.operands) || - parser.resolveOperand(addr, type, result.operands)) + // Extract the element type from the pointer type. + type = elementType; + std::optional pointerElementType = + getLoadStoreElementType(parser, type, trailingTypeLoc); + if (!pointerElementType) return failure(); - + elementType = *pointerElementType; return success(); } +/// Prints the StoreOp type either using the typed or opaque pointer format. +// TODO: Drop once the typed pointer assembly format is not needed anymore. +static void printStoreType(OpAsmPrinter &printer, Operation *op, + Type elementType, Type type) { + auto pointerType = cast(type); + if (pointerType.isOpaque()) + printer << elementType << ", "; + printer << type; +} + //===----------------------------------------------------------------------===// // CallOp //===----------------------------------------------------------------------===// diff --git a/mlir/test/Target/LLVMIR/Import/instructions.ll b/mlir/test/Target/LLVMIR/Import/instructions.ll index 14dbf07371bf..cd54411de4ef 100644 --- a/mlir/test/Target/LLVMIR/Import/instructions.ll +++ b/mlir/test/Target/LLVMIR/Import/instructions.ll @@ -251,7 +251,7 @@ define void @integer_arith(i32 %arg1, i32 %arg2, i64 %arg3, i64 %arg4) { ; CHECK-SAME: %[[VEC:[a-zA-Z0-9]+]] ; CHECK-SAME: %[[IDX:[a-zA-Z0-9]+]] define half @extract_element(ptr %vec, i32 %idx) { - ; CHECK: %[[V1:.+]] = llvm.load %[[VEC]] : !llvm.ptr -> vector<4xf16> + ; CHECK: %[[V1:.+]] = llvm.load %[[VEC]] {{.*}} : !llvm.ptr -> vector<4xf16> ; CHECK: %[[V2:.+]] = llvm.extractelement %[[V1]][%[[IDX]] : i32] : vector<4xf16> ; CHECK: llvm.return %[[V2]] %1 = load <4 x half>, ptr %vec @@ -266,7 +266,7 @@ define half @extract_element(ptr %vec, i32 %idx) { ; CHECK-SAME: %[[VAL:[a-zA-Z0-9]+]] ; CHECK-SAME: %[[IDX:[a-zA-Z0-9]+]] define <4 x half> @insert_element(ptr %vec, half %val, i32 %idx) { - ; CHECK: %[[V1:.+]] = llvm.load %[[VEC]] : !llvm.ptr -> vector<4xf16> + ; CHECK: %[[V1:.+]] = llvm.load %[[VEC]] {{.*}} : !llvm.ptr -> vector<4xf16> ; CHECK: %[[V2:.+]] = llvm.insertelement %[[VAL]], %[[V1]][%[[IDX]] : i32] : vector<4xf16> ; CHECK: llvm.return %[[V2]] %1 = load <4 x half>, ptr %vec @@ -352,13 +352,20 @@ define ptr @alloca(i64 %size) { ; CHECK-LABEL: @load_store ; CHECK-SAME: %[[PTR:[a-zA-Z0-9]+]] define void @load_store(ptr %ptr) { - ; CHECK: %[[V1:[0-9]+]] = llvm.load %[[PTR]] : !llvm.ptr -> f64 - ; CHECK: llvm.store %[[V1]], %[[PTR]] : f64, !llvm.ptr + ; CHECK: %[[V1:[0-9]+]] = llvm.load %[[PTR]] {alignment = 8 : i64} : !llvm.ptr -> f64 + ; CHECK: %[[V2:[0-9]+]] = llvm.load volatile %[[PTR]] {alignment = 16 : i64, nontemporal} : !llvm.ptr -> f64 %1 = load double, ptr %ptr + %2 = load volatile double, ptr %ptr, align 16, !nontemporal !0 + + ; CHECK: llvm.store %[[V1]], %[[PTR]] {alignment = 8 : i64} : f64, !llvm.ptr + ; CHECK: llvm.store volatile %[[V2]], %[[PTR]] {alignment = 16 : i64, nontemporal} : f64, !llvm.ptr store double %1, ptr %ptr + store volatile double %2, ptr %ptr, align 16, !nontemporal !0 ret void } +!0 = !{i32 1} + ; // ----- ; CHECK-LABEL: @atomic_rmw diff --git a/mlir/test/Target/LLVMIR/Import/metadata-loop.ll b/mlir/test/Target/LLVMIR/Import/metadata-loop.ll index 75ef607d9f18..315852663dd0 100644 --- a/mlir/test/Target/LLVMIR/Import/metadata-loop.ll +++ b/mlir/test/Target/LLVMIR/Import/metadata-loop.ll @@ -8,13 +8,12 @@ ; CHECK: } ; CHECK-LABEL: llvm.func @access_group -; CHECK-SAME: %[[ARG1:[a-zA-Z0-9]+]] define void @access_group(ptr %arg1) { - ; CHECK: llvm.load %[[ARG1]] {access_groups = [@__llvm_global_metadata::@[[$GROUP0]], @__llvm_global_metadata::@[[$GROUP1]]]} + ; CHECK: access_groups = [@__llvm_global_metadata::@[[$GROUP0]], @__llvm_global_metadata::@[[$GROUP1]]] %1 = load i32, ptr %arg1, !llvm.access.group !0 - ; CHECK: llvm.load %[[ARG1]] {access_groups = [@__llvm_global_metadata::@[[$GROUP2]], @__llvm_global_metadata::@[[$GROUP0]]]} + ; CHECK: access_groups = [@__llvm_global_metadata::@[[$GROUP2]], @__llvm_global_metadata::@[[$GROUP0]]] %2 = load i32, ptr %arg1, !llvm.access.group !1 - ; CHECK: llvm.load %[[ARG1]] {access_groups = [@__llvm_global_metadata::@[[$GROUP3]]]} + ; CHECK: access_groups = [@__llvm_global_metadata::@[[$GROUP3]]] %3 = load i32, ptr %arg1, !llvm.access.group !2 ret void } From 62db45af98c7a3cd47996ae1e9ea207b6b1a36b4 Mon Sep 17 00:00:00 2001 From: Tobias Gysi Date: Mon, 6 Mar 2023 10:57:32 +0100 Subject: [PATCH 075/147] [mlir][llvm] Reintroduce string based attribute setting. Reintroduce string based attribute setting in the translation from LLVM dialect to LLVM IR. The TypeSwitch based implementation introduced in https://reviews.llvm.org/D143654 does not work for intrinsics that set the requiresAccessGroup or requiresAliasScope flag. Reviewed By: hgreving Differential Revision: https://reviews.llvm.org/D143936 (cherry picked from commit bfea837676038ef3e312563b15811b524cbfb6aa) --- mlir/lib/Target/LLVMIR/ModuleTranslation.cpp | 36 +++++++++----------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp index 3834bf02dcbc..c928d78fcc55 100644 --- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp @@ -27,7 +27,6 @@ #include "mlir/Support/LLVM.h" #include "mlir/Target/LLVMIR/LLVMTranslationInterface.h" #include "mlir/Target/LLVMIR/TypeToLLVM.h" -#include "llvm/ADT/TypeSwitch.h" #include "llvm/ADT/PostOrderIterator.h" #include "llvm/ADT/SetVector.h" @@ -1010,13 +1009,13 @@ LogicalResult ModuleTranslation::createAccessGroupMetadata() { void ModuleTranslation::setAccessGroupsMetadata(Operation *op, llvm::Instruction *inst) { - auto populateGroupsMetadata = [&](std::optional groupRefs) { - if (!groupRefs || groupRefs->empty()) + auto populateGroupsMetadata = [&](ArrayAttr groupRefs) { + if (!groupRefs || groupRefs.empty()) return; llvm::Module *module = inst->getModule(); SmallVector groupMDs; - for (SymbolRefAttr groupRef : groupRefs->getAsRange()) + for (SymbolRefAttr groupRef : groupRefs.getAsRange()) groupMDs.push_back(getAccessGroup(op, groupRef)); llvm::MDNode *node = nullptr; @@ -1028,10 +1027,9 @@ void ModuleTranslation::setAccessGroupsMetadata(Operation *op, inst->setMetadata(llvm::LLVMContext::MD_access_group, node); }; - llvm::TypeSwitch(op) - .Case( - [&](auto memOp) { populateGroupsMetadata(memOp.getAccessGroups()); }) - .Default([](auto) { llvm_unreachable("expected LoadOp or StoreOp"); }); + auto groupRefs = + op->getAttrOfType(LLVMDialect::getAccessGroupsAttrName()); + populateGroupsMetadata(groupRefs); } LogicalResult ModuleTranslation::createAliasScopeMetadata() { @@ -1085,26 +1083,24 @@ ModuleTranslation::getAliasScope(Operation *op, void ModuleTranslation::setAliasScopeMetadata(Operation *op, llvm::Instruction *inst) { - auto populateScopeMetadata = [&](std::optional scopeRefs, - unsigned kind) { - if (!scopeRefs || scopeRefs->empty()) + auto populateScopeMetadata = [&](ArrayAttr scopeRefs, unsigned kind) { + if (!scopeRefs || scopeRefs.empty()) return; llvm::Module *module = inst->getModule(); SmallVector scopeMDs; - for (SymbolRefAttr scopeRef : scopeRefs->getAsRange()) + for (SymbolRefAttr scopeRef : scopeRefs.getAsRange()) scopeMDs.push_back(getAliasScope(op, scopeRef)); llvm::MDNode *node = llvm::MDNode::get(module->getContext(), scopeMDs); inst->setMetadata(kind, node); }; - llvm::TypeSwitch(op) - .Case([&](auto memOp) { - populateScopeMetadata(memOp.getAliasScopes(), - llvm::LLVMContext::MD_alias_scope); - populateScopeMetadata(memOp.getNoaliasScopes(), - llvm::LLVMContext::MD_noalias); - }) - .Default([](auto) { llvm_unreachable("expected LoadOp or StoreOp"); }); + auto aliasScopeRefs = + op->getAttrOfType(LLVMDialect::getAliasScopesAttrName()); + populateScopeMetadata(aliasScopeRefs, llvm::LLVMContext::MD_alias_scope); + + auto noaliasScopeRefs = + op->getAttrOfType(LLVMDialect::getNoAliasScopesAttrName()); + populateScopeMetadata(noaliasScopeRefs, llvm::LLVMContext::MD_noalias); } llvm::MDNode *ModuleTranslation::getTBAANode(Operation *op, From 65c1d76a0aa89a7fe923372db87fcd25de516fd0 Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Mon, 6 Mar 2023 10:58:23 +0100 Subject: [PATCH 076/147] [mlir] Drop unused arith conversion class (NFC) This commit removes an old helper class for fastmath attribute conversion that is no longer used. The last usage of this class was dropped in https://reviews.llvm.org/D137456. Reviewed By: gysit Differential Revision: https://reviews.llvm.org/D143912 (cherry picked from commit 544831ab9f3575e5c3a23ac9c324e2a801ff8436) --- .../ArithCommon/AttrToLLVMConverter.h | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/mlir/include/mlir/Conversion/ArithCommon/AttrToLLVMConverter.h b/mlir/include/mlir/Conversion/ArithCommon/AttrToLLVMConverter.h index f27f7bb5975e..95b12b69153a 100644 --- a/mlir/include/mlir/Conversion/ArithCommon/AttrToLLVMConverter.h +++ b/mlir/include/mlir/Conversion/ArithCommon/AttrToLLVMConverter.h @@ -50,28 +50,6 @@ class AttrConvertFastMathToLLVM { ArrayRef getAttrs() const { return convertedAttr.getAttrs(); } -private: - NamedAttrList convertedAttr; -}; - -// Attribute converter that populates a NamedAttrList by removing the fastmath -// attribute from the source operation attributes. This may be useful for -// target operations that do not require the fastmath attribute, or for targets -// that do not yet support the LLVM fastmath attribute. -template -class AttrDropFastMath { -public: - AttrDropFastMath(SourceOp srcOp) { - // Copy the source attributes. - convertedAttr = NamedAttrList{srcOp->getAttrs()}; - // Get the name of the arith fastmath attribute. - llvm::StringRef arithFMFAttrName = SourceOp::getFastMathAttrName(); - // Remove the source fastmath attribute. - convertedAttr.erase(arithFMFAttrName); - } - - ArrayRef getAttrs() const { return convertedAttr.getAttrs(); } - private: NamedAttrList convertedAttr; }; From e233161610ca44c9c74b28f39a5b2007bab9a034 Mon Sep 17 00:00:00 2001 From: Frederik Gossen Date: Mon, 6 Mar 2023 10:59:03 +0100 Subject: [PATCH 077/147] [MLIR] Reintroduce shared attribute names in the LLVM dialect Differential Revision: https://reviews.llvm.org/D143774 (cherry picked from commit 3de0bc4c3d0284354b0c0ec8ca1536ee080193e2) --- mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td index fbc0e6d0a596..dda8f22a0f9b 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td @@ -36,7 +36,10 @@ def LLVM_Dialect : Dialect { let extraClassDeclaration = [{ /// Name of the data layout attributes. static StringRef getDataLayoutAttrName() { return "llvm.data_layout"; } + static StringRef getNoAliasScopesAttrName() { return "noalias_scopes"; } + static StringRef getAliasScopesAttrName() { return "alias_scopes"; } static StringRef getLoopAttrName() { return "llvm.loop"; } + static StringRef getAccessGroupsAttrName() { return "access_groups"; } /// Names of llvm parameter attributes. static StringRef getAlignAttrName() { return "llvm.align"; } From c8e8bb132f025831ffb44ec9d4554e8c31cff4ad Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Mon, 6 Mar 2023 10:59:56 +0100 Subject: [PATCH 078/147] [mlir][llvm] Make LoopAnnotations non-discardable This commit adds the loop annotation attribute to LLVM::Br and LLVM::CondBr to ensure it is non-discardable. Furthermore, the name is changed from "llvm.loop" to "loop-annotation". Reviewed By: gysit Differential Revision: https://reviews.llvm.org/D143986 (cherry picked from commit d94399c641518802f6e1f3568a9982483f1eafa5) --- .../include/mlir/Dialect/LLVMIR/LLVMOpBase.td | 1 - mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td | 31 ++++---- mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 75 ++++++++++++------- .../LLVMIR/LLVMIRToLLVMTranslation.cpp | 8 +- mlir/lib/Target/LLVMIR/ModuleTranslation.cpp | 6 +- mlir/test/Dialect/LLVMIR/invalid.mlir | 15 +--- .../Target/LLVMIR/Import/metadata-loop.ll | 26 +++---- mlir/test/Target/LLVMIR/loop-metadata.mlir | 32 ++++---- 8 files changed, 108 insertions(+), 86 deletions(-) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td index dda8f22a0f9b..754656f3dec2 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td @@ -38,7 +38,6 @@ def LLVM_Dialect : Dialect { static StringRef getDataLayoutAttrName() { return "llvm.data_layout"; } static StringRef getNoAliasScopesAttrName() { return "noalias_scopes"; } static StringRef getAliasScopesAttrName() { return "alias_scopes"; } - static StringRef getLoopAttrName() { return "llvm.loop"; } static StringRef getAccessGroupsAttrName() { return "access_groups"; } /// Names of llvm parameter attributes. diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td index d79e511b210d..0a40ea9218b2 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td @@ -777,7 +777,10 @@ def LLVM_FreezeOp : LLVM_Op<"freeze", [SameOperandsAndResultType]> { // Terminators. def LLVM_BrOp : LLVM_TerminatorOp<"br", [DeclareOpInterfaceMethods, Pure]> { - let arguments = (ins Variadic:$destOperands); + let arguments = (ins + Variadic:$destOperands, + OptionalAttr:$loop_annotation + ); let successors = (successor AnySuccessor:$dest); let assemblyFormat = [{ $dest (`(` $destOperands^ `:` type($destOperands) `)`)? attr-dict @@ -786,8 +789,12 @@ def LLVM_BrOp : LLVM_TerminatorOp<"br", OpBuilder<(ins "Block *":$dest), [{ build($_builder, $_state, ValueRange(), dest); }]>, + OpBuilder<(ins "ValueRange":$operands, "Block *":$dest), [{ + build($_builder, $_state, operands, /*loop_annotation=*/{}, dest); + }]>, LLVM_TerminatorPassthroughOpBuilder ]; + let hasVerifier = 1; } def LLVM_CondBrOp : LLVM_TerminatorOp<"cond_br", [AttrSizedOperandSegments, DeclareOpInterfaceMethods, @@ -795,7 +802,8 @@ def LLVM_CondBrOp : LLVM_TerminatorOp<"cond_br", let arguments = (ins I1:$condition, Variadic:$trueDestOperands, Variadic:$falseDestOperands, - OptionalAttr:$branch_weights); + OptionalAttr:$branch_weights, + OptionalAttr:$loop_annotation); let successors = (successor AnySuccessor:$trueDest, AnySuccessor:$falseDest); let assemblyFormat = [{ $condition ( `weights` `(` $branch_weights^ `)` )? `,` @@ -808,23 +816,20 @@ def LLVM_CondBrOp : LLVM_TerminatorOp<"cond_br", OpBuilder<(ins "Value":$condition, "Block *":$trueDest, "ValueRange":$trueOperands, "Block *":$falseDest, "ValueRange":$falseOperands, - CArg<"std::optional>", "{}">:$weights), - [{ - ElementsAttr weightsAttr; - if (weights) { - weightsAttr = - $_builder.getI32VectorAttr({static_cast(weights->first), - static_cast(weights->second)}); - } - build($_builder, $_state, condition, trueOperands, falseOperands, weightsAttr, - trueDest, falseDest); - }]>, + CArg<"std::optional>", "{}">:$weights)>, OpBuilder<(ins "Value":$condition, "Block *":$trueDest, "Block *":$falseDest, CArg<"ValueRange", "{}">:$falseOperands), [{ build($_builder, $_state, condition, trueDest, ValueRange(), falseDest, falseOperands); + }]>, + OpBuilder<(ins "Value":$condition, "ValueRange":$trueOperands, "ValueRange":$falseOperands, + "ElementsAttr":$branchWeights, "Block *":$trueDest, "Block *":$falseDest), + [{ + build($_builder, $_state, condition, trueOperands, falseOperands, branchWeights, + {}, trueDest, falseDest); }]>, LLVM_TerminatorPassthroughOpBuilder]; + let hasVerifier = 1; } //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp index 1b890a4a49e7..6d3d0fbad191 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -264,11 +264,42 @@ LogicalResult AllocaOp::verify() { // LLVM::BrOp //===----------------------------------------------------------------------===// +/// Check if the `loopAttr` references correct symbols. +static LogicalResult verifyLoopAnnotationAttr(LoopAnnotationAttr loopAttr, + Operation *op) { + if (!loopAttr) + return success(); + // If the `llvm.loop` attribute is present, enforce the following structure, + // which the module translation can assume. + ArrayRef parallelAccesses = loopAttr.getParallelAccesses(); + if (parallelAccesses.empty()) + return success(); + for (SymbolRefAttr accessGroupRef : parallelAccesses) { + StringAttr metadataName = accessGroupRef.getRootReference(); + auto metadataOp = SymbolTable::lookupNearestSymbolFrom( + op->getParentOp(), metadataName); + if (!metadataOp) + return op->emitOpError() << "expected '" << accessGroupRef + << "' to reference a metadata op"; + StringAttr accessGroupName = accessGroupRef.getLeafReference(); + Operation *accessGroupOp = + SymbolTable::lookupNearestSymbolFrom(metadataOp, accessGroupName); + if (!accessGroupOp) + return op->emitOpError() << "expected '" << accessGroupRef + << "' to reference an access_group op"; + } + return success(); +} + SuccessorOperands BrOp::getSuccessorOperands(unsigned index) { assert(index == 0 && "invalid successor index"); return SuccessorOperands(getDestOperandsMutable()); } +LogicalResult BrOp::verify() { + return verifyLoopAnnotationAttr(getLoopAnnotationAttr(), *this); +} + //===----------------------------------------------------------------------===// // LLVM::CondBrOp //===----------------------------------------------------------------------===// @@ -279,6 +310,24 @@ SuccessorOperands CondBrOp::getSuccessorOperands(unsigned index) { : getFalseDestOperandsMutable()); } +LogicalResult CondBrOp::verify() { + return verifyLoopAnnotationAttr(getLoopAnnotationAttr(), *this); +} + +void CondBrOp::build(OpBuilder &builder, OperationState &result, + Value condition, Block *trueDest, ValueRange trueOperands, + Block *falseDest, ValueRange falseOperands, + std::optional> weights) { + ElementsAttr weightsAttr; + if (weights) + weightsAttr = + builder.getI32VectorAttr({static_cast(weights->first), + static_cast(weights->second)}); + + build(builder, result, condition, trueOperands, falseOperands, weightsAttr, + /*loop_annotation=*/{}, trueDest, falseDest); +} + //===----------------------------------------------------------------------===// // LLVM::SwitchOp //===----------------------------------------------------------------------===// @@ -2856,32 +2905,6 @@ LogicalResult LLVMDialect::verifyDataLayoutString( /// Verify LLVM dialect attributes. LogicalResult LLVMDialect::verifyOperationAttribute(Operation *op, NamedAttribute attr) { - // If the `llvm.loop` attribute is present, enforce the following structure, - // which the module translation can assume. - if (attr.getName() == LLVMDialect::getLoopAttrName()) { - auto loopAttr = attr.getValue().dyn_cast(); - if (!loopAttr) - return op->emitOpError() << "expected '" << LLVMDialect::getLoopAttrName() - << "' to be a loop annotation attribute"; - ArrayRef parallelAccesses = loopAttr.getParallelAccesses(); - if (parallelAccesses.empty()) - return success(); - for (SymbolRefAttr accessGroupRef : parallelAccesses) { - StringAttr metadataName = accessGroupRef.getRootReference(); - auto metadataOp = SymbolTable::lookupNearestSymbolFrom( - op->getParentOp(), metadataName); - if (!metadataOp) - return op->emitOpError() << "expected '" << accessGroupRef - << "' to reference a metadata op"; - StringAttr accessGroupName = accessGroupRef.getLeafReference(); - Operation *accessGroupOp = - SymbolTable::lookupNearestSymbolFrom(metadataOp, accessGroupName); - if (!accessGroupOp) - return op->emitOpError() << "expected '" << accessGroupRef - << "' to reference an access_group op"; - } - } - // If the data layout attribute is present, it must use the LLVM data layout // syntax. Try parsing it and report errors in case of failure. Users of this // attribute may assume it is well-formed and can pass it to the (asserting) diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp index 7d36d40c7049..e58d267e3171 100644 --- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp @@ -203,8 +203,12 @@ static LogicalResult setLoopAttr(const llvm::MDNode *node, Operation *op, if (!attr) return failure(); - op->setAttr(LLVMDialect::getLoopAttrName(), attr); - return success(); + return TypeSwitch(op) + .Case([&](auto branchOp) { + branchOp.setLoopAnnotationAttr(attr); + return success(); + }) + .Default([](auto) { return failure(); }); } namespace { diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp index c928d78fcc55..b49fc383d73a 100644 --- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp @@ -1234,8 +1234,10 @@ LogicalResult ModuleTranslation::createTBAAMetadata() { void ModuleTranslation::setLoopMetadata(Operation *op, llvm::Instruction *inst) { - auto attr = - op->getAttrOfType(LLVMDialect::getLoopAttrName()); + LoopAnnotationAttr attr = + TypeSwitch(op) + .Case( + [](auto branchOp) { return branchOp.getLoopAnnotationAttr(); }); if (!attr) return; llvm::MDNode *loopMD = loopAnnotationTranslation->translate(attr, op); diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir index 4547d13dc38b..23c4cf5e97e7 100644 --- a/mlir/test/Dialect/LLVMIR/invalid.mlir +++ b/mlir/test/Dialect/LLVMIR/invalid.mlir @@ -840,21 +840,10 @@ llvm.mlir.global appending @non_array_type_global_appending_linkage() : i32 // ----- -module { - llvm.func @loopOptions() { - // expected-error@below {{expected 'llvm.loop' to be a loop annotation attribute}} - llvm.br ^bb4 {llvm.loop = "test"} - ^bb4: - llvm.return - } -} - -// ----- - module { llvm.func @loopOptions() { // expected-error@below {{expected '@func1' to reference a metadata op}} - llvm.br ^bb4 {llvm.loop = #llvm.loop_annotation} + llvm.br ^bb4 {loop_annotation = #llvm.loop_annotation} ^bb4: llvm.return } @@ -868,7 +857,7 @@ module { module { llvm.func @loopOptions() { // expected-error@below {{expected '@metadata' to reference an access_group op}} - llvm.br ^bb4 {llvm.loop = #llvm.loop_annotation} + llvm.br ^bb4 {loop_annotation = #llvm.loop_annotation} ^bb4: llvm.return } diff --git a/mlir/test/Target/LLVMIR/Import/metadata-loop.ll b/mlir/test/Target/LLVMIR/Import/metadata-loop.ll index 315852663dd0..ff2d9c84fc13 100644 --- a/mlir/test/Target/LLVMIR/Import/metadata-loop.ll +++ b/mlir/test/Target/LLVMIR/Import/metadata-loop.ll @@ -32,7 +32,7 @@ define void @access_group(ptr %arg1) { ; CHECK-LABEL: @simple define void @simple(i64 %n, ptr %A) { entry: -; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]} +; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]} br label %end, !llvm.loop !1 end: ret void @@ -52,7 +52,7 @@ end: ; CHECK-LABEL: @vectorize define void @vectorize(i64 %n, ptr %A) { entry: -; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]} +; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]} br label %end, !llvm.loop !1 end: ret void @@ -78,7 +78,7 @@ end: ; CHECK-LABEL: @interleave define void @interleave(i64 %n, ptr %A) { entry: -; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]} +; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]} br label %end, !llvm.loop !1 end: ret void @@ -96,7 +96,7 @@ end: ; CHECK-LABEL: @unroll define void @unroll(i64 %n, ptr %A) { entry: -; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]} +; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]} br label %end, !llvm.loop !1 end: ret void @@ -122,7 +122,7 @@ end: ; CHECK-LABEL: @unroll_disable define void @unroll_disable(i64 %n, ptr %A) { entry: -; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]} +; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]} br label %end, !llvm.loop !1 end: ret void @@ -140,7 +140,7 @@ end: ; CHECK-LABEL: @unroll_and_jam define void @unroll_and_jam(i64 %n, ptr %A) { entry: -; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]} +; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]} br label %end, !llvm.loop !1 end: ret void @@ -166,7 +166,7 @@ end: ; CHECK-LABEL: @licm define void @licm(i64 %n, ptr %A) { entry: -; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]} +; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]} br label %end, !llvm.loop !1 end: ret void @@ -185,7 +185,7 @@ end: ; CHECK-LABEL: @distribute define void @distribute(i64 %n, ptr %A) { entry: -; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]} +; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]} br label %end, !llvm.loop !1 end: ret void @@ -209,7 +209,7 @@ end: ; CHECK-LABEL: @pipeline define void @pipeline(i64 %n, ptr %A) { entry: -; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]} +; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]} br label %end, !llvm.loop !1 end: ret void @@ -227,7 +227,7 @@ end: ; CHECK-LABEL: @peeled define void @peeled(i64 %n, ptr %A) { entry: -; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]} +; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]} br label %end, !llvm.loop !1 end: ret void @@ -244,7 +244,7 @@ end: ; CHECK-LABEL: @unswitched define void @unswitched(i64 %n, ptr %A) { entry: -; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]} +; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]} br label %end, !llvm.loop !1 end: ret void @@ -264,7 +264,7 @@ end: define void @parallel_accesses(ptr %arg) { entry: %0 = load i32, ptr %arg, !llvm.access.group !0 -; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]} +; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]} br label %end, !llvm.loop !1 end: ret void @@ -287,7 +287,7 @@ define void @multiple_parallel_accesses(ptr %arg) { entry: %0 = load i32, ptr %arg, !llvm.access.group !0 %1 = load i32, ptr %arg, !llvm.access.group !3 -; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]} +; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]} br label %end, !llvm.loop !1 end: ret void diff --git a/mlir/test/Target/LLVMIR/loop-metadata.mlir b/mlir/test/Target/LLVMIR/loop-metadata.mlir index e7d06775bf9f..c0ea4b7812c4 100644 --- a/mlir/test/Target/LLVMIR/loop-metadata.mlir +++ b/mlir/test/Target/LLVMIR/loop-metadata.mlir @@ -3,7 +3,7 @@ // CHECK-LABEL: @disableNonForced llvm.func @disableNonForced() { // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] - llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation} + llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation} ^bb1: llvm.return } @@ -16,7 +16,7 @@ llvm.func @disableNonForced() { // CHECK-LABEL: @mustprogress llvm.func @mustprogress() { // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] - llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation} + llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation} ^bb1: llvm.return } @@ -29,7 +29,7 @@ llvm.func @mustprogress() { // CHECK-LABEL: @isvectorized llvm.func @isvectorized() { // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] - llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation} + llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation} ^bb1: llvm.return } @@ -44,7 +44,7 @@ llvm.func @isvectorized() { // CHECK-LABEL: @vectorizeOptions llvm.func @vectorizeOptions() { // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] - llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation >} @@ -68,7 +68,7 @@ llvm.func @vectorizeOptions() { // CHECK-LABEL: @interleaveOptions llvm.func @interleaveOptions() { // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] - llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation>} + llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation>} ^bb1: llvm.return } @@ -83,7 +83,7 @@ llvm.func @interleaveOptions() { // CHECK-LABEL: @unrollOptions llvm.func @unrollOptions() { // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] - llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation >} @@ -106,7 +106,7 @@ llvm.func @unrollOptions() { // CHECK-LABEL: @unrollOptions2 llvm.func @unrollOptions2() { // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] - llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation>} + llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation>} ^bb1: llvm.return } @@ -122,7 +122,7 @@ llvm.func @unrollOptions2() { // CHECK-LABEL: @unrollAndJamOptions llvm.func @unrollAndJamOptions() { // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] - llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation >} @@ -146,7 +146,7 @@ llvm.func @unrollAndJamOptions() { // CHECK-LABEL: @licmOptions llvm.func @licmOptions() { // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] - llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation>} + llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation>} ^bb1: llvm.return } @@ -159,7 +159,7 @@ llvm.func @licmOptions() { // CHECK-LABEL: @licmOptions2 llvm.func @licmOptions2() { // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] - llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation>} + llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation>} ^bb1: llvm.return } @@ -174,7 +174,7 @@ llvm.func @licmOptions2() { // CHECK-LABEL: @distributeOptions llvm.func @distributeOptions() { // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] - llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation >} @@ -196,7 +196,7 @@ llvm.func @distributeOptions() { // CHECK-LABEL: @pipelineOptions llvm.func @pipelineOptions() { // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] - llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation>} + llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation>} ^bb1: llvm.return } @@ -210,7 +210,7 @@ llvm.func @pipelineOptions() { // CHECK-LABEL: @peeledOptions llvm.func @peeledOptions() { // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] - llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation>} + llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation>} ^bb1: llvm.return } @@ -223,7 +223,7 @@ llvm.func @peeledOptions() { // CHECK-LABEL: @unswitchOptions llvm.func @unswitchOptions() { // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] - llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation>} + llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation>} ^bb1: llvm.return } @@ -241,7 +241,7 @@ llvm.func @loopOptions(%arg1 : i32, %arg2 : i32) { ^bb3(%1: i32): %2 = llvm.icmp "slt" %1, %arg1 : i32 // CHECK: br i1 {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] - llvm.cond_br %2, ^bb4, ^bb5 {llvm.loop = #llvm.loop_annotation< + llvm.cond_br %2, ^bb4, ^bb5 {loop_annotation = #llvm.loop_annotation< licm = , interleave = , unroll = , pipeline = , @@ -251,7 +251,7 @@ llvm.func @loopOptions(%arg1 : i32, %arg2 : i32) { // CHECK: = load i32, ptr %{{.*}} !llvm.access.group ![[ACCESS_GROUPS_NODE:[0-9]+]] %5 = llvm.load %4 { access_groups = [@metadata::@group1, @metadata::@group2] } : !llvm.ptr // CHECK: br label {{.*}} !llvm.loop ![[LOOP_NODE]] - llvm.br ^bb3(%3 : i32) {llvm.loop = #llvm.loop_annotation< + llvm.br ^bb3(%3 : i32) {loop_annotation = #llvm.loop_annotation< licm = , interleave = , unroll = , pipeline = , From 467460d344fa6862f8973f7c556cf09bea0f779d Mon Sep 17 00:00:00 2001 From: Tobias Gysi Date: Mon, 6 Mar 2023 11:00:56 +0100 Subject: [PATCH 079/147] [mlir][llvm] Import alias scope metadata from LLVM IR. The revision adds support for importing alias.scope and noalias metadata from LLVM IR into LLVM dialect. It also adds a verifier to the AliasScopeMetadataOp to check that the associated domain exists and is of type AliasScopeDomainMetadataOp. Reviewed By: Dinistro Differential Revision: https://reviews.llvm.org/D143923 (cherry picked from commit 4469ec1d196ff737e9e18320a124023bb420dfed) --- mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td | 1 + .../include/mlir/Target/LLVMIR/ModuleImport.h | 20 +++- mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 15 +++ .../LLVMIR/LLVMIRToLLVMTranslation.cpp | 55 +++++++++- mlir/lib/Target/LLVMIR/ModuleImport.cpp | 101 +++++++++++++++++- mlir/test/Dialect/LLVMIR/invalid.mlir | 22 ++++ .../Target/LLVMIR/Import/import-failure.ll | 36 +++++++ .../LLVMIR/Import/metadata-alias-scopes.ll | 61 +++++++++++ 8 files changed, 301 insertions(+), 10 deletions(-) create mode 100644 mlir/test/Target/LLVMIR/Import/metadata-alias-scopes.ll diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td index 0a40ea9218b2..011231c15be5 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td @@ -1090,6 +1090,7 @@ def LLVM_AliasScopeMetadataOp : LLVM_Op<"alias_scope", [ https://llvm.org/docs/LangRef.html#noalias-and-alias-scope-metadata }]; let assemblyFormat = "$sym_name attr-dict"; + let hasVerifier = 1; } def LLVM_AccessGroupMetadataOp : LLVM_Op<"access_group", [ diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h index 3265c3237241..aaf9b2cf55ef 100644 --- a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h +++ b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h @@ -176,6 +176,12 @@ class ModuleImport { LoopAnnotationAttr translateLoopAnnotationAttr(const llvm::MDNode *node, Location loc) const; + /// Returns the symbol references pointing to the alias scope operations that + /// map to the alias scope nodes starting from the metadata `node`. Returns + /// failure, if any of the symbol references cannot be found. + FailureOr> + lookupAliasScopeAttrs(const llvm::MDNode *node) const; + private: /// Clears the block and value mapping before processing a new region. void clearBlockAndValueMapping() { @@ -268,6 +274,12 @@ class ModuleImport { /// symbol pointing to the translated operation. Returns success if all /// conversions succeed and failure otherwise. LogicalResult processAccessGroupMetadata(const llvm::MDNode *node); + /// Converts all LLVM alias scopes and domains starting from `node` to MLIR + /// alias scope and domain operations and stores a mapping from every nested + /// alias scope or alias domain node to the symbol pointing to the translated + /// operation. Returns success if all conversions succeed and failure + /// otherwise. + LogicalResult processAliasScopeMetadata(const llvm::MDNode *node); /// Builder pointing at where the next instruction should be generated. OpBuilder builder; @@ -298,9 +310,11 @@ class ModuleImport { /// operations for all operations that return no result. All operations that /// return a result have a valueMapping entry instead. DenseMap noResultOpMapping; - /// Mapping between LLVM TBAA metadata nodes and symbol references - /// to the LLVMIR dialect TBAA operations corresponding to these - /// nodes. + /// Mapping between LLVM alias scope and domain metadata nodes and symbol + /// references to the LLVM dialect operations corresponding to these nodes. + DenseMap aliasScopeMapping; + /// Mapping between LLVM TBAA metadata nodes and symbol references to the LLVM + /// dialect TBAA operations corresponding to these nodes. DenseMap tbaaMapping; /// The stateful type translator (contains named structs). LLVM::TypeFromLLVMIRTranslator typeTranslator; diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp index 6d3d0fbad191..2dd6e40fd9db 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -2699,6 +2699,21 @@ LogicalResult TBAATypeDescriptorOp::verify() { return success(); } +//===----------------------------------------------------------------------===// +// AliasScopeMetadataOp +//===----------------------------------------------------------------------===// + +LogicalResult AliasScopeMetadataOp::verify() { + Operation *domainOp = SymbolTable::lookupNearestSymbolFrom( + this->getOperation(), getDomainAttr()); + if (!isa_and_nonnull(domainOp)) { + return this->emitOpError() + << "expected '" << getDomain() + << "' to reference a domain operation in the same region"; + } + return success(); +} + //===----------------------------------------------------------------------===// // OpAsmDialectInterface //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp index e58d267e3171..88f0688180a3 100644 --- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp @@ -71,8 +71,9 @@ static LogicalResult convertIntrinsicImpl(OpBuilder &odsBuilder, /// dialect attributes. static ArrayRef getSupportedMetadataImpl() { static const SmallVector convertibleMetadata = { - llvm::LLVMContext::MD_prof, llvm::LLVMContext::MD_tbaa, - llvm::LLVMContext::MD_access_group, llvm::LLVMContext::MD_loop}; + llvm::LLVMContext::MD_prof, llvm::LLVMContext::MD_tbaa, + llvm::LLVMContext::MD_access_group, llvm::LLVMContext::MD_loop, + llvm::LLVMContext::MD_noalias, llvm::LLVMContext::MD_alias_scope}; return convertibleMetadata; } @@ -178,8 +179,9 @@ static LogicalResult setTBAAAttr(const llvm::MDNode *node, Operation *op, /// that map to the access group nodes starting from the access group metadata /// `node`, and attaches all of them to the imported operation if the lookups /// succeed. Returns failure otherwise. -static LogicalResult setAccessGroupAttr(const llvm::MDNode *node, Operation *op, - LLVM::ModuleImport &moduleImport) { +static LogicalResult setAccessGroupsAttr(const llvm::MDNode *node, + Operation *op, + LLVM::ModuleImport &moduleImport) { FailureOr> accessGroups = moduleImport.lookupAccessGroupAttrs(node); if (failed(accessGroups)) @@ -211,6 +213,45 @@ static LogicalResult setLoopAttr(const llvm::MDNode *node, Operation *op, .Default([](auto) { return failure(); }); } +/// Looks up all the symbol references pointing to the alias scope operations +/// that map to the alias scope nodes starting from the alias scope metadata +/// `node`, and attaches all of them to the imported operation if the lookups +/// succeed. Returns failure otherwise. +static LogicalResult setAliasScopesAttr(const llvm::MDNode *node, Operation *op, + LLVM::ModuleImport &moduleImport) { + FailureOr> aliasScopes = + moduleImport.lookupAliasScopeAttrs(node); + if (failed(aliasScopes)) + return failure(); + + SmallVector aliasScopeAttrs(aliasScopes->begin(), + aliasScopes->end()); + return AttributeSetter(op).apply([&](auto memOp) { + memOp.setAliasScopesAttr( + ArrayAttr::get(memOp.getContext(), aliasScopeAttrs)); + }); +} + +/// Looks up all the symbol references pointing to the alias scope operations +/// that map to the alias scope nodes starting from the noalias metadata `node`, +/// and attaches all of them to the imported operation if the lookups succeed. +/// Returns failure otherwise. +static LogicalResult setNoaliasScopesAttr(const llvm::MDNode *node, + Operation *op, + LLVM::ModuleImport &moduleImport) { + FailureOr> noaliasScopes = + moduleImport.lookupAliasScopeAttrs(node); + if (failed(noaliasScopes)) + return failure(); + + SmallVector noaliasScopeAttrs(noaliasScopes->begin(), + noaliasScopes->end()); + return AttributeSetter(op).apply([&](auto memOp) { + memOp.setNoaliasScopesAttr( + ArrayAttr::get(memOp.getContext(), noaliasScopeAttrs)); + }); +} + namespace { /// Implementation of the dialect interface that converts operations belonging @@ -238,9 +279,13 @@ class LLVMDialectLLVMIRImportInterface : public LLVMImportDialectInterface { if (kind == llvm::LLVMContext::MD_tbaa) return setTBAAAttr(node, op, moduleImport); if (kind == llvm::LLVMContext::MD_access_group) - return setAccessGroupAttr(node, op, moduleImport); + return setAccessGroupsAttr(node, op, moduleImport); if (kind == llvm::LLVMContext::MD_loop) return setLoopAttr(node, op, moduleImport); + if (kind == llvm::LLVMContext::MD_alias_scope) + return setAliasScopesAttr(node, op, moduleImport); + if (kind == llvm::LLVMContext::MD_noalias) + return setNoaliasScopesAttr(node, op, moduleImport); // A handler for a supported metadata kind is missing. llvm_unreachable("unknown metadata type"); diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp index 7bcef5d85e62..94dd030ebc0f 100644 --- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp @@ -521,6 +521,99 @@ ModuleImport::processAccessGroupMetadata(const llvm::MDNode *node) { return success(); } +LogicalResult +ModuleImport::processAliasScopeMetadata(const llvm::MDNode *node) { + Location loc = mlirModule.getLoc(); + // Helper that verifies the node has a self reference operand. + auto verifySelfRef = [](const llvm::MDNode *node) { + return node->getNumOperands() != 0 && + node == dyn_cast(node->getOperand(0)); + }; + // Helper that verifies the given operand is a string or does not exist. + auto verifyDescription = [](const llvm::MDNode *node, unsigned idx) { + return idx >= node->getNumOperands() || + isa(node->getOperand(idx)); + }; + // Helper that creates an alias scope domain operation. + auto createAliasScopeDomainOp = [&](const llvm::MDNode *aliasDomain) { + StringAttr description = nullptr; + if (aliasDomain->getNumOperands() >= 2) + if (auto *operand = dyn_cast(aliasDomain->getOperand(1))) + description = builder.getStringAttr(operand->getString()); + std::string name = llvm::formatv("domain_{0}", aliasScopeMapping.size()); + return builder.create(loc, name, description); + }; + + // Collect the alias scopes and domains to translate them. + for (const llvm::MDOperand &operand : node->operands()) { + if (const auto *scope = dyn_cast(operand)) { + llvm::AliasScopeNode aliasScope(scope); + const llvm::MDNode *domain = aliasScope.getDomain(); + + // Verify the scope node points to valid scope metadata which includes + // verifying its domain. Perform the verification before looking it up in + // the alias scope mapping since it could have been inserted as a domain + // node before. + if (!verifySelfRef(scope) || !domain || !verifyDescription(scope, 2)) + return emitError(loc) << "unsupported alias scope node: " + << diagMD(scope, llvmModule.get()); + if (!verifySelfRef(domain) || !verifyDescription(domain, 1)) + return emitError(loc) << "unsupported alias domain node: " + << diagMD(domain, llvmModule.get()); + + if (aliasScopeMapping.count(scope)) + continue; + + // Set the insertion point to the end of the global metadata operation. + MetadataOp metadataOp = getGlobalMetadataOp(); + StringAttr metadataOpName = + SymbolTable::getSymbolName(getGlobalMetadataOp()); + OpBuilder::InsertionGuard guard(builder); + builder.setInsertionPointToEnd(&metadataOp.getBody().back()); + + // Convert the domain metadata node if it has not been translated before. + auto it = aliasScopeMapping.find(aliasScope.getDomain()); + if (it == aliasScopeMapping.end()) { + auto aliasScopeDomainOp = createAliasScopeDomainOp(domain); + auto symbolRef = SymbolRefAttr::get( + builder.getContext(), metadataOpName, + FlatSymbolRefAttr::get(builder.getContext(), + aliasScopeDomainOp.getSymName())); + it = aliasScopeMapping.try_emplace(domain, symbolRef).first; + } + + // Convert the scope metadata node if it has not been converted before. + StringAttr description = nullptr; + if (!aliasScope.getName().empty()) + description = builder.getStringAttr(aliasScope.getName()); + std::string name = llvm::formatv("scope_{0}", aliasScopeMapping.size()); + auto aliasScopeOp = builder.create( + loc, name, it->getSecond().getLeafReference().getValue(), + description); + auto symbolRef = + SymbolRefAttr::get(builder.getContext(), metadataOpName, + FlatSymbolRefAttr::get(builder.getContext(), + aliasScopeOp.getSymName())); + aliasScopeMapping.try_emplace(aliasScope.getNode(), symbolRef); + } + } + return success(); +} + +FailureOr> +ModuleImport::lookupAliasScopeAttrs(const llvm::MDNode *node) const { + SmallVector aliasScopes; + aliasScopes.reserve(node->getNumOperands()); + for (const llvm::MDOperand &operand : node->operands()) { + auto *node = cast(operand.get()); + aliasScopes.push_back(aliasScopeMapping.lookup(node)); + } + // Return failure if one of the alias scope lookups failed. + if (llvm::is_contained(aliasScopes, nullptr)) + return failure(); + return aliasScopes; +} + LogicalResult ModuleImport::convertMetadata() { OpBuilder::InsertionGuard guard(builder); builder.setInsertionPointToEnd(mlirModule.getBody()); @@ -539,8 +632,12 @@ LogicalResult ModuleImport::convertMetadata() { if (aliasAnalysisNodes.TBAA) if (failed(processTBAAMetadata(aliasAnalysisNodes.TBAA))) return failure(); - - // TODO: Support noalias and scope metadata nodes. + if (aliasAnalysisNodes.Scope) + if (failed(processAliasScopeMetadata(aliasAnalysisNodes.Scope))) + return failure(); + if (aliasAnalysisNodes.NoAlias) + if (failed(processAliasScopeMetadata(aliasAnalysisNodes.NoAlias))) + return failure(); } } return success(); diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir index 23c4cf5e97e7..18dfc4ce1ff0 100644 --- a/mlir/test/Dialect/LLVMIR/invalid.mlir +++ b/mlir/test/Dialect/LLVMIR/invalid.mlir @@ -964,6 +964,28 @@ module { // ----- +module { + llvm.metadata @metadata { + llvm.access_group @group + // expected-error@below {{expected 'group' to reference a domain operation in the same region}} + llvm.alias_scope @scope { domain = @group } + } +} + +// ----- + +module { + llvm.metadata @metadata { + // expected-error@below {{expected 'domain' to reference a domain operation in the same region}} + llvm.alias_scope @scope { domain = @domain } + } + llvm.metadata @other_metadata { + llvm.alias_scope_domain @domain + } +} + +// ----- + llvm.func @wmmaLoadOp_invalid_mem_space(%arg0: !llvm.ptr, %arg1: i32) { // expected-error@+1 {{'nvvm.wmma.load' op expected source pointer in memory space 0, 1, 3}} %0 = nvvm.wmma.load %arg0, %arg1 diff --git a/mlir/test/Target/LLVMIR/Import/import-failure.ll b/mlir/test/Target/LLVMIR/Import/import-failure.ll index 1163d703256f..8454b511f2e6 100644 --- a/mlir/test/Target/LLVMIR/Import/import-failure.ll +++ b/mlir/test/Target/LLVMIR/Import/import-failure.ll @@ -572,3 +572,39 @@ define void @cond_br(i1 %arg) !prof !0 { } !0 = !{!"branch_weights", i32 64} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: error: unsupported alias scope node: ![[NODE:[0-9]+]] = distinct !{![[NODE]]} +define void @alias_scope(ptr %arg1) { + %1 = load i32, ptr %arg1, !alias.scope !0 + ret void +} + +!0 = !{!0} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: error: unsupported alias scope node: ![[NODE:[0-9]+]] = distinct !{![[NODE]], !"The domain"} +define void @alias_scope(ptr %arg1) { + %1 = load i32, ptr %arg1, !alias.scope !1 + ret void +} + +!0 = distinct !{!0, !"The domain"} +!1 = !{!1, !0} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: error: unsupported alias domain node: ![[NODE:[0-9]+]] = distinct !{![[NODE]], ![[NODE]]} +define void @alias_scope_domain(ptr %arg1) { + %1 = load i32, ptr %arg1, !alias.scope !2 + ret void +} + +!0 = distinct !{!0, !0} +!1 = !{!1, !0} +!2 = !{!1} diff --git a/mlir/test/Target/LLVMIR/Import/metadata-alias-scopes.ll b/mlir/test/Target/LLVMIR/Import/metadata-alias-scopes.ll new file mode 100644 index 000000000000..9db045dde27a --- /dev/null +++ b/mlir/test/Target/LLVMIR/Import/metadata-alias-scopes.ll @@ -0,0 +1,61 @@ +; RUN: mlir-translate -import-llvm -split-input-file %s | FileCheck %s + +; CHECK: llvm.metadata @__llvm_global_metadata { +; CHECK: llvm.alias_scope_domain @[[DOMAIN:.*]] {description = "The domain"} +; CHECK: llvm.alias_scope @[[$SCOPE0:.*]] {description = "The first scope", domain = @[[DOMAIN]]} +; CHECK: llvm.alias_scope @[[$SCOPE1:.*]] {domain = @[[DOMAIN]]} +; CHECK: llvm.alias_scope @[[$SCOPE2:.*]] {domain = @[[DOMAIN]]} +; CHECK: } + +; CHECK-LABEL: llvm.func @alias_scope +define void @alias_scope(ptr %arg1) { + ; CHECK: llvm.load + ; CHECK-SAME: alias_scopes = [@__llvm_global_metadata::@[[$SCOPE0]]] + ; CHECK-SAME: noalias_scopes = [@__llvm_global_metadata::@[[$SCOPE1]], @__llvm_global_metadata::@[[$SCOPE2]]] + %1 = load i32, ptr %arg1, !alias.scope !4, !noalias !7 + ; CHECK: llvm.load + ; CHECK-SAME: alias_scopes = [@__llvm_global_metadata::@[[$SCOPE1]]] + ; CHECK-SAME: noalias_scopes = [@__llvm_global_metadata::@[[$SCOPE0]], @__llvm_global_metadata::@[[$SCOPE2]]] + %2 = load i32, ptr %arg1, !alias.scope !5, !noalias !8 + ; CHECK: llvm.load + ; CHECK-SAME: alias_scopes = [@__llvm_global_metadata::@[[$SCOPE2]]] + ; CHECK-SAME: noalias_scopes = [@__llvm_global_metadata::@[[$SCOPE0]], @__llvm_global_metadata::@[[$SCOPE1]]] + %3 = load i32, ptr %arg1, !alias.scope !6, !noalias !9 + ret void +} + +!0 = distinct !{!0, !"The domain"} +!1 = distinct !{!1, !0, !"The first scope"} +!2 = distinct !{!2, !0} +!3 = distinct !{!3, !0} +!4 = !{!1} +!5 = !{!2} +!6 = !{!3} +!7 = !{!2, !3} +!8 = !{!1, !3} +!9 = !{!1, !2} + +; // ----- + +; CHECK: llvm.metadata @__llvm_global_metadata { +; CHECK: llvm.alias_scope_domain @[[DOMAIN0:.*]] {description = "The domain"} +; CHECK: llvm.alias_scope @[[$SCOPE0:.*]] {domain = @[[DOMAIN0]]} +; CHECK: llvm.alias_scope_domain @[[DOMAIN1:.*]] +; CHECK: llvm.alias_scope @[[$SCOPE1:.*]] {domain = @[[DOMAIN1]]} +; CHECK: } + +; CHECK-LABEL: llvm.func @two_domains +define void @two_domains(ptr %arg1) { + ; CHECK: llvm.load + ; CHECK-SAME: alias_scopes = [@__llvm_global_metadata::@[[$SCOPE0]]] + ; CHECK-SAME: noalias_scopes = [@__llvm_global_metadata::@[[$SCOPE1]]] + %1 = load i32, ptr %arg1, !alias.scope !4, !noalias !5 + ret void +} + +!0 = distinct !{!0, !"The domain"} +!1 = distinct !{!1} +!2 = !{!2, !0} +!3 = !{!3, !1} +!4 = !{!2} +!5 = !{!3} From 9cbf47c6267ffbbc9da183a0f48d64aad0c60d6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20B=C3=B6ck?= Date: Mon, 6 Mar 2023 11:01:37 +0100 Subject: [PATCH 080/147] [mlir] Port Conversion Passes to LLVM to use TableGen generated constructors and options See https://github.com/llvm/llvm-project/issues/57475 for more context. Using auto-generated constructors and options has significant advantages: * It forces a uniform style and expectation for consuming a pass * It allows to very easily add, remove or change options to a pass by simply making the changes in TableGen * Its less code This patch in particular ports all the conversion passes which lower to LLVM to use the auto generated constructors and options. For the most part, care was taken so that auto generated constructor functions have the same name as they previously did. Only following slight breaking changes (which I consider as worth the churn) have been made: * `mlir::cf::createConvertControlFlowToLLVMPass` has been moved to the `mlir` namespace. This is consistent with basically all conversion passes * `createGpuToLLVMConversionPass` now takes a proper options struct array for its pass options. The pass options are now also autogenerated. * `LowerVectorToLLVMOptions` has been replaced by the autogenerated `ConvertVectorToLLVMPassOptions` which is automatically kept up to date by TableGen * I had to move one function in the GPU to LLVM lowering as it is used as default value for an option. * All passes that previously returned `unique_ptr>` now simply return `unique_ptr` Differential Revision: https://reviews.llvm.org/D143773 (cherry picked from commit cd4ca2d7f991177b64db9df3c17986dc8e250f1d --- .../mlir/Conversion/AsyncToLLVM/AsyncToLLVM.h | 9 +-- .../Conversion/ComplexToLLVM/ComplexToLLVM.h | 6 +- .../ControlFlowToLLVM/ControlFlowToLLVM.h | 6 +- .../mlir/Conversion/GPUCommon/GPUCommonPass.h | 13 +--- .../Conversion/LinalgToLLVM/LinalgToLLVM.h | 9 +-- .../mlir/Conversion/MathToLLVM/MathToLLVM.h | 4 +- .../OpenACCToLLVM/ConvertOpenACCToLLVM.h | 10 +-- .../OpenMPToLLVM/ConvertOpenMPToLLVM.h | 10 +-- mlir/include/mlir/Conversion/Passes.td | 65 +++++++++++++------ .../Conversion/SPIRVToLLVM/SPIRVToLLVMPass.h | 21 +----- .../VectorToLLVM/ConvertVectorToLLVM.h | 50 +------------- .../mlir/Dialect/GPU/Transforms/Passes.h | 1 + .../mlir/Dialect/GPU/Transforms/Utils.h | 5 ++ .../Dialect/SparseTensor/Pipelines/Passes.h | 16 ++--- .../Conversion/AsyncToLLVM/AsyncToLLVM.cpp | 10 ++- .../ComplexToLLVM/ComplexToLLVM.cpp | 10 ++- .../ControlFlowToLLVM/ControlFlowToLLVM.cpp | 11 ++-- .../GPUCommon/GPUToLLVMConversion.cpp | 27 +------- .../Conversion/LinalgToLLVM/LinalgToLLVM.cpp | 8 +-- mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp | 10 +-- .../OpenACCToLLVM/OpenACCToLLVM.cpp | 11 ++-- .../Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp | 10 ++- .../ConvertLaunchFuncToLLVMCalls.cpp | 12 ++-- .../SPIRVToLLVM/SPIRVToLLVMPass.cpp | 11 ++-- .../VectorToLLVM/ConvertVectorToLLVMPass.cpp | 20 ++---- .../test/lib/Dialect/LLVM/TestLowerToLLVM.cpp | 3 +- 26 files changed, 119 insertions(+), 249 deletions(-) diff --git a/mlir/include/mlir/Conversion/AsyncToLLVM/AsyncToLLVM.h b/mlir/include/mlir/Conversion/AsyncToLLVM/AsyncToLLVM.h index d487f661a01a..60441f9faaa6 100644 --- a/mlir/include/mlir/Conversion/AsyncToLLVM/AsyncToLLVM.h +++ b/mlir/include/mlir/Conversion/AsyncToLLVM/AsyncToLLVM.h @@ -14,19 +14,14 @@ namespace mlir { class ConversionTarget; -class ModuleOp; -template -class OperationPass; +class Pass; class MLIRContext; class TypeConverter; class RewritePatternSet; -#define GEN_PASS_DECL_CONVERTASYNCTOLLVM +#define GEN_PASS_DECL_CONVERTASYNCTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" -/// Create a pass to convert Async operations to the LLVM dialect. -std::unique_ptr> createConvertAsyncToLLVMPass(); - /// Populates patterns for async structural type conversions. /// /// A "structural" type conversion is one where the underlying ops are diff --git a/mlir/include/mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h b/mlir/include/mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h index 7779321ca46c..1385618c0980 100644 --- a/mlir/include/mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h +++ b/mlir/include/mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h @@ -15,7 +15,7 @@ class LLVMTypeConverter; class Pass; class RewritePatternSet; -#define GEN_PASS_DECL_CONVERTCOMPLEXTOLLVM +#define GEN_PASS_DECL_CONVERTCOMPLEXTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" class ComplexStructBuilder : public StructBuilder { @@ -40,10 +40,6 @@ class ComplexStructBuilder : public StructBuilder { /// Populate the given list with patterns that convert from Complex to LLVM. void populateComplexToLLVMConversionPatterns(LLVMTypeConverter &converter, RewritePatternSet &patterns); - -/// Create a pass to convert Complex operations to the LLVMIR dialect. -std::unique_ptr createConvertComplexToLLVMPass(); - } // namespace mlir #endif // MLIR_CONVERSION_COMPLEXTOLLVM_COMPLEXTOLLVM_H_ diff --git a/mlir/include/mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h b/mlir/include/mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h index b9dfaafcf5a5..9cc867238a00 100644 --- a/mlir/include/mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h +++ b/mlir/include/mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h @@ -20,10 +20,11 @@ class LLVMTypeConverter; class RewritePatternSet; class Pass; -#define GEN_PASS_DECL_CONVERTCONTROLFLOWTOLLVM +#define GEN_PASS_DECL_CONVERTCONTROLFLOWTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" namespace cf { + /// Collect the patterns to convert from the ControlFlow dialect to LLVM. The /// conversion patterns capture the LLVMTypeConverter by reference meaning the /// references have to remain alive during the entire pattern lifetime. @@ -36,9 +37,6 @@ void populateControlFlowToLLVMConversionPatterns(LLVMTypeConverter &converter, void populateAssertToLLVMConversionPattern(LLVMTypeConverter &converter, RewritePatternSet &patterns, bool abortOnFailure = true); - -/// Creates a pass to convert the ControlFlow dialect into the LLVMIR dialect. -std::unique_ptr createConvertControlFlowToLLVMPass(); } // namespace cf } // namespace mlir diff --git a/mlir/include/mlir/Conversion/GPUCommon/GPUCommonPass.h b/mlir/include/mlir/Conversion/GPUCommon/GPUCommonPass.h index 2c73f43c9541..64a9614ee439 100644 --- a/mlir/include/mlir/Conversion/GPUCommon/GPUCommonPass.h +++ b/mlir/include/mlir/Conversion/GPUCommon/GPUCommonPass.h @@ -8,6 +8,7 @@ #ifndef MLIR_CONVERSION_GPUCOMMON_GPUCOMMONPASS_H_ #define MLIR_CONVERSION_GPUCOMMON_GPUCOMMONPASS_H_ +#include "mlir/Dialect/GPU/Transforms/Utils.h" #include "mlir/Support/LLVM.h" #include "llvm/ADT/StringRef.h" #include @@ -27,8 +28,7 @@ class ModuleOp; class Operation; class RewritePatternSet; -template -class OperationPass; +class Pass; namespace gpu { class GPUModuleOp; @@ -47,15 +47,6 @@ using BlobGenerator = using LoweringCallback = std::function( Operation *, llvm::LLVMContext &, StringRef)>; -/// Creates a pass to convert a GPU operations into a sequence of GPU runtime -/// calls. -/// -/// This pass does not generate code to call GPU runtime APIs directly but -/// instead uses a small wrapper library that exports a stable and conveniently -/// typed ABI on top of GPU runtimes such as CUDA or ROCm (HIP). -std::unique_ptr> -createGpuToLLVMConversionPass(bool kernelBarePtrCallConv = false); - /// Collect a set of patterns to convert from the GPU dialect to LLVM and /// populate converter for gpu types. void populateGpuToLLVMConversionPatterns(LLVMTypeConverter &converter, diff --git a/mlir/include/mlir/Conversion/LinalgToLLVM/LinalgToLLVM.h b/mlir/include/mlir/Conversion/LinalgToLLVM/LinalgToLLVM.h index b75aa4b5e428..72d263c488dc 100644 --- a/mlir/include/mlir/Conversion/LinalgToLLVM/LinalgToLLVM.h +++ b/mlir/include/mlir/Conversion/LinalgToLLVM/LinalgToLLVM.h @@ -13,21 +13,16 @@ namespace mlir { class LLVMTypeConverter; class MLIRContext; -class ModuleOp; -template -class OperationPass; +class Pass; class RewritePatternSet; -#define GEN_PASS_DECL_CONVERTLINALGTOLLVM +#define GEN_PASS_DECL_CONVERTLINALGTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" /// Populate the given list with patterns that convert from Linalg to LLVM. void populateLinalgToLLVMConversionPatterns(LLVMTypeConverter &converter, RewritePatternSet &patterns); -/// Create a pass to convert Linalg operations to the LLVMIR dialect. -std::unique_ptr> createConvertLinalgToLLVMPass(); - } // namespace mlir #endif // MLIR_CONVERSION_LINALGTOLLVM_LINALGTOLLVM_H_ diff --git a/mlir/include/mlir/Conversion/MathToLLVM/MathToLLVM.h b/mlir/include/mlir/Conversion/MathToLLVM/MathToLLVM.h index 0cb1fe33cadb..d0fc2e390ed7 100644 --- a/mlir/include/mlir/Conversion/MathToLLVM/MathToLLVM.h +++ b/mlir/include/mlir/Conversion/MathToLLVM/MathToLLVM.h @@ -17,13 +17,11 @@ class LLVMTypeConverter; class RewritePatternSet; class Pass; -#define GEN_PASS_DECL_CONVERTMATHTOLLVM +#define GEN_PASS_DECL_CONVERTMATHTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" void populateMathToLLVMConversionPatterns(LLVMTypeConverter &converter, RewritePatternSet &patterns); - -std::unique_ptr createConvertMathToLLVMPass(); } // namespace mlir #endif // MLIR_CONVERSION_MATHTOLLVM_MATHTOLLVM_H diff --git a/mlir/include/mlir/Conversion/OpenACCToLLVM/ConvertOpenACCToLLVM.h b/mlir/include/mlir/Conversion/OpenACCToLLVM/ConvertOpenACCToLLVM.h index 811533d2775d..0cbea0c9ef5b 100644 --- a/mlir/include/mlir/Conversion/OpenACCToLLVM/ConvertOpenACCToLLVM.h +++ b/mlir/include/mlir/Conversion/OpenACCToLLVM/ConvertOpenACCToLLVM.h @@ -13,12 +13,10 @@ namespace mlir { class LLVMTypeConverter; -class ModuleOp; -template -class OperationPass; +class Pass; class RewritePatternSet; -#define GEN_PASS_DECL_CONVERTOPENACCTOLLVM +#define GEN_PASS_DECL_CONVERTOPENACCTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" static constexpr unsigned kPtrBasePosInDataDescriptor = 0; @@ -69,10 +67,6 @@ class DataDescriptor : public StructBuilder { /// Collect the patterns to convert from the OpenACC dialect LLVMIR dialect. void populateOpenACCToLLVMConversionPatterns(LLVMTypeConverter &converter, RewritePatternSet &patterns); - -/// Create a pass to convert the OpenACC dialect into the LLVMIR dialect. -std::unique_ptr> createConvertOpenACCToLLVMPass(); - } // namespace mlir #endif // MLIR_CONVERSION_OPENACCTOLLVM_CONVERTOPENACCTOLLVM_H diff --git a/mlir/include/mlir/Conversion/OpenMPToLLVM/ConvertOpenMPToLLVM.h b/mlir/include/mlir/Conversion/OpenMPToLLVM/ConvertOpenMPToLLVM.h index 6827055b1c10..2ed077f8f83e 100644 --- a/mlir/include/mlir/Conversion/OpenMPToLLVM/ConvertOpenMPToLLVM.h +++ b/mlir/include/mlir/Conversion/OpenMPToLLVM/ConvertOpenMPToLLVM.h @@ -14,12 +14,10 @@ namespace mlir { class LLVMTypeConverter; class ConversionTarget; class MLIRContext; -class ModuleOp; -template -class OperationPass; +class Pass; class RewritePatternSet; -#define GEN_PASS_DECL_CONVERTOOPENMPTOLLVM +#define GEN_PASS_DECL_CONVERTOPENMPTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" /// Configure dynamic conversion legality of regionless operations from OpenMP @@ -30,10 +28,6 @@ void configureOpenMPToLLVMConversionLegality(ConversionTarget &target, /// Populate the given list with patterns that convert from OpenMP to LLVM. void populateOpenMPToLLVMConversionPatterns(LLVMTypeConverter &converter, RewritePatternSet &patterns); - -/// Create a pass to convert OpenMP operations to the LLVMIR dialect. -std::unique_ptr> createConvertOpenMPToLLVMPass(); - } // namespace mlir #endif // MLIR_CONVERSION_OPENMPTOLLVM_CONVERTOPENMPTOLLVM_H diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td index a2865922e7a0..767b6b981e8b 100644 --- a/mlir/include/mlir/Conversion/Passes.td +++ b/mlir/include/mlir/Conversion/Passes.td @@ -143,14 +143,13 @@ def ConvertArmNeon2dToIntr : Pass<"arm-neon-2d-to-intr"> { // AsyncToLLVM //===----------------------------------------------------------------------===// -def ConvertAsyncToLLVM : Pass<"convert-async-to-llvm", "ModuleOp"> { +def ConvertAsyncToLLVMPass : Pass<"convert-async-to-llvm", "ModuleOp"> { let summary = "Convert the operations from the async dialect into the LLVM " "dialect"; let description = [{ Convert `async.execute` operations to LLVM coroutines and use async runtime API to execute them. }]; - let constructor = "mlir::createConvertAsyncToLLVMPass()"; let dependentDialects = [ "arith::ArithDialect", "async::AsyncDialect", @@ -198,9 +197,8 @@ def ConvertBufferizationToMemRef : Pass<"convert-bufferization-to-memref"> { // ComplexToLLVM //===----------------------------------------------------------------------===// -def ConvertComplexToLLVM : Pass<"convert-complex-to-llvm"> { +def ConvertComplexToLLVMPass : Pass<"convert-complex-to-llvm"> { let summary = "Convert Complex dialect to LLVM dialect"; - let constructor = "mlir::createConvertComplexToLLVMPass()"; let dependentDialects = ["LLVM::LLVMDialect"]; } @@ -233,7 +231,7 @@ def ConvertComplexToStandard : Pass<"convert-complex-to-standard"> { // ControlFlowToLLVM //===----------------------------------------------------------------------===// -def ConvertControlFlowToLLVM : Pass<"convert-cf-to-llvm", "ModuleOp"> { +def ConvertControlFlowToLLVMPass : Pass<"convert-cf-to-llvm", "ModuleOp"> { let summary = "Convert ControlFlow operations to the LLVM dialect"; let description = [{ Convert ControlFlow operations into LLVM IR dialect operations. @@ -242,7 +240,6 @@ def ConvertControlFlowToLLVM : Pass<"convert-cf-to-llvm", "ModuleOp"> { IR dialect operations, the pass will fail. Any LLVM IR operations or types already present in the IR will be kept as is. }]; - let constructor = "mlir::cf::createConvertControlFlowToLLVMPass()"; let dependentDialects = ["LLVM::LLVMDialect"]; let options = [ Option<"indexBitwidth", "index-bitwidth", "unsigned", @@ -333,16 +330,48 @@ def ConvertFuncToSPIRV : Pass<"convert-func-to-spirv"> { def GpuToLLVMConversionPass : Pass<"gpu-to-llvm", "ModuleOp"> { let summary = "Convert GPU dialect to LLVM dialect with GPU runtime calls"; - let constructor = "mlir::createGpuToLLVMConversionPass()"; + + let description = [{ + Creates a pass to convert a GPU operations into a sequence of GPU runtime + calls. + + This pass does not generate code to call GPU runtime APIs directly but + instead uses a small wrapper library that exports a stable and conveniently + typed ABI on top of GPU runtimes such as CUDA or ROCm (HIP). + }]; + + let options = [ + Option<"kernelBarePtrCallConv", "use-bare-pointers-for-kernels", "bool", + /*default=*/"false", + "Use bare pointers to pass memref arguments to kernels. " + "The kernel must use the same setting for this option." + >, + Option<"gpuBinaryAnnotation", "gpu-binary-annotation", "std::string", + /*default=*/"gpu::getDefaultGpuBinaryAnnotation()", + "Annotation attribute string for GPU binary" + >, + ]; + let dependentDialects = [ "LLVM::LLVMDialect", "memref::MemRefDialect", ]; } -def LowerHostCodeToLLVM : Pass<"lower-host-to-llvm", "ModuleOp"> { +def LowerHostCodeToLLVMPass : Pass<"lower-host-to-llvm", "ModuleOp"> { let summary = "Lowers the host module code and `gpu.launch_func` to LLVM"; - let constructor = "mlir::createLowerHostCodeToLLVMPass()"; + + let description = [{ + Creates a pass to emulate `gpu.launch_func` call in LLVM dialect and lower + the host module code to LLVM. + + This transformation creates a sequence of global variables that are later + linked to the variables in the kernel module, and a series of copies to/from + them to emulate the memory transfer from the host or to the device sides. It + also converts the remaining Arithmetic, Func, and MemRef dialects into LLVM + dialect, emitting C wrappers. + }]; + let dependentDialects = ["LLVM::LLVMDialect"]; } @@ -475,10 +504,9 @@ def ConvertIndexToLLVMPass : Pass<"convert-index-to-llvm"> { // LinalgToLLVM //===----------------------------------------------------------------------===// -def ConvertLinalgToLLVM : Pass<"convert-linalg-to-llvm", "ModuleOp"> { +def ConvertLinalgToLLVMPass : Pass<"convert-linalg-to-llvm", "ModuleOp"> { let summary = "Convert the operations from the linalg dialect into the LLVM " "dialect"; - let constructor = "mlir::createConvertLinalgToLLVMPass()"; let dependentDialects = ["scf::SCFDialect", "LLVM::LLVMDialect"]; } @@ -514,12 +542,11 @@ def ConvertMathToLibm : Pass<"convert-math-to-libm", "ModuleOp"> { // MathToLLVM //===----------------------------------------------------------------------===// -def ConvertMathToLLVM : Pass<"convert-math-to-llvm"> { +def ConvertMathToLLVMPass : Pass<"convert-math-to-llvm"> { let summary = "Convert Math dialect to LLVM dialect"; let description = [{ This pass converts supported Math ops to LLVM dialect intrinsics. }]; - let constructor = "mlir::createConvertMathToLLVMPass()"; let dependentDialects = ["LLVM::LLVMDialect"]; } @@ -636,9 +663,8 @@ def ConvertOpenACCToSCF : Pass<"convert-openacc-to-scf", "ModuleOp"> { // OpenACCToLLVM //===----------------------------------------------------------------------===// -def ConvertOpenACCToLLVM : Pass<"convert-openacc-to-llvm", "ModuleOp"> { +def ConvertOpenACCToLLVMPass : Pass<"convert-openacc-to-llvm", "ModuleOp"> { let summary = "Convert the OpenACC ops to LLVM dialect"; - let constructor = "mlir::createConvertOpenACCToLLVMPass()"; let dependentDialects = ["LLVM::LLVMDialect"]; } @@ -646,9 +672,8 @@ def ConvertOpenACCToLLVM : Pass<"convert-openacc-to-llvm", "ModuleOp"> { // OpenMPToLLVM //===----------------------------------------------------------------------===// -def ConvertOpenMPToLLVM : Pass<"convert-openmp-to-llvm", "ModuleOp"> { +def ConvertOpenMPToLLVMPass : Pass<"convert-openmp-to-llvm", "ModuleOp"> { let summary = "Convert the OpenMP ops to OpenMP ops with LLVM dialect"; - let constructor = "mlir::createConvertOpenMPToLLVMPass()"; let dependentDialects = ["LLVM::LLVMDialect"]; } @@ -783,13 +808,12 @@ def ConvertShapeConstraints : Pass<"convert-shape-constraints"> { // SPIRVToLLVM //===----------------------------------------------------------------------===// -def ConvertSPIRVToLLVM : Pass<"convert-spirv-to-llvm", "ModuleOp"> { +def ConvertSPIRVToLLVMPass : Pass<"convert-spirv-to-llvm", "ModuleOp"> { let summary = "Convert SPIR-V dialect to LLVM dialect"; let description = [{ See https://mlir.llvm.org/docs/SPIRVToLLVMDialectConversion/ for more details. }]; - let constructor = "mlir::createConvertSPIRVToLLVMPass()"; let dependentDialects = ["LLVM::LLVMDialect"]; } @@ -957,7 +981,7 @@ def ConvertVectorToSCF : Pass<"convert-vector-to-scf"> { // VectorToLLVM //===----------------------------------------------------------------------===// -def ConvertVectorToLLVM : Pass<"convert-vector-to-llvm", "ModuleOp"> { +def ConvertVectorToLLVMPass : Pass<"convert-vector-to-llvm", "ModuleOp"> { let summary = "Lower the operations from the vector dialect into the LLVM " "dialect"; let description = [{ @@ -970,7 +994,6 @@ def ConvertVectorToLLVM : Pass<"convert-vector-to-llvm", "ModuleOp"> { architectural-neutral vector dialect lowering. }]; - let constructor = "mlir::createConvertVectorToLLVMPass()"; // Override explicitly in C++ to allow conditional dialect dependence. // let dependentDialects; let options = [ diff --git a/mlir/include/mlir/Conversion/SPIRVToLLVM/SPIRVToLLVMPass.h b/mlir/include/mlir/Conversion/SPIRVToLLVM/SPIRVToLLVMPass.h index 1e6b322512c6..c845a745e89d 100644 --- a/mlir/include/mlir/Conversion/SPIRVToLLVM/SPIRVToLLVMPass.h +++ b/mlir/include/mlir/Conversion/SPIRVToLLVM/SPIRVToLLVMPass.h @@ -16,27 +16,12 @@ #include namespace mlir { -class ModuleOp; -template -class OperationPass; +class Pass; -#define GEN_PASS_DECL_LOWERHOSTCODETOLLVM -#define GEN_PASS_DECL_CONVERTSPIRVTOLLVM +#define GEN_PASS_DECL_LOWERHOSTCODETOLLVMPASS +#define GEN_PASS_DECL_CONVERTSPIRVTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" -/// Creates a pass to emulate `gpu.launch_func` call in LLVM dialect and lower -/// the host module code to LLVM. -/// -/// This transformation creates a sequence of global variables that are later -/// linked to the variables in the kernel module, and a series of copies to/from -/// them to emulate the memory transfer from the host or to the device sides. It -/// also converts the remaining Arithmetic, Func, and MemRef dialects into LLVM -/// dialect, emitting C wrappers. -std::unique_ptr> createLowerHostCodeToLLVMPass(); - -/// Creates a pass to convert SPIR-V operations to the LLVMIR dialect. -std::unique_ptr> createConvertSPIRVToLLVMPass(); - } // namespace mlir #endif // MLIR_CONVERSION_SPIRVTOLLVM_SPIRVTOLLVMPASS_H diff --git a/mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h b/mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h index 0931af0774fe..0d5d1c8b5ffe 100644 --- a/mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h +++ b/mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h @@ -12,53 +12,11 @@ namespace mlir { class LLVMTypeConverter; -class ModuleOp; -template -class OperationPass; +class Pass; -#define GEN_PASS_DECL_CONVERTVECTORTOLLVM +#define GEN_PASS_DECL_CONVERTVECTORTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" -/// Options to control Vector to LLVM lowering. -/// -/// This should kept in sync with VectorToLLVM options defined for the -/// ConvertVectorToLLVM pass in include/mlir/Conversion/Passes.td -struct LowerVectorToLLVMOptions { - LowerVectorToLLVMOptions() {} - - LowerVectorToLLVMOptions &enableReassociateFPReductions(bool b = true) { - reassociateFPReductions = b; - return *this; - } - LowerVectorToLLVMOptions &enableIndexOptimizations(bool b = true) { - force32BitVectorIndices = b; - return *this; - } - LowerVectorToLLVMOptions &enableArmNeon(bool b = true) { - armNeon = b; - return *this; - } - LowerVectorToLLVMOptions &enableArmSVE(bool b = true) { - armSVE = b; - return *this; - } - LowerVectorToLLVMOptions &enableAMX(bool b = true) { - amx = b; - return *this; - } - LowerVectorToLLVMOptions &enableX86Vector(bool b = true) { - x86Vector = b; - return *this; - } - - bool reassociateFPReductions{false}; - bool force32BitVectorIndices{true}; - bool armNeon{false}; - bool armSVE{false}; - bool amx{false}; - bool x86Vector{false}; -}; - /// Collect a set of patterns to convert from Vector contractions to LLVM Matrix /// Intrinsics. To lower to assembly, the LLVM flag -lower-matrix-intrinsics /// will be needed when invoking LLVM. @@ -70,10 +28,6 @@ void populateVectorToLLVMConversionPatterns( LLVMTypeConverter &converter, RewritePatternSet &patterns, bool reassociateFPReductions = false, bool force32BitVectorIndices = false); -/// Create a pass to convert vector operations to the LLVMIR dialect. -std::unique_ptr> createConvertVectorToLLVMPass( - const LowerVectorToLLVMOptions &options = LowerVectorToLLVMOptions()); - } // namespace mlir #endif // MLIR_CONVERSION_VECTORTOLLVM_CONVERTVECTORTOLLVM_H_ diff --git a/mlir/include/mlir/Dialect/GPU/Transforms/Passes.h b/mlir/include/mlir/Dialect/GPU/Transforms/Passes.h index 594d8be0838b..657fb6dd3eea 100644 --- a/mlir/include/mlir/Dialect/GPU/Transforms/Passes.h +++ b/mlir/include/mlir/Dialect/GPU/Transforms/Passes.h @@ -13,6 +13,7 @@ #ifndef MLIR_DIALECT_GPU_TRANSFORMS_PASSES_H_ #define MLIR_DIALECT_GPU_TRANSFORMS_PASSES_H_ +#include "Utils.h" #include "mlir/Dialect/GPU/IR/GPUDialect.h" #include "mlir/Pass/Pass.h" #include diff --git a/mlir/include/mlir/Dialect/GPU/Transforms/Utils.h b/mlir/include/mlir/Dialect/GPU/Transforms/Utils.h index e72d2bc41c52..a426bee7686d 100644 --- a/mlir/include/mlir/Dialect/GPU/Transforms/Utils.h +++ b/mlir/include/mlir/Dialect/GPU/Transforms/Utils.h @@ -15,6 +15,8 @@ #include "mlir/Support/LLVM.h" +#include + namespace mlir { struct LogicalResult; class Operation; @@ -23,6 +25,9 @@ class Value; namespace gpu { class GPUFuncOp; class LaunchOp; + +/// Returns the default annotation name for GPU binary blobs. +std::string getDefaultGpuBinaryAnnotation(); } // namespace gpu /// Get a gpu.func created from outlining the region of a gpu.launch op with the diff --git a/mlir/include/mlir/Dialect/SparseTensor/Pipelines/Passes.h b/mlir/include/mlir/Dialect/SparseTensor/Pipelines/Passes.h index 4059cd4cc401..d3a16de8c551 100644 --- a/mlir/include/mlir/Dialect/SparseTensor/Pipelines/Passes.h +++ b/mlir/include/mlir/Dialect/SparseTensor/Pipelines/Passes.h @@ -118,14 +118,14 @@ struct SparseCompilerOptions } /// Projects out the options for `createConvertVectorToLLVMPass`. - LowerVectorToLLVMOptions lowerVectorToLLVMOptions() const { - LowerVectorToLLVMOptions opts{}; - opts.enableReassociateFPReductions(reassociateFPReductions); - opts.enableIndexOptimizations(force32BitVectorIndices); - opts.enableArmNeon(armNeon); - opts.enableArmSVE(armSVE); - opts.enableAMX(amx); - opts.enableX86Vector(x86Vector); + ConvertVectorToLLVMPassOptions lowerVectorToLLVMOptions() const { + ConvertVectorToLLVMPassOptions opts{}; + opts.reassociateFPReductions = reassociateFPReductions; + opts.force32BitVectorIndices = force32BitVectorIndices; + opts.armNeon = armNeon; + opts.armSVE = armSVE; + opts.amx = amx; + opts.x86Vector = x86Vector; return opts; } }; diff --git a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp index 5f00c3da77ea..7abc2141b6dc 100644 --- a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp +++ b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp @@ -24,7 +24,7 @@ #include "llvm/ADT/TypeSwitch.h" namespace mlir { -#define GEN_PASS_DEF_CONVERTASYNCTOLLVM +#define GEN_PASS_DEF_CONVERTASYNCTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" } // namespace mlir @@ -990,7 +990,9 @@ class ReturnOpOpConversion : public OpConversionPattern { namespace { struct ConvertAsyncToLLVMPass - : public impl::ConvertAsyncToLLVMBase { + : public impl::ConvertAsyncToLLVMPassBase { + using Base::Base; + void runOnOperation() override; }; } // namespace @@ -1121,10 +1123,6 @@ class ConvertYieldOpTypes : public OpConversionPattern { }; } // namespace -std::unique_ptr> mlir::createConvertAsyncToLLVMPass() { - return std::make_unique(); -} - void mlir::populateAsyncStructuralTypeConversionsAndLegality( TypeConverter &typeConverter, RewritePatternSet &patterns, ConversionTarget &target) { diff --git a/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp b/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp index 14f32d66d244..7185b01afdc0 100644 --- a/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp +++ b/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp @@ -16,7 +16,7 @@ #include "mlir/Pass/Pass.h" namespace mlir { -#define GEN_PASS_DEF_CONVERTCOMPLEXTOLLVM +#define GEN_PASS_DEF_CONVERTCOMPLEXTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" } // namespace mlir @@ -323,7 +323,9 @@ void mlir::populateComplexToLLVMConversionPatterns( namespace { struct ConvertComplexToLLVMPass - : public impl::ConvertComplexToLLVMBase { + : public impl::ConvertComplexToLLVMPassBase { + using Base::Base; + void runOnOperation() override; }; } // namespace @@ -340,7 +342,3 @@ void ConvertComplexToLLVMPass::runOnOperation() { applyPartialConversion(getOperation(), target, std::move(patterns)))) signalPassFailure(); } - -std::unique_ptr mlir::createConvertComplexToLLVMPass() { - return std::make_unique(); -} diff --git a/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp b/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp index c51daeb05ac5..2be87e6490f5 100644 --- a/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp +++ b/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp @@ -27,7 +27,7 @@ #include namespace mlir { -#define GEN_PASS_DEF_CONVERTCONTROLFLOWTOLLVM +#define GEN_PASS_DEF_CONVERTCONTROLFLOWTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" } // namespace mlir @@ -262,8 +262,9 @@ void mlir::cf::populateAssertToLLVMConversionPattern( namespace { /// A pass converting MLIR operations into the LLVM IR dialect. struct ConvertControlFlowToLLVM - : public impl::ConvertControlFlowToLLVMBase { - ConvertControlFlowToLLVM() = default; + : public impl::ConvertControlFlowToLLVMPassBase { + + using Base::Base; /// Run the dialect converter on the module. void runOnOperation() override { @@ -283,7 +284,3 @@ struct ConvertControlFlowToLLVM } }; } // namespace - -std::unique_ptr mlir::cf::createConvertControlFlowToLLVMPass() { - return std::make_unique(); -} diff --git a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp index c4dd47a762cf..98bdc7902e59 100644 --- a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp +++ b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp @@ -52,30 +52,10 @@ namespace { class GpuToLLVMConversionPass : public impl::GpuToLLVMConversionPassBase { public: - GpuToLLVMConversionPass() = default; - - GpuToLLVMConversionPass(bool kernelBarePtrCallConv) - : GpuToLLVMConversionPass() { - if (this->kernelBarePtrCallConv.getNumOccurrences() == 0) - this->kernelBarePtrCallConv = kernelBarePtrCallConv; - } - - GpuToLLVMConversionPass(const GpuToLLVMConversionPass &other) - : GpuToLLVMConversionPassBase(other) {} + using Base::Base; // Run the dialect converter on the module. void runOnOperation() override; - -private: - Option gpuBinaryAnnotation{ - *this, "gpu-binary-annotation", - llvm::cl::desc("Annotation attribute string for GPU binary"), - llvm::cl::init(gpu::getDefaultGpuBinaryAnnotation())}; - Option kernelBarePtrCallConv{ - *this, "use-bare-pointers-for-kernels", - llvm::cl::desc("Use bare pointers to pass memref arguments to kernels. " - "The kernel must use the same setting for this option."), - llvm::cl::init(false)}; }; struct FunctionCallBuilder { @@ -905,11 +885,6 @@ LogicalResult ConvertSetDefaultDeviceOpToGpuRuntimeCallPattern::matchAndRewrite( return success(); } -std::unique_ptr> -mlir::createGpuToLLVMConversionPass(bool kernelBarePtrCallConv) { - return std::make_unique(kernelBarePtrCallConv); -} - void mlir::populateGpuToLLVMConversionPatterns(LLVMTypeConverter &converter, RewritePatternSet &patterns, StringRef gpuBinaryAnnotation, diff --git a/mlir/lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp b/mlir/lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp index 7bc3680c2eae..de55a0da75d2 100644 --- a/mlir/lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp +++ b/mlir/lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp @@ -41,7 +41,7 @@ #include "llvm/Support/ErrorHandling.h" namespace mlir { -#define GEN_PASS_DEF_CONVERTLINALGTOLLVM +#define GEN_PASS_DEF_CONVERTLINALGTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" } // namespace mlir @@ -78,7 +78,7 @@ void mlir::populateLinalgToLLVMConversionPatterns(LLVMTypeConverter &converter, namespace { struct ConvertLinalgToLLVMPass - : public impl::ConvertLinalgToLLVMBase { + : public impl::ConvertLinalgToLLVMPassBase { void runOnOperation() override; }; } // namespace @@ -97,7 +97,3 @@ void ConvertLinalgToLLVMPass::runOnOperation() { if (failed(applyPartialConversion(module, target, std::move(patterns)))) signalPassFailure(); } - -std::unique_ptr> mlir::createConvertLinalgToLLVMPass() { - return std::make_unique(); -} diff --git a/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp b/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp index ece80921e29b..888c51238d06 100644 --- a/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp +++ b/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp @@ -18,7 +18,7 @@ #include "mlir/Pass/Pass.h" namespace mlir { -#define GEN_PASS_DEF_CONVERTMATHTOLLVM +#define GEN_PASS_DEF_CONVERTMATHTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" } // namespace mlir @@ -285,8 +285,8 @@ struct RsqrtOpLowering : public ConvertOpToLLVMPattern { }; struct ConvertMathToLLVMPass - : public impl::ConvertMathToLLVMBase { - ConvertMathToLLVMPass() = default; + : public impl::ConvertMathToLLVMPassBase { + using Base::Base; void runOnOperation() override { RewritePatternSet patterns(&getContext()); @@ -332,7 +332,3 @@ void mlir::populateMathToLLVMConversionPatterns(LLVMTypeConverter &converter, >(converter); // clang-format on } - -std::unique_ptr mlir::createConvertMathToLLVMPass() { - return std::make_unique(); -} diff --git a/mlir/lib/Conversion/OpenACCToLLVM/OpenACCToLLVM.cpp b/mlir/lib/Conversion/OpenACCToLLVM/OpenACCToLLVM.cpp index 54f3e44f0c17..16863206ac05 100644 --- a/mlir/lib/Conversion/OpenACCToLLVM/OpenACCToLLVM.cpp +++ b/mlir/lib/Conversion/OpenACCToLLVM/OpenACCToLLVM.cpp @@ -15,7 +15,7 @@ #include "mlir/Pass/Pass.h" namespace mlir { -#define GEN_PASS_DEF_CONVERTOPENACCTOLLVM +#define GEN_PASS_DEF_CONVERTOPENACCTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" } // namespace mlir @@ -154,7 +154,9 @@ void mlir::populateOpenACCToLLVMConversionPatterns( namespace { struct ConvertOpenACCToLLVMPass - : public impl::ConvertOpenACCToLLVMBase { + : public impl::ConvertOpenACCToLLVMPassBase { + using Base::Base; + void runOnOperation() override; }; } // namespace @@ -238,8 +240,3 @@ void ConvertOpenACCToLLVMPass::runOnOperation() { if (failed(applyPartialConversion(op, target, std::move(patterns)))) signalPassFailure(); } - -std::unique_ptr> -mlir::createConvertOpenACCToLLVMPass() { - return std::make_unique(); -} diff --git a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp index 6a2d375c69f7..df27c487be77 100644 --- a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp +++ b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp @@ -20,7 +20,7 @@ #include "mlir/Pass/Pass.h" namespace mlir { -#define GEN_PASS_DEF_CONVERTOPENMPTOLLVM +#define GEN_PASS_DEF_CONVERTOPENMPTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" } // namespace mlir @@ -137,7 +137,9 @@ void mlir::populateOpenMPToLLVMConversionPatterns(LLVMTypeConverter &converter, namespace { struct ConvertOpenMPToLLVMPass - : public impl::ConvertOpenMPToLLVMBase { + : public impl::ConvertOpenMPToLLVMPassBase { + using Base::Base; + void runOnOperation() override; }; } // namespace @@ -161,7 +163,3 @@ void ConvertOpenMPToLLVMPass::runOnOperation() { if (failed(applyPartialConversion(module, target, std::move(patterns)))) signalPassFailure(); } - -std::unique_ptr> mlir::createConvertOpenMPToLLVMPass() { - return std::make_unique(); -} diff --git a/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp b/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp index 08512e9f31e9..bfc1bb65dce9 100644 --- a/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp +++ b/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp @@ -33,7 +33,7 @@ #include "llvm/Support/FormatVariadic.h" namespace mlir { -#define GEN_PASS_DEF_LOWERHOSTCODETOLLVM +#define GEN_PASS_DEF_LOWERHOSTCODETOLLVMPASS #include "mlir/Conversion/Passes.h.inc" } // namespace mlir @@ -281,8 +281,11 @@ class GPULaunchLowering : public ConvertOpToLLVMPattern { }; class LowerHostCodeToLLVM - : public impl::LowerHostCodeToLLVMBase { + : public impl::LowerHostCodeToLLVMPassBase { public: + + using Base::Base; + void runOnOperation() override { ModuleOp module = getOperation(); @@ -327,8 +330,3 @@ class LowerHostCodeToLLVM } }; } // namespace - -std::unique_ptr> -mlir::createLowerHostCodeToLLVMPass() { - return std::make_unique(); -} diff --git a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVMPass.cpp b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVMPass.cpp index a2f3c8adf395..7766d8d9a0c9 100644 --- a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVMPass.cpp +++ b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVMPass.cpp @@ -19,7 +19,7 @@ #include "mlir/Pass/Pass.h" namespace mlir { -#define GEN_PASS_DEF_CONVERTSPIRVTOLLVM +#define GEN_PASS_DEF_CONVERTSPIRVTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" } // namespace mlir @@ -28,8 +28,11 @@ using namespace mlir; namespace { /// A pass converting MLIR SPIR-V operations into LLVM dialect. class ConvertSPIRVToLLVMPass - : public impl::ConvertSPIRVToLLVMBase { + : public impl::ConvertSPIRVToLLVMPassBase { void runOnOperation() override; + +public: + using Base::Base; }; } // namespace @@ -58,7 +61,3 @@ void ConvertSPIRVToLLVMPass::runOnOperation() { if (failed(applyPartialConversion(module, target, std::move(patterns)))) signalPassFailure(); } - -std::unique_ptr> mlir::createConvertSPIRVToLLVMPass() { - return std::make_unique(); -} diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp index a1667ff25b12..dcf04b7ed58d 100644 --- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp +++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp @@ -26,7 +26,7 @@ #include "mlir/Transforms/GreedyPatternRewriteDriver.h" namespace mlir { -#define GEN_PASS_DEF_CONVERTVECTORTOLLVM +#define GEN_PASS_DEF_CONVERTVECTORTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" } // namespace mlir @@ -35,15 +35,10 @@ using namespace mlir::vector; namespace { struct LowerVectorToLLVMPass - : public impl::ConvertVectorToLLVMBase { - LowerVectorToLLVMPass(const LowerVectorToLLVMOptions &options) { - this->reassociateFPReductions = options.reassociateFPReductions; - this->force32BitVectorIndices = options.force32BitVectorIndices; - this->armNeon = options.armNeon; - this->armSVE = options.armSVE; - this->amx = options.amx; - this->x86Vector = options.x86Vector; - } + : public impl::ConvertVectorToLLVMPassBase { + + using Base::Base; + // Override explicitly to allow conditional dialect dependence. void getDependentDialects(DialectRegistry ®istry) const override { registry.insert(); @@ -116,8 +111,3 @@ void LowerVectorToLLVMPass::runOnOperation() { applyPartialConversion(getOperation(), target, std::move(patterns)))) signalPassFailure(); } - -std::unique_ptr> -mlir::createConvertVectorToLLVMPass(const LowerVectorToLLVMOptions &options) { - return std::make_unique(options); -} diff --git a/mlir/test/lib/Dialect/LLVM/TestLowerToLLVM.cpp b/mlir/test/lib/Dialect/LLVM/TestLowerToLLVM.cpp index f863240f3465..ed84a466121f 100644 --- a/mlir/test/lib/Dialect/LLVM/TestLowerToLLVM.cpp +++ b/mlir/test/lib/Dialect/LLVM/TestLowerToLLVM.cpp @@ -65,8 +65,7 @@ void buildTestLowerToLLVM(OpPassManager &pm, // Convert vector to LLVM (always needed). pm.addPass(createConvertVectorToLLVMPass( // TODO: add more options on a per-need basis. - LowerVectorToLLVMOptions().enableReassociateFPReductions( - options.reassociateFPReductions))); + ConvertVectorToLLVMPassOptions{options.reassociateFPReductions})); // Convert Math to LLVM (always needed). pm.addNestedPass(createConvertMathToLLVMPass()); // Expand complicated MemRef operations before lowering them. From 8a09c6eee2a5ef25920d56314f27fbdd51afe4c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20B=C3=B6ck?= Date: Mon, 6 Mar 2023 11:02:43 +0100 Subject: [PATCH 081/147] [mlir][MemRef] Add required address space cast when lowering alloc to LLVM alloc uses either `malloc` or a plugable allocation function for allocating the required memory. Both of these functions always return a `llvm.ptr`, aka a pointer in the default address space. When allocating for a memref in a different memory space however, no address space cast is created, leading to invalid LLVM IR being generated. This is currently not caught by the verifier since the pointer to the memory is always bitcast which currently lacks a verifier disallowing address space casts. Translating to actual LLVM IR would cause the verifier to go off, since bitcast cannot translate from one address space to another: https://godbolt.org/z/3a1z97rc9 This patch fixes that issue by generating an address space cast if the address space of the allocation function does not match the address space of the resulting memref. Not sure whether this is actually a real life problem. I found this issue while converting the pass to using opaque pointers which gets rid of all the bitcasts and hence caused type errors without the address space cast. Differential Revision: https://reviews.llvm.org/D143341 (cherry picked from commit eafca2303769800f5da4bc4cbf9e842c6a8cde9f) --- .../MemRefToLLVM/AllocLikeConversion.cpp | 28 +++++++++++++++---- .../MemRefToLLVM/memref-to-llvm.mlir | 5 ++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp b/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp index 4a5be4870709..8d99e1ffed9b 100644 --- a/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp +++ b/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp @@ -50,6 +50,23 @@ Value AllocationOpLLVMLowering::createAligned( return rewriter.create(loc, bumped, mod); } +static Value castAllocFuncResult(ConversionPatternRewriter &rewriter, + Location loc, Value allocatedPtr, + MemRefType memRefType, Type elementPtrType, + LLVMTypeConverter &typeConverter) { + auto allocatedPtrTy = allocatedPtr.getType().cast(); + if (allocatedPtrTy.getAddressSpace() != memRefType.getMemorySpaceAsInt()) + allocatedPtr = rewriter.create( + loc, + LLVM::LLVMPointerType::get(allocatedPtrTy.getElementType(), + memRefType.getMemorySpaceAsInt()), + allocatedPtr); + + allocatedPtr = + rewriter.create(loc, elementPtrType, allocatedPtr); + return allocatedPtr; +} + std::tuple AllocationOpLLVMLowering::allocateBufferManuallyAlign( ConversionPatternRewriter &rewriter, Location loc, Value sizeBytes, Operation *op, Value alignment) const { @@ -64,8 +81,10 @@ std::tuple AllocationOpLLVMLowering::allocateBufferManuallyAlign( LLVM::LLVMFuncOp allocFuncOp = getNotalignedAllocFn( getTypeConverter(), op->getParentOfType(), getIndexType()); auto results = rewriter.create(loc, allocFuncOp, sizeBytes); - Value allocatedPtr = rewriter.create(loc, elementPtrType, - results.getResult()); + + Value allocatedPtr = + castAllocFuncResult(rewriter, loc, results.getResult(), memRefType, + elementPtrType, *getTypeConverter()); Value alignedPtr = allocatedPtr; if (alignment) { @@ -126,10 +145,9 @@ Value AllocationOpLLVMLowering::allocateBufferAutoAlign( getTypeConverter(), op->getParentOfType(), getIndexType()); auto results = rewriter.create( loc, allocFuncOp, ValueRange({allocAlignment, sizeBytes})); - Value allocatedPtr = rewriter.create(loc, elementPtrType, - results.getResult()); - return allocatedPtr; + return castAllocFuncResult(rewriter, loc, results.getResult(), memRefType, + elementPtrType, *getTypeConverter()); } LogicalResult AllocLikeOpLLVMLowering::matchAndRewrite( diff --git a/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir b/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir index 0adda9731548..a92015fdbe70 100644 --- a/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir +++ b/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir @@ -182,6 +182,11 @@ func.func @dim_of_unranked(%unranked: memref<*xi32>) -> index { // CHECK-LABEL: func @address_space( func.func @address_space(%arg0 : memref<32xf32, affine_map<(d0) -> (d0)>, 7>) { + // CHECK: %[[MEMORY:.*]] = llvm.call @malloc(%{{.*}}) + // CHECK: %[[CAST:.*]] = llvm.addrspacecast %[[MEMORY]] : !llvm.ptr to !llvm.ptr + // CHECK: %[[BCAST:.*]] = llvm.bitcast %[[CAST]] + // CHECK: llvm.insertvalue %[[BCAST]], %{{[[:alnum:]]+}}[0] + // CHECK: llvm.insertvalue %[[BCAST]], %{{[[:alnum:]]+}}[1] %0 = memref.alloc() : memref<32xf32, affine_map<(d0) -> (d0)>, 5> %1 = arith.constant 7 : index // CHECK: llvm.load %{{.*}} : !llvm.ptr From e0e3ea9e3955a990fc16daf44c139576ddf58365 Mon Sep 17 00:00:00 2001 From: Quentin Colombet Date: Mon, 6 Mar 2023 11:03:32 +0100 Subject: [PATCH 082/147] [mlir][Conversion] Rename the MemRefToLLVM pass Since the recent MemRef refactoring that centralizes the lowering of complex MemRef operations outside of the conversion framework, the MemRefToLLVM pass doesn't directly convert these complex operations. Instead, to fully convert the whole MemRef dialect space, MemRefToLLVM needs to run after `expand-strided-metadata`. Make this more obvious by changing the name of the pass and the option associated with it from `convert-memref-to-llvm` to `finalize-memref-to-llvm`. The word "finalize" conveys that this pass needs to run after something else and that something else is documented in its tablegen description. This is a follow-up patch related to the conversation at: https://discourse.llvm.org/t/psa-you-need-to-run-expand-strided-metadata-before-memref-to-llvm-now/66956/14 Differential Revision: https://reviews.llvm.org/D142463 (cherry picked from commit cb4ccd38fa0407928feed7e39d197169eabc95b2) --- .../mlir/Dialect/MemRef/IR/MemRefOps.td | 6 +++-- .../Conversion/MemRefToLLVM/MemRefToLLVM.cpp | 6 +++-- .../MemRef/Transforms/EmulateWideInt.cpp | 6 +++-- .../MemRef/Transforms/FoldMemRefAliasOps.cpp | 17 ++++++++++---- .../MemRefToLLVM/memref-to-llvm.mlir | 21 ++++++++++++++++++ mlir/test/Dialect/MemRef/canonicalize.mlir | 12 ++++++++++ .../test/Dialect/MemRef/emulate-wide-int.mlir | 16 ++++++++++++++ .../Dialect/MemRef/fold-memref-alias-ops.mlir | 22 +++++++++++++++++++ 8 files changed, 96 insertions(+), 10 deletions(-) diff --git a/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td b/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td index c2c0d0afe4f1..8fe96d16d5ec 100644 --- a/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td +++ b/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td @@ -1155,7 +1155,8 @@ def LoadOp : MemRef_Op<"load", let arguments = (ins Arg:$memref, - Variadic:$indices); + Variadic:$indices, + DefaultValuedOptionalAttr:$nontemporal); let results = (outs AnyType:$result); let extraClassDeclaration = [{ @@ -1690,7 +1691,8 @@ def MemRef_StoreOp : MemRef_Op<"store", let arguments = (ins AnyType:$value, Arg:$memref, - Variadic:$indices); + Variadic:$indices, + DefaultValuedOptionalAttr:$nontemporal); let builders = [ OpBuilder<(ins "Value":$valueToStore, "Value":$memref), [{ diff --git a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp index f5da371f550d..a1ea03f52ef6 100644 --- a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp +++ b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp @@ -727,7 +727,8 @@ struct LoadOpLowering : public LoadStoreOpLowering { Value dataPtr = getStridedElementPtr(loadOp.getLoc(), type, adaptor.getMemref(), adaptor.getIndices(), rewriter); - rewriter.replaceOpWithNewOp(loadOp, dataPtr); + rewriter.replaceOpWithNewOp(loadOp, dataPtr, 0, false, + loadOp.getNontemporal()); return success(); } }; @@ -744,7 +745,8 @@ struct StoreOpLowering : public LoadStoreOpLowering { Value dataPtr = getStridedElementPtr(op.getLoc(), type, adaptor.getMemref(), adaptor.getIndices(), rewriter); - rewriter.replaceOpWithNewOp(op, adaptor.getValue(), dataPtr); + rewriter.replaceOpWithNewOp(op, adaptor.getValue(), dataPtr, + 0, false, op.getNontemporal()); return success(); } }; diff --git a/mlir/lib/Dialect/MemRef/Transforms/EmulateWideInt.cpp b/mlir/lib/Dialect/MemRef/Transforms/EmulateWideInt.cpp index b0e884c285f0..81b9b279a650 100644 --- a/mlir/lib/Dialect/MemRef/Transforms/EmulateWideInt.cpp +++ b/mlir/lib/Dialect/MemRef/Transforms/EmulateWideInt.cpp @@ -66,7 +66,8 @@ struct ConvertMemRefLoad final : OpConversionPattern { op.getMemRefType())); rewriter.replaceOpWithNewOp( - op, newResTy, adaptor.getMemref(), adaptor.getIndices()); + op, newResTy, adaptor.getMemref(), adaptor.getIndices(), + op.getNontemporal()); return success(); } }; @@ -88,7 +89,8 @@ struct ConvertMemRefStore final : OpConversionPattern { op.getMemRefType())); rewriter.replaceOpWithNewOp( - op, adaptor.getValue(), adaptor.getMemref(), adaptor.getIndices()); + op, adaptor.getValue(), adaptor.getMemref(), adaptor.getIndices(), + op.getNontemporal()); return success(); } }; diff --git a/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp b/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp index 92f02c068d2b..ff297a9cd98f 100644 --- a/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp +++ b/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp @@ -384,10 +384,14 @@ LogicalResult LoadOpOfSubViewOpFolder::matchAndRewrite( return failure(); llvm::TypeSwitch(loadOp) - .Case([&](auto op) { - rewriter.replaceOpWithNewOp(loadOp, subViewOp.getSource(), + .Case([&](AffineLoadOp op) { + rewriter.replaceOpWithNewOp(loadOp, subViewOp.getSource(), sourceIndices); }) + .Case([&](memref::LoadOp op) { + rewriter.replaceOpWithNewOp( + loadOp, subViewOp.getSource(), sourceIndices, op.getNontemporal()); + }) .Case([&](vector::TransferReadOp transferReadOp) { rewriter.replaceOpWithNewOp( transferReadOp, transferReadOp.getVectorType(), @@ -490,10 +494,15 @@ LogicalResult StoreOpOfSubViewOpFolder::matchAndRewrite( return failure(); llvm::TypeSwitch(storeOp) - .Case([&](auto op) { - rewriter.replaceOpWithNewOp( + .Case([&](AffineStoreOp op) { + rewriter.replaceOpWithNewOp( storeOp, storeOp.getValue(), subViewOp.getSource(), sourceIndices); }) + .Case([&](memref::StoreOp op) { + rewriter.replaceOpWithNewOp( + storeOp, storeOp.getValue(), subViewOp.getSource(), sourceIndices, + op.getNontemporal()); + }) .Case([&](vector::TransferWriteOp op) { rewriter.replaceOpWithNewOp( op, op.getValue(), subViewOp.getSource(), sourceIndices, diff --git a/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir b/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir index a92015fdbe70..eb3a9482392a 100644 --- a/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir +++ b/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir @@ -542,3 +542,24 @@ func.func @extract_strided_metadata( return } + +// ----- + +// CHECK-LABEL: func @load_non_temporal( +func.func @load_non_temporal(%arg0 : memref<32xf32, affine_map<(d0) -> (d0)>>) { + %1 = arith.constant 7 : index + // CHECK: llvm.load %{{.*}} {nontemporal} : !llvm.ptr + %2 = memref.load %arg0[%1] {nontemporal = true} : memref<32xf32, affine_map<(d0) -> (d0)>> + func.return +} + +// ----- + +// CHECK-LABEL: func @store_non_temporal( +func.func @store_non_temporal(%input : memref<32xf32, affine_map<(d0) -> (d0)>>, %output : memref<32xf32, affine_map<(d0) -> (d0)>>) { + %1 = arith.constant 7 : index + %2 = memref.load %input[%1] {nontemporal = true} : memref<32xf32, affine_map<(d0) -> (d0)>> + // CHECK: llvm.store %{{.*}}, %{{.*}} {nontemporal} : !llvm.ptr + memref.store %2, %output[%1] {nontemporal = true} : memref<32xf32, affine_map<(d0) -> (d0)>> + func.return +} diff --git a/mlir/test/Dialect/MemRef/canonicalize.mlir b/mlir/test/Dialect/MemRef/canonicalize.mlir index 3d9f71e26055..14c570759b4c 100644 --- a/mlir/test/Dialect/MemRef/canonicalize.mlir +++ b/mlir/test/Dialect/MemRef/canonicalize.mlir @@ -894,3 +894,15 @@ func.func @fold_trivial_subviews(%m: memref>, to memref> return %1 : memref> } + +// ----- + +// CHECK-LABEL: func @load_store_nontemporal( +func.func @load_store_nontemporal(%input : memref<32xf32, affine_map<(d0) -> (d0)>>, %output : memref<32xf32, affine_map<(d0) -> (d0)>>) { + %1 = arith.constant 7 : index + // CHECK: memref.load %{{.*}}[%{{.*}}] {nontemporal = true} : memref<32xf32> + %2 = memref.load %input[%1] {nontemporal = true} : memref<32xf32, affine_map<(d0) -> (d0)>> + // CHECK: memref.store %{{.*}}, %{{.*}}[%{{.*}}] {nontemporal = true} : memref<32xf32> + memref.store %2, %output[%1] {nontemporal = true} : memref<32xf32, affine_map<(d0) -> (d0)>> + func.return +} diff --git a/mlir/test/Dialect/MemRef/emulate-wide-int.mlir b/mlir/test/Dialect/MemRef/emulate-wide-int.mlir index de1cba5c0477..65ac5beed0a1 100644 --- a/mlir/test/Dialect/MemRef/emulate-wide-int.mlir +++ b/mlir/test/Dialect/MemRef/emulate-wide-int.mlir @@ -44,3 +44,19 @@ func.func @alloc_load_store_i64() { memref.store %c1, %m[%c0] : memref<4xi64, 1> return } + + +// CHECK-LABEL: func @alloc_load_store_i64_nontemporal +// CHECK: [[C1:%.+]] = arith.constant dense<[1, 0]> : vector<2xi32> +// CHECK-NEXT: [[M:%.+]] = memref.alloc() : memref<4xvector<2xi32>, 1> +// CHECK-NEXT: [[V:%.+]] = memref.load [[M]][{{%.+}}] {nontemporal = true} : memref<4xvector<2xi32>, 1> +// CHECK-NEXT: memref.store [[C1]], [[M]][{{%.+}}] {nontemporal = true} : memref<4xvector<2xi32>, 1> +// CHECK-NEXT: return +func.func @alloc_load_store_i64_nontemporal() { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : i64 + %m = memref.alloc() : memref<4xi64, 1> + %v = memref.load %m[%c0] {nontemporal = true} : memref<4xi64, 1> + memref.store %c1, %m[%c0] {nontemporal = true} : memref<4xi64, 1> + return +} diff --git a/mlir/test/Dialect/MemRef/fold-memref-alias-ops.mlir b/mlir/test/Dialect/MemRef/fold-memref-alias-ops.mlir index c2ecc90be8dd..f0d5008f991a 100644 --- a/mlir/test/Dialect/MemRef/fold-memref-alias-ops.mlir +++ b/mlir/test/Dialect/MemRef/fold-memref-alias-ops.mlir @@ -502,3 +502,25 @@ func.func @subview_of_subview_rank_reducing(%m: memref, to memref> return %1 : memref> } + +// ----- + +// CHECK-LABEL: func @fold_load_keep_nontemporal( +// CHECK: memref.load %{{.+}}[%{{.+}}, %{{.+}}] {nontemporal = true} +func.func @fold_load_keep_nontemporal(%arg0 : memref<12x32xf32>, %arg1 : index, %arg2 : index, %arg3 : index, %arg4 : index) -> f32 { + %0 = memref.subview %arg0[%arg1, %arg2][4, 4][2, 3] : memref<12x32xf32> to memref<4x4xf32, strided<[64, 3], offset: ?>> + %1 = memref.load %0[%arg3, %arg4] {nontemporal = true }: memref<4x4xf32, strided<[64, 3], offset: ?>> + return %1 : f32 +} + + +// ----- + +// CHECK-LABEL: func @fold_store_keep_nontemporal( +// CHECK: memref.store %{{.+}}, %{{.+}}[%{{.+}}, %{{.+}}] {nontemporal = true} : memref<12x32xf32> +func.func @fold_store_keep_nontemporal(%arg0 : memref<12x32xf32>, %arg1 : index, %arg2 : index, %arg3 : index, %arg4 : index, %arg5 : f32) { + %0 = memref.subview %arg0[%arg1, %arg2][4, 4][2, 3] : + memref<12x32xf32> to memref<4x4xf32, strided<[64, 3], offset: ?>> + memref.store %arg5, %0[%arg3, %arg4] {nontemporal=true}: memref<4x4xf32, strided<[64, 3], offset: ?>> + return +} From 620d2c86eb63d0214c1b7ca752279e4c09d386e6 Mon Sep 17 00:00:00 2001 From: Guray Ozen Date: Mon, 6 Mar 2023 11:04:38 +0100 Subject: [PATCH 083/147] [mlir] Add nontemporal field to memref.load/store and convey to llvm.load/store `llvm.load` op has nonTemporal field which is missing for `memref.load` and `memref.store`. This revision first adds nonTemporal field to memref's load/store op, then it lowers the field to llvm.load/store ops. Reviewed By: nicolasvasilache Differential Revision: https://reviews.llvm.org/D142616 (cherry picked from commit 1cb91b421e7d73029c0c4bc44d33d006371f7adb) --- mlir/docs/TargetLLVMIR.md | 2 +- mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp | 2 +- mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp | 2 +- .../mlir/Conversion/MemRefToLLVM/MemRefToLLVM.h | 6 +++--- mlir/include/mlir/Conversion/Passes.td | 10 +++++++--- .../Conversion/GPUCommon/GPUToLLVMConversion.cpp | 2 +- .../GPUToNVVM/LowerGpuOpsToNVVMOps.cpp | 2 +- .../GPUToROCDL/LowerGpuOpsToROCDLOps.cpp | 2 +- .../lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp | 2 +- .../lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp | 16 +++++++++------- .../lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp | 2 +- .../SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp | 2 +- .../Pipelines/SparseTensorPipelines.cpp | 2 +- .../FuncToLLVM/calling-convention.mlir | 4 ++-- .../MemRefToLLVM/convert-alloca-scope.mlir | 2 +- .../MemRefToLLVM/convert-dynamic-memref-ops.mlir | 6 +++--- .../MemRefToLLVM/convert-static-memref-ops.mlir | 2 +- .../expand-then-convert-to-llvm.mlir | 4 ++-- .../MemRefToLLVM/generic-functions.mlir | 4 ++-- .../Conversion/MemRefToLLVM/memref-to-llvm.mlir | 4 ++-- .../microbench-linalg-async-parallel-for.mlir | 4 ++-- .../CPU/microbench-scf-async-parallel-for.mlir | 6 +++--- .../Async/CPU/test-async-parallel-for-1d.mlir | 6 +++--- .../Async/CPU/test-async-parallel-for-2d.mlir | 6 +++--- .../Dialect/Complex/CPU/correctness.mlir | 2 +- .../LLVMIR/CPU/test-complex-sparse-constant.mlir | 2 +- .../Dialect/LLVMIR/CPU/test-vp-intrinsic.mlir | 2 +- .../Dialect/Linalg/CPU/matmul-vs-matvec.mlir | 2 +- .../Linalg/CPU/rank-reducing-subview.mlir | 2 +- .../Dialect/Linalg/CPU/test-collapse-tensor.mlir | 2 +- .../Dialect/Linalg/CPU/test-conv-1d-call.mlir | 4 ++-- .../Linalg/CPU/test-conv-1d-nwc-wcf-call.mlir | 4 ++-- .../Dialect/Linalg/CPU/test-conv-2d-call.mlir | 4 ++-- .../Linalg/CPU/test-conv-2d-nhwc-hwcf-call.mlir | 4 ++-- .../Dialect/Linalg/CPU/test-conv-3d-call.mlir | 4 ++-- .../CPU/test-conv-3d-ndhwc-dhwcf-call.mlir | 4 ++-- .../Dialect/Linalg/CPU/test-elementwise.mlir | 2 +- .../Dialect/Linalg/CPU/test-expand-tensor.mlir | 2 +- .../Linalg/CPU/test-one-shot-bufferize.mlir | 2 +- .../Dialect/Linalg/CPU/test-padtensor.mlir | 2 +- .../CPU/test-subtensor-insert-multiple-uses.mlir | 2 +- .../Linalg/CPU/test-subtensor-insert.mlir | 2 +- .../Dialect/Linalg/CPU/test-tensor-e2e.mlir | 2 +- .../Dialect/Linalg/CPU/test-tensor-matmul.mlir | 4 ++-- .../Memref/cast-runtime-verification.mlir | 2 +- .../test/Integration/Dialect/Memref/memref_abi.c | 2 +- .../Standard/CPU/test-ceil-floor-pos-neg.mlir | 2 +- .../Dialect/Standard/CPU/test_subview.mlir | 2 +- .../Dialect/Vector/CPU/AMX/test-mulf-full.mlir | 2 +- .../Dialect/Vector/CPU/AMX/test-mulf.mlir | 2 +- .../Dialect/Vector/CPU/AMX/test-muli-ext.mlir | 2 +- .../Dialect/Vector/CPU/AMX/test-muli-full.mlir | 2 +- .../Dialect/Vector/CPU/AMX/test-muli.mlir | 2 +- .../Vector/CPU/AMX/test-tilezero-block.mlir | 2 +- .../Dialect/Vector/CPU/AMX/test-tilezero.mlir | 2 +- .../Dialect/Vector/CPU/ArmSVE/test-sve.mlir | 2 +- .../X86Vector/test-inline-asm-vector-avx512.mlir | 2 +- .../CPU/X86Vector/test-sparse-dot-product.mlir | 2 +- .../Dialect/Vector/CPU/test-0-d-vectors.mlir | 2 +- .../Dialect/Vector/CPU/test-compress.mlir | 2 +- .../Dialect/Vector/CPU/test-expand.mlir | 2 +- .../Dialect/Vector/CPU/test-gather.mlir | 2 +- .../Dialect/Vector/CPU/test-maskedload.mlir | 2 +- .../Dialect/Vector/CPU/test-maskedstore.mlir | 2 +- .../Dialect/Vector/CPU/test-realloc.mlir | 4 ++-- .../Dialect/Vector/CPU/test-scatter.mlir | 2 +- .../Vector/CPU/test-sparse-dot-matvec.mlir | 2 +- .../CPU/test-sparse-saxpy-jagged-matvec.mlir | 2 +- .../Vector/CPU/test-transfer-read-1d.mlir | 4 ++-- .../Vector/CPU/test-transfer-read-2d.mlir | 4 ++-- .../Vector/CPU/test-transfer-read-3d.mlir | 4 ++-- .../Dialect/Vector/CPU/test-transfer-read.mlir | 4 ++-- .../Vector/CPU/test-transfer-to-loops.mlir | 4 ++-- .../Dialect/Vector/CPU/test-transfer-write.mlir | 2 +- mlir/test/lib/Dialect/LLVM/TestLowerToLLVM.cpp | 2 +- mlir/test/mlir-cpu-runner/async-value.mlir | 2 +- mlir/test/mlir-cpu-runner/async.mlir | 2 +- .../test/mlir-cpu-runner/bare-ptr-call-conv.mlir | 2 +- mlir/test/mlir-cpu-runner/copy.mlir | 2 +- mlir/test/mlir-cpu-runner/global-memref.mlir | 2 +- .../mlir-cpu-runner/memref-reinterpret-cast.mlir | 2 +- mlir/test/mlir-cpu-runner/memref-reshape.mlir | 2 +- .../mlir-cpu-runner/sgemm-naive-codegen.mlir | 2 +- mlir/test/mlir-cpu-runner/unranked-memref.mlir | 2 +- mlir/test/mlir-cpu-runner/utils.mlir | 8 ++++---- mlir/test/mlir-opt/async.mlir | 2 +- mlir/test/python/execution_engine.py | 2 +- .../python/integration/dialects/linalg/opsrun.py | 2 +- mlir/unittests/ExecutionEngine/Invoke.cpp | 4 ++-- 89 files changed, 135 insertions(+), 129 deletions(-) diff --git a/mlir/docs/TargetLLVMIR.md b/mlir/docs/TargetLLVMIR.md index 5ba7adc2463a..03bf920a04a0 100644 --- a/mlir/docs/TargetLLVMIR.md +++ b/mlir/docs/TargetLLVMIR.md @@ -35,7 +35,7 @@ Conversion to the LLVM dialect from other dialects is the first step to produce LLVM IR. All non-trivial IR modifications are expected to happen at this stage or before. The conversion is *progressive*: most passes convert one dialect to the LLVM dialect and keep operations from other dialects intact. For example, -the `-convert-memref-to-llvm` pass will only convert operations from the +the `-finalize-memref-to-llvm` pass will only convert operations from the `memref` dialect but will not convert operations from other dialects even if they use or produce `memref`-typed values. diff --git a/mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp b/mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp index 9b5d1a737dbf..06e509639a91 100644 --- a/mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp +++ b/mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp @@ -206,7 +206,7 @@ void ToyToLLVMLoweringPass::runOnOperation() { populateAffineToStdConversionPatterns(patterns); populateSCFToControlFlowConversionPatterns(patterns); mlir::arith::populateArithToLLVMConversionPatterns(typeConverter, patterns); - populateMemRefToLLVMConversionPatterns(typeConverter, patterns); + populateFinalizeMemRefToLLVMConversionPatterns(typeConverter, patterns); cf::populateControlFlowToLLVMConversionPatterns(typeConverter, patterns); populateFuncToLLVMConversionPatterns(typeConverter, patterns); diff --git a/mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp b/mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp index 9b5d1a737dbf..06e509639a91 100644 --- a/mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp +++ b/mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp @@ -206,7 +206,7 @@ void ToyToLLVMLoweringPass::runOnOperation() { populateAffineToStdConversionPatterns(patterns); populateSCFToControlFlowConversionPatterns(patterns); mlir::arith::populateArithToLLVMConversionPatterns(typeConverter, patterns); - populateMemRefToLLVMConversionPatterns(typeConverter, patterns); + populateFinalizeMemRefToLLVMConversionPatterns(typeConverter, patterns); cf::populateControlFlowToLLVMConversionPatterns(typeConverter, patterns); populateFuncToLLVMConversionPatterns(typeConverter, patterns); diff --git a/mlir/include/mlir/Conversion/MemRefToLLVM/MemRefToLLVM.h b/mlir/include/mlir/Conversion/MemRefToLLVM/MemRefToLLVM.h index 34d01392e260..943e2108eb97 100644 --- a/mlir/include/mlir/Conversion/MemRefToLLVM/MemRefToLLVM.h +++ b/mlir/include/mlir/Conversion/MemRefToLLVM/MemRefToLLVM.h @@ -16,13 +16,13 @@ class Pass; class LLVMTypeConverter; class RewritePatternSet; -#define GEN_PASS_DECL_MEMREFTOLLVMCONVERSIONPASS +#define GEN_PASS_DECL_FINALIZEMEMREFTOLLVMCONVERSIONPASS #include "mlir/Conversion/Passes.h.inc" /// Collect a set of patterns to convert memory-related operations from the /// MemRef dialect to the LLVM dialect. -void populateMemRefToLLVMConversionPatterns(LLVMTypeConverter &converter, - RewritePatternSet &patterns); +void populateFinalizeMemRefToLLVMConversionPatterns( + LLVMTypeConverter &converter, RewritePatternSet &patterns); } // namespace mlir #endif // MLIR_CONVERSION_MEMREFTOLLVM_MEMREFTOLLVM_H diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td index 767b6b981e8b..86b9a208bbeb 100644 --- a/mlir/include/mlir/Conversion/Passes.td +++ b/mlir/include/mlir/Conversion/Passes.td @@ -590,9 +590,13 @@ def ConvertMathToFuncs : Pass<"convert-math-to-funcs", "ModuleOp"> { // MemRefToLLVM //===----------------------------------------------------------------------===// -def MemRefToLLVMConversionPass : Pass<"convert-memref-to-llvm", "ModuleOp"> { - let summary = "Convert operations from the MemRef dialect to the LLVM " - "dialect"; +def FinalizeMemRefToLLVMConversionPass : + Pass<"finalize-memref-to-llvm", "ModuleOp"> { + let summary = "Finalize the conversion of the operations from the MemRef " + "dialect to the LLVM dialect." + "This conversion will not convert some complex MemRef " + "operations. Make sure to run `expand-strided-metadata` " + "beforehand for these."; let dependentDialects = ["LLVM::LLVMDialect"]; let options = [ Option<"useAlignedAlloc", "use-aligned-alloc", "bool", /*default=*/"false", diff --git a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp index 98bdc7902e59..3f61be5471e7 100644 --- a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp +++ b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp @@ -372,7 +372,7 @@ void GpuToLLVMConversionPass::runOnOperation() { mlir::arith::populateArithToLLVMConversionPatterns(converter, patterns); mlir::cf::populateControlFlowToLLVMConversionPatterns(converter, patterns); populateVectorToLLVMConversionPatterns(converter, patterns); - populateMemRefToLLVMConversionPatterns(converter, patterns); + populateFinalizeMemRefToLLVMConversionPatterns(converter, patterns); populateFuncToLLVMConversionPatterns(converter, patterns); populateAsyncStructuralTypeConversionsAndLegality(converter, patterns, target); diff --git a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp index cefe493ecbc0..a80c024ceb24 100644 --- a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp +++ b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp @@ -281,7 +281,7 @@ struct LowerGpuOpsToNVVMOpsPass arith::populateArithToLLVMConversionPatterns(converter, llvmPatterns); cf::populateControlFlowToLLVMConversionPatterns(converter, llvmPatterns); populateFuncToLLVMConversionPatterns(converter, llvmPatterns); - populateMemRefToLLVMConversionPatterns(converter, llvmPatterns); + populateFinalizeMemRefToLLVMConversionPatterns(converter, llvmPatterns); populateGpuToNVVMConversionPatterns(converter, llvmPatterns); populateGpuWMMAToNVVMConversionPatterns(converter, llvmPatterns); if (this->hasRedux) diff --git a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp index daf6583ef6e6..233e2fb3681d 100644 --- a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp +++ b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp @@ -167,7 +167,7 @@ struct LowerGpuOpsToROCDLOpsPass populateVectorToLLVMConversionPatterns(converter, llvmPatterns); cf::populateControlFlowToLLVMConversionPatterns(converter, llvmPatterns); populateFuncToLLVMConversionPatterns(converter, llvmPatterns); - populateMemRefToLLVMConversionPatterns(converter, llvmPatterns); + populateFinalizeMemRefToLLVMConversionPatterns(converter, llvmPatterns); populateGpuToROCDLConversionPatterns(converter, llvmPatterns, runtime); LLVMConversionTarget target(getContext()); configureGpuToROCDLConversionLegality(target); diff --git a/mlir/lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp b/mlir/lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp index de55a0da75d2..8eb7bbe3e503 100644 --- a/mlir/lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp +++ b/mlir/lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp @@ -90,7 +90,7 @@ void ConvertLinalgToLLVMPass::runOnOperation() { RewritePatternSet patterns(&getContext()); LLVMTypeConverter converter(&getContext()); populateLinalgToLLVMConversionPatterns(converter, patterns); - populateMemRefToLLVMConversionPatterns(converter, patterns); + populateFinalizeMemRefToLLVMConversionPatterns(converter, patterns); LLVMConversionTarget target(getContext()); target.addLegalOp(); diff --git a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp index a1ea03f52ef6..d425658a0ecf 100644 --- a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp +++ b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp @@ -25,7 +25,7 @@ #include namespace mlir { -#define GEN_PASS_DEF_MEMREFTOLLVMCONVERSIONPASS +#define GEN_PASS_DEF_FINALIZEMEMREFTOLLVMCONVERSIONPASS #include "mlir/Conversion/Passes.h.inc" } // namespace mlir @@ -1700,8 +1700,8 @@ class ExtractStridedMetadataOpLowering } // namespace -void mlir::populateMemRefToLLVMConversionPatterns(LLVMTypeConverter &converter, - RewritePatternSet &patterns) { +void mlir::populateFinalizeMemRefToLLVMConversionPatterns( + LLVMTypeConverter &converter, RewritePatternSet &patterns) { // clang-format off patterns.add< AllocaOpLowering, @@ -1738,9 +1738,11 @@ void mlir::populateMemRefToLLVMConversionPatterns(LLVMTypeConverter &converter, } namespace { -struct MemRefToLLVMConversionPass - : public impl::MemRefToLLVMConversionPassBase { - using MemRefToLLVMConversionPassBase::MemRefToLLVMConversionPassBase; +struct FinalizeMemRefToLLVMConversionPass + : public impl::FinalizeMemRefToLLVMConversionPassBase< + FinalizeMemRefToLLVMConversionPass> { + using FinalizeMemRefToLLVMConversionPassBase:: + FinalizeMemRefToLLVMConversionPassBase; void runOnOperation() override { Operation *op = getOperation(); @@ -1759,7 +1761,7 @@ struct MemRefToLLVMConversionPass LLVMTypeConverter typeConverter(&getContext(), options, &dataLayoutAnalysis); RewritePatternSet patterns(&getContext()); - populateMemRefToLLVMConversionPatterns(typeConverter, patterns); + populateFinalizeMemRefToLLVMConversionPatterns(typeConverter, patterns); LLVMConversionTarget target(getContext()); target.addLegalOp(); if (failed(applyPartialConversion(op, target, std::move(patterns)))) diff --git a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp index df27c487be77..249e21e90803 100644 --- a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp +++ b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp @@ -152,7 +152,7 @@ void ConvertOpenMPToLLVMPass::runOnOperation() { LLVMTypeConverter converter(&getContext()); arith::populateArithToLLVMConversionPatterns(converter, patterns); cf::populateControlFlowToLLVMConversionPatterns(converter, patterns); - populateMemRefToLLVMConversionPatterns(converter, patterns); + populateFinalizeMemRefToLLVMConversionPatterns(converter, patterns); populateFuncToLLVMConversionPatterns(converter, patterns); populateOpenMPToLLVMConversionPatterns(converter, patterns); diff --git a/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp b/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp index bfc1bb65dce9..7803593d73a7 100644 --- a/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp +++ b/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp @@ -306,7 +306,7 @@ class LowerHostCodeToLLVM RewritePatternSet patterns(context); LLVMTypeConverter typeConverter(context, options); mlir::arith::populateArithToLLVMConversionPatterns(typeConverter, patterns); - populateMemRefToLLVMConversionPatterns(typeConverter, patterns); + populateFinalizeMemRefToLLVMConversionPatterns(typeConverter, patterns); populateFuncToLLVMConversionPatterns(typeConverter, patterns); patterns.add(typeConverter); diff --git a/mlir/lib/Dialect/SparseTensor/Pipelines/SparseTensorPipelines.cpp b/mlir/lib/Dialect/SparseTensor/Pipelines/SparseTensorPipelines.cpp index 8590c56a49d4..44ccf05ff267 100644 --- a/mlir/lib/Dialect/SparseTensor/Pipelines/SparseTensorPipelines.cpp +++ b/mlir/lib/Dialect/SparseTensor/Pipelines/SparseTensorPipelines.cpp @@ -73,7 +73,7 @@ void mlir::sparse_tensor::buildSparseCompiler( pm.addPass(memref::createExpandStridedMetadataPass()); pm.addPass(createLowerAffinePass()); pm.addPass(createConvertVectorToLLVMPass(options.lowerVectorToLLVMOptions())); - pm.addPass(createMemRefToLLVMConversionPass()); + pm.addPass(createFinalizeMemRefToLLVMConversionPass()); pm.addNestedPass(createConvertComplexToStandardPass()); pm.addNestedPass(mlir::arith::createArithExpandOpsPass()); pm.addNestedPass(createConvertMathToLLVMPass()); diff --git a/mlir/test/Conversion/FuncToLLVM/calling-convention.mlir b/mlir/test/Conversion/FuncToLLVM/calling-convention.mlir index 24d85e8596a4..3324dff9d997 100644 --- a/mlir/test/Conversion/FuncToLLVM/calling-convention.mlir +++ b/mlir/test/Conversion/FuncToLLVM/calling-convention.mlir @@ -1,5 +1,5 @@ -// RUN: mlir-opt -convert-memref-to-llvm -llvm-request-c-wrappers -convert-func-to-llvm -reconcile-unrealized-casts %s | FileCheck %s -// RUN: mlir-opt -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts %s | FileCheck %s --check-prefix=EMIT_C_ATTRIBUTE +// RUN: mlir-opt -finalize-memref-to-llvm -llvm-request-c-wrappers -convert-func-to-llvm -reconcile-unrealized-casts %s | FileCheck %s +// RUN: mlir-opt -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts %s | FileCheck %s --check-prefix=EMIT_C_ATTRIBUTE // This tests the default memref calling convention and the emission of C // wrappers. We don't need to separate runs because the wrapper-emission diff --git a/mlir/test/Conversion/MemRefToLLVM/convert-alloca-scope.mlir b/mlir/test/Conversion/MemRefToLLVM/convert-alloca-scope.mlir index f443f05795d2..7553d726cf71 100644 --- a/mlir/test/Conversion/MemRefToLLVM/convert-alloca-scope.mlir +++ b/mlir/test/Conversion/MemRefToLLVM/convert-alloca-scope.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt -convert-memref-to-llvm %s | FileCheck %s +// RUN: mlir-opt -finalize-memref-to-llvm %s | FileCheck %s // CHECK-LABEL: @empty func.func @empty() { diff --git a/mlir/test/Conversion/MemRefToLLVM/convert-dynamic-memref-ops.mlir b/mlir/test/Conversion/MemRefToLLVM/convert-dynamic-memref-ops.mlir index 821f298ba4ce..6a9bd2645d09 100644 --- a/mlir/test/Conversion/MemRefToLLVM/convert-dynamic-memref-ops.mlir +++ b/mlir/test/Conversion/MemRefToLLVM/convert-dynamic-memref-ops.mlir @@ -1,6 +1,6 @@ -// RUN: mlir-opt -split-input-file -convert-memref-to-llvm %s | FileCheck %s -// RUN: mlir-opt -split-input-file -convert-memref-to-llvm='use-aligned-alloc=1' %s | FileCheck %s --check-prefix=ALIGNED-ALLOC -// RUN: mlir-opt -split-input-file -convert-memref-to-llvm='index-bitwidth=32' %s | FileCheck --check-prefix=CHECK32 %s +// RUN: mlir-opt -split-input-file -finalize-memref-to-llvm %s | FileCheck %s +// RUN: mlir-opt -split-input-file -finalize-memref-to-llvm='use-aligned-alloc=1' %s | FileCheck %s --check-prefix=ALIGNED-ALLOC +// RUN: mlir-opt -split-input-file -finalize-memref-to-llvm='index-bitwidth=32' %s | FileCheck --check-prefix=CHECK32 %s // CHECK-LABEL: func @mixed_alloc( // CHECK: %[[Marg:.*]]: index, %[[Narg:.*]]: index) diff --git a/mlir/test/Conversion/MemRefToLLVM/convert-static-memref-ops.mlir b/mlir/test/Conversion/MemRefToLLVM/convert-static-memref-ops.mlir index 023955987fb5..8617deccd6fa 100644 --- a/mlir/test/Conversion/MemRefToLLVM/convert-static-memref-ops.mlir +++ b/mlir/test/Conversion/MemRefToLLVM/convert-static-memref-ops.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt -convert-memref-to-llvm -split-input-file %s | FileCheck %s +// RUN: mlir-opt -finalize-memref-to-llvm -split-input-file %s | FileCheck %s // CHECK-LABEL: func @zero_d_alloc() func.func @zero_d_alloc() -> memref { diff --git a/mlir/test/Conversion/MemRefToLLVM/expand-then-convert-to-llvm.mlir b/mlir/test/Conversion/MemRefToLLVM/expand-then-convert-to-llvm.mlir index 51650adc9a9f..f2be1b525696 100644 --- a/mlir/test/Conversion/MemRefToLLVM/expand-then-convert-to-llvm.mlir +++ b/mlir/test/Conversion/MemRefToLLVM/expand-then-convert-to-llvm.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt -expand-strided-metadata -convert-memref-to-llvm -lower-affine -convert-arith-to-llvm -cse %s -split-input-file | FileCheck %s +// RUN: mlir-opt -expand-strided-metadata -finalize-memref-to-llvm -lower-affine -convert-arith-to-llvm -cse %s -split-input-file | FileCheck %s // // This test demonstrates a full "memref to llvm" pipeline where // we first expand some of the memref operations (using affine, @@ -7,7 +7,7 @@ // // Note: We run CSE in that test to get rid of duplicated // unrealized_conversion_cast that are inserted with -// convert-memref-to-llvm and then convert-arith-to-llvm. +// finalize-memref-to-llvm and then convert-arith-to-llvm. // The final code is still not perfect, because we have // noop unrealized_conversion_cast from i64 to index // and back. diff --git a/mlir/test/Conversion/MemRefToLLVM/generic-functions.mlir b/mlir/test/Conversion/MemRefToLLVM/generic-functions.mlir index 5aa844c5afc5..4de7fbe59581 100644 --- a/mlir/test/Conversion/MemRefToLLVM/generic-functions.mlir +++ b/mlir/test/Conversion/MemRefToLLVM/generic-functions.mlir @@ -1,7 +1,7 @@ -// RUN: mlir-opt -pass-pipeline="builtin.module(convert-memref-to-llvm{use-generic-functions=1})" -split-input-file %s \ +// RUN: mlir-opt -pass-pipeline="builtin.module(finalize-memref-to-llvm{use-generic-functions=1})" -split-input-file %s \ // RUN: | FileCheck %s --check-prefix="CHECK-NOTALIGNED" -// RUN: mlir-opt -pass-pipeline="builtin.module(convert-memref-to-llvm{use-generic-functions=1 use-aligned-alloc=1})" -split-input-file %s \ +// RUN: mlir-opt -pass-pipeline="builtin.module(finalize-memref-to-llvm{use-generic-functions=1 use-aligned-alloc=1})" -split-input-file %s \ // RUN: | FileCheck %s --check-prefix="CHECK-ALIGNED" // CHECK-LABEL: func @alloc() diff --git a/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir b/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir index eb3a9482392a..3f61f6d78a7b 100644 --- a/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir +++ b/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir @@ -1,5 +1,5 @@ -// RUN: mlir-opt -convert-memref-to-llvm %s -split-input-file | FileCheck %s -// RUN: mlir-opt -convert-memref-to-llvm='index-bitwidth=32' %s -split-input-file | FileCheck --check-prefix=CHECK32 %s +// RUN: mlir-opt -finalize-memref-to-llvm %s -split-input-file | FileCheck %s +// RUN: mlir-opt -finalize-memref-to-llvm='index-bitwidth=32' %s -split-input-file | FileCheck --check-prefix=CHECK32 %s // CHECK-LABEL: func @view( // CHECK: %[[ARG0F:.*]]: index, %[[ARG1F:.*]]: index, %[[ARG2F:.*]]: index diff --git a/mlir/test/Integration/Dialect/Async/CPU/microbench-linalg-async-parallel-for.mlir b/mlir/test/Integration/Dialect/Async/CPU/microbench-linalg-async-parallel-for.mlir index 23853acfa898..53f19b5f95dd 100644 --- a/mlir/test/Integration/Dialect/Async/CPU/microbench-linalg-async-parallel-for.mlir +++ b/mlir/test/Integration/Dialect/Async/CPU/microbench-linalg-async-parallel-for.mlir @@ -9,7 +9,7 @@ // RUN: -arith-expand \ // RUN: -memref-expand \ // RUN: -convert-vector-to-llvm \ -// RUN: -convert-memref-to-llvm \ +// RUN: -finalize-memref-to-llvm \ // RUN: -convert-func-to-llvm \ // RUN: -reconcile-unrealized-casts \ // RUN: | mlir-cpu-runner \ @@ -23,7 +23,7 @@ // RUN: -convert-linalg-to-loops \ // RUN: -convert-scf-to-cf \ // RUN: -convert-vector-to-llvm \ -// RUN: -convert-memref-to-llvm \ +// RUN: -finalize-memref-to-llvm \ // RUN: -convert-func-to-llvm \ // RUN: -reconcile-unrealized-casts \ // RUN: | mlir-cpu-runner \ diff --git a/mlir/test/Integration/Dialect/Async/CPU/microbench-scf-async-parallel-for.mlir b/mlir/test/Integration/Dialect/Async/CPU/microbench-scf-async-parallel-for.mlir index 046e419d9e1d..74531befff18 100644 --- a/mlir/test/Integration/Dialect/Async/CPU/microbench-scf-async-parallel-for.mlir +++ b/mlir/test/Integration/Dialect/Async/CPU/microbench-scf-async-parallel-for.mlir @@ -9,7 +9,7 @@ // RUN: -arith-expand \ // RUN: -memref-expand \ // RUN: -convert-vector-to-llvm \ -// RUN: -convert-memref-to-llvm \ +// RUN: -finalize-memref-to-llvm \ // RUN: -convert-func-to-llvm \ // RUN: -reconcile-unrealized-casts \ // RUN: | mlir-cpu-runner \ @@ -30,7 +30,7 @@ // RUN: -arith-expand \ // RUN: -memref-expand \ // RUN: -convert-vector-to-llvm \ -// RUN: -convert-memref-to-llvm \ +// RUN: -finalize-memref-to-llvm \ // RUN: -convert-func-to-llvm \ // RUN: -reconcile-unrealized-casts \ // RUN: | mlir-cpu-runner \ @@ -44,7 +44,7 @@ // RUN: -convert-linalg-to-loops \ // RUN: -convert-scf-to-cf \ // RUN: -convert-vector-to-llvm \ -// RUN: -convert-memref-to-llvm \ +// RUN: -finalize-memref-to-llvm \ // RUN: -convert-func-to-llvm \ // RUN: -reconcile-unrealized-casts \ // RUN: | mlir-cpu-runner \ diff --git a/mlir/test/Integration/Dialect/Async/CPU/test-async-parallel-for-1d.mlir b/mlir/test/Integration/Dialect/Async/CPU/test-async-parallel-for-1d.mlir index 27e143388363..61a92aa1d790 100644 --- a/mlir/test/Integration/Dialect/Async/CPU/test-async-parallel-for-1d.mlir +++ b/mlir/test/Integration/Dialect/Async/CPU/test-async-parallel-for-1d.mlir @@ -4,7 +4,7 @@ // RUN: -async-runtime-ref-counting-opt \ // RUN: -convert-async-to-llvm \ // RUN: -convert-scf-to-cf \ -// RUN: -convert-memref-to-llvm \ +// RUN: -finalize-memref-to-llvm \ // RUN: -arith-expand \ // RUN: -memref-expand \ // RUN: -convert-func-to-llvm \ @@ -20,7 +20,7 @@ // RUN: -async-runtime-policy-based-ref-counting \ // RUN: -convert-async-to-llvm \ // RUN: -convert-scf-to-cf \ -// RUN: -convert-memref-to-llvm \ +// RUN: -finalize-memref-to-llvm \ // RUN: -arith-expand \ // RUN: -memref-expand \ // RUN: -convert-func-to-llvm \ @@ -39,7 +39,7 @@ // RUN: -async-runtime-ref-counting-opt \ // RUN: -convert-async-to-llvm \ // RUN: -convert-scf-to-cf \ -// RUN: -convert-memref-to-llvm \ +// RUN: -finalize-memref-to-llvm \ // RUN: -arith-expand \ // RUN: -memref-expand \ // RUN: -convert-func-to-llvm \ diff --git a/mlir/test/Integration/Dialect/Async/CPU/test-async-parallel-for-2d.mlir b/mlir/test/Integration/Dialect/Async/CPU/test-async-parallel-for-2d.mlir index 4e089135aa8b..46da16985ff6 100644 --- a/mlir/test/Integration/Dialect/Async/CPU/test-async-parallel-for-2d.mlir +++ b/mlir/test/Integration/Dialect/Async/CPU/test-async-parallel-for-2d.mlir @@ -5,7 +5,7 @@ // RUN: -arith-expand \ // RUN: -convert-async-to-llvm \ // RUN: -convert-scf-to-cf \ -// RUN: -convert-memref-to-llvm \ +// RUN: -finalize-memref-to-llvm \ // RUN: -convert-func-to-llvm \ // RUN: -reconcile-unrealized-casts \ // RUN: | mlir-cpu-runner \ @@ -20,7 +20,7 @@ // RUN: -arith-expand \ // RUN: -convert-async-to-llvm \ // RUN: -convert-scf-to-cf \ -// RUN: -convert-memref-to-llvm \ +// RUN: -finalize-memref-to-llvm \ // RUN: -convert-func-to-llvm \ // RUN: -reconcile-unrealized-casts \ // RUN: | mlir-cpu-runner \ @@ -38,7 +38,7 @@ // RUN: -arith-expand \ // RUN: -convert-async-to-llvm \ // RUN: -convert-scf-to-cf \ -// RUN: -convert-memref-to-llvm \ +// RUN: -finalize-memref-to-llvm \ // RUN: -convert-func-to-llvm \ // RUN: -reconcile-unrealized-casts \ // RUN: | mlir-cpu-runner \ diff --git a/mlir/test/Integration/Dialect/Complex/CPU/correctness.mlir b/mlir/test/Integration/Dialect/Complex/CPU/correctness.mlir index c89a146bd0a7..dfefb0b14918 100644 --- a/mlir/test/Integration/Dialect/Complex/CPU/correctness.mlir +++ b/mlir/test/Integration/Dialect/Complex/CPU/correctness.mlir @@ -1,7 +1,7 @@ // RUN: mlir-opt %s \ // RUN: -func-bufferize -tensor-bufferize -arith-bufferize --canonicalize \ // RUN: -convert-scf-to-cf --convert-complex-to-standard \ -// RUN: -convert-memref-to-llvm -convert-math-to-llvm -convert-math-to-libm \ +// RUN: -finalize-memref-to-llvm -convert-math-to-llvm -convert-math-to-libm \ // RUN: -convert-vector-to-llvm -convert-complex-to-llvm \ // RUN: -convert-func-to-llvm -reconcile-unrealized-casts |\ // RUN: mlir-cpu-runner \ diff --git a/mlir/test/Integration/Dialect/LLVMIR/CPU/test-complex-sparse-constant.mlir b/mlir/test/Integration/Dialect/LLVMIR/CPU/test-complex-sparse-constant.mlir index a42c64d57a87..470614e97325 100644 --- a/mlir/test/Integration/Dialect/LLVMIR/CPU/test-complex-sparse-constant.mlir +++ b/mlir/test/Integration/Dialect/LLVMIR/CPU/test-complex-sparse-constant.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s --convert-memref-to-llvm | \ +// RUN: mlir-opt %s --finalize-memref-to-llvm | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void // diff --git a/mlir/test/Integration/Dialect/LLVMIR/CPU/test-vp-intrinsic.mlir b/mlir/test/Integration/Dialect/LLVMIR/CPU/test-vp-intrinsic.mlir index e8e5ef392762..e39d97ae2933 100644 --- a/mlir/test/Integration/Dialect/LLVMIR/CPU/test-vp-intrinsic.mlir +++ b/mlir/test/Integration/Dialect/LLVMIR/CPU/test-vp-intrinsic.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-vector-to-llvm -convert-memref-to-llvm \ +// RUN: mlir-opt %s -convert-vector-to-llvm -finalize-memref-to-llvm \ // RUN: -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-translate -mlir-to-llvmir | \ // RUN: %lli --entry-function=entry \ diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/matmul-vs-matvec.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/matmul-vs-matvec.mlir index e801a85c87d3..ae37b2195f6a 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/matmul-vs-matvec.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/matmul-vs-matvec.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -O3 -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/rank-reducing-subview.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/rank-reducing-subview.mlir index f6617f145bc1..351394ad3188 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/rank-reducing-subview.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/rank-reducing-subview.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -O3 -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-collapse-tensor.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-collapse-tensor.mlir index c4714089c2f1..b0529c8b97ad 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-collapse-tensor.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-collapse-tensor.mlir @@ -2,7 +2,7 @@ // RUN: -arith-bufferize -tensor-bufferize -func-bufferize \ // RUN: -finalizing-bufferize -buffer-deallocation -convert-linalg-to-llvm \ // RUN: -expand-strided-metadata -lower-affine \ -// RUN: -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-1d-call.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-1d-call.mlir index 6e66dec6a9fe..cb09816c1efd 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-1d-call.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-1d-call.mlir @@ -1,10 +1,10 @@ -// RUN: mlir-opt %s -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s // RUN: mlir-opt %s -test-transform-dialect-interpreter -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf \ -// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-1d-nwc-wcf-call.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-1d-nwc-wcf-call.mlir index cbb55895b1ac..315914bf29d5 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-1d-nwc-wcf-call.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-1d-nwc-wcf-call.mlir @@ -1,10 +1,10 @@ -// RUN: mlir-opt %s -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s // RUN: mlir-opt %s -test-transform-dialect-interpreter -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf \ -// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-2d-call.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-2d-call.mlir index ca5d0c9213d3..dbdfd35864f1 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-2d-call.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-2d-call.mlir @@ -1,10 +1,10 @@ -// RUN: mlir-opt %s -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s // RUN: mlir-opt %s -test-transform-dialect-interpreter -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf \ -// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-2d-nhwc-hwcf-call.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-2d-nhwc-hwcf-call.mlir index 410de4bbd575..0b009d0540a6 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-2d-nhwc-hwcf-call.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-2d-nhwc-hwcf-call.mlir @@ -1,10 +1,10 @@ -// RUN: mlir-opt %s -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s // RUN: mlir-opt %s -test-transform-dialect-interpreter -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf \ -// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-3d-call.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-3d-call.mlir index a5a89a5d7606..9e510ffd36c3 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-3d-call.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-3d-call.mlir @@ -1,10 +1,10 @@ -// RUN: mlir-opt %s -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s // RUN: mlir-opt %s -test-transform-dialect-interpreter -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf \ -// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-3d-ndhwc-dhwcf-call.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-3d-ndhwc-dhwcf-call.mlir index 4a3e35659d4a..f8372fb5b474 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-3d-ndhwc-dhwcf-call.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-3d-ndhwc-dhwcf-call.mlir @@ -1,10 +1,10 @@ -// RUN: mlir-opt %s -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s // RUN: mlir-opt %s -test-transform-dialect-interpreter -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf \ -// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-elementwise.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-elementwise.mlir index 9e14c971049c..de0babae03bc 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-elementwise.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-elementwise.mlir @@ -1,7 +1,7 @@ // RUN: mlir-opt %s -convert-elementwise-to-linalg \ // RUN: -arith-bufferize -linalg-bufferize -tensor-bufferize \ // RUN: -func-bufferize -buffer-deallocation -convert-linalg-to-loops \ -// RUN: -convert-linalg-to-llvm --convert-memref-to-llvm -convert-func-to-llvm \ +// RUN: -convert-linalg-to-llvm --finalize-memref-to-llvm -convert-func-to-llvm \ // RUN: -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-expand-tensor.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-expand-tensor.mlir index 868d940d233d..819589be27b5 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-expand-tensor.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-expand-tensor.mlir @@ -2,7 +2,7 @@ // RUN: -arith-bufferize -tensor-bufferize -func-bufferize \ // RUN: -finalizing-bufferize -buffer-deallocation -convert-linalg-to-llvm \ // RUN: -expand-strided-metadata -lower-affine \ -// RUN: -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-one-shot-bufferize.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-one-shot-bufferize.mlir index d41a96428306..40a5cc452e2a 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-one-shot-bufferize.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-one-shot-bufferize.mlir @@ -1,6 +1,6 @@ // RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(canonicalize,cse),one-shot-bufferize{bufferize-function-boundaries})" |\ // RUN: mlir-opt -pass-pipeline="builtin.module(func.func(buffer-deallocation,convert-vector-to-scf,lower-affine,convert-linalg-to-loops))" |\ -// RUN: mlir-opt -pass-pipeline="builtin.module(func.func(canonicalize,convert-scf-to-cf),convert-vector-to-llvm,expand-strided-metadata,lower-affine,convert-arith-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-opt -pass-pipeline="builtin.module(func.func(canonicalize,convert-scf-to-cf),convert-vector-to-llvm,expand-strided-metadata,lower-affine,convert-arith-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ // RUN: mlir-cpu-runner -O3 -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext,%mlir_lib_dir/libmlir_c_runner_utils%shlibext |\ diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-padtensor.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-padtensor.mlir index 6a255b8cd8a4..fd035881d008 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-padtensor.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-padtensor.mlir @@ -2,7 +2,7 @@ // RUN: -empty-tensor-to-alloc-tensor -linalg-bufferize -arith-bufferize \ // RUN: -bufferization-bufferize -tensor-bufferize -func-bufferize \ // RUN: -finalizing-bufferize -buffer-deallocation \ -// RUN: -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext,%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-subtensor-insert-multiple-uses.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-subtensor-insert-multiple-uses.mlir index 2f3d971c3df7..9f23f55bf66b 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-subtensor-insert-multiple-uses.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-subtensor-insert-multiple-uses.mlir @@ -1,7 +1,7 @@ // RUN: mlir-opt %s -linalg-bufferize \ // RUN: -arith-bufferize -tensor-bufferize -func-bufferize \ // RUN: -finalizing-bufferize -buffer-deallocation \ -// RUN: -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext,%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-subtensor-insert.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-subtensor-insert.mlir index 0fc72ba6618d..4d05ede756d6 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-subtensor-insert.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-subtensor-insert.mlir @@ -1,7 +1,7 @@ // RUN: mlir-opt %s -linalg-bufferize \ // RUN: -arith-bufferize -tensor-bufferize -func-bufferize \ // RUN: -finalizing-bufferize -buffer-deallocation \ -// RUN: -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-tensor-e2e.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-tensor-e2e.mlir index 2c9718cc1e26..b371421262cc 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-tensor-e2e.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-tensor-e2e.mlir @@ -1,6 +1,6 @@ // RUN: mlir-opt %s -arith-bufferize -linalg-bufferize \ // RUN: -tensor-bufferize -func-bufferize -finalizing-bufferize -buffer-deallocation -convert-linalg-to-loops \ -// RUN: -convert-linalg-to-llvm --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -convert-linalg-to-llvm --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-tensor-matmul.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-tensor-matmul.mlir index 80fb41b96d11..af51df42eb4b 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-tensor-matmul.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-tensor-matmul.mlir @@ -1,7 +1,7 @@ // UNSUPPORTED: asan // RUN: mlir-opt %s -test-transform-dialect-erase-schedule -linalg-bufferize -arith-bufferize \ // RUN: -tensor-bufferize -func-bufferize -finalizing-bufferize -buffer-deallocation -convert-linalg-to-loops -convert-scf-to-cf \ -// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext,%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s @@ -10,7 +10,7 @@ // RUN: -scf-bufferize -arith-bufferize -tensor-bufferize \ // RUN: -func-bufferize \ // RUN: -finalizing-bufferize -convert-linalg-to-loops -convert-scf-to-cf -convert-scf-to-cf \ -// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext,%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Memref/cast-runtime-verification.mlir b/mlir/test/Integration/Dialect/Memref/cast-runtime-verification.mlir index 7da0b00337f2..9221d8eb3586 100644 --- a/mlir/test/Integration/Dialect/Memref/cast-runtime-verification.mlir +++ b/mlir/test/Integration/Dialect/Memref/cast-runtime-verification.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -generate-runtime-verification -convert-memref-to-llvm \ +// RUN: mlir-opt %s -generate-runtime-verification -finalize-memref-to-llvm \ // RUN: -test-cf-assert \ // RUN: -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ diff --git a/mlir/test/Integration/Dialect/Memref/memref_abi.c b/mlir/test/Integration/Dialect/Memref/memref_abi.c index 12079171f549..8862727a539a 100644 --- a/mlir/test/Integration/Dialect/Memref/memref_abi.c +++ b/mlir/test/Integration/Dialect/Memref/memref_abi.c @@ -3,7 +3,7 @@ // Compile the MLIR file to LLVM: // RUN: mlir-opt %t/input.mlir \ -// RUN: -lower-affine -convert-scf-to-cf -convert-memref-to-llvm \ +// RUN: -lower-affine -convert-scf-to-cf -finalize-memref-to-llvm \ // RUN: -convert-func-to-llvm -reconcile-unrealized-casts \ // RUN: | mlir-translate --mlir-to-llvmir -o %t.ll diff --git a/mlir/test/Integration/Dialect/Standard/CPU/test-ceil-floor-pos-neg.mlir b/mlir/test/Integration/Dialect/Standard/CPU/test-ceil-floor-pos-neg.mlir index 5372280e8a01..d289be9799cf 100644 --- a/mlir/test/Integration/Dialect/Standard/CPU/test-ceil-floor-pos-neg.mlir +++ b/mlir/test/Integration/Dialect/Standard/CPU/test-ceil-floor-pos-neg.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf,memref-expand,arith-expand),convert-vector-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf,memref-expand,arith-expand),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Standard/CPU/test_subview.mlir b/mlir/test/Integration/Dialect/Standard/CPU/test_subview.mlir index 55a98a8958cb..50cac3269891 100644 --- a/mlir/test/Integration/Dialect/Standard/CPU/test_subview.mlir +++ b/mlir/test/Integration/Dialect/Standard/CPU/test_subview.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -expand-strided-metadata -lower-affine -convert-arith-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-mulf-full.mlir b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-mulf-full.mlir index b0d22c5cc986..0e8440e4cfaf 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-mulf-full.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-mulf-full.mlir @@ -1,6 +1,6 @@ // RUN: mlir-opt %s -convert-vector-to-scf -lower-affine -convert-scf-to-cf \ // RUN: -arith-bufferize -convert-vector-to-llvm="enable-amx" \ -// RUN: -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-translate -mlir-to-llvmir | \ // RUN: %lli --entry-function=entry --mattr="+amx-tile,+amx-int8,+amx-bf16" \ // RUN: --dlopen=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ diff --git a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-mulf.mlir b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-mulf.mlir index f346fc203930..ff77efea416c 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-mulf.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-mulf.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-vector-to-scf -lower-affine -convert-scf-to-cf -convert-vector-to-llvm="enable-amx" -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-vector-to-scf -lower-affine -convert-scf-to-cf -convert-vector-to-llvm="enable-amx" -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-translate -mlir-to-llvmir | \ // RUN: %lli --entry-function=entry --mattr="+amx-tile,+amx-int8,+amx-bf16" --dlopen=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli-ext.mlir b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli-ext.mlir index dabed01de2b2..086d6af1049f 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli-ext.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli-ext.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-vector-to-scf -lower-affine -convert-scf-to-cf -convert-vector-to-llvm="enable-amx" -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-vector-to-scf -lower-affine -convert-scf-to-cf -convert-vector-to-llvm="enable-amx" -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-translate -mlir-to-llvmir | \ // RUN: %lli --entry-function=entry --mattr="+amx-tile,+amx-int8,+amx-bf16" --dlopen=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli-full.mlir b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli-full.mlir index 14c79b7387b5..b2c51234249c 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli-full.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli-full.mlir @@ -1,6 +1,6 @@ // RUN: mlir-opt %s -convert-vector-to-scf -lower-affine -convert-scf-to-cf \ // RUN: -arith-bufferize -convert-vector-to-llvm="enable-amx" \ -// RUN: -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-translate -mlir-to-llvmir | \ // RUN: %lli --entry-function=entry --mattr="+amx-tile,+amx-int8,+amx-bf16" \ // RUN: --dlopen=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ diff --git a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli.mlir b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli.mlir index 6c86372062f5..ec431d4b9bf2 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-vector-to-scf -lower-affine -convert-scf-to-cf -convert-vector-to-llvm="enable-amx" -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-vector-to-scf -lower-affine -convert-scf-to-cf -convert-vector-to-llvm="enable-amx" -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-translate -mlir-to-llvmir | \ // RUN: %lli --entry-function=entry --mattr="+amx-tile,+amx-int8,+amx-bf16" --dlopen=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero-block.mlir b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero-block.mlir index 10e34e0d0edf..0b108fbb12b1 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero-block.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero-block.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-vector-to-scf -lower-affine -convert-scf-to-cf -convert-vector-to-llvm="enable-amx" -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-vector-to-scf -lower-affine -convert-scf-to-cf -convert-vector-to-llvm="enable-amx" -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-translate -mlir-to-llvmir | \ // RUN: %lli --entry-function=entry --mattr="+amx-tile,+amx-int8,+amx-bf16" --dlopen=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero.mlir b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero.mlir index 0c8f086ebabb..418c3e86097c 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-vector-to-scf -lower-affine -convert-scf-to-cf -convert-vector-to-llvm="enable-amx" -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-vector-to-scf -lower-affine -convert-scf-to-cf -convert-vector-to-llvm="enable-amx" -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-translate -mlir-to-llvmir | \ // RUN: %lli --entry-function=entry --mattr="+amx-tile,+amx-int8,+amx-bf16" --dlopen=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-sve.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-sve.mlir index a0688dcf16d9..794ed761e65b 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-sve.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-sve.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -lower-affine -convert-scf-to-cf -convert-vector-to-llvm="enable-arm-sve" -convert-memref-to-llvm -convert-func-to-llvm -convert-arith-to-llvm -canonicalize | \ +// RUN: mlir-opt %s -lower-affine -convert-scf-to-cf -convert-vector-to-llvm="enable-arm-sve" -finalize-memref-to-llvm -convert-func-to-llvm -convert-arith-to-llvm -canonicalize | \ // RUN: mlir-translate -mlir-to-llvmir | \ // RUN: %lli --entry-function=entry --march=aarch64 --mattr="+sve" --dlopen=%mlir_native_utils_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/X86Vector/test-inline-asm-vector-avx512.mlir b/mlir/test/Integration/Dialect/Vector/CPU/X86Vector/test-inline-asm-vector-avx512.mlir index 30a2e40aadc2..6295dfaa4ec3 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/X86Vector/test-inline-asm-vector-avx512.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/X86Vector/test-inline-asm-vector-avx512.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-linalg-to-loops -convert-vector-to-scf='full-unroll=true' -lower-affine -convert-scf-to-cf -convert-vector-to-llvm -convert-memref-to-llvm -convert-func-to-llvm='use-bare-ptr-memref-call-conv=1' -convert-arith-to-llvm -reconcile-unrealized-casts |\ +// RUN: mlir-opt %s -convert-linalg-to-loops -convert-vector-to-scf='full-unroll=true' -lower-affine -convert-scf-to-cf -convert-vector-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm='use-bare-ptr-memref-call-conv=1' -convert-arith-to-llvm -reconcile-unrealized-casts |\ // RUN: mlir-translate --mlir-to-llvmir |\ // RUN: %lli --entry-function=entry --mattr="avx512f" --dlopen=%mlir_lib_dir/libmlir_c_runner_utils%shlibext |\ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/X86Vector/test-sparse-dot-product.mlir b/mlir/test/Integration/Dialect/Vector/CPU/X86Vector/test-sparse-dot-product.mlir index abfc871925bb..cde451cb48e9 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/X86Vector/test-sparse-dot-product.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/X86Vector/test-sparse-dot-product.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm="enable-x86vector" -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm="enable-x86vector" -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-translate --mlir-to-llvmir | \ // RUN: %lli --entry-function=entry --mattr="avx512bw,avx512vp2intersect" --dlopen=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-0-d-vectors.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-0-d-vectors.mlir index 1e3f65354e89..7fc4e4b95dd1 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-0-d-vectors.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-0-d-vectors.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-compress.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-compress.mlir index 4ef83ddf8f02..60f653d0f8d4 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-compress.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-compress.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-expand.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-expand.mlir index b7507025e2c7..381a3d6719c7 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-expand.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-expand.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-gather.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-gather.mlir index 0b177497bb4f..5d9d4d88d2e0 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-gather.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-gather.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-maskedload.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-maskedload.mlir index b7a0de72a5c9..160c2a4ebecd 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-maskedload.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-maskedload.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-maskedstore.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-maskedstore.mlir index 93b355aca36c..717300f785bd 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-maskedstore.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-maskedstore.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-realloc.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-realloc.mlir index 518b1554505f..62975a93e452 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-realloc.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-realloc.mlir @@ -1,7 +1,7 @@ -// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts |\ +// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts |\ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext -// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -convert-memref-to-llvm='use-aligned-alloc=1' -convert-func-to-llvm -arith-expand -reconcile-unrealized-casts |\ +// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -finalize-memref-to-llvm='use-aligned-alloc=1' -convert-func-to-llvm -arith-expand -reconcile-unrealized-casts |\ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-scatter.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-scatter.mlir index 1358840bf2e6..34f8bbd61230 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-scatter.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-scatter.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-sparse-dot-matvec.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-sparse-dot-matvec.mlir index 29aab234014f..6c8a24d54b8a 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-sparse-dot-matvec.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-sparse-dot-matvec.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-sparse-saxpy-jagged-matvec.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-sparse-saxpy-jagged-matvec.mlir index 457ddb33214f..2ff69ad8f72c 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-sparse-saxpy-jagged-matvec.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-sparse-saxpy-jagged-matvec.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read-1d.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read-1d.mlir index 7e622da7aa79..c3df2665ca3f 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read-1d.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read-1d.mlir @@ -1,9 +1,9 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,expand-strided-metadata,lower-affine,convert-arith-to-llvm,convert-scf-to-cf),convert-vector-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,expand-strided-metadata,lower-affine,convert-arith-to-llvm,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true},expand-strided-metadata,lower-affine,convert-arith-to-llvm,convert-scf-to-cf),convert-vector-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true},expand-strided-metadata,lower-affine,convert-arith-to-llvm,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read-2d.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read-2d.mlir index 2f38ef674dd3..60020c0993fd 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read-2d.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read-2d.mlir @@ -1,9 +1,9 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf),convert-vector-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true},lower-affine,convert-scf-to-cf),convert-vector-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true},lower-affine,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read-3d.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read-3d.mlir index a49349b0f49c..f60303b9cdf2 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read-3d.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read-3d.mlir @@ -1,9 +1,9 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf),convert-vector-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true},lower-affine,convert-scf-to-cf),convert-vector-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true},lower-affine,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read.mlir index ccafa5b36a7c..27c22f5070f4 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read.mlir @@ -1,9 +1,9 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf),convert-vector-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true},lower-affine,convert-scf-to-cf),convert-vector-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true},lower-affine,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-to-loops.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-to-loops.mlir index 929a30a4d478..2e25c9fee3cb 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-to-loops.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-to-loops.mlir @@ -1,9 +1,9 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf),convert-vector-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext,%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true},lower-affine,convert-scf-to-cf),convert-vector-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true},lower-affine,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext,%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-write.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-write.mlir index 4dbdc69fdb4d..9bf3b46fbc33 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-write.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-write.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-vector-to-scf -convert-scf-to-cf -convert-vector-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-vector-to-scf -convert-scf-to-cf -convert-vector-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/lib/Dialect/LLVM/TestLowerToLLVM.cpp b/mlir/test/lib/Dialect/LLVM/TestLowerToLLVM.cpp index ed84a466121f..f3506226b16c 100644 --- a/mlir/test/lib/Dialect/LLVM/TestLowerToLLVM.cpp +++ b/mlir/test/lib/Dialect/LLVM/TestLowerToLLVM.cpp @@ -73,7 +73,7 @@ void buildTestLowerToLLVM(OpPassManager &pm, // The expansion may create affine expressions. Get rid of them. pm.addPass(createLowerAffinePass()); // Convert MemRef to LLVM (always needed). - pm.addPass(createMemRefToLLVMConversionPass()); + pm.addPass(createFinalizeMemRefToLLVMConversionPass()); // Convert Func to LLVM (always needed). pm.addPass(createConvertFuncToLLVMPass()); // Convert Index to LLVM (always needed). diff --git a/mlir/test/mlir-cpu-runner/async-value.mlir b/mlir/test/mlir-cpu-runner/async-value.mlir index cafdc9f57a7a..c9e0b8c9dc8b 100644 --- a/mlir/test/mlir-cpu-runner/async-value.mlir +++ b/mlir/test/mlir-cpu-runner/async-value.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(async-to-async-runtime,func.func(async-runtime-ref-counting,async-runtime-ref-counting-opt),convert-async-to-llvm,func.func(convert-arith-to-llvm),convert-vector-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(async-to-async-runtime,func.func(async-runtime-ref-counting,async-runtime-ref-counting-opt),convert-async-to-llvm,func.func(convert-arith-to-llvm),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" \ // RUN: | mlir-cpu-runner \ // RUN: -e main -entry-point-result=void -O0 \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext \ diff --git a/mlir/test/mlir-cpu-runner/async.mlir b/mlir/test/mlir-cpu-runner/async.mlir index 8f387121ad41..e63b878903de 100644 --- a/mlir/test/mlir-cpu-runner/async.mlir +++ b/mlir/test/mlir-cpu-runner/async.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(async-to-async-runtime,func.func(async-runtime-ref-counting,async-runtime-ref-counting-opt),convert-async-to-llvm,func.func(convert-linalg-to-loops,convert-scf-to-cf),convert-linalg-to-llvm,convert-memref-to-llvm,func.func(convert-arith-to-llvm),convert-func-to-llvm,reconcile-unrealized-casts)" \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(async-to-async-runtime,func.func(async-runtime-ref-counting,async-runtime-ref-counting-opt),convert-async-to-llvm,func.func(convert-linalg-to-loops,convert-scf-to-cf),convert-linalg-to-llvm,finalize-memref-to-llvm,func.func(convert-arith-to-llvm),convert-func-to-llvm,reconcile-unrealized-casts)" \ // RUN: | mlir-cpu-runner \ // RUN: -e main -entry-point-result=void -O0 \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext \ diff --git a/mlir/test/mlir-cpu-runner/bare-ptr-call-conv.mlir b/mlir/test/mlir-cpu-runner/bare-ptr-call-conv.mlir index 3720402f8cae..ab14b17e16ec 100644 --- a/mlir/test/mlir-cpu-runner/bare-ptr-call-conv.mlir +++ b/mlir/test/mlir-cpu-runner/bare-ptr-call-conv.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-scf-to-cf,convert-arith-to-llvm),convert-memref-to-llvm,convert-func-to-llvm{use-bare-ptr-memref-call-conv=1},reconcile-unrealized-casts)" | mlir-cpu-runner -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext -entry-point-result=void | FileCheck %s +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-scf-to-cf,convert-arith-to-llvm),finalize-memref-to-llvm,convert-func-to-llvm{use-bare-ptr-memref-call-conv=1},reconcile-unrealized-casts)" | mlir-cpu-runner -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext -entry-point-result=void | FileCheck %s // Verify bare pointer memref calling convention. `simple_add1_add2_test` // gets two 2xf32 memrefs, adds 1.0f to the first one and 2.0f to the second diff --git a/mlir/test/mlir-cpu-runner/copy.mlir b/mlir/test/mlir-cpu-runner/copy.mlir index d1769dccef3e..742fd5018ce7 100644 --- a/mlir/test/mlir-cpu-runner/copy.mlir +++ b/mlir/test/mlir-cpu-runner/copy.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-scf-to-cf,convert-arith-to-llvm),convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-scf-to-cf,convert-arith-to-llvm),finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" \ // RUN: | mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext,%mlir_lib_dir/libmlir_c_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/mlir-cpu-runner/global-memref.mlir b/mlir/test/mlir-cpu-runner/global-memref.mlir index 937d8d3afb24..5a75781781c3 100644 --- a/mlir/test/mlir-cpu-runner/global-memref.mlir +++ b/mlir/test/mlir-cpu-runner/global-memref.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-arith-to-llvm),convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | mlir-cpu-runner -e main -entry-point-result=void -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext,%mlir_lib_dir/libmlir_c_runner_utils%shlibext | FileCheck %s +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-arith-to-llvm),finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | mlir-cpu-runner -e main -entry-point-result=void -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext,%mlir_lib_dir/libmlir_c_runner_utils%shlibext | FileCheck %s func.func private @printMemrefF32(memref<*xf32>) attributes { llvm.emit_c_interface } func.func private @printMemrefI32(memref<*xi32>) attributes { llvm.emit_c_interface } diff --git a/mlir/test/mlir-cpu-runner/memref-reinterpret-cast.mlir b/mlir/test/mlir-cpu-runner/memref-reinterpret-cast.mlir index 134a1293d28c..685af8d3e063 100644 --- a/mlir/test/mlir-cpu-runner/memref-reinterpret-cast.mlir +++ b/mlir/test/mlir-cpu-runner/memref-reinterpret-cast.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-scf-to-cf),convert-memref-to-llvm,func.func(convert-arith-to-llvm),convert-func-to-llvm,reconcile-unrealized-casts)" \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-scf-to-cf),finalize-memref-to-llvm,func.func(convert-arith-to-llvm),convert-func-to-llvm,reconcile-unrealized-casts)" \ // RUN: | mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext,%mlir_lib_dir/libmlir_c_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/mlir-cpu-runner/memref-reshape.mlir b/mlir/test/mlir-cpu-runner/memref-reshape.mlir index 47ee21a7784a..94fe7d499b4e 100644 --- a/mlir/test/mlir-cpu-runner/memref-reshape.mlir +++ b/mlir/test/mlir-cpu-runner/memref-reshape.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-scf-to-cf,memref-expand,convert-arith-to-llvm),convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-scf-to-cf,memref-expand,convert-arith-to-llvm),finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" \ // RUN: | mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext,%mlir_lib_dir/libmlir_c_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/mlir-cpu-runner/sgemm-naive-codegen.mlir b/mlir/test/mlir-cpu-runner/sgemm-naive-codegen.mlir index 8a427ddf4b39..ccfea1cfedba 100644 --- a/mlir/test/mlir-cpu-runner/sgemm-naive-codegen.mlir +++ b/mlir/test/mlir-cpu-runner/sgemm-naive-codegen.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt -pass-pipeline="builtin.module(func.func(convert-linalg-to-loops,lower-affine,convert-scf-to-cf,convert-arith-to-llvm),convert-vector-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" %s | mlir-cpu-runner -O3 -e main -entry-point-result=void -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | FileCheck %s +// RUN: mlir-opt -pass-pipeline="builtin.module(func.func(convert-linalg-to-loops,lower-affine,convert-scf-to-cf,convert-arith-to-llvm),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" %s | mlir-cpu-runner -O3 -e main -entry-point-result=void -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | FileCheck %s func.func @main() { %A = memref.alloc() : memref<16x16xf32> diff --git a/mlir/test/mlir-cpu-runner/unranked-memref.mlir b/mlir/test/mlir-cpu-runner/unranked-memref.mlir index f8a37f12c051..24562030aeea 100644 --- a/mlir/test/mlir-cpu-runner/unranked-memref.mlir +++ b/mlir/test/mlir-cpu-runner/unranked-memref.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-linalg-to-loops,convert-scf-to-cf,convert-arith-to-llvm),convert-linalg-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-linalg-to-loops,convert-scf-to-cf,convert-arith-to-llvm),convert-linalg-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext,%mlir_lib_dir/libmlir_c_runner_utils%shlibext | FileCheck %s diff --git a/mlir/test/mlir-cpu-runner/utils.mlir b/mlir/test/mlir-cpu-runner/utils.mlir index 565495b802f2..d902430c887a 100644 --- a/mlir/test/mlir-cpu-runner/utils.mlir +++ b/mlir/test/mlir-cpu-runner/utils.mlir @@ -1,7 +1,7 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-linalg-to-loops,convert-scf-to-cf,convert-arith-to-llvm),convert-linalg-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | mlir-cpu-runner -e print_0d -entry-point-result=void -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | FileCheck %s --check-prefix=PRINT-0D -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-linalg-to-loops,convert-scf-to-cf,convert-arith-to-llvm),convert-linalg-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | mlir-cpu-runner -e print_1d -entry-point-result=void -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | FileCheck %s --check-prefix=PRINT-1D -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-linalg-to-loops,convert-scf-to-cf,convert-arith-to-llvm),convert-linalg-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | mlir-cpu-runner -e print_3d -entry-point-result=void -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | FileCheck %s --check-prefix=PRINT-3D -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-linalg-to-loops,convert-scf-to-cf,convert-arith-to-llvm),convert-linalg-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | mlir-cpu-runner -e vector_splat_2d -entry-point-result=void -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | FileCheck %s --check-prefix=PRINT-VECTOR-SPLAT-2D +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-linalg-to-loops,convert-scf-to-cf,convert-arith-to-llvm),convert-linalg-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | mlir-cpu-runner -e print_0d -entry-point-result=void -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | FileCheck %s --check-prefix=PRINT-0D +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-linalg-to-loops,convert-scf-to-cf,convert-arith-to-llvm),convert-linalg-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | mlir-cpu-runner -e print_1d -entry-point-result=void -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | FileCheck %s --check-prefix=PRINT-1D +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-linalg-to-loops,convert-scf-to-cf,convert-arith-to-llvm),convert-linalg-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | mlir-cpu-runner -e print_3d -entry-point-result=void -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | FileCheck %s --check-prefix=PRINT-3D +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-linalg-to-loops,convert-scf-to-cf,convert-arith-to-llvm),convert-linalg-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | mlir-cpu-runner -e vector_splat_2d -entry-point-result=void -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | FileCheck %s --check-prefix=PRINT-VECTOR-SPLAT-2D func.func @print_0d() { %f = arith.constant 2.00000e+00 : f32 diff --git a/mlir/test/mlir-opt/async.mlir b/mlir/test/mlir-opt/async.mlir index d5eafd1c6aef..b1c16b794d8b 100644 --- a/mlir/test/mlir-opt/async.mlir +++ b/mlir/test/mlir-opt/async.mlir @@ -1,6 +1,6 @@ // Check if mlir marks the corresponding function with required coroutine attribute. // -// RUN: mlir-opt %s -pass-pipeline="builtin.module(async-to-async-runtime,func.func(async-runtime-ref-counting,async-runtime-ref-counting-opt),convert-async-to-llvm,func.func(convert-linalg-to-loops,convert-scf-to-cf),convert-linalg-to-llvm,convert-memref-to-llvm,func.func(convert-arith-to-llvm),convert-func-to-llvm,reconcile-unrealized-casts)" \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(async-to-async-runtime,func.func(async-runtime-ref-counting,async-runtime-ref-counting-opt),convert-async-to-llvm,func.func(convert-linalg-to-loops,convert-scf-to-cf),convert-linalg-to-llvm,finalize-memref-to-llvm,func.func(convert-arith-to-llvm),convert-func-to-llvm,reconcile-unrealized-casts)" \ // RUN: | FileCheck %s // CHECK: llvm.func @async_execute_fn{{.*}}attributes{{.*}}presplitcoroutine diff --git a/mlir/test/python/execution_engine.py b/mlir/test/python/execution_engine.py index 7b7ee953ea19..c40d522b8ca2 100644 --- a/mlir/test/python/execution_engine.py +++ b/mlir/test/python/execution_engine.py @@ -63,7 +63,7 @@ def testInvalidModule(): def lowerToLLVM(module): pm = PassManager.parse( - "builtin.module(convert-complex-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)") + "builtin.module(convert-complex-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)") pm.run(module) return module diff --git a/mlir/test/python/integration/dialects/linalg/opsrun.py b/mlir/test/python/integration/dialects/linalg/opsrun.py index 585741ae9336..2a890eafa80f 100644 --- a/mlir/test/python/integration/dialects/linalg/opsrun.py +++ b/mlir/test/python/integration/dialects/linalg/opsrun.py @@ -199,7 +199,7 @@ def transform(module, boilerplate): pm.add("func.func(arith-expand)") pm.add("func.func(memref-expand)") pm.add("convert-vector-to-llvm") - pm.add("convert-memref-to-llvm") + pm.add("finalize-memref-to-llvm") pm.add("convert-func-to-llvm") pm.add("reconcile-unrealized-casts") pm.run(mod) diff --git a/mlir/unittests/ExecutionEngine/Invoke.cpp b/mlir/unittests/ExecutionEngine/Invoke.cpp index 57a260bfc5c8..eeefbfe258a8 100644 --- a/mlir/unittests/ExecutionEngine/Invoke.cpp +++ b/mlir/unittests/ExecutionEngine/Invoke.cpp @@ -52,8 +52,8 @@ static struct LLVMInitializer { /// Simple conversion pipeline for the purpose of testing sources written in /// dialects lowering to LLVM Dialect. static LogicalResult lowerToLLVMDialect(ModuleOp module) { - PassManager pm(module.getContext()); - pm.addPass(mlir::createMemRefToLLVMConversionPass()); + PassManager pm(module->getContext()); + pm.addPass(mlir::createFinalizeMemRefToLLVMConversionPass()); pm.addNestedPass(mlir::createArithToLLVMConversionPass()); pm.addPass(mlir::createConvertFuncToLLVMPass()); pm.addPass(mlir::createReconcileUnrealizedCastsPass()); From f07ea8d9bb3082458fcdd5a682ea501f4f7671b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20B=C3=B6ck?= Date: Mon, 6 Mar 2023 11:11:13 +0100 Subject: [PATCH 084/147] [mlir][MemRef] Add option to to emit opaque pointers This is the first patch in a series of patches part of this RFC: https://discourse.llvm.org/t/rfc-switching-the-llvm-dialect-and-dialect-lowerings-to-opaque-pointers/68179 This patch adds the ability to lower the memref dialect to the LLVM Dialect with the use of opaque pointers instead of typed pointers. The latter are being phased out of LLVM and this patch is part of an effort to phase them out of MLIR as well. To do this, we'll need to support both typed and opaque pointers in lowering passes, to allow downstream projects to change without breakage. The gist of changes required to change a conversion pass are: * Change any `LLVM::LLVMPointerType::get` calls to NOT use an element type if opaque pointers are to be used. * Use the `build` method of `llvm.load` with the explicit result type. Since the pointer does not have an element type anymore it has to be specified explicitly. * Use the `build` method of `llvm.getelementptr` with the explicit `basePtrType`. Ditto to above, we have to now specify what the element type is so that GEP can do its indexing calculations * Use the `build` method of `llvm.alloca` with the explicit `elementType`. Ditto to the above, alloca needs to know how many bytes to allocate through the element type. * Get rid of any `llvm.bitcast`s * Adapt the tests to the above. Note that `llvm.store` changes syntax as well when using opaque pointers (cherry picked from commit 50ea17b849a601bad634d98186155e7526cd4b5c) --- .../Conversion/LLVMCommon/LoweringOptions.h | 1 + .../Conversion/LLVMCommon/MemRefBuilder.h | 19 +- .../Conversion/LLVMCommon/TypeConverter.h | 12 + mlir/include/mlir/Conversion/Passes.td | 5 +- .../mlir/Dialect/LLVMIR/FunctionCallUtils.h | 22 +- .../Conversion/AsyncToLLVM/AsyncToLLVM.cpp | 7 +- .../ControlFlowToLLVM/ControlFlowToLLVM.cpp | 3 +- .../Conversion/LLVMCommon/MemRefBuilder.cpp | 215 ++++++--- mlir/lib/Conversion/LLVMCommon/Pattern.cpp | 36 +- .../Conversion/LLVMCommon/TypeConverter.cpp | 23 +- .../MemRefToLLVM/AllocLikeConversion.cpp | 21 +- .../Conversion/MemRefToLLVM/MemRefToLLVM.cpp | 194 +++++--- .../Dialect/LLVMIR/IR/FunctionCallUtils.cpp | 57 ++- .../MemRefToLLVM/convert-alloca-scope.mlir | 2 +- .../convert-dynamic-memref-ops.mlir | 378 ++++++--------- .../convert-static-memref-ops.mlir | 247 +++++----- .../expand-then-convert-to-llvm.mlir | 434 +++++++++--------- .../MemRefToLLVM/generic-functions.mlir | 12 +- .../MemRefToLLVM/memref-to-llvm.mlir | 318 +++++++------ .../MemRefToLLVM/typed-pointers.mlir | 415 +++++++++++++++++ 20 files changed, 1446 insertions(+), 975 deletions(-) create mode 100644 mlir/test/Conversion/MemRefToLLVM/typed-pointers.mlir diff --git a/mlir/include/mlir/Conversion/LLVMCommon/LoweringOptions.h b/mlir/include/mlir/Conversion/LLVMCommon/LoweringOptions.h index 240a144bc6cd..a3338ebf982c 100644 --- a/mlir/include/mlir/Conversion/LLVMCommon/LoweringOptions.h +++ b/mlir/include/mlir/Conversion/LLVMCommon/LoweringOptions.h @@ -33,6 +33,7 @@ class LowerToLLVMOptions { LowerToLLVMOptions(MLIRContext *ctx, const DataLayout &dl); bool useBarePtrCallConv = false; + bool useOpaquePointers = false; enum class AllocLowering { /// Use malloc for for heap allocations. diff --git a/mlir/include/mlir/Conversion/LLVMCommon/MemRefBuilder.h b/mlir/include/mlir/Conversion/LLVMCommon/MemRefBuilder.h index 2b7735da8466..bf1bd8f7f325 100644 --- a/mlir/include/mlir/Conversion/LLVMCommon/MemRefBuilder.h +++ b/mlir/include/mlir/Conversion/LLVMCommon/MemRefBuilder.h @@ -111,6 +111,8 @@ class MemRefDescriptor : public StructBuilder { static unsigned getNumUnpackedValues(MemRefType type); private: + bool useOpaquePointers(); + // Cached index type. Type indexType; }; @@ -194,36 +196,39 @@ class UnrankedMemRefDescriptor : public StructBuilder { /// Builds IR extracting the allocated pointer from the descriptor. static Value allocatedPtr(OpBuilder &builder, Location loc, - Value memRefDescPtr, Type elemPtrPtrType); + Value memRefDescPtr, + LLVM::LLVMPointerType elemPtrType); /// Builds IR inserting the allocated pointer into the descriptor. static void setAllocatedPtr(OpBuilder &builder, Location loc, - Value memRefDescPtr, Type elemPtrPtrType, + Value memRefDescPtr, + LLVM::LLVMPointerType elemPtrType, Value allocatedPtr); /// Builds IR extracting the aligned pointer from the descriptor. static Value alignedPtr(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, Value memRefDescPtr, - Type elemPtrPtrType); + LLVM::LLVMPointerType elemPtrType); /// Builds IR inserting the aligned pointer into the descriptor. static void setAlignedPtr(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, - Value memRefDescPtr, Type elemPtrPtrType, + Value memRefDescPtr, + LLVM::LLVMPointerType elemPtrType, Value alignedPtr); /// Builds IR extracting the offset from the descriptor. static Value offset(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, Value memRefDescPtr, - Type elemPtrPtrType); + LLVM::LLVMPointerType elemPtrType); /// Builds IR inserting the offset into the descriptor. static void setOffset(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, Value memRefDescPtr, - Type elemPtrPtrType, Value offset); + LLVM::LLVMPointerType elemPtrType, Value offset); /// Builds IR extracting the pointer to the first element of the size array. static Value sizeBasePtr(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, Value memRefDescPtr, - LLVM::LLVMPointerType elemPtrPtrType); + LLVM::LLVMPointerType elemPtrType); /// Builds IR extracting the size[index] from the descriptor. static Value size(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, Value sizeBasePtr, diff --git a/mlir/include/mlir/Conversion/LLVMCommon/TypeConverter.h b/mlir/include/mlir/Conversion/LLVMCommon/TypeConverter.h index f9c1511948a2..57ef6f48ba09 100644 --- a/mlir/include/mlir/Conversion/LLVMCommon/TypeConverter.h +++ b/mlir/include/mlir/Conversion/LLVMCommon/TypeConverter.h @@ -24,6 +24,7 @@ class LowerToLLVMOptions; namespace LLVM { class LLVMDialect; +class LLVMPointerType; } // namespace LLVM /// Conversion from types to the LLVM IR dialect. @@ -119,6 +120,17 @@ class LLVMTypeConverter : public TypeConverter { /// integer type with the size configured for this type converter. Type getIndexType(); + /// Returns true if using opaque pointers was enabled in the lowering options. + bool useOpaquePointers() const { return getOptions().useOpaquePointers; } + + /// Creates an LLVM pointer type with the given element type and address + /// space. + /// This function is meant to be used in code supporting both typed and opaque + /// pointers, as it will create an opaque pointer with the given address space + /// if opaque pointers are enabled in the lowering options. + LLVM::LLVMPointerType getPointerType(Type elementType, + unsigned addressSpace = 0); + /// Gets the bitwidth of the index type when converted to LLVM. unsigned getIndexTypeBitwidth() { return options.getIndexBitwidth(); } diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td index 86b9a208bbeb..d7d60126ce78 100644 --- a/mlir/include/mlir/Conversion/Passes.td +++ b/mlir/include/mlir/Conversion/Passes.td @@ -608,7 +608,10 @@ def FinalizeMemRefToLLVMConversionPass : "bool", /*default=*/"false", "Use generic allocation and deallocation functions instead of the " - "classic 'malloc', 'aligned_alloc' and 'free' functions"> + "classic 'malloc', 'aligned_alloc' and 'free' functions">, + Option<"useOpaquePointers", "use-opaque-pointers", "bool", + /*default=*/"false", "Generate LLVM IR using opaque pointers " + "instead of typed pointers"> ]; } diff --git a/mlir/include/mlir/Dialect/LLVMIR/FunctionCallUtils.h b/mlir/include/mlir/Dialect/LLVMIR/FunctionCallUtils.h index 3421e9bb67b6..76838ddf7a63 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/FunctionCallUtils.h +++ b/mlir/include/mlir/Dialect/LLVMIR/FunctionCallUtils.h @@ -36,20 +36,24 @@ LLVM::LLVMFuncOp lookupOrCreatePrintI64Fn(ModuleOp moduleOp); LLVM::LLVMFuncOp lookupOrCreatePrintU64Fn(ModuleOp moduleOp); LLVM::LLVMFuncOp lookupOrCreatePrintF32Fn(ModuleOp moduleOp); LLVM::LLVMFuncOp lookupOrCreatePrintF64Fn(ModuleOp moduleOp); -LLVM::LLVMFuncOp lookupOrCreatePrintStrFn(ModuleOp moduleOp); +LLVM::LLVMFuncOp lookupOrCreatePrintStrFn(ModuleOp moduleOp, + bool opaquePointers); LLVM::LLVMFuncOp lookupOrCreatePrintOpenFn(ModuleOp moduleOp); LLVM::LLVMFuncOp lookupOrCreatePrintCloseFn(ModuleOp moduleOp); LLVM::LLVMFuncOp lookupOrCreatePrintCommaFn(ModuleOp moduleOp); LLVM::LLVMFuncOp lookupOrCreatePrintNewlineFn(ModuleOp moduleOp); -LLVM::LLVMFuncOp lookupOrCreateMallocFn(ModuleOp moduleOp, Type indexType); -LLVM::LLVMFuncOp lookupOrCreateAlignedAllocFn(ModuleOp moduleOp, - Type indexType); -LLVM::LLVMFuncOp lookupOrCreateFreeFn(ModuleOp moduleOp); -LLVM::LLVMFuncOp lookupOrCreateGenericAllocFn(ModuleOp moduleOp, - Type indexType); +LLVM::LLVMFuncOp lookupOrCreateMallocFn(ModuleOp moduleOp, Type indexType, + bool opaquePointers); +LLVM::LLVMFuncOp lookupOrCreateAlignedAllocFn(ModuleOp moduleOp, Type indexType, + bool opaquePointers); +LLVM::LLVMFuncOp lookupOrCreateFreeFn(ModuleOp moduleOp, bool opaquePointers); +LLVM::LLVMFuncOp lookupOrCreateGenericAllocFn(ModuleOp moduleOp, Type indexType, + bool opaquePointers); LLVM::LLVMFuncOp lookupOrCreateGenericAlignedAllocFn(ModuleOp moduleOp, - Type indexType); -LLVM::LLVMFuncOp lookupOrCreateGenericFreeFn(ModuleOp moduleOp); + Type indexType, + bool opaquePointers); +LLVM::LLVMFuncOp lookupOrCreateGenericFreeFn(ModuleOp moduleOp, + bool opaquePointers); LLVM::LLVMFuncOp lookupOrCreateMemRefCopyFn(ModuleOp moduleOp, Type indexType, Type unrankedDescriptorType); diff --git a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp index 7abc2141b6dc..628127757cdf 100644 --- a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp +++ b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp @@ -378,7 +378,8 @@ class CoroBeginOpConversion : public OpConversionPattern { // Allocate memory for the coroutine frame. auto allocFuncOp = LLVM::lookupOrCreateAlignedAllocFn( - op->getParentOfType(), rewriter.getI64Type()); + op->getParentOfType(), rewriter.getI64Type(), + /*TODO: opaquePointers=*/false); auto coroAlloc = rewriter.create( loc, allocFuncOp, ValueRange{coroAlign, coroSize}); @@ -412,8 +413,8 @@ class CoroFreeOpConversion : public OpConversionPattern { rewriter.create(loc, i8Ptr, adaptor.getOperands()); // Free the memory. - auto freeFuncOp = - LLVM::lookupOrCreateFreeFn(op->getParentOfType()); + auto freeFuncOp = LLVM::lookupOrCreateFreeFn( + op->getParentOfType(), /*TODO: opaquePointers=*/false); rewriter.replaceOpWithNewOp(op, freeFuncOp, ValueRange(coroMem.getResult())); diff --git a/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp b/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp index 2be87e6490f5..bcfbfdc20af3 100644 --- a/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp +++ b/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp @@ -72,7 +72,8 @@ static void createPrintMsg(OpBuilder &builder, Location loc, ModuleOp moduleOp, SmallVector indices(1, 0); Value gep = builder.create( loc, LLVM::LLVMPointerType::get(builder.getI8Type()), msgAddr, indices); - Operation *printer = LLVM::lookupOrCreatePrintStrFn(moduleOp); + Operation *printer = + LLVM::lookupOrCreatePrintStrFn(moduleOp, /*TODO: opaquePointers=*/false); builder.create(loc, TypeRange(), SymbolRefAttr::get(printer), gep); } diff --git a/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp b/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp index f808e3dabf5c..8ce9fb69ee6d 100644 --- a/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp +++ b/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp @@ -132,22 +132,30 @@ Value MemRefDescriptor::size(OpBuilder &builder, Location loc, unsigned pos) { Value MemRefDescriptor::size(OpBuilder &builder, Location loc, Value pos, int64_t rank) { - auto indexPtrTy = LLVM::LLVMPointerType::get(indexType); auto arrayTy = LLVM::LLVMArrayType::get(indexType, rank); - auto arrayPtrTy = LLVM::LLVMPointerType::get(arrayTy); + + LLVM::LLVMPointerType indexPtrTy; + LLVM::LLVMPointerType arrayPtrTy; + + if (useOpaquePointers()) { + arrayPtrTy = indexPtrTy = LLVM::LLVMPointerType::get(builder.getContext()); + } else { + indexPtrTy = LLVM::LLVMPointerType::get(indexType); + arrayPtrTy = LLVM::LLVMPointerType::get(arrayTy); + } // Copy size values to stack-allocated memory. auto one = createIndexAttrConstant(builder, loc, indexType, 1); auto sizes = builder.create( loc, value, llvm::ArrayRef({kSizePosInMemRefDescriptor})); - auto sizesPtr = - builder.create(loc, arrayPtrTy, one, /*alignment=*/0); + auto sizesPtr = builder.create(loc, arrayPtrTy, arrayTy, one, + /*alignment=*/0); builder.create(loc, sizes, sizesPtr); // Load an return size value of interest. - auto resultPtr = builder.create(loc, indexPtrTy, sizesPtr, - ArrayRef{0, pos}); - return builder.create(loc, resultPtr); + auto resultPtr = builder.create( + loc, indexPtrTy, arrayTy, sizesPtr, ArrayRef{0, pos}); + return builder.create(loc, indexType, resultPtr); } /// Builds IR inserting the pos-th size into the descriptor @@ -242,6 +250,10 @@ unsigned MemRefDescriptor::getNumUnpackedValues(MemRefType type) { return 3 + 2 * type.getRank(); } +bool MemRefDescriptor::useOpaquePointers() { + return getElementPtrType().isOpaque(); +} + //===----------------------------------------------------------------------===// // MemRefDescriptorView implementation. //===----------------------------------------------------------------------===// @@ -372,134 +384,195 @@ void UnrankedMemRefDescriptor::computeSizes( } } -Value UnrankedMemRefDescriptor::allocatedPtr(OpBuilder &builder, Location loc, - Value memRefDescPtr, - Type elemPtrPtrType) { +Value UnrankedMemRefDescriptor::allocatedPtr( + OpBuilder &builder, Location loc, Value memRefDescPtr, + LLVM::LLVMPointerType elemPtrType) { + + Value elementPtrPtr; + if (elemPtrType.isOpaque()) + elementPtrPtr = memRefDescPtr; + else + elementPtrPtr = builder.create( + loc, LLVM::LLVMPointerType::get(elemPtrType), memRefDescPtr); - Value elementPtrPtr = - builder.create(loc, elemPtrPtrType, memRefDescPtr); - return builder.create(loc, elementPtrPtr); + return builder.create(loc, elemPtrType, elementPtrPtr); } -void UnrankedMemRefDescriptor::setAllocatedPtr(OpBuilder &builder, Location loc, - Value memRefDescPtr, - Type elemPtrPtrType, - Value allocatedPtr) { - Value elementPtrPtr = - builder.create(loc, elemPtrPtrType, memRefDescPtr); +void UnrankedMemRefDescriptor::setAllocatedPtr( + OpBuilder &builder, Location loc, Value memRefDescPtr, + LLVM::LLVMPointerType elemPtrType, Value allocatedPtr) { + Value elementPtrPtr; + if (elemPtrType.isOpaque()) + elementPtrPtr = memRefDescPtr; + else + elementPtrPtr = builder.create( + loc, LLVM::LLVMPointerType::get(elemPtrType), memRefDescPtr); + builder.create(loc, allocatedPtr, elementPtrPtr); } +static std::pair +castToElemPtrPtr(OpBuilder &builder, Location loc, Value memRefDescPtr, + LLVM::LLVMPointerType elemPtrType) { + Value elementPtrPtr; + Type elemPtrPtrType; + if (elemPtrType.isOpaque()) { + elementPtrPtr = memRefDescPtr; + elemPtrPtrType = LLVM::LLVMPointerType::get(builder.getContext()); + } else { + elemPtrPtrType = LLVM::LLVMPointerType::get(elemPtrType); + elementPtrPtr = + builder.create(loc, elemPtrPtrType, memRefDescPtr); + } + return {elementPtrPtr, elemPtrPtrType}; +} + Value UnrankedMemRefDescriptor::alignedPtr(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, Value memRefDescPtr, - Type elemPtrPtrType) { - Value elementPtrPtr = - builder.create(loc, elemPtrPtrType, memRefDescPtr); + LLVM::LLVMPointerType elemPtrType) { + auto [elementPtrPtr, elemPtrPtrType] = + castToElemPtrPtr(builder, loc, memRefDescPtr, elemPtrType); - Value alignedGep = builder.create( - loc, elemPtrPtrType, elementPtrPtr, ArrayRef{1}); - return builder.create(loc, alignedGep); + Value alignedGep = + builder.create(loc, elemPtrPtrType, elemPtrType, + elementPtrPtr, ArrayRef{1}); + return builder.create(loc, elemPtrType, alignedGep); } void UnrankedMemRefDescriptor::setAlignedPtr(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, Value memRefDescPtr, - Type elemPtrPtrType, + LLVM::LLVMPointerType elemPtrType, Value alignedPtr) { - Value elementPtrPtr = - builder.create(loc, elemPtrPtrType, memRefDescPtr); + auto [elementPtrPtr, elemPtrPtrType] = + castToElemPtrPtr(builder, loc, memRefDescPtr, elemPtrType); - Value alignedGep = builder.create( - loc, elemPtrPtrType, elementPtrPtr, ArrayRef{1}); + Value alignedGep = + builder.create(loc, elemPtrPtrType, elemPtrType, + elementPtrPtr, ArrayRef{1}); builder.create(loc, alignedPtr, alignedGep); } Value UnrankedMemRefDescriptor::offset(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, Value memRefDescPtr, - Type elemPtrPtrType) { - Value elementPtrPtr = - builder.create(loc, elemPtrPtrType, memRefDescPtr); + LLVM::LLVMPointerType elemPtrType) { + auto [elementPtrPtr, elemPtrPtrType] = + castToElemPtrPtr(builder, loc, memRefDescPtr, elemPtrType); + + Value offsetGep = + builder.create(loc, elemPtrPtrType, elemPtrType, + elementPtrPtr, ArrayRef{2}); + + if (!elemPtrType.isOpaque()) { + offsetGep = builder.create( + loc, LLVM::LLVMPointerType::get(typeConverter.getIndexType()), + offsetGep); + } - Value offsetGep = builder.create( - loc, elemPtrPtrType, elementPtrPtr, ArrayRef{2}); - offsetGep = builder.create( - loc, LLVM::LLVMPointerType::get(typeConverter.getIndexType()), offsetGep); - return builder.create(loc, offsetGep); + return builder.create(loc, typeConverter.getIndexType(), + offsetGep); } void UnrankedMemRefDescriptor::setOffset(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, Value memRefDescPtr, - Type elemPtrPtrType, Value offset) { - Value elementPtrPtr = - builder.create(loc, elemPtrPtrType, memRefDescPtr); - - Value offsetGep = builder.create( - loc, elemPtrPtrType, elementPtrPtr, ArrayRef{2}); - offsetGep = builder.create( - loc, LLVM::LLVMPointerType::get(typeConverter.getIndexType()), offsetGep); + LLVM::LLVMPointerType elemPtrType, + Value offset) { + auto [elementPtrPtr, elemPtrPtrType] = + castToElemPtrPtr(builder, loc, memRefDescPtr, elemPtrType); + + Value offsetGep = + builder.create(loc, elemPtrPtrType, elemPtrType, + elementPtrPtr, ArrayRef{2}); + + if (!elemPtrType.isOpaque()) { + offsetGep = builder.create( + loc, LLVM::LLVMPointerType::get(typeConverter.getIndexType()), + offsetGep); + } + builder.create(loc, offset, offsetGep); } -Value UnrankedMemRefDescriptor::sizeBasePtr( - OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, - Value memRefDescPtr, LLVM::LLVMPointerType elemPtrPtrType) { - Type elemPtrTy = elemPtrPtrType.getElementType(); +Value UnrankedMemRefDescriptor::sizeBasePtr(OpBuilder &builder, Location loc, + LLVMTypeConverter &typeConverter, + Value memRefDescPtr, + LLVM::LLVMPointerType elemPtrType) { Type indexTy = typeConverter.getIndexType(); - Type structPtrTy = - LLVM::LLVMPointerType::get(LLVM::LLVMStructType::getLiteral( - indexTy.getContext(), {elemPtrTy, elemPtrTy, indexTy, indexTy})); - Value structPtr = - builder.create(loc, structPtrTy, memRefDescPtr); + Type structTy = LLVM::LLVMStructType::getLiteral( + indexTy.getContext(), {elemPtrType, elemPtrType, indexTy, indexTy}); + Value structPtr; + if (elemPtrType.isOpaque()) { + structPtr = memRefDescPtr; + } else { + Type structPtrTy = LLVM::LLVMPointerType::get(structTy); + structPtr = + builder.create(loc, structPtrTy, memRefDescPtr); + } - return builder.create(loc, LLVM::LLVMPointerType::get(indexTy), - structPtr, ArrayRef{0, 3}); + auto resultType = elemPtrType.isOpaque() + ? LLVM::LLVMPointerType::get(indexTy.getContext()) + : LLVM::LLVMPointerType::get(indexTy); + return builder.create(loc, resultType, structTy, structPtr, + ArrayRef{0, 3}); } Value UnrankedMemRefDescriptor::size(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, Value sizeBasePtr, Value index) { - Type indexPtrTy = LLVM::LLVMPointerType::get(typeConverter.getIndexType()); + + Type indexTy = typeConverter.getIndexType(); + Type indexPtrTy = typeConverter.getPointerType(indexTy); + Value sizeStoreGep = - builder.create(loc, indexPtrTy, sizeBasePtr, index); - return builder.create(loc, sizeStoreGep); + builder.create(loc, indexPtrTy, indexTy, sizeBasePtr, index); + return builder.create(loc, indexTy, sizeStoreGep); } void UnrankedMemRefDescriptor::setSize(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, Value sizeBasePtr, Value index, Value size) { - Type indexPtrTy = LLVM::LLVMPointerType::get(typeConverter.getIndexType()); + Type indexTy = typeConverter.getIndexType(); + Type indexPtrTy = typeConverter.getPointerType(indexTy); + Value sizeStoreGep = - builder.create(loc, indexPtrTy, sizeBasePtr, index); + builder.create(loc, indexPtrTy, indexTy, sizeBasePtr, index); builder.create(loc, size, sizeStoreGep); } Value UnrankedMemRefDescriptor::strideBasePtr(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, Value sizeBasePtr, Value rank) { - Type indexPtrTy = LLVM::LLVMPointerType::get(typeConverter.getIndexType()); - return builder.create(loc, indexPtrTy, sizeBasePtr, rank); + Type indexTy = typeConverter.getIndexType(); + Type indexPtrTy = typeConverter.getPointerType(indexTy); + + return builder.create(loc, indexPtrTy, indexTy, sizeBasePtr, + rank); } Value UnrankedMemRefDescriptor::stride(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, Value strideBasePtr, Value index, Value stride) { - Type indexPtrTy = LLVM::LLVMPointerType::get(typeConverter.getIndexType()); - Value strideStoreGep = - builder.create(loc, indexPtrTy, strideBasePtr, index); - return builder.create(loc, strideStoreGep); + Type indexTy = typeConverter.getIndexType(); + Type indexPtrTy = typeConverter.getPointerType(indexTy); + + Value strideStoreGep = builder.create(loc, indexPtrTy, indexTy, + strideBasePtr, index); + return builder.create(loc, indexTy, strideStoreGep); } void UnrankedMemRefDescriptor::setStride(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, Value strideBasePtr, Value index, Value stride) { - Type indexPtrTy = LLVM::LLVMPointerType::get(typeConverter.getIndexType()); - Value strideStoreGep = - builder.create(loc, indexPtrTy, strideBasePtr, index); + Type indexTy = typeConverter.getIndexType(); + Type indexPtrTy = typeConverter.getPointerType(indexTy); + + Value strideStoreGep = builder.create(loc, indexPtrTy, indexTy, + strideBasePtr, index); builder.create(loc, stride, strideStoreGep); } diff --git a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp index 6c28022e1a07..f66f2e21c705 100644 --- a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp +++ b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp @@ -48,7 +48,7 @@ Type ConvertToLLVMPattern::getVoidType() const { } Type ConvertToLLVMPattern::getVoidPtrType() const { - return LLVM::LLVMPointerType::get( + return getTypeConverter()->getPointerType( IntegerType::get(&getTypeConverter()->getContext(), 8)); } @@ -93,7 +93,10 @@ Value ConvertToLLVMPattern::getStridedElementPtr( } Type elementPtrType = memRefDescriptor.getElementPtrType(); - return index ? rewriter.create(loc, elementPtrType, base, index) + return index ? rewriter.create( + loc, elementPtrType, + getTypeConverter()->convertType(type.getElementType()), + base, index) : base; } @@ -109,8 +112,8 @@ bool ConvertToLLVMPattern::isConvertibleAndHasIdentityMaps( Type ConvertToLLVMPattern::getElementPtrType(MemRefType type) const { auto elementType = type.getElementType(); auto structElementType = typeConverter->convertType(elementType); - return LLVM::LLVMPointerType::get(structElementType, - type.getMemorySpaceAsInt()); + return getTypeConverter()->getPointerType(structElementType, + type.getMemorySpaceAsInt()); } void ConvertToLLVMPattern::getMemRefDescriptorSizes( @@ -157,10 +160,11 @@ void ConvertToLLVMPattern::getMemRefDescriptorSizes( } // Buffer size in bytes. - Type elementPtrType = getElementPtrType(memRefType); + Type elementType = typeConverter->convertType(memRefType.getElementType()); + Type elementPtrType = getTypeConverter()->getPointerType(elementType); Value nullPtr = rewriter.create(loc, elementPtrType); - Value gepPtr = - rewriter.create(loc, elementPtrType, nullPtr, runningStride); + Value gepPtr = rewriter.create(loc, elementPtrType, elementType, + nullPtr, runningStride); sizeBytes = rewriter.create(loc, getIndexType(), gepPtr); } @@ -171,11 +175,11 @@ Value ConvertToLLVMPattern::getSizeInBytes( // %0 = getelementptr %elementType* null, %indexType 1 // %1 = ptrtoint %elementType* %0 to %indexType // which is a common pattern of getting the size of a type in bytes. - auto convertedPtrType = - LLVM::LLVMPointerType::get(typeConverter->convertType(type)); + Type llvmType = typeConverter->convertType(type); + auto convertedPtrType = getTypeConverter()->getPointerType(llvmType); auto nullPtr = rewriter.create(loc, convertedPtrType); - auto gep = rewriter.create(loc, convertedPtrType, nullPtr, - ArrayRef{1}); + auto gep = rewriter.create(loc, convertedPtrType, llvmType, + nullPtr, ArrayRef{1}); return rewriter.create(loc, getIndexType(), gep); } @@ -241,7 +245,6 @@ LogicalResult ConvertToLLVMPattern::copyUnrankedDescriptors( // Get frequently used types. MLIRContext *context = builder.getContext(); - Type voidPtrType = LLVM::LLVMPointerType::get(IntegerType::get(context, 8)); auto i1Type = IntegerType::get(context, 1); Type indexType = getTypeConverter()->getIndexType(); @@ -249,9 +252,11 @@ LogicalResult ConvertToLLVMPattern::copyUnrankedDescriptors( auto module = builder.getInsertionPoint()->getParentOfType(); LLVM::LLVMFuncOp freeFunc, mallocFunc; if (toDynamic) - mallocFunc = LLVM::lookupOrCreateMallocFn(module, indexType); + mallocFunc = LLVM::lookupOrCreateMallocFn( + module, indexType, getTypeConverter()->useOpaquePointers()); if (!toDynamic) - freeFunc = LLVM::lookupOrCreateFreeFn(module); + freeFunc = LLVM::lookupOrCreateFreeFn( + module, getTypeConverter()->useOpaquePointers()); // Initialize shared constants. Value zero = @@ -270,7 +275,8 @@ LogicalResult ConvertToLLVMPattern::copyUnrankedDescriptors( toDynamic ? builder.create(loc, mallocFunc, allocationSize) .getResult() - : builder.create(loc, voidPtrType, allocationSize, + : builder.create(loc, getVoidPtrType(), + getVoidType(), allocationSize, /*alignment=*/0); Value source = desc.memRefDescPtr(builder, loc); builder.create(loc, memory, source, allocationSize, zero); diff --git a/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp b/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp index f5b35b794bf1..84496d998457 100644 --- a/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp +++ b/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp @@ -169,6 +169,13 @@ Type LLVMTypeConverter::getIndexType() { return IntegerType::get(&getContext(), getIndexTypeBitwidth()); } +LLVM::LLVMPointerType +LLVMTypeConverter::getPointerType(Type elementType, unsigned int addressSpace) { + if (useOpaquePointers()) + return LLVM::LLVMPointerType::get(&getContext(), addressSpace); + return LLVM::LLVMPointerType::get(elementType, addressSpace); +} + unsigned LLVMTypeConverter::getPointerBitwidth(unsigned addressSpace) { return options.dataLayout.getPointerSizeInBits(addressSpace); } @@ -201,7 +208,7 @@ Type LLVMTypeConverter::convertFunctionType(FunctionType type) { convertFunctionSignature(type, /*isVariadic=*/false, conversion); if (!converted) return {}; - return LLVM::LLVMPointerType::get(converted); + return getPointerType(converted); } // Function types are converted to LLVM Function types by recursively converting @@ -311,8 +318,9 @@ LLVMTypeConverter::getMemRefDescriptorFields(MemRefType type, Type elementType = convertType(type.getElementType()); if (!elementType) return {}; - auto ptrTy = - LLVM::LLVMPointerType::get(elementType, type.getMemorySpaceAsInt()); + + LLVM::LLVMPointerType ptrTy = + getPointerType(elementType, type.getMemorySpaceAsInt()); auto indexTy = getIndexType(); SmallVector results = {ptrTy, ptrTy, indexTy}; @@ -355,8 +363,7 @@ Type LLVMTypeConverter::convertMemRefType(MemRefType type) { /// stack allocated (alloca) copy of a MemRef descriptor that got casted to /// be unranked. SmallVector LLVMTypeConverter::getUnrankedMemRefDescriptorFields() { - return {getIndexType(), - LLVM::LLVMPointerType::get(IntegerType::get(&getContext(), 8))}; + return {getIndexType(), getPointerType(IntegerType::get(&getContext(), 8))}; } unsigned @@ -406,7 +413,7 @@ Type LLVMTypeConverter::convertMemRefToBarePtr(BaseMemRefType type) { Type elementType = convertType(type.getElementType()); if (!elementType) return {}; - return LLVM::LLVMPointerType::get(elementType, type.getMemorySpaceAsInt()); + return getPointerType(elementType, type.getMemorySpaceAsInt()); } /// Convert an n-D vector type to an LLVM vector type: @@ -483,11 +490,11 @@ Value LLVMTypeConverter::promoteOneMemRefDescriptor(Location loc, Value operand, OpBuilder &builder) { // Alloca with proper alignment. We do not expect optimizations of this // alloca op and so we omit allocating at the entry block. - auto ptrType = LLVM::LLVMPointerType::get(operand.getType()); + auto ptrType = getPointerType(operand.getType()); Value one = builder.create(loc, builder.getI64Type(), builder.getIndexAttr(1)); Value allocated = - builder.create(loc, ptrType, one, /*alignment=*/0); + builder.create(loc, ptrType, operand.getType(), one); // Store into the alloca'ed descriptor. builder.create(loc, operand, allocated); return allocated; diff --git a/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp b/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp index 8d99e1ffed9b..4afe2ffebb34 100644 --- a/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp +++ b/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp @@ -23,9 +23,11 @@ LLVM::LLVMFuncOp getNotalignedAllocFn(LLVMTypeConverter *typeConverter, bool useGenericFn = typeConverter->getOptions().useGenericFunctions; if (useGenericFn) - return LLVM::lookupOrCreateGenericAllocFn(module, indexType); + return LLVM::lookupOrCreateGenericAllocFn( + module, indexType, typeConverter->useOpaquePointers()); - return LLVM::lookupOrCreateMallocFn(module, indexType); + return LLVM::lookupOrCreateMallocFn(module, indexType, + typeConverter->useOpaquePointers()); } LLVM::LLVMFuncOp getAlignedAllocFn(LLVMTypeConverter *typeConverter, @@ -33,9 +35,11 @@ LLVM::LLVMFuncOp getAlignedAllocFn(LLVMTypeConverter *typeConverter, bool useGenericFn = typeConverter->getOptions().useGenericFunctions; if (useGenericFn) - return LLVM::lookupOrCreateGenericAlignedAllocFn(module, indexType); + return LLVM::lookupOrCreateGenericAlignedAllocFn( + module, indexType, typeConverter->useOpaquePointers()); - return LLVM::lookupOrCreateAlignedAllocFn(module, indexType); + return LLVM::lookupOrCreateAlignedAllocFn(module, indexType, + typeConverter->useOpaquePointers()); } } // end namespace @@ -58,12 +62,13 @@ static Value castAllocFuncResult(ConversionPatternRewriter &rewriter, if (allocatedPtrTy.getAddressSpace() != memRefType.getMemorySpaceAsInt()) allocatedPtr = rewriter.create( loc, - LLVM::LLVMPointerType::get(allocatedPtrTy.getElementType(), - memRefType.getMemorySpaceAsInt()), + typeConverter.getPointerType(allocatedPtrTy.getElementType(), + memRefType.getMemorySpaceAsInt()), allocatedPtr); - allocatedPtr = - rewriter.create(loc, elementPtrType, allocatedPtr); + if (!typeConverter.useOpaquePointers()) + allocatedPtr = + rewriter.create(loc, elementPtrType, allocatedPtr); return allocatedPtr; } diff --git a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp index d425658a0ecf..d6097f057db4 100644 --- a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp +++ b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp @@ -41,9 +41,10 @@ LLVM::LLVMFuncOp getFreeFn(LLVMTypeConverter *typeConverter, ModuleOp module) { bool useGenericFn = typeConverter->getOptions().useGenericFunctions; if (useGenericFn) - return LLVM::lookupOrCreateGenericFreeFn(module); + return LLVM::lookupOrCreateGenericFreeFn( + module, typeConverter->useOpaquePointers()); - return LLVM::lookupOrCreateFreeFn(module); + return LLVM::lookupOrCreateFreeFn(module, typeConverter->useOpaquePointers()); } struct AllocOpLowering : public AllocLikeOpLLVMLowering { @@ -93,10 +94,14 @@ struct AllocaOpLowering : public AllocLikeOpLLVMLowering { // With alloca, one gets a pointer to the element type right away. // For stack allocations. auto allocaOp = cast(op); - auto elementPtrType = this->getElementPtrType(allocaOp.getType()); + auto elementType = + typeConverter->convertType(allocaOp.getType().getElementType()); + auto elementPtrType = getTypeConverter()->getPointerType( + elementType, allocaOp.getType().getMemorySpaceAsInt()); auto allocatedElementPtr = rewriter.create( - loc, elementPtrType, sizeBytes, allocaOp.getAlignment().value_or(0)); + loc, elementPtrType, elementType, sizeBytes, + allocaOp.getAlignment().value_or(0)); return std::make_tuple(allocatedElementPtr, allocatedElementPtr); } @@ -205,6 +210,8 @@ struct ReallocOpLoweringBase : public AllocationOpLLVMLowering { Value isVolatile = rewriter.create(loc, rewriter.getBoolAttr(false)); auto toVoidPtr = [&](Value ptr) -> Value { + if (getTypeConverter()->useOpaquePointers()) + return ptr; return rewriter.create(loc, getVoidPtrType(), ptr); }; rewriter.create(loc, toVoidPtr(dstAlignedPtr), @@ -372,9 +379,12 @@ struct DeallocOpLowering : public ConvertOpToLLVMPattern { LLVM::LLVMFuncOp freeFunc = getFreeFn(getTypeConverter(), op->getParentOfType()); MemRefDescriptor memref(adaptor.getMemref()); - Value casted = rewriter.create( - op.getLoc(), getVoidPtrType(), - memref.allocatedPtr(rewriter, op.getLoc())); + Value allocatedPtr = memref.allocatedPtr(rewriter, op.getLoc()); + Value casted = allocatedPtr; + if (!getTypeConverter()->useOpaquePointers()) + casted = rewriter.create(op.getLoc(), getVoidPtrType(), + allocatedPtr); + rewriter.replaceOpWithNewOp(op, freeFunc, casted); return success(); } @@ -421,25 +431,32 @@ struct DimOpLowering : public ConvertOpToLLVMPattern { // operations. UnrankedMemRefDescriptor unrankedDesc(adaptor.getSource()); Value underlyingRankedDesc = unrankedDesc.memRefDescPtr(rewriter, loc); - Value scalarMemRefDescPtr = rewriter.create( - loc, - LLVM::LLVMPointerType::get(typeConverter->convertType(scalarMemRefType), - addressSpace), - underlyingRankedDesc); + + Type elementType = typeConverter->convertType(scalarMemRefType); + Value scalarMemRefDescPtr; + if (getTypeConverter()->useOpaquePointers()) + scalarMemRefDescPtr = underlyingRankedDesc; + else + scalarMemRefDescPtr = rewriter.create( + loc, LLVM::LLVMPointerType::get(elementType, addressSpace), + underlyingRankedDesc); // Get pointer to offset field of memref descriptor. - Type indexPtrTy = LLVM::LLVMPointerType::get( + Type indexPtrTy = getTypeConverter()->getPointerType( getTypeConverter()->getIndexType(), addressSpace); Value offsetPtr = rewriter.create( - loc, indexPtrTy, scalarMemRefDescPtr, ArrayRef{0, 2}); + loc, indexPtrTy, elementType, scalarMemRefDescPtr, + ArrayRef{0, 2}); // The size value that we have to extract can be obtained using GEPop with // `dimOp.index() + 1` index argument. Value idxPlusOne = rewriter.create( loc, createIndexConstant(rewriter, loc, 1), adaptor.getIndex()); - Value sizePtr = - rewriter.create(loc, indexPtrTy, offsetPtr, idxPlusOne); - return rewriter.create(loc, sizePtr); + Value sizePtr = rewriter.create( + loc, indexPtrTy, getTypeConverter()->getIndexType(), offsetPtr, + idxPlusOne); + return rewriter.create( + loc, getTypeConverter()->getIndexType(), sizePtr); } std::optional getConstantDimIndex(memref::DimOp dimOp) const { @@ -549,7 +566,8 @@ struct GenericAtomicRMWOpLowering auto memRefType = atomicOp.getMemref().getType().cast(); auto dataPtr = getStridedElementPtr(loc, memRefType, adaptor.getMemref(), adaptor.getIndices(), rewriter); - Value init = rewriter.create(loc, dataPtr); + Value init = rewriter.create( + loc, typeConverter->convertType(memRefType.getElementType()), dataPtr); rewriter.create(loc, init, loopBlock); // Prepare the body of the loop block. @@ -686,17 +704,18 @@ struct GetGlobalMemrefOpLowering : public AllocLikeOpLLVMLowering { unsigned memSpace = type.getMemorySpaceAsInt(); Type arrayTy = convertGlobalMemrefTypeToLLVM(type, *getTypeConverter()); - auto addressOf = rewriter.create( - loc, LLVM::LLVMPointerType::get(arrayTy, memSpace), - getGlobalOp.getName()); + Type resTy = getTypeConverter()->getPointerType(arrayTy, memSpace); + auto addressOf = + rewriter.create(loc, resTy, getGlobalOp.getName()); // Get the address of the first element in the array by creating a GEP with // the address of the GV as the base, and (rank + 1) number of 0 indices. Type elementType = typeConverter->convertType(type.getElementType()); - Type elementPtrType = LLVM::LLVMPointerType::get(elementType, memSpace); + Type elementPtrType = + getTypeConverter()->getPointerType(elementType, memSpace); auto gep = rewriter.create( - loc, elementPtrType, addressOf, + loc, elementPtrType, arrayTy, addressOf, SmallVector(type.getRank() + 1, 0)); // We do not expect the memref obtained using `memref.get_global` to be @@ -727,8 +746,9 @@ struct LoadOpLowering : public LoadStoreOpLowering { Value dataPtr = getStridedElementPtr(loadOp.getLoc(), type, adaptor.getMemref(), adaptor.getIndices(), rewriter); - rewriter.replaceOpWithNewOp(loadOp, dataPtr, 0, false, - loadOp.getNontemporal()); + rewriter.replaceOpWithNewOp( + loadOp, typeConverter->convertType(type.getElementType()), dataPtr, 0, + false, loadOp.getNontemporal()); return success(); } }; @@ -850,10 +870,14 @@ struct MemRefCastOpLowering : public ConvertOpToLLVMPattern { // ptr = AllocaOp sizeof(MemRefDescriptor) auto ptr = getTypeConverter()->promoteOneMemRefDescriptor( loc, adaptor.getSource(), rewriter); + // voidptr = BitCastOp srcType* to void* - auto voidPtr = - rewriter.create(loc, getVoidPtrType(), ptr) - .getResult(); + Value voidPtr; + if (getTypeConverter()->useOpaquePointers()) + voidPtr = ptr; + else + voidPtr = rewriter.create(loc, getVoidPtrType(), ptr); + // rank = ConstantOp srcRank auto rankVal = rewriter.create( loc, getIndexType(), rewriter.getIndexAttr(rank)); @@ -874,13 +898,16 @@ struct MemRefCastOpLowering : public ConvertOpToLLVMPattern { // ptr = ExtractValueOp src, 1 auto ptr = memRefDesc.memRefDescPtr(rewriter, loc); // castPtr = BitCastOp i8* to structTy* - auto castPtr = - rewriter - .create( - loc, LLVM::LLVMPointerType::get(targetStructType), ptr) - .getResult(); + Value castPtr; + if (getTypeConverter()->useOpaquePointers()) + castPtr = ptr; + else + castPtr = rewriter.create( + loc, LLVM::LLVMPointerType::get(targetStructType), ptr); + // struct = LoadOp castPtr - auto loadOp = rewriter.create(loc, castPtr); + auto loadOp = + rewriter.create(loc, targetStructType, castPtr); rewriter.replaceOp(memRefCastOp, loadOp.getResult()); } else { llvm_unreachable("Unsupported unranked memref to unranked memref cast"); @@ -918,15 +945,17 @@ struct MemRefCopyOpLowering : public ConvertOpToLLVMPattern { Value totalSize = rewriter.create(loc, numElements, sizeInBytes); + Type elementType = typeConverter->convertType(srcType.getElementType()); + Value srcBasePtr = srcDesc.alignedPtr(rewriter, loc); Value srcOffset = srcDesc.offset(rewriter, loc); - Value srcPtr = rewriter.create(loc, srcBasePtr.getType(), - srcBasePtr, srcOffset); + Value srcPtr = rewriter.create( + loc, srcBasePtr.getType(), elementType, srcBasePtr, srcOffset); MemRefDescriptor targetDesc(adaptor.getTarget()); Value targetBasePtr = targetDesc.alignedPtr(rewriter, loc); Value targetOffset = targetDesc.offset(rewriter, loc); - Value targetPtr = rewriter.create(loc, targetBasePtr.getType(), - targetBasePtr, targetOffset); + Value targetPtr = rewriter.create( + loc, targetBasePtr.getType(), elementType, targetBasePtr, targetOffset); Value isVolatile = rewriter.create(loc, rewriter.getBoolAttr(false)); rewriter.create(loc, targetPtr, srcPtr, totalSize, @@ -950,9 +979,13 @@ struct MemRefCopyOpLowering : public ConvertOpToLLVMPattern { auto *typeConverter = getTypeConverter(); auto ptr = typeConverter->promoteOneMemRefDescriptor(loc, ranked, rewriter); - auto voidPtr = - rewriter.create(loc, getVoidPtrType(), ptr) - .getResult(); + + Value voidPtr; + if (getTypeConverter()->useOpaquePointers()) + voidPtr = ptr; + else + voidPtr = rewriter.create(loc, getVoidPtrType(), ptr); + auto unrankedType = UnrankedMemRefType::get(type.getElementType(), type.getMemorySpace()); return UnrankedMemRefDescriptor::pack(rewriter, loc, *typeConverter, @@ -975,9 +1008,9 @@ struct MemRefCopyOpLowering : public ConvertOpToLLVMPattern { auto one = rewriter.create(loc, getIndexType(), rewriter.getIndexAttr(1)); auto promote = [&](Value desc) { - auto ptrType = LLVM::LLVMPointerType::get(desc.getType()); + Type ptrType = getTypeConverter()->getPointerType(desc.getType()); auto allocated = - rewriter.create(loc, ptrType, ValueRange{one}); + rewriter.create(loc, ptrType, desc.getType(), one); rewriter.create(loc, desc, allocated); return allocated; }; @@ -1068,8 +1101,8 @@ static void extractPointersAndOffset(Location loc, operandType.cast().getMemorySpaceAsInt(); Type elementType = operandType.cast().getElementType(); Type llvmElementType = typeConverter.convertType(elementType); - Type elementPtrPtrType = LLVM::LLVMPointerType::get( - LLVM::LLVMPointerType::get(llvmElementType, memorySpace)); + LLVM::LLVMPointerType elementPtrType = + typeConverter.getPointerType(llvmElementType, memorySpace); // Extract pointer to the underlying ranked memref descriptor and cast it to // ElemType**. @@ -1077,12 +1110,12 @@ static void extractPointersAndOffset(Location loc, Value underlyingDescPtr = unrankedDesc.memRefDescPtr(rewriter, loc); *allocatedPtr = UnrankedMemRefDescriptor::allocatedPtr( - rewriter, loc, underlyingDescPtr, elementPtrPtrType); + rewriter, loc, underlyingDescPtr, elementPtrType); *alignedPtr = UnrankedMemRefDescriptor::alignedPtr( - rewriter, loc, typeConverter, underlyingDescPtr, elementPtrPtrType); + rewriter, loc, typeConverter, underlyingDescPtr, elementPtrType); if (offset != nullptr) { *offset = UnrankedMemRefDescriptor::offset( - rewriter, loc, typeConverter, underlyingDescPtr, elementPtrPtrType); + rewriter, loc, typeConverter, underlyingDescPtr, elementPtrType); } } @@ -1277,7 +1310,8 @@ struct MemRefReshapeOpLowering UnrankedMemRefDescriptor::computeSizes(rewriter, loc, *getTypeConverter(), targetDesc, sizes); Value underlyingDescPtr = rewriter.create( - loc, getVoidPtrType(), sizes.front(), std::nullopt); + loc, getVoidPtrType(), IntegerType::get(getContext(), 8), + sizes.front()); targetDesc.setMemRefDescPtr(rewriter, loc, underlyingDescPtr); // Extract pointers and offset from the source memref. @@ -1288,22 +1322,22 @@ struct MemRefReshapeOpLowering // Set pointers and offset. Type llvmElementType = typeConverter->convertType(elementType); - auto elementPtrPtrType = LLVM::LLVMPointerType::get( - LLVM::LLVMPointerType::get(llvmElementType, addressSpace)); + LLVM::LLVMPointerType elementPtrType = + getTypeConverter()->getPointerType(llvmElementType, addressSpace); + UnrankedMemRefDescriptor::setAllocatedPtr(rewriter, loc, underlyingDescPtr, - elementPtrPtrType, allocatedPtr); + elementPtrType, allocatedPtr); UnrankedMemRefDescriptor::setAlignedPtr(rewriter, loc, *getTypeConverter(), - underlyingDescPtr, - elementPtrPtrType, alignedPtr); + underlyingDescPtr, elementPtrType, + alignedPtr); UnrankedMemRefDescriptor::setOffset(rewriter, loc, *getTypeConverter(), - underlyingDescPtr, elementPtrPtrType, + underlyingDescPtr, elementPtrType, offset); // Use the offset pointer as base for further addressing. Copy over the new // shape and compute strides. For this, we create a loop from rank-1 to 0. Value targetSizesBase = UnrankedMemRefDescriptor::sizeBasePtr( - rewriter, loc, *getTypeConverter(), underlyingDescPtr, - elementPtrPtrType); + rewriter, loc, *getTypeConverter(), underlyingDescPtr, elementPtrType); Value targetStridesBase = UnrankedMemRefDescriptor::strideBasePtr( rewriter, loc, *getTypeConverter(), targetSizesBase, resultRank); Value shapeOperandPtr = shapeDesc.alignedPtr(rewriter, loc); @@ -1339,10 +1373,12 @@ struct MemRefReshapeOpLowering rewriter.setInsertionPointToStart(bodyBlock); // Copy size from shape to descriptor. - Type llvmIndexPtrType = LLVM::LLVMPointerType::get(indexType); - Value sizeLoadGep = rewriter.create(loc, llvmIndexPtrType, - shapeOperandPtr, indexArg); - Value size = rewriter.create(loc, sizeLoadGep); + Type llvmIndexPtrType = getTypeConverter()->getPointerType(indexType); + Value sizeLoadGep = rewriter.create( + loc, llvmIndexPtrType, + typeConverter->convertType(shapeMemRefType.getElementType()), + shapeOperandPtr, indexArg); + Value size = rewriter.create(loc, indexType, sizeLoadGep); UnrankedMemRefDescriptor::setSize(rewriter, loc, *getTypeConverter(), targetSizesBase, indexArg, size); @@ -1528,22 +1564,35 @@ struct ViewOpLowering : public ConvertOpToLLVMPattern { // Field 1: Copy the allocated pointer, used for malloc/free. Value allocatedPtr = sourceMemRef.allocatedPtr(rewriter, loc); auto srcMemRefType = viewOp.getSource().getType().cast(); - Value bitcastPtr = rewriter.create( - loc, - LLVM::LLVMPointerType::get(targetElementTy, - srcMemRefType.getMemorySpaceAsInt()), - allocatedPtr); + Value bitcastPtr; + if (getTypeConverter()->useOpaquePointers()) + bitcastPtr = allocatedPtr; + else + bitcastPtr = rewriter.create( + loc, + LLVM::LLVMPointerType::get(targetElementTy, + srcMemRefType.getMemorySpaceAsInt()), + allocatedPtr); + targetMemRef.setAllocatedPtr(rewriter, loc, bitcastPtr); // Field 2: Copy the actual aligned pointer to payload. Value alignedPtr = sourceMemRef.alignedPtr(rewriter, loc); alignedPtr = rewriter.create( - loc, alignedPtr.getType(), alignedPtr, adaptor.getByteShift()); - bitcastPtr = rewriter.create( - loc, - LLVM::LLVMPointerType::get(targetElementTy, - srcMemRefType.getMemorySpaceAsInt()), - alignedPtr); + loc, alignedPtr.getType(), + typeConverter->convertType(srcMemRefType.getElementType()), alignedPtr, + adaptor.getByteShift()); + + if (getTypeConverter()->useOpaquePointers()) { + bitcastPtr = alignedPtr; + } else { + bitcastPtr = rewriter.create( + loc, + LLVM::LLVMPointerType::get(targetElementTy, + srcMemRefType.getMemorySpaceAsInt()), + alignedPtr); + } + targetMemRef.setAlignedPtr(rewriter, loc, bitcastPtr); // Field 3: The offset in the resulting type must be 0. This is because of @@ -1754,6 +1803,7 @@ struct FinalizeMemRefToLLVMConversionPass : LowerToLLVMOptions::AllocLowering::Malloc); options.useGenericFunctions = useGenericFunctions; + options.useOpaquePointers = useOpaquePointers; if (indexBitwidth != kDeriveIndexBitwidthFromDataLayout) options.overrideIndexBitwidth(indexBitwidth); diff --git a/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp b/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp index ff590d4e7070..b38f849c186a 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp @@ -79,11 +79,25 @@ LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintF64Fn(ModuleOp moduleOp) { LLVM::LLVMVoidType::get(moduleOp->getContext())); } -LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintStrFn(ModuleOp moduleOp) { - return lookupOrCreateFn( - moduleOp, kPrintStr, - LLVM::LLVMPointerType::get(IntegerType::get(moduleOp->getContext(), 8)), - LLVM::LLVMVoidType::get(moduleOp->getContext())); +static LLVM::LLVMPointerType getCharPtr(MLIRContext *context, + bool opaquePointers) { + if (opaquePointers) + return LLVM::LLVMPointerType::get(context); + + return LLVM::LLVMPointerType::get(IntegerType::get(context, 8)); +} + +static LLVM::LLVMPointerType getVoidPtr(MLIRContext *context, + bool opaquePointers) { + // A char pointer and void ptr are the same in LLVM IR. + return getCharPtr(context, opaquePointers); +} + +LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintStrFn(ModuleOp moduleOp, + bool opaquePointers) { + return lookupOrCreateFn(moduleOp, kPrintStr, + getCharPtr(moduleOp->getContext(), opaquePointers), + LLVM::LLVMVoidType::get(moduleOp->getContext())); } LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintOpenFn(ModuleOp moduleOp) { @@ -107,45 +121,48 @@ LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintNewlineFn(ModuleOp moduleOp) { } LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateMallocFn(ModuleOp moduleOp, - Type indexType) { + Type indexType, + bool opaquePointers) { return LLVM::lookupOrCreateFn( moduleOp, kMalloc, indexType, - LLVM::LLVMPointerType::get(IntegerType::get(moduleOp->getContext(), 8))); + getVoidPtr(moduleOp->getContext(), opaquePointers)); } LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateAlignedAllocFn(ModuleOp moduleOp, - Type indexType) { + Type indexType, + bool opaquePointers) { return LLVM::lookupOrCreateFn( moduleOp, kAlignedAlloc, {indexType, indexType}, - LLVM::LLVMPointerType::get(IntegerType::get(moduleOp->getContext(), 8))); + getVoidPtr(moduleOp->getContext(), opaquePointers)); } -LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateFreeFn(ModuleOp moduleOp) { +LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateFreeFn(ModuleOp moduleOp, + bool opaquePointers) { return LLVM::lookupOrCreateFn( - moduleOp, kFree, - LLVM::LLVMPointerType::get(IntegerType::get(moduleOp->getContext(), 8)), + moduleOp, kFree, getVoidPtr(moduleOp->getContext(), opaquePointers), LLVM::LLVMVoidType::get(moduleOp->getContext())); } LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateGenericAllocFn(ModuleOp moduleOp, - Type indexType) { + Type indexType, + bool opaquePointers) { return LLVM::lookupOrCreateFn( moduleOp, kGenericAlloc, indexType, - LLVM::LLVMPointerType::get(IntegerType::get(moduleOp->getContext(), 8))); + getVoidPtr(moduleOp->getContext(), opaquePointers)); } -LLVM::LLVMFuncOp -mlir::LLVM::lookupOrCreateGenericAlignedAllocFn(ModuleOp moduleOp, - Type indexType) { +LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateGenericAlignedAllocFn( + ModuleOp moduleOp, Type indexType, bool opaquePointers) { return LLVM::lookupOrCreateFn( moduleOp, kGenericAlignedAlloc, {indexType, indexType}, - LLVM::LLVMPointerType::get(IntegerType::get(moduleOp->getContext(), 8))); + getVoidPtr(moduleOp->getContext(), opaquePointers)); } -LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateGenericFreeFn(ModuleOp moduleOp) { +LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateGenericFreeFn(ModuleOp moduleOp, + bool opaquePointers) { return LLVM::lookupOrCreateFn( moduleOp, kGenericFree, - LLVM::LLVMPointerType::get(IntegerType::get(moduleOp->getContext(), 8)), + getVoidPtr(moduleOp->getContext(), opaquePointers), LLVM::LLVMVoidType::get(moduleOp->getContext())); } diff --git a/mlir/test/Conversion/MemRefToLLVM/convert-alloca-scope.mlir b/mlir/test/Conversion/MemRefToLLVM/convert-alloca-scope.mlir index 7553d726cf71..5cc9b33d21f1 100644 --- a/mlir/test/Conversion/MemRefToLLVM/convert-alloca-scope.mlir +++ b/mlir/test/Conversion/MemRefToLLVM/convert-alloca-scope.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt -finalize-memref-to-llvm %s | FileCheck %s +// RUN: mlir-opt -finalize-memref-to-llvm='use-opaque-pointers=1' %s | FileCheck %s // CHECK-LABEL: @empty func.func @empty() { diff --git a/mlir/test/Conversion/MemRefToLLVM/convert-dynamic-memref-ops.mlir b/mlir/test/Conversion/MemRefToLLVM/convert-dynamic-memref-ops.mlir index 6a9bd2645d09..6987bdcc104d 100644 --- a/mlir/test/Conversion/MemRefToLLVM/convert-dynamic-memref-ops.mlir +++ b/mlir/test/Conversion/MemRefToLLVM/convert-dynamic-memref-ops.mlir @@ -1,6 +1,6 @@ -// RUN: mlir-opt -split-input-file -finalize-memref-to-llvm %s | FileCheck %s -// RUN: mlir-opt -split-input-file -finalize-memref-to-llvm='use-aligned-alloc=1' %s | FileCheck %s --check-prefix=ALIGNED-ALLOC -// RUN: mlir-opt -split-input-file -finalize-memref-to-llvm='index-bitwidth=32' %s | FileCheck --check-prefix=CHECK32 %s +// RUN: mlir-opt -split-input-file -finalize-memref-to-llvm='use-opaque-pointers=1' %s | FileCheck %s +// RUN: mlir-opt -split-input-file -finalize-memref-to-llvm='use-aligned-alloc=1 use-opaque-pointers=1' %s | FileCheck %s --check-prefix=ALIGNED-ALLOC +// RUN: mlir-opt -split-input-file -finalize-memref-to-llvm='index-bitwidth=32 use-opaque-pointers=1' %s | FileCheck --check-prefix=CHECK32 %s // CHECK-LABEL: func @mixed_alloc( // CHECK: %[[Marg:.*]]: index, %[[Narg:.*]]: index) @@ -11,22 +11,21 @@ func.func @mixed_alloc(%arg0: index, %arg1: index) -> memref { // CHECK-NEXT: %[[one:.*]] = llvm.mlir.constant(1 : index) : i64 // CHECK-NEXT: %[[st0:.*]] = llvm.mul %[[N]], %[[c42]] : i64 // CHECK-NEXT: %[[sz:.*]] = llvm.mul %[[st0]], %[[M]] : i64 -// CHECK-NEXT: %[[null:.*]] = llvm.mlir.null : !llvm.ptr -// CHECK-NEXT: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[sz]]] : (!llvm.ptr, i64) -> !llvm.ptr -// CHECK-NEXT: %[[sz_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 -// CHECK-NEXT: llvm.call @malloc(%[[sz_bytes]]) : (i64) -> !llvm.ptr -// CHECK-NEXT: llvm.bitcast %{{.*}} : !llvm.ptr to !llvm.ptr -// CHECK-NEXT: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK-NEXT: llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK-NEXT: llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: %[[null:.*]] = llvm.mlir.null : !llvm.ptr +// CHECK-NEXT: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[sz]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 +// CHECK-NEXT: %[[sz_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 +// CHECK-NEXT: llvm.call @malloc(%[[sz_bytes]]) : (i64) -> !llvm.ptr +// CHECK-NEXT: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // CHECK-NEXT: %[[off:.*]] = llvm.mlir.constant(0 : index) : i64 -// CHECK-NEXT: llvm.insertvalue %[[off]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[M]], %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[c42]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[st0]], %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[one]], %{{.*}}[4, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[off]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[M]], %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[c42]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[st0]], %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[one]], %{{.*}}[4, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> %0 = memref.alloc(%arg0, %arg1) : memref return %0 : memref } @@ -35,9 +34,8 @@ func.func @mixed_alloc(%arg0: index, %arg1: index) -> memref { // CHECK-LABEL: func @mixed_dealloc func.func @mixed_dealloc(%arg0: memref) { -// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK-NEXT: %[[ptri8:.*]] = llvm.bitcast %[[ptr]] : !llvm.ptr to !llvm.ptr -// CHECK-NEXT: llvm.call @free(%[[ptri8]]) : (!llvm.ptr) -> () +// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.call @free(%[[ptr]]) : (!llvm.ptr) -> () memref.dealloc %arg0 : memref return } @@ -51,20 +49,19 @@ func.func @dynamic_alloc(%arg0: index, %arg1: index) -> memref { // CHECK-DAG: %[[N:.*]] = builtin.unrealized_conversion_cast %[[Narg]] // CHECK-NEXT: %[[one:.*]] = llvm.mlir.constant(1 : index) : i64 // CHECK-NEXT: %[[sz:.*]] = llvm.mul %[[N]], %[[M]] : i64 -// CHECK-NEXT: %[[null:.*]] = llvm.mlir.null : !llvm.ptr -// CHECK-NEXT: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[sz]]] : (!llvm.ptr, i64) -> !llvm.ptr -// CHECK-NEXT: %[[sz_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 -// CHECK-NEXT: llvm.call @malloc(%[[sz_bytes]]) : (i64) -> !llvm.ptr -// CHECK-NEXT: llvm.bitcast %{{.*}} : !llvm.ptr to !llvm.ptr -// CHECK-NEXT: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: %[[null:.*]] = llvm.mlir.null : !llvm.ptr +// CHECK-NEXT: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[sz]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 +// CHECK-NEXT: %[[sz_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 +// CHECK-NEXT: llvm.call @malloc(%[[sz_bytes]]) : (i64) -> !llvm.ptr +// CHECK-NEXT: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK-NEXT: %[[off:.*]] = llvm.mlir.constant(0 : index) : i64 -// CHECK-NEXT: llvm.insertvalue %[[off]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[M]], %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[one]], %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[off]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[M]], %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[one]], %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> %0 = memref.alloc(%arg0, %arg1) : memref return %0 : memref } @@ -78,28 +75,28 @@ func.func @dynamic_alloca(%arg0: index, %arg1: index) -> memref { // CHECK-DAG: %[[N:.*]] = builtin.unrealized_conversion_cast %[[Narg]] // CHECK-NEXT: %[[st1:.*]] = llvm.mlir.constant(1 : index) : i64 // CHECK-NEXT: %[[num_elems:.*]] = llvm.mul %[[N]], %[[M]] : i64 -// CHECK-NEXT: %[[null:.*]] = llvm.mlir.null : !llvm.ptr -// CHECK-NEXT: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[num_elems]]] : (!llvm.ptr, i64) -> !llvm.ptr -// CHECK-NEXT: %[[sz_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 -// CHECK-NEXT: %[[allocated:.*]] = llvm.alloca %[[sz_bytes]] x f32 : (i64) -> !llvm.ptr -// CHECK-NEXT: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[allocated]], %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[allocated]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: %[[null:.*]] = llvm.mlir.null : !llvm.ptr +// CHECK-NEXT: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[num_elems]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 +// CHECK-NEXT: %[[sz_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 +// CHECK-NEXT: %[[allocated:.*]] = llvm.alloca %[[sz_bytes]] x f32 : (i64) -> !llvm.ptr +// CHECK-NEXT: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[allocated]], %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[allocated]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK-NEXT: %[[off:.*]] = llvm.mlir.constant(0 : index) : i64 -// CHECK-NEXT: llvm.insertvalue %[[off]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[M]], %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[st1]], %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[off]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[M]], %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[st1]], %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> %0 = memref.alloca(%arg0, %arg1) : memref // Test with explicitly specified alignment. llvm.alloca takes care of the // alignment. The same pointer is thus used for allocation and aligned // accesses. -// CHECK: %[[alloca_aligned:.*]] = llvm.alloca %{{.*}} x f32 {alignment = 32 : i64} : (i64) -> !llvm.ptr -// CHECK: %[[desc:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[desc1:.*]] = llvm.insertvalue %[[alloca_aligned]], %[[desc]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: llvm.insertvalue %[[alloca_aligned]], %[[desc1]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[alloca_aligned:.*]] = llvm.alloca %{{.*}} x f32 {alignment = 32 : i64} : (i64) -> !llvm.ptr +// CHECK: %[[desc:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[desc1:.*]] = llvm.insertvalue %[[alloca_aligned]], %[[desc]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: llvm.insertvalue %[[alloca_aligned]], %[[desc1]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> memref.alloca(%arg0, %arg1) {alignment = 32} : memref return %0 : memref } @@ -108,9 +105,8 @@ func.func @dynamic_alloca(%arg0: index, %arg1: index) -> memref { // CHECK-LABEL: func @dynamic_dealloc func.func @dynamic_dealloc(%arg0: memref) { -// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: %[[ptri8:.*]] = llvm.bitcast %[[ptr]] : !llvm.ptr to !llvm.ptr -// CHECK-NEXT: llvm.call @free(%[[ptri8]]) : (!llvm.ptr) -> () +// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.call @free(%[[ptr]]) : (!llvm.ptr) -> () memref.dealloc %arg0 : memref return } @@ -124,12 +120,11 @@ func.func @stdlib_aligned_alloc(%N : index) -> memref<32x18xf32> { // ALIGNED-ALLOC-NEXT: %[[sz2:.*]] = llvm.mlir.constant(18 : index) : i64 // ALIGNED-ALLOC-NEXT: %[[one:.*]] = llvm.mlir.constant(1 : index) : i64 // ALIGNED-ALLOC-NEXT: %[[num_elems:.*]] = llvm.mlir.constant(576 : index) : i64 -// ALIGNED-ALLOC-NEXT: %[[null:.*]] = llvm.mlir.null : !llvm.ptr -// ALIGNED-ALLOC-NEXT: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[num_elems]]] : (!llvm.ptr, i64) -> !llvm.ptr -// ALIGNED-ALLOC-NEXT: %[[bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 +// ALIGNED-ALLOC-NEXT: %[[null:.*]] = llvm.mlir.null : !llvm.ptr +// ALIGNED-ALLOC-NEXT: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[num_elems]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 +// ALIGNED-ALLOC-NEXT: %[[bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 // ALIGNED-ALLOC-NEXT: %[[alignment:.*]] = llvm.mlir.constant(32 : index) : i64 -// ALIGNED-ALLOC-NEXT: %[[allocated:.*]] = llvm.call @aligned_alloc(%[[alignment]], %[[bytes]]) : (i64, i64) -> !llvm.ptr -// ALIGNED-ALLOC-NEXT: llvm.bitcast %[[allocated]] : !llvm.ptr to !llvm.ptr +// ALIGNED-ALLOC-NEXT: %[[allocated:.*]] = llvm.call @aligned_alloc(%[[alignment]], %[[bytes]]) : (i64, i64) -> !llvm.ptr %0 = memref.alloc() {alignment = 32} : memref<32x18xf32> // Do another alloc just to test that we have a unique declaration for // aligned_alloc. @@ -170,12 +165,12 @@ func.func @stdlib_aligned_alloc(%N : index) -> memref<32x18xf32> { func.func @mixed_load(%mixed : memref<42x?xf32>, %i : index, %j : index) { // CHECK-DAG: %[[I:.*]] = builtin.unrealized_conversion_cast %[[Iarg]] // CHECK-DAG: %[[J:.*]] = builtin.unrealized_conversion_cast %[[Jarg]] -// CHECK: %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK-NEXT: %[[offI:.*]] = llvm.mul %[[I]], %[[st0]] : i64 // CHECK-NEXT: %[[off1:.*]] = llvm.add %[[offI]], %[[J]] : i64 -// CHECK-NEXT: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr, i64) -> !llvm.ptr -// CHECK-NEXT: llvm.load %[[addr]] : !llvm.ptr +// CHECK-NEXT: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 +// CHECK-NEXT: llvm.load %[[addr]] : !llvm.ptr -> f32 %0 = memref.load %mixed[%i, %j] : memref<42x?xf32> return } @@ -187,12 +182,12 @@ func.func @mixed_load(%mixed : memref<42x?xf32>, %i : index, %j : index) { func.func @dynamic_load(%dynamic : memref, %i : index, %j : index) { // CHECK-DAG: %[[I:.*]] = builtin.unrealized_conversion_cast %[[Iarg]] // CHECK-DAG: %[[J:.*]] = builtin.unrealized_conversion_cast %[[Jarg]] -// CHECK: %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK-NEXT: %[[offI:.*]] = llvm.mul %[[I]], %[[st0]] : i64 // CHECK-NEXT: %[[off1:.*]] = llvm.add %[[offI]], %[[J]] : i64 -// CHECK-NEXT: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr, i64) -> !llvm.ptr -// CHECK-NEXT: llvm.load %[[addr]] : !llvm.ptr +// CHECK-NEXT: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 +// CHECK-NEXT: llvm.load %[[addr]] : !llvm.ptr -> f32 %0 = memref.load %dynamic[%i, %j] : memref return } @@ -204,25 +199,25 @@ func.func @dynamic_load(%dynamic : memref, %i : index, %j : index) { func.func @prefetch(%A : memref, %i : index, %j : index) { // CHECK-DAG: %[[I:.*]] = builtin.unrealized_conversion_cast %[[Iarg]] // CHECK-DAG: %[[J:.*]] = builtin.unrealized_conversion_cast %[[Jarg]] -// CHECK: %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK-NEXT: %[[offI:.*]] = llvm.mul %[[I]], %[[st0]] : i64 // CHECK-NEXT: %[[off1:.*]] = llvm.add %[[offI]], %[[J]] : i64 -// CHECK-NEXT: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr, i64) -> !llvm.ptr +// CHECK-NEXT: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 // CHECK-NEXT: [[C1:%.*]] = llvm.mlir.constant(1 : i32) : i32 // CHECK-NEXT: [[C3:%.*]] = llvm.mlir.constant(3 : i32) : i32 // CHECK-NEXT: [[C1_1:%.*]] = llvm.mlir.constant(1 : i32) : i32 -// CHECK-NEXT: "llvm.intr.prefetch"(%[[addr]], [[C1]], [[C3]], [[C1_1]]) : (!llvm.ptr, i32, i32, i32) -> () +// CHECK-NEXT: "llvm.intr.prefetch"(%[[addr]], [[C1]], [[C3]], [[C1_1]]) : (!llvm.ptr, i32, i32, i32) -> () memref.prefetch %A[%i, %j], write, locality<3>, data : memref // CHECK: [[C0:%.*]] = llvm.mlir.constant(0 : i32) : i32 // CHECK: [[C0_1:%.*]] = llvm.mlir.constant(0 : i32) : i32 // CHECK: [[C1_2:%.*]] = llvm.mlir.constant(1 : i32) : i32 -// CHECK: "llvm.intr.prefetch"(%{{.*}}, [[C0]], [[C0_1]], [[C1_2]]) : (!llvm.ptr, i32, i32, i32) -> () +// CHECK: "llvm.intr.prefetch"(%{{.*}}, [[C0]], [[C0_1]], [[C1_2]]) : (!llvm.ptr, i32, i32, i32) -> () memref.prefetch %A[%i, %j], read, locality<0>, data : memref // CHECK: [[C0_2:%.*]] = llvm.mlir.constant(0 : i32) : i32 // CHECK: [[C2:%.*]] = llvm.mlir.constant(2 : i32) : i32 // CHECK: [[C0_3:%.*]] = llvm.mlir.constant(0 : i32) : i32 -// CHECK: "llvm.intr.prefetch"(%{{.*}}, [[C0_2]], [[C2]], [[C0_3]]) : (!llvm.ptr, i32, i32, i32) -> () +// CHECK: "llvm.intr.prefetch"(%{{.*}}, [[C0_2]], [[C2]], [[C0_3]]) : (!llvm.ptr, i32, i32, i32) -> () memref.prefetch %A[%i, %j], read, locality<2>, instr : memref return } @@ -234,12 +229,12 @@ func.func @prefetch(%A : memref, %i : index, %j : index) { func.func @dynamic_store(%dynamic : memref, %i : index, %j : index, %val : f32) { // CHECK-DAG: %[[I:.*]] = builtin.unrealized_conversion_cast %[[Iarg]] // CHECK-DAG: %[[J:.*]] = builtin.unrealized_conversion_cast %[[Jarg]] -// CHECK: %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK-NEXT: %[[offI:.*]] = llvm.mul %[[I]], %[[st0]] : i64 // CHECK-NEXT: %[[off1:.*]] = llvm.add %[[offI]], %[[J]] : i64 -// CHECK-NEXT: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr, i64) -> !llvm.ptr -// CHECK-NEXT: llvm.store %{{.*}}, %[[addr]] : !llvm.ptr +// CHECK-NEXT: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 +// CHECK-NEXT: llvm.store %{{.*}}, %[[addr]] : f32, !llvm.ptr memref.store %val, %dynamic[%i, %j] : memref return } @@ -251,100 +246,35 @@ func.func @dynamic_store(%dynamic : memref, %i : index, %j : index, %va func.func @mixed_store(%mixed : memref<42x?xf32>, %i : index, %j : index, %val : f32) { // CHECK-DAG: %[[I:.*]] = builtin.unrealized_conversion_cast %[[Iarg]] // CHECK-DAG: %[[J:.*]] = builtin.unrealized_conversion_cast %[[Jarg]] -// CHECK: %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK-NEXT: %[[offI:.*]] = llvm.mul %[[I]], %[[st0]] : i64 // CHECK-NEXT: %[[off1:.*]] = llvm.add %[[offI]], %[[J]] : i64 -// CHECK-NEXT: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr, i64) -> !llvm.ptr -// CHECK-NEXT: llvm.store %{{.*}}, %[[addr]] : !llvm.ptr +// CHECK-NEXT: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 +// CHECK-NEXT: llvm.store %{{.*}}, %[[addr]] : f32, !llvm.ptr memref.store %val, %mixed[%i, %j] : memref<42x?xf32> return } // ----- -// CHECK-LABEL: func @memref_cast_static_to_dynamic -func.func @memref_cast_static_to_dynamic(%static : memref<10x42xf32>) { -// CHECK-NOT: llvm.bitcast - %0 = memref.cast %static : memref<10x42xf32> to memref - return -} - -// ----- - -// CHECK-LABEL: func @memref_cast_static_to_mixed -func.func @memref_cast_static_to_mixed(%static : memref<10x42xf32>) { -// CHECK-NOT: llvm.bitcast - %0 = memref.cast %static : memref<10x42xf32> to memref - return -} - -// ----- - -// CHECK-LABEL: func @memref_cast_dynamic_to_static -func.func @memref_cast_dynamic_to_static(%dynamic : memref) { -// CHECK-NOT: llvm.bitcast - %0 = memref.cast %dynamic : memref to memref<10x12xf32> - return -} - -// ----- - -// CHECK-LABEL: func @memref_cast_dynamic_to_mixed -func.func @memref_cast_dynamic_to_mixed(%dynamic : memref) { -// CHECK-NOT: llvm.bitcast - %0 = memref.cast %dynamic : memref to memref - return -} - -// ----- - -// CHECK-LABEL: func @memref_cast_mixed_to_dynamic -func.func @memref_cast_mixed_to_dynamic(%mixed : memref<42x?xf32>) { -// CHECK-NOT: llvm.bitcast - %0 = memref.cast %mixed : memref<42x?xf32> to memref - return -} - -// ----- - -// CHECK-LABEL: func @memref_cast_mixed_to_static -func.func @memref_cast_mixed_to_static(%mixed : memref<42x?xf32>) { -// CHECK-NOT: llvm.bitcast - %0 = memref.cast %mixed : memref<42x?xf32> to memref<42x1xf32> - return -} - -// ----- - -// CHECK-LABEL: func @memref_cast_mixed_to_mixed -func.func @memref_cast_mixed_to_mixed(%mixed : memref<42x?xf32>) { -// CHECK-NOT: llvm.bitcast - %0 = memref.cast %mixed : memref<42x?xf32> to memref - return -} - -// ----- - // CHECK-LABEL: func @memref_cast_ranked_to_unranked // CHECK32-LABEL: func @memref_cast_ranked_to_unranked func.func @memref_cast_ranked_to_unranked(%arg : memref<42x2x?xf32>) { // CHECK-DAG: %[[c:.*]] = llvm.mlir.constant(1 : index) : i64 -// CHECK-DAG: %[[p:.*]] = llvm.alloca %[[c]] x !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> : (i64) -> !llvm.ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>> -// CHECK-DAG: llvm.store %{{.*}}, %[[p]] : !llvm.ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>> -// CHECK-DAG: %[[p2:.*]] = llvm.bitcast %[[p]] : !llvm.ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>> to !llvm.ptr +// CHECK-DAG: %[[p:.*]] = llvm.alloca %[[c]] x !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> : (i64) -> !llvm.ptr +// CHECK-DAG: llvm.store %{{.*}}, %[[p]] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>, !llvm.ptr // CHECK-DAG: %[[r:.*]] = llvm.mlir.constant(3 : index) : i64 -// CHECK : llvm.mlir.undef : !llvm.struct<(i64, ptr)> -// CHECK-DAG: llvm.insertvalue %[[r]], %{{.*}}[0] : !llvm.struct<(i64, ptr)> -// CHECK-DAG: llvm.insertvalue %[[p2]], %{{.*}}[1] : !llvm.struct<(i64, ptr)> +// CHECK : llvm.mlir.undef : !llvm.struct<(i64, ptr)> +// CHECK-DAG: llvm.insertvalue %[[r]], %{{.*}}[0] : !llvm.struct<(i64, ptr)> +// CHECK-DAG: llvm.insertvalue %[[p]], %{{.*}}[1] : !llvm.struct<(i64, ptr)> // CHECK32-DAG: %[[c:.*]] = llvm.mlir.constant(1 : index) : i64 -// CHECK32-DAG: %[[p:.*]] = llvm.alloca %[[c]] x !llvm.struct<(ptr, ptr, i32, array<3 x i32>, array<3 x i32>)> : (i64) -> !llvm.ptr, ptr, i32, array<3 x i32>, array<3 x i32>)>> -// CHECK32-DAG: llvm.store %{{.*}}, %[[p]] : !llvm.ptr, ptr, i32, array<3 x i32>, array<3 x i32>)>> -// CHECK32-DAG: %[[p2:.*]] = llvm.bitcast %[[p]] : !llvm.ptr, ptr, i32, array<3 x i32>, array<3 x i32>)>> to !llvm.ptr +// CHECK32-DAG: %[[p:.*]] = llvm.alloca %[[c]] x !llvm.struct<(ptr, ptr, i32, array<3 x i32>, array<3 x i32>)> : (i64) -> !llvm.ptr +// CHECK32-DAG: llvm.store %{{.*}}, %[[p]] : !llvm.struct<(ptr, ptr, i32, array<3 x i32>, array<3 x i32>)>, !llvm.ptr // CHECK32-DAG: %[[r:.*]] = llvm.mlir.constant(3 : index) : i32 -// CHECK32 : llvm.mlir.undef : !llvm.struct<(i32, ptr)> -// CHECK32-DAG: llvm.insertvalue %[[r]], %{{.*}}[0] : !llvm.struct<(i32, ptr)> -// CHECK32-DAG: llvm.insertvalue %[[p2]], %{{.*}}[1] : !llvm.struct<(i32, ptr)> +// CHECK32 : llvm.mlir.undef : !llvm.struct<(i32, ptr)> +// CHECK32-DAG: llvm.insertvalue %[[r]], %{{.*}}[0] : !llvm.struct<(i32, ptr)> +// CHECK32-DAG: llvm.insertvalue %[[p]], %{{.*}}[1] : !llvm.struct<(i32, ptr)> %0 = memref.cast %arg : memref<42x2x?xf32> to memref<*xf32> return } @@ -353,8 +283,7 @@ func.func @memref_cast_ranked_to_unranked(%arg : memref<42x2x?xf32>) { // CHECK-LABEL: func @memref_cast_unranked_to_ranked func.func @memref_cast_unranked_to_ranked(%arg : memref<*xf32>) { -// CHECK: %[[p:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(i64, ptr)> -// CHECK-NEXT: llvm.bitcast %[[p]] : !llvm.ptr to !llvm.ptr, ptr, i64, array<4 x i64>, array<4 x i64>)>> +// CHECK: %[[p:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(i64, ptr)> %0 = memref.cast %arg : memref<*xf32> to memref return } @@ -366,16 +295,16 @@ func.func @mixed_memref_dim(%mixed : memref<42x?x?x13x?xf32>) { // CHECK: llvm.mlir.constant(42 : index) : i64 %c0 = arith.constant 0 : index %0 = memref.dim %mixed, %c0 : memref<42x?x?x13x?xf32> -// CHECK: llvm.extractvalue %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: llvm.extractvalue %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> %c1 = arith.constant 1 : index %1 = memref.dim %mixed, %c1 : memref<42x?x?x13x?xf32> -// CHECK: llvm.extractvalue %{{.*}}[3, 2] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: llvm.extractvalue %{{.*}}[3, 2] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> %c2 = arith.constant 2 : index %2 = memref.dim %mixed, %c2 : memref<42x?x?x13x?xf32> // CHECK: llvm.mlir.constant(13 : index) : i64 %c3 = arith.constant 3 : index %3 = memref.dim %mixed, %c3 : memref<42x?x?x13x?xf32> -// CHECK: llvm.extractvalue %{{.*}}[3, 4] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: llvm.extractvalue %{{.*}}[3, 4] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> %c4 = arith.constant 4 : index %4 = memref.dim %mixed, %c4 : memref<42x?x?x13x?xf32> return @@ -389,10 +318,10 @@ func.func @memref_dim_with_dyn_index(%arg : memref<3x?xf32>, %idx : index) -> in // CHECK-DAG: %[[IDX:.*]] = builtin.unrealized_conversion_cast %[[IDXarg]] // CHECK-DAG: %[[C1:.*]] = llvm.mlir.constant(1 : index) : i64 // CHECK-DAG: %[[SIZES:.*]] = llvm.extractvalue %{{.*}}[3] : ![[DESCR_TY:.*]] - // CHECK-DAG: %[[SIZES_PTR:.*]] = llvm.alloca %[[C1]] x !llvm.array<2 x i64> : (i64) -> !llvm.ptr> - // CHECK-DAG: llvm.store %[[SIZES]], %[[SIZES_PTR]] : !llvm.ptr> - // CHECK-DAG: %[[RESULT_PTR:.*]] = llvm.getelementptr %[[SIZES_PTR]][0, %[[IDX]]] : (!llvm.ptr>, i64) -> !llvm.ptr - // CHECK-DAG: %[[RESULT:.*]] = llvm.load %[[RESULT_PTR]] : !llvm.ptr + // CHECK-DAG: %[[SIZES_PTR:.*]] = llvm.alloca %[[C1]] x !llvm.array<2 x i64> : (i64) -> !llvm.ptr + // CHECK-DAG: llvm.store %[[SIZES]], %[[SIZES_PTR]] : !llvm.array<2 x i64>, !llvm.ptr + // CHECK-DAG: %[[RESULT_PTR:.*]] = llvm.getelementptr %[[SIZES_PTR]][0, %[[IDX]]] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.array<2 x i64> + // CHECK-DAG: %[[RESULT:.*]] = llvm.load %[[RESULT_PTR]] : !llvm.ptr -> i64 %result = memref.dim %arg, %idx : memref<3x?xf32> return %result : index } @@ -449,13 +378,11 @@ func.func @memref_reinterpret_cast_unranked_to_dynamic_shape(%offset: index, // CHECK-DAG: [[STRIDE_1:%.*]] = builtin.unrealized_conversion_cast [[STRIDE_1arg]] // CHECK-DAG: [[INPUT:%.*]] = builtin.unrealized_conversion_cast // CHECK: [[OUT_0:%.*]] = llvm.mlir.undef : [[TY:!.*]] -// CHECK: [[DESCRIPTOR:%.*]] = llvm.extractvalue [[INPUT]][1] : !llvm.struct<(i64, ptr)> -// CHECK: [[BASE_PTR_PTR:%.*]] = llvm.bitcast [[DESCRIPTOR]] : !llvm.ptr to !llvm.ptr> -// CHECK: [[BASE_PTR:%.*]] = llvm.load [[BASE_PTR_PTR]] : !llvm.ptr> -// CHECK: [[BASE_PTR_PTR_:%.*]] = llvm.bitcast [[DESCRIPTOR]] : !llvm.ptr to !llvm.ptr> -// CHECK: [[ALIGNED_PTR_PTR:%.*]] = llvm.getelementptr [[BASE_PTR_PTR_]]{{\[}}1] -// CHECK-SAME: : (!llvm.ptr>) -> !llvm.ptr> -// CHECK: [[ALIGNED_PTR:%.*]] = llvm.load [[ALIGNED_PTR_PTR]] : !llvm.ptr> +// CHECK: [[DESCRIPTOR:%.*]] = llvm.extractvalue [[INPUT]][1] : !llvm.struct<(i64, ptr)> +// CHECK: [[BASE_PTR:%.*]] = llvm.load [[DESCRIPTOR]] : !llvm.ptr -> !llvm.ptr +// CHECK: [[ALIGNED_PTR_PTR:%.*]] = llvm.getelementptr [[DESCRIPTOR]]{{\[}}1] +// CHECK-SAME: : (!llvm.ptr) -> !llvm.ptr, !llvm.ptr +// CHECK: [[ALIGNED_PTR:%.*]] = llvm.load [[ALIGNED_PTR_PTR]] : !llvm.ptr -> !llvm.ptr // CHECK: [[OUT_1:%.*]] = llvm.insertvalue [[BASE_PTR]], [[OUT_0]][0] : [[TY]] // CHECK: [[OUT_2:%.*]] = llvm.insertvalue [[ALIGNED_PTR]], [[OUT_1]][1] : [[TY]] // CHECK: [[OUT_3:%.*]] = llvm.insertvalue [[OFFSET]], [[OUT_2]][2] : [[TY]] @@ -475,8 +402,8 @@ func.func @memref_reshape(%input : memref<2x3xf32>, %shape : memref) { // CHECK: [[INPUT:%.*]] = builtin.unrealized_conversion_cast %{{.*}} to [[INPUT_TY:!.*]] // CHECK: [[SHAPE:%.*]] = builtin.unrealized_conversion_cast %{{.*}} to [[SHAPE_TY:!.*]] // CHECK: [[RANK:%.*]] = llvm.extractvalue [[SHAPE]][3, 0] : [[SHAPE_TY]] -// CHECK: [[UNRANKED_OUT_O:%.*]] = llvm.mlir.undef : !llvm.struct<(i64, ptr)> -// CHECK: [[UNRANKED_OUT_1:%.*]] = llvm.insertvalue [[RANK]], [[UNRANKED_OUT_O]][0] : !llvm.struct<(i64, ptr)> +// CHECK: [[UNRANKED_OUT_O:%.*]] = llvm.mlir.undef : !llvm.struct<(i64, ptr)> +// CHECK: [[UNRANKED_OUT_1:%.*]] = llvm.insertvalue [[RANK]], [[UNRANKED_OUT_O]][0] : !llvm.struct<(i64, ptr)> // Compute size in bytes to allocate result ranked descriptor // CHECK: [[C1:%.*]] = llvm.mlir.constant(1 : index) : i64 @@ -492,21 +419,14 @@ func.func @memref_reshape(%input : memref<2x3xf32>, %shape : memref) { // CHECK: [[ALLOC_PTR:%.*]] = llvm.extractvalue [[INPUT]][0] : [[INPUT_TY]] // CHECK: [[ALIGN_PTR:%.*]] = llvm.extractvalue [[INPUT]][1] : [[INPUT_TY]] // CHECK: [[OFFSET:%.*]] = llvm.extractvalue [[INPUT]][2] : [[INPUT_TY]] -// CHECK: [[BASE_PTR_PTR:%.*]] = llvm.bitcast [[UNDERLYING_DESC]] -// CHECK-SAME: !llvm.ptr to !llvm.ptr> -// CHECK: llvm.store [[ALLOC_PTR]], [[BASE_PTR_PTR]] : !llvm.ptr> -// CHECK: [[BASE_PTR_PTR_:%.*]] = llvm.bitcast [[UNDERLYING_DESC]] : !llvm.ptr to !llvm.ptr> -// CHECK: [[ALIGNED_PTR_PTR:%.*]] = llvm.getelementptr [[BASE_PTR_PTR_]]{{\[}}1] -// CHECK: llvm.store [[ALIGN_PTR]], [[ALIGNED_PTR_PTR]] : !llvm.ptr> -// CHECK: [[BASE_PTR_PTR__:%.*]] = llvm.bitcast [[UNDERLYING_DESC]] : !llvm.ptr to !llvm.ptr> -// CHECK: [[OFFSET_PTR_:%.*]] = llvm.getelementptr [[BASE_PTR_PTR__]]{{\[}}2] -// CHECK: [[OFFSET_PTR:%.*]] = llvm.bitcast [[OFFSET_PTR_]] -// CHECK: llvm.store [[OFFSET]], [[OFFSET_PTR]] : !llvm.ptr +// CHECK: llvm.store [[ALLOC_PTR]], [[UNDERLYING_DESC]] : !llvm.ptr, !llvm.ptr +// CHECK: [[ALIGNED_PTR_PTR:%.*]] = llvm.getelementptr [[UNDERLYING_DESC]]{{\[}}1] +// CHECK: llvm.store [[ALIGN_PTR]], [[ALIGNED_PTR_PTR]] : !llvm.ptr, !llvm.ptr +// CHECK: [[OFFSET_PTR:%.*]] = llvm.getelementptr [[UNDERLYING_DESC]]{{\[}}2] +// CHECK: llvm.store [[OFFSET]], [[OFFSET_PTR]] : i64, !llvm.ptr // Iterate over shape operand in reverse order and set sizes and strides. -// CHECK: [[STRUCT_PTR:%.*]] = llvm.bitcast [[UNDERLYING_DESC]] -// CHECK-SAME: !llvm.ptr to !llvm.ptr, ptr, i64, i64)>> -// CHECK: [[SIZES_PTR:%.*]] = llvm.getelementptr [[STRUCT_PTR]]{{\[}}0, 3] +// CHECK: [[SIZES_PTR:%.*]] = llvm.getelementptr [[UNDERLYING_DESC]]{{\[}}0, 3] // CHECK: [[STRIDES_PTR:%.*]] = llvm.getelementptr [[SIZES_PTR]]{{\[}}[[RANK]]] // CHECK: [[SHAPE_IN_PTR:%.*]] = llvm.extractvalue [[SHAPE]][1] : [[SHAPE_TY]] // CHECK: [[C1_:%.*]] = llvm.mlir.constant(1 : index) : i64 @@ -520,11 +440,11 @@ func.func @memref_reshape(%input : memref<2x3xf32>, %shape : memref) { // CHECK: ^bb2: // CHECK: [[SIZE_PTR:%.*]] = llvm.getelementptr [[SHAPE_IN_PTR]]{{\[}}[[DIM]]] -// CHECK: [[SIZE:%.*]] = llvm.load [[SIZE_PTR]] : !llvm.ptr +// CHECK: [[SIZE:%.*]] = llvm.load [[SIZE_PTR]] : !llvm.ptr -> i64 // CHECK: [[TARGET_SIZE_PTR:%.*]] = llvm.getelementptr [[SIZES_PTR]]{{\[}}[[DIM]]] -// CHECK: llvm.store [[SIZE]], [[TARGET_SIZE_PTR]] : !llvm.ptr +// CHECK: llvm.store [[SIZE]], [[TARGET_SIZE_PTR]] : i64, !llvm.ptr // CHECK: [[TARGET_STRIDE_PTR:%.*]] = llvm.getelementptr [[STRIDES_PTR]]{{\[}}[[DIM]]] -// CHECK: llvm.store [[CUR_STRIDE]], [[TARGET_STRIDE_PTR]] : !llvm.ptr +// CHECK: llvm.store [[CUR_STRIDE]], [[TARGET_STRIDE_PTR]] : i64, !llvm.ptr // CHECK: [[UPDATE_STRIDE:%.*]] = llvm.mul [[CUR_STRIDE]], [[SIZE]] : i64 // CHECK: [[STRIDE_COND:%.*]] = llvm.sub [[DIM]], [[C1_]] : i64 // CHECK: llvm.br ^bb1([[STRIDE_COND]], [[UPDATE_STRIDE]] : i64, i64) @@ -547,11 +467,8 @@ func.func @memref_of_memref() { // Check that the types are converted as expected. // ALIGNED-ALLOC: llvm.call @aligned_alloc - // ALIGNED-ALLOC: llvm.bitcast %{{.*}} : !llvm.ptr to - // ALIGNED-ALLOC-SAME: !llvm. - // ALIGNED-ALLOC-SAME: [[INNER:ptr, ptr, i64, array<1 x i64>, array<1 x i64>\)>>]] // ALIGNED-ALLOC: llvm.mlir.undef - // ALIGNED-ALLOC-SAME: !llvm.struct<([[INNER]], [[INNER]], i64, array<1 x i64>, array<1 x i64>)> + // ALIGNED-ALLOC-SAME: !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> %0 = memref.alloc() : memref<1xmemref<1xf32>> return } @@ -572,11 +489,8 @@ module attributes { dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry> } { // Check that the types are converted as expected. // ALIGNED-ALLOC: llvm.call @aligned_alloc - // ALIGNED-ALLOC: llvm.bitcast %{{.*}} : !llvm.ptr to - // ALIGNED-ALLOC-SAME: !llvm. - // ALIGNED-ALLOC-SAME: [[INNER:ptr, ptr, i32, array<1 x i32>, array<1 x i32>\)>>]] // ALIGNED-ALLOC: llvm.mlir.undef - // ALIGNED-ALLOC-SAME: !llvm.struct<([[INNER]], [[INNER]], i32, array<1 x i32>, array<1 x i32>)> + // ALIGNED-ALLOC-SAME: !llvm.struct<(ptr, ptr, i32, array<1 x i32>, array<1 x i32>)> %0 = memref.alloc() : memref<1xmemref<1xf32>> return } @@ -588,13 +502,7 @@ module attributes { dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry> } { // ALIGNED-ALLOC-LABEL: @memref_of_memref_of_memref func.func @memref_of_memref_of_memref() { // Sizeof computation is as usual, also check the type. - // ALIGNED-ALLOC: %[[NULL:.*]] = llvm.mlir.null : !llvm.ptr< - // ALIGNED-ALLOC-SAME: struct<( - // ALIGNED-ALLOC-SAME: [[INNER:ptr, ptr, i64, array<1 x i64>, array<1 x i64>\)>>]], - // ALIGNED-ALLOC-SAME: [[INNER]], - // ALIGNED-ALLOC-SAME: i64, array<1 x i64>, array<1 x i64> - // ALIGNED-ALLOC-SAME: )> - // ALIGNED-ALLOC-SAME: > + // ALIGNED-ALLOC: %[[NULL:.*]] = llvm.mlir.null : !llvm.ptr // ALIGNED-ALLOC: %[[PTR:.*]] = llvm.getelementptr // ALIGNED-ALLOC: %[[SIZEOF:.*]] = llvm.ptrtoint @@ -611,7 +519,7 @@ func.func @memref_of_memref_of_memref() { // ALIGNED-ALLOC-LABEL: @ranked_unranked func.func @ranked_unranked() { // ALIGNED-ALLOC: llvm.mlir.null - // ALIGNED-ALLOC-SAME: !llvm.[[INNER:ptr\)>>]] + // ALIGNED-ALLOC-SAME: !llvm.ptr // ALIGNED-ALLOC: llvm.getelementptr // ALIGNED-ALLOC: llvm.ptrtoint @@ -619,8 +527,6 @@ func.func @ranked_unranked() { // sizeof(pointer)) = 16. // ALIGNED-ALLOC: llvm.mlir.constant(16 : index) // ALIGNED-ALLOC: llvm.call @aligned_alloc - // ALIGNED-ALLOC: llvm.bitcast - // ALIGNED-ALLOC-SAME: !llvm.ptr to !llvm.[[INNER]] %0 = memref.alloc() : memref<1 x memref<* x f32>> memref.cast %0 : memref<1 x memref<* x f32>> to memref<* x memref<* x f32>> return @@ -638,25 +544,21 @@ func.func @realloc_dynamic(%in: memref, %d: index) -> memref{ // CHECK: %[[cond:.*]] = llvm.icmp "ugt" %[[dst_dim]], %[[src_dim]] : i64 // CHECK: llvm.cond_br %[[cond]], ^bb1, ^bb2(%[[descriptor]] // CHECK: ^bb1: -// CHECK: %[[dst_null:.*]] = llvm.mlir.null : !llvm.ptr +// CHECK: %[[dst_null:.*]] = llvm.mlir.null : !llvm.ptr // CHECK: %[[dst_gep:.*]] = llvm.getelementptr %[[dst_null]][1] -// CHECK: %[[dst_es:.*]] = llvm.ptrtoint %[[dst_gep]] : !llvm.ptr to i64 +// CHECK: %[[dst_es:.*]] = llvm.ptrtoint %[[dst_gep]] : !llvm.ptr to i64 // CHECK: %[[dst_size:.*]] = llvm.mul %[[dst_dim]], %[[dst_es]] // CHECK: %[[src_size:.*]] = llvm.mul %[[src_dim]], %[[dst_es]] // CHECK: %[[new_buffer_raw:.*]] = llvm.call @malloc(%[[dst_size]]) -// CHECK: %[[new_buffer:.*]] = llvm.bitcast %[[new_buffer_raw]] : !llvm.ptr to !llvm.ptr // CHECK: %[[old_buffer_aligned:.*]] = llvm.extractvalue %[[descriptor]][1] // CHECK: %[[volatile:.*]] = llvm.mlir.constant(false) : i1 -// CHECK-DAG: %[[new_buffer_void:.*]] = llvm.bitcast %[[new_buffer]] : !llvm.ptr to !llvm.ptr -// CHECK-DAG: %[[old_buffer_void:.*]] = llvm.bitcast %[[old_buffer_aligned]] : !llvm.ptr to !llvm.ptr -// CHECK: "llvm.intr.memcpy"(%[[new_buffer_void]], %[[old_buffer_void]], %[[src_size]], %[[volatile]]) +// CHECK: "llvm.intr.memcpy"(%[[new_buffer_raw]], %[[old_buffer_aligned]], %[[src_size]], %[[volatile]]) // CHECK: %[[old_buffer_unaligned:.*]] = llvm.extractvalue %[[descriptor]][0] -// CHECK: %[[old_buffer_unaligned_void:.*]] = llvm.bitcast %[[old_buffer_unaligned]] : !llvm.ptr to !llvm.ptr -// CHECK: llvm.call @free(%[[old_buffer_unaligned_void]]) -// CHECK: %[[descriptor_update1:.*]] = llvm.insertvalue %[[new_buffer]], %[[descriptor]][0] -// CHECK: %[[descriptor_update2:.*]] = llvm.insertvalue %[[new_buffer]], %[[descriptor_update1]][1] +// CHECK: llvm.call @free(%[[old_buffer_unaligned]]) +// CHECK: %[[descriptor_update1:.*]] = llvm.insertvalue %[[new_buffer_raw]], %[[descriptor]][0] +// CHECK: %[[descriptor_update2:.*]] = llvm.insertvalue %[[new_buffer_raw]], %[[descriptor_update1]][1] // CHECK: llvm.br ^bb2(%[[descriptor_update2]] -// CHECK: ^bb2(%[[descriptor_update3:.*]]: !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>): +// CHECK: ^bb2(%[[descriptor_update3:.*]]: !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>): // CHECK: %[[descriptor_update4:.*]] = llvm.insertvalue %[[dst_dim]], %[[descriptor_update3]][3, 0] // CHECK: %[[descriptor_update5:.*]] = builtin.unrealized_conversion_cast %[[descriptor_update4]] // CHECK: return %[[descriptor_update5]] : memref @@ -680,34 +582,30 @@ func.func @realloc_dynamic_alignment(%in: memref, %d: index) -> memref +// CHECK: %[[dst_null:.*]] = llvm.mlir.null : !llvm.ptr // CHECK: %[[dst_gep:.*]] = llvm.getelementptr %[[dst_null]][1] -// CHECK: %[[dst_es:.*]] = llvm.ptrtoint %[[dst_gep]] : !llvm.ptr to i64 +// CHECK: %[[dst_es:.*]] = llvm.ptrtoint %[[dst_gep]] : !llvm.ptr to i64 // CHECK: %[[dst_size:.*]] = llvm.mul %[[dst_dim]], %[[dst_es]] // CHECK: %[[src_size:.*]] = llvm.mul %[[drc_dim]], %[[dst_es]] // CHECK: %[[alignment:.*]] = llvm.mlir.constant(8 : index) : i64 // CHECK: %[[adjust_dst_size:.*]] = llvm.add %[[dst_size]], %[[alignment]] // CHECK: %[[new_buffer_raw:.*]] = llvm.call @malloc(%[[adjust_dst_size]]) -// CHECK: %[[new_buffer_unaligned:.*]] = llvm.bitcast %[[new_buffer_raw]] : !llvm.ptr to !llvm.ptr -// CHECK: %[[new_buffer_int:.*]] = llvm.ptrtoint %[[new_buffer_unaligned]] : !llvm.ptr +// CHECK: %[[new_buffer_int:.*]] = llvm.ptrtoint %[[new_buffer_raw]] : !llvm.ptr // CHECK: %[[const_1:.*]] = llvm.mlir.constant(1 : index) : i64 // CHECK: %[[alignment_m1:.*]] = llvm.sub %[[alignment]], %[[const_1]] // CHECK: %[[ptr_alignment_m1:.*]] = llvm.add %[[new_buffer_int]], %[[alignment_m1]] // CHECK: %[[padding:.*]] = llvm.urem %[[ptr_alignment_m1]], %[[alignment]] // CHECK: %[[new_buffer_aligned_int:.*]] = llvm.sub %[[ptr_alignment_m1]], %[[padding]] -// CHECK: %[[new_buffer_aligned:.*]] = llvm.inttoptr %[[new_buffer_aligned_int]] : i64 to !llvm.ptr +// CHECK: %[[new_buffer_aligned:.*]] = llvm.inttoptr %[[new_buffer_aligned_int]] : i64 to !llvm.ptr // CHECK: %[[old_buffer_aligned:.*]] = llvm.extractvalue %[[descriptor]][1] // CHECK: %[[volatile:.*]] = llvm.mlir.constant(false) : i1 -// CHECK-DAG: %[[new_buffer_void:.*]] = llvm.bitcast %[[new_buffer_aligned]] : !llvm.ptr to !llvm.ptr -// CHECK-DAG: %[[old_buffer_void:.*]] = llvm.bitcast %[[old_buffer_aligned]] : !llvm.ptr to !llvm.ptr -// CHECK: "llvm.intr.memcpy"(%[[new_buffer_void]], %[[old_buffer_void]], %[[src_size]], %[[volatile]]) +// CHECK: "llvm.intr.memcpy"(%[[new_buffer_aligned]], %[[old_buffer_aligned]], %[[src_size]], %[[volatile]]) // CHECK: %[[old_buffer_unaligned:.*]] = llvm.extractvalue %[[descriptor]][0] -// CHECK: %[[old_buffer_unaligned_void:.*]] = llvm.bitcast %[[old_buffer_unaligned]] : !llvm.ptr to !llvm.ptr -// CHECK: llvm.call @free(%[[old_buffer_unaligned_void]]) -// CHECK: %[[descriptor_update1:.*]] = llvm.insertvalue %[[new_buffer_unaligned]], %[[descriptor]][0] +// CHECK: llvm.call @free(%[[old_buffer_unaligned]]) +// CHECK: %[[descriptor_update1:.*]] = llvm.insertvalue %[[new_buffer_raw]], %[[descriptor]][0] // CHECK: %[[descriptor_update2:.*]] = llvm.insertvalue %[[new_buffer_aligned]], %[[descriptor_update1]][1] // CHECK: llvm.br ^bb2(%[[descriptor_update2]] -// CHECK: ^bb2(%[[descriptor_update3:.*]]: !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>): +// CHECK: ^bb2(%[[descriptor_update3:.*]]: !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>): // CHECK: %[[descriptor_update4:.*]] = llvm.insertvalue %[[dst_dim]], %[[descriptor_update3]][3, 0] // CHECK: %[[descriptor_update5:.*]] = builtin.unrealized_conversion_cast %[[descriptor_update4]] // CHECK: return %[[descriptor_update5]] : memref @@ -718,9 +616,9 @@ func.func @realloc_dynamic_alignment(%in: memref, %d: index) -> memref +// ALIGNED-ALLOC: %[[dst_null:.*]] = llvm.mlir.null : !llvm.ptr // ALIGNED-ALLOC: %[[dst_gep:.*]] = llvm.getelementptr %[[dst_null]][1] -// ALIGNED-ALLOC: %[[dst_es:.*]] = llvm.ptrtoint %[[dst_gep]] : !llvm.ptr to i64 +// ALIGNED-ALLOC: %[[dst_es:.*]] = llvm.ptrtoint %[[dst_gep]] : !llvm.ptr to i64 // ALIGNED-ALLOC: %[[dst_size:.*]] = llvm.mul %[[dst_dim]], %[[dst_es]] // ALIGNED-ALLOC: %[[src_size:.*]] = llvm.mul %[[drc_dim]], %[[dst_es]] // ALIGNED-ALLOC-DAG: %[[alignment:.*]] = llvm.mlir.constant(8 : index) : i64 @@ -730,19 +628,15 @@ func.func @realloc_dynamic_alignment(%in: memref, %d: index) -> memref to !llvm.ptr // ALIGNED-ALLOC: %[[old_buffer_aligned:.*]] = llvm.extractvalue %[[descriptor]][1] // ALIGNED-ALLOC: %[[volatile:.*]] = llvm.mlir.constant(false) : i1 -// ALIGNED-ALLOC-DAG: %[[new_buffer_void:.*]] = llvm.bitcast %[[new_buffer_aligned]] : !llvm.ptr to !llvm.ptr -// ALIGNED-ALLOC-DAG: %[[old_buffer_void:.*]] = llvm.bitcast %[[old_buffer_aligned]] : !llvm.ptr to !llvm.ptr -// ALIGNED-ALLOC: "llvm.intr.memcpy"(%[[new_buffer_void]], %[[old_buffer_void]], %[[src_size]], %[[volatile]]) +// ALIGNED-ALLOC: "llvm.intr.memcpy"(%[[new_buffer_raw]], %[[old_buffer_aligned]], %[[src_size]], %[[volatile]]) // ALIGNED-ALLOC: %[[old_buffer_unaligned:.*]] = llvm.extractvalue %[[descriptor]][0] -// ALIGNED-ALLOC: %[[old_buffer_unaligned_void:.*]] = llvm.bitcast %[[old_buffer_unaligned]] : !llvm.ptr to !llvm.ptr -// ALIGNED-ALLOC: llvm.call @free(%[[old_buffer_unaligned_void]]) -// ALIGNED-ALLOC: %[[descriptor_update1:.*]] = llvm.insertvalue %[[new_buffer_aligned]], %[[descriptor]][0] -// ALIGNED-ALLOC: %[[descriptor_update2:.*]] = llvm.insertvalue %[[new_buffer_aligned]], %[[descriptor_update1]][1] -// ALIGNED-ALLOC: llvm.br ^bb2(%[[descriptor_update2]] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>) -// ALIGNED-ALLOC: ^bb2(%[[descriptor_update3:.*]]: !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>): +// ALIGNED-ALLOC: llvm.call @free(%[[old_buffer_unaligned]]) +// ALIGNED-ALLOC: %[[descriptor_update1:.*]] = llvm.insertvalue %[[new_buffer_raw]], %[[descriptor]][0] +// ALIGNED-ALLOC: %[[descriptor_update2:.*]] = llvm.insertvalue %[[new_buffer_raw]], %[[descriptor_update1]][1] +// ALIGNED-ALLOC: llvm.br ^bb2(%[[descriptor_update2]] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>) +// ALIGNED-ALLOC: ^bb2(%[[descriptor_update3:.*]]: !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>): // ALIGNED-ALLOC: %[[descriptor_update4:.*]] = llvm.insertvalue %[[dst_dim]], %[[descriptor_update3]][3, 0] // ALIGNED-ALLOC: %[[descriptor_update5:.*]] = builtin.unrealized_conversion_cast %[[descriptor_update4]] // ALIGNED-ALLOC: return %[[descriptor_update5]] : memref diff --git a/mlir/test/Conversion/MemRefToLLVM/convert-static-memref-ops.mlir b/mlir/test/Conversion/MemRefToLLVM/convert-static-memref-ops.mlir index 8617deccd6fa..d9552972e2d9 100644 --- a/mlir/test/Conversion/MemRefToLLVM/convert-static-memref-ops.mlir +++ b/mlir/test/Conversion/MemRefToLLVM/convert-static-memref-ops.mlir @@ -1,18 +1,17 @@ -// RUN: mlir-opt -finalize-memref-to-llvm -split-input-file %s | FileCheck %s +// RUN: mlir-opt -finalize-memref-to-llvm='use-opaque-pointers=1' -split-input-file %s | FileCheck %s // CHECK-LABEL: func @zero_d_alloc() func.func @zero_d_alloc() -> memref { // CHECK: %[[one:.*]] = llvm.mlir.constant(1 : index) : i64 -// CHECK: %[[null:.*]] = llvm.mlir.null : !llvm.ptr -// CHECK: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm.ptr, i64) -> !llvm.ptr -// CHECK: %[[size_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 -// CHECK: llvm.call @malloc(%[[size_bytes]]) : (i64) -> !llvm.ptr -// CHECK: %[[ptr:.*]] = llvm.bitcast %{{.*}} : !llvm.ptr to !llvm.ptr -// CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64)> -// CHECK: llvm.insertvalue %[[ptr]], %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64)> -// CHECK: llvm.insertvalue %[[ptr]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64)> +// CHECK: %[[null:.*]] = llvm.mlir.null : !llvm.ptr +// CHECK: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 +// CHECK: %[[size_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 +// CHECK: %[[ptr:.*]] = llvm.call @malloc(%[[size_bytes]]) : (i64) -> !llvm.ptr +// CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64)> +// CHECK: llvm.insertvalue %[[ptr]], %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64)> +// CHECK: llvm.insertvalue %[[ptr]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64)> // CHECK: %[[c0:.*]] = llvm.mlir.constant(0 : index) : i64 -// CHECK: llvm.insertvalue %[[c0]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64)> +// CHECK: llvm.insertvalue %[[c0]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64)> // CHECK: unrealized_conversion_cast %{{.*}} %0 = memref.alloc() : memref @@ -24,9 +23,8 @@ func.func @zero_d_alloc() -> memref { // CHECK-LABEL: func @zero_d_dealloc func.func @zero_d_dealloc(%arg0: memref) { // CHECK: unrealized_conversion_cast -// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64)> -// CHECK: %[[bc:.*]] = llvm.bitcast %[[ptr]] : !llvm.ptr to !llvm.ptr -// CHECK: llvm.call @free(%[[bc]]) : (!llvm.ptr) -> () +// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64)> +// CHECK: llvm.call @free(%[[ptr]]) : (!llvm.ptr) -> () memref.dealloc %arg0 : memref return @@ -38,25 +36,24 @@ func.func @zero_d_dealloc(%arg0: memref) { func.func @aligned_1d_alloc() -> memref<42xf32> { // CHECK: %[[sz1:.*]] = llvm.mlir.constant(42 : index) : i64 // CHECK: %[[st1:.*]] = llvm.mlir.constant(1 : index) : i64 -// CHECK: %[[null:.*]] = llvm.mlir.null : !llvm.ptr -// CHECK: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[sz1]]] : (!llvm.ptr, i64) -> !llvm.ptr -// CHECK: %[[size_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 +// CHECK: %[[null:.*]] = llvm.mlir.null : !llvm.ptr +// CHECK: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[sz1]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 +// CHECK: %[[size_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 // CHECK: %[[alignment:.*]] = llvm.mlir.constant(8 : index) : i64 // CHECK: %[[allocsize:.*]] = llvm.add %[[size_bytes]], %[[alignment]] : i64 -// CHECK: %[[allocated:.*]] = llvm.call @malloc(%[[allocsize]]) : (i64) -> !llvm.ptr -// CHECK: %[[ptr:.*]] = llvm.bitcast %{{.*}} : !llvm.ptr to !llvm.ptr -// CHECK: %[[allocatedAsInt:.*]] = llvm.ptrtoint %[[ptr]] : !llvm.ptr to i64 +// CHECK: %[[ptr:.*]] = llvm.call @malloc(%[[allocsize]]) : (i64) -> !llvm.ptr +// CHECK: %[[allocatedAsInt:.*]] = llvm.ptrtoint %[[ptr]] : !llvm.ptr to i64 // CHECK: %[[one_1:.*]] = llvm.mlir.constant(1 : index) : i64 // CHECK: %[[bump:.*]] = llvm.sub %[[alignment]], %[[one_1]] : i64 // CHECK: %[[bumped:.*]] = llvm.add %[[allocatedAsInt]], %[[bump]] : i64 // CHECK: %[[mod:.*]] = llvm.urem %[[bumped]], %[[alignment]] : i64 // CHECK: %[[aligned:.*]] = llvm.sub %[[bumped]], %[[mod]] : i64 -// CHECK: %[[alignedBitCast:.*]] = llvm.inttoptr %[[aligned]] : i64 to !llvm.ptr -// CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> -// CHECK: llvm.insertvalue %[[ptr]], %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> -// CHECK: llvm.insertvalue %[[alignedBitCast]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> +// CHECK: %[[alignedBitCast:.*]] = llvm.inttoptr %[[aligned]] : i64 to !llvm.ptr +// CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> +// CHECK: llvm.insertvalue %[[ptr]], %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> +// CHECK: llvm.insertvalue %[[alignedBitCast]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // CHECK: %[[c0:.*]] = llvm.mlir.constant(0 : index) : i64 -// CHECK: llvm.insertvalue %[[c0]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> +// CHECK: llvm.insertvalue %[[c0]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> %0 = memref.alloc() {alignment = 8} : memref<42xf32> return %0 : memref<42xf32> } @@ -66,11 +63,10 @@ func.func @aligned_1d_alloc() -> memref<42xf32> { // CHECK-LABEL: func @static_alloc() func.func @static_alloc() -> memref<32x18xf32> { // CHECK: %[[num_elems:.*]] = llvm.mlir.constant(576 : index) : i64 -// CHECK: %[[null:.*]] = llvm.mlir.null : !llvm.ptr -// CHECK: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[num_elems]]] : (!llvm.ptr, i64) -> !llvm.ptr -// CHECK: %[[size_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 -// CHECK: %[[allocated:.*]] = llvm.call @malloc(%[[size_bytes]]) : (i64) -> !llvm.ptr -// CHECK: llvm.bitcast %[[allocated]] : !llvm.ptr to !llvm.ptr +// CHECK: %[[null:.*]] = llvm.mlir.null : !llvm.ptr +// CHECK: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[num_elems]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 +// CHECK: %[[size_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 +// CHECK: llvm.call @malloc(%[[size_bytes]]) : (i64) -> !llvm.ptr %0 = memref.alloc() : memref<32x18xf32> return %0 : memref<32x18xf32> } @@ -83,19 +79,19 @@ func.func @static_alloca() -> memref<32x18xf32> { // CHECK: %[[sz2:.*]] = llvm.mlir.constant(18 : index) : i64 // CHECK: %[[st2:.*]] = llvm.mlir.constant(1 : index) : i64 // CHECK: %[[num_elems:.*]] = llvm.mlir.constant(576 : index) : i64 -// CHECK: %[[null:.*]] = llvm.mlir.null : !llvm.ptr -// CHECK: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[num_elems]]] : (!llvm.ptr, i64) -> !llvm.ptr -// CHECK: %[[size_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 -// CHECK: %[[allocated:.*]] = llvm.alloca %[[size_bytes]] x f32 : (i64) -> !llvm.ptr +// CHECK: %[[null:.*]] = llvm.mlir.null : !llvm.ptr +// CHECK: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[num_elems]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 +// CHECK: %[[size_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 +// CHECK: %[[allocated:.*]] = llvm.alloca %[[size_bytes]] x f32 : (i64) -> !llvm.ptr %0 = memref.alloca() : memref<32x18xf32> // Test with explicitly specified alignment. llvm.alloca takes care of the // alignment. The same pointer is thus used for allocation and aligned // accesses. - // CHECK: %[[alloca_aligned:.*]] = llvm.alloca %{{.*}} x f32 {alignment = 32 : i64} : (i64) -> !llvm.ptr - // CHECK: %[[desc:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[desc1:.*]] = llvm.insertvalue %[[alloca_aligned]], %[[desc]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: llvm.insertvalue %[[alloca_aligned]], %[[desc1]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[alloca_aligned:.*]] = llvm.alloca %{{.*}} x f32 {alignment = 32 : i64} : (i64) -> !llvm.ptr + // CHECK: %[[desc:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[desc1:.*]] = llvm.insertvalue %[[alloca_aligned]], %[[desc]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[alloca_aligned]], %[[desc1]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> memref.alloca() {alignment = 32} : memref<32x18xf32> return %0 : memref<32x18xf32> } @@ -104,9 +100,8 @@ func.func @static_alloca() -> memref<32x18xf32> { // CHECK-LABEL: func @static_dealloc func.func @static_dealloc(%static: memref<10x8xf32>) { -// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[bc:.*]] = llvm.bitcast %[[ptr]] : !llvm.ptr to !llvm.ptr -// CHECK: llvm.call @free(%[[bc]]) : (!llvm.ptr) -> () +// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: llvm.call @free(%[[ptr]]) : (!llvm.ptr) -> () memref.dealloc %static : memref<10x8xf32> return } @@ -115,8 +110,8 @@ func.func @static_dealloc(%static: memref<10x8xf32>) { // CHECK-LABEL: func @zero_d_load func.func @zero_d_load(%arg0: memref) -> f32 { -// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64)> -// CHECK: %{{.*}} = llvm.load %[[ptr]] : !llvm.ptr +// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64)> +// CHECK: %{{.*}} = llvm.load %[[ptr]] : !llvm.ptr -> f32 %0 = memref.load %arg0[] : memref return %0 : f32 } @@ -130,12 +125,12 @@ func.func @zero_d_load(%arg0: memref) -> f32 { func.func @static_load(%static : memref<10x42xf32>, %i : index, %j : index) { // CHECK: %[[II:.*]] = builtin.unrealized_conversion_cast %[[I]] // CHECK: %[[JJ:.*]] = builtin.unrealized_conversion_cast %[[J]] -// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[st0:.*]] = llvm.mlir.constant(42 : index) : i64 // CHECK: %[[offI:.*]] = llvm.mul %[[II]], %[[st0]] : i64 // CHECK: %[[off1:.*]] = llvm.add %[[offI]], %[[JJ]] : i64 -// CHECK: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr, i64) -> !llvm.ptr -// CHECK: llvm.load %[[addr]] : !llvm.ptr +// CHECK: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 +// CHECK: llvm.load %[[addr]] : !llvm.ptr -> f32 %0 = memref.load %static[%i, %j] : memref<10x42xf32> return } @@ -144,8 +139,8 @@ func.func @static_load(%static : memref<10x42xf32>, %i : index, %j : index) { // CHECK-LABEL: func @zero_d_store func.func @zero_d_store(%arg0: memref, %arg1: f32) { -// CHECK: %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr, ptr, i64)> -// CHECK: llvm.store %{{.*}}, %[[ptr]] : !llvm.ptr +// CHECK: %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr, ptr, i64)> +// CHECK: llvm.store %{{.*}}, %[[ptr]] : f32, !llvm.ptr memref.store %arg1, %arg0[] : memref return } @@ -158,12 +153,12 @@ func.func @zero_d_store(%arg0: memref, %arg1: f32) { func.func @static_store(%static : memref<10x42xf32>, %i : index, %j : index, %val : f32) { // CHECK: %[[II:.*]] = builtin.unrealized_conversion_cast %[[I]] // CHECK: %[[JJ:.*]] = builtin.unrealized_conversion_cast %[[J]] -// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[st0:.*]] = llvm.mlir.constant(42 : index) : i64 // CHECK: %[[offI:.*]] = llvm.mul %[[II]], %[[st0]] : i64 // CHECK: %[[off1:.*]] = llvm.add %[[offI]], %[[JJ]] : i64 -// CHECK: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr, i64) -> !llvm.ptr -// CHECK: llvm.store %{{.*}}, %[[addr]] : !llvm.ptr +// CHECK: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 +// CHECK: llvm.store %{{.*}}, %[[addr]] : f32, !llvm.ptr memref.store %val, %static[%i, %j] : memref<10x42xf32> return @@ -203,11 +198,11 @@ module attributes { dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry> } { // CHECK: %[[CST:.*]] = builtin.unrealized_conversion_cast // CHECK: llvm.mlir.null // CHECK: llvm.getelementptr %{{.*}}[[CST]] - // CHECK: llvm.ptrtoint %{{.*}} : !llvm.ptr<{{.*}}> to i32 - // CHECK: llvm.ptrtoint %{{.*}} : !llvm.ptr<{{.*}}> to i32 + // CHECK: llvm.ptrtoint %{{.*}} : !llvm.ptr to i32 + // CHECK: llvm.ptrtoint %{{.*}} : !llvm.ptr to i32 // CHECK: llvm.add %{{.*}} : i32 // CHECK: llvm.call @malloc(%{{.*}}) : (i32) -> !llvm.ptr - // CHECK: llvm.ptrtoint %{{.*}} : !llvm.ptr<{{.*}}> to i32 + // CHECK: llvm.ptrtoint %{{.*}} : !llvm.ptr to i32 // CHECK: llvm.sub {{.*}} : i32 // CHECK: llvm.add {{.*}} : i32 // CHECK: llvm.urem {{.*}} : i32 @@ -224,34 +219,34 @@ memref.global "private" constant @__constant_3xi64 : memref<3xi64> = dense<[2, 6 // CHECK-LABEL: func @memref.reshape // CHECK-SAME: %[[arg0:.*]]: memref<4x5x6xf32>) -> memref<2x6x20xf32> func.func @memref.reshape(%arg0: memref<4x5x6xf32>) -> memref<2x6x20xf32> { - // CHECK: %[[cast0:.*]] = builtin.unrealized_conversion_cast %arg0 : memref<4x5x6xf32> to !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[cast0:.*]] = builtin.unrealized_conversion_cast %arg0 : memref<4x5x6xf32> to !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> %0 = memref.get_global @__constant_3xi64 : memref<3xi64> - // CHECK: %[[undef:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK: %[[elem0:.*]] = llvm.extractvalue %[[cast0]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK: %[[elem1:.*]] = llvm.extractvalue %[[cast0]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK: %[[insert0:.*]] = llvm.insertvalue %[[elem0]], %[[undef]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK: %[[insert1:.*]] = llvm.insertvalue %[[elem1]], %[[insert0:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[undef:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[elem0:.*]] = llvm.extractvalue %[[cast0]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[elem1:.*]] = llvm.extractvalue %[[cast0]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[insert0:.*]] = llvm.insertvalue %[[elem0]], %[[undef]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[insert1:.*]] = llvm.insertvalue %[[elem1]], %[[insert0:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // CHECK: %[[zero:.*]] = llvm.mlir.constant(0 : index) : i64 - // CHECK: %[[insert2:.*]] = llvm.insertvalue %[[zero]], %[[insert1]][2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[insert2:.*]] = llvm.insertvalue %[[zero]], %[[insert1]][2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // CHECK: %[[one:.*]] = llvm.mlir.constant(1 : index) : i64 // CHECK: %[[twenty0:.*]] = llvm.mlir.constant(20 : index) : i64 - // CHECK: %[[insert3:.*]] = llvm.insertvalue %[[twenty0]], %[[insert2]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK: %[[insert4:.*]] = llvm.insertvalue %[[one]], %[[insert3]][4, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[insert3:.*]] = llvm.insertvalue %[[twenty0]], %[[insert2]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[insert4:.*]] = llvm.insertvalue %[[one]], %[[insert3]][4, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // CHECK: %[[twenty1:.*]] = llvm.mlir.constant(20 : index) : i64 // CHECK: %[[six:.*]] = llvm.mlir.constant(6 : index) : i64 - // CHECK: %[[insert5:.*]] = llvm.insertvalue %[[six]], %[[insert4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK: %[[insert6:.*]] = llvm.insertvalue %[[twenty1]], %[[insert5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[insert5:.*]] = llvm.insertvalue %[[six]], %[[insert4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[insert6:.*]] = llvm.insertvalue %[[twenty1]], %[[insert5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // CHECK: %[[hundred_and_twenty:.*]] = llvm.mlir.constant(120 : index) : i64 // CHECK: %[[two:.*]] = llvm.mlir.constant(2 : index) : i64 - // CHECK: %[[insert7:.*]] = llvm.insertvalue %[[two]], %[[insert6]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK: %[[insert8:.*]] = llvm.insertvalue %[[hundred_and_twenty]], %[[insert7]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[insert7:.*]] = llvm.insertvalue %[[two]], %[[insert6]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[insert8:.*]] = llvm.insertvalue %[[hundred_and_twenty]], %[[insert7]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK: %[[cast1:.*]] = builtin.unrealized_conversion_cast %[[insert8]] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> to memref<2x6x20xf32> + // CHECK: %[[cast1:.*]] = builtin.unrealized_conversion_cast %[[insert8]] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> to memref<2x6x20xf32> %1 = memref.reshape %arg0(%0) : (memref<4x5x6xf32>, memref<3xi64>) -> memref<2x6x20xf32> // CHECK: return %[[cast1]] : memref<2x6x20xf32> @@ -263,44 +258,44 @@ func.func @memref.reshape(%arg0: memref<4x5x6xf32>) -> memref<2x6x20xf32> { // CHECK-LABEL: func @memref.reshape.dynamic.dim // CHECK-SAME: %[[arg:.*]]: memref, %[[shape:.*]]: memref<4xi64>) -> memref func.func @memref.reshape.dynamic.dim(%arg: memref, %shape: memref<4xi64>) -> memref { - // CHECK: %[[arg_cast:.*]] = builtin.unrealized_conversion_cast %[[arg]] : memref to !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK: %[[shape_cast:.*]] = builtin.unrealized_conversion_cast %[[shape]] : memref<4xi64> to !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[undef:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> - // CHECK: %[[alloc_ptr:.*]] = llvm.extractvalue %[[arg_cast]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK: %[[align_ptr:.*]] = llvm.extractvalue %[[arg_cast]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK: %[[insert0:.*]] = llvm.insertvalue %[[alloc_ptr]], %[[undef]][0] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> - // CHECK: %[[insert1:.*]] = llvm.insertvalue %[[align_ptr]], %[[insert0]][1] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> + // CHECK: %[[arg_cast:.*]] = builtin.unrealized_conversion_cast %[[arg]] : memref to !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[shape_cast:.*]] = builtin.unrealized_conversion_cast %[[shape]] : memref<4xi64> to !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[undef:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> + // CHECK: %[[alloc_ptr:.*]] = llvm.extractvalue %[[arg_cast]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[align_ptr:.*]] = llvm.extractvalue %[[arg_cast]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[insert0:.*]] = llvm.insertvalue %[[alloc_ptr]], %[[undef]][0] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> + // CHECK: %[[insert1:.*]] = llvm.insertvalue %[[align_ptr]], %[[insert0]][1] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> // CHECK: %[[zero0:.*]] = llvm.mlir.constant(0 : index) : i64 - // CHECK: %[[insert2:.*]] = llvm.insertvalue %[[zero0]], %[[insert1]][2] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> + // CHECK: %[[insert2:.*]] = llvm.insertvalue %[[zero0]], %[[insert1]][2] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> // CHECK: %[[one0:.*]] = llvm.mlir.constant(1 : index) : i64 // CHECK: %[[thirty_two0:.*]] = llvm.mlir.constant(32 : index) : i64 - // CHECK: %[[insert3:.*]] = llvm.insertvalue %[[thirty_two0]], %[[insert2]][3, 3] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> - // CHECK: %[[insert4:.*]] = llvm.insertvalue %[[one0]], %[[insert3]][4, 3] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> + // CHECK: %[[insert3:.*]] = llvm.insertvalue %[[thirty_two0]], %[[insert2]][3, 3] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> + // CHECK: %[[insert4:.*]] = llvm.insertvalue %[[one0]], %[[insert3]][4, 3] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> // CHECK: %[[thirty_two1:.*]] = llvm.mlir.constant(32 : index) : i64 // CHECK: %[[twelve:.*]] = llvm.mlir.constant(12 : index) : i64 - // CHECK: %[[insert5:.*]] = llvm.insertvalue %[[twelve]], %[[insert4]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> - // CHECK: %[[insert6:.*]] = llvm.insertvalue %[[thirty_two1]], %[[insert5]][4, 2] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> + // CHECK: %[[insert5:.*]] = llvm.insertvalue %[[twelve]], %[[insert4]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> + // CHECK: %[[insert6:.*]] = llvm.insertvalue %[[thirty_two1]], %[[insert5]][4, 2] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> // CHECK: %[[three_hundred_and_eighty_four:.*]] = llvm.mlir.constant(384 : index) : i64 // CHECK: %[[one1:.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK: %[[shape_ptr0:.*]] = llvm.extractvalue %[[shape_cast]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[shape_gep0:.*]] = llvm.getelementptr %[[shape_ptr0]][%[[one1]]] : (!llvm.ptr, i64) -> !llvm.ptr - // CHECK: %[[shape_load0:.*]] = llvm.load %[[shape_gep0]] : !llvm.ptr - // CHECK: %[[insert7:.*]] = llvm.insertvalue %[[shape_load0]], %[[insert6]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> - // CHECK: %[[insert8:.*]] = llvm.insertvalue %[[three_hundred_and_eighty_four]], %[[insert7]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> + // CHECK: %[[shape_ptr0:.*]] = llvm.extractvalue %[[shape_cast]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[shape_gep0:.*]] = llvm.getelementptr %[[shape_ptr0]][%[[one1]]] : (!llvm.ptr, i64) -> !llvm.ptr, i64 + // CHECK: %[[shape_load0:.*]] = llvm.load %[[shape_gep0]] : !llvm.ptr -> i64 + // CHECK: %[[insert7:.*]] = llvm.insertvalue %[[shape_load0]], %[[insert6]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> + // CHECK: %[[insert8:.*]] = llvm.insertvalue %[[three_hundred_and_eighty_four]], %[[insert7]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> // CHECK: %[[mul:.*]] = llvm.mul %19, %23 : i64 // CHECK: %[[zero1:.*]] = llvm.mlir.constant(0 : index) : i64 - // CHECK: %[[shape_ptr1:.*]] = llvm.extractvalue %[[shape_cast]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[shape_gep1:.*]] = llvm.getelementptr %[[shape_ptr1]][%[[zero1]]] : (!llvm.ptr, i64) -> !llvm.ptr - // CHECK: %[[shape_load1:.*]] = llvm.load %[[shape_gep1]] : !llvm.ptr - // CHECK: %[[insert9:.*]] = llvm.insertvalue %[[shape_load1]], %[[insert8]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> - // CHECK: %[[insert10:.*]] = llvm.insertvalue %[[mul]], %[[insert9]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> + // CHECK: %[[shape_ptr1:.*]] = llvm.extractvalue %[[shape_cast]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[shape_gep1:.*]] = llvm.getelementptr %[[shape_ptr1]][%[[zero1]]] : (!llvm.ptr, i64) -> !llvm.ptr, i64 + // CHECK: %[[shape_load1:.*]] = llvm.load %[[shape_gep1]] : !llvm.ptr -> i64 + // CHECK: %[[insert9:.*]] = llvm.insertvalue %[[shape_load1]], %[[insert8]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> + // CHECK: %[[insert10:.*]] = llvm.insertvalue %[[mul]], %[[insert9]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> - // CHECK: %[[result_cast:.*]] = builtin.unrealized_conversion_cast %[[insert10]] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> to memref + // CHECK: %[[result_cast:.*]] = builtin.unrealized_conversion_cast %[[insert10]] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> to memref %0 = memref.reshape %arg(%shape) : (memref, memref<4xi64>) -> memref return %0 : memref @@ -312,27 +307,27 @@ func.func @memref.reshape.dynamic.dim(%arg: memref, %shape: memref<4x // CHECK-LABEL: func @memref.reshape_index // CHECK-SAME: %[[arg:.*]]: memref, %[[shape:.*]]: memref<1xindex> func.func @memref.reshape_index(%arg0: memref, %shape: memref<1xindex>) -> memref { - // CHECK: %[[arg_cast:.*]] = builtin.unrealized_conversion_cast %[[arg]] : memref to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[shape_cast:.*]] = builtin.unrealized_conversion_cast %[[shape]] : memref<1xindex> to !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[undef:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[alloc_ptr:.*]] = llvm.extractvalue %[[arg_cast]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[align_ptr:.*]] = llvm.extractvalue %[[arg_cast]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[insert0:.*]] = llvm.insertvalue %[[alloc_ptr]], %[[undef:.*]][0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[insert1:.*]] = llvm.insertvalue %[[align_ptr]], %[[insert0:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[arg_cast:.*]] = builtin.unrealized_conversion_cast %[[arg]] : memref to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[shape_cast:.*]] = builtin.unrealized_conversion_cast %[[shape]] : memref<1xindex> to !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[undef:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[alloc_ptr:.*]] = llvm.extractvalue %[[arg_cast]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[align_ptr:.*]] = llvm.extractvalue %[[arg_cast]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[insert0:.*]] = llvm.insertvalue %[[alloc_ptr]], %[[undef:.*]][0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[insert1:.*]] = llvm.insertvalue %[[align_ptr]], %[[insert0:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // CHECK: %[[zero0:.*]] = llvm.mlir.constant(0 : index) : i64 - // CHECK: %[[insert2:.*]] = llvm.insertvalue %[[zero0]], %[[insert1:.*]][2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[insert2:.*]] = llvm.insertvalue %[[zero0]], %[[insert1:.*]][2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // CHECK: %[[one0:.*]] = llvm.mlir.constant(1 : index) : i64 // CHECK: %[[zero1:.*]] = llvm.mlir.constant(0 : index) : i64 - // CHECK: %[[shape_ptr0:.*]] = llvm.extractvalue %[[shape_cast:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[shape_gep0:.*]] = llvm.getelementptr %[[shape_ptr0:.*]][%[[zero1:.*]]] : (!llvm.ptr, i64) -> !llvm.ptr - // CHECK: %[[shape_load0:.*]] = llvm.load %[[shape_gep0:.*]] : !llvm.ptr - // CHECK: %[[insert3:.*]] = llvm.insertvalue %[[shape_load0:.*]], %[[insert2:.*]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[insert4:.*]] = llvm.insertvalue %[[one0:.*]], %[[insert3:.*]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[shape_ptr0:.*]] = llvm.extractvalue %[[shape_cast:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[shape_gep0:.*]] = llvm.getelementptr %[[shape_ptr0:.*]][%[[zero1:.*]]] : (!llvm.ptr, i64) -> !llvm.ptr, i64 + // CHECK: %[[shape_load0:.*]] = llvm.load %[[shape_gep0:.*]] : !llvm.ptr -> i64 + // CHECK: %[[insert3:.*]] = llvm.insertvalue %[[shape_load0:.*]], %[[insert2:.*]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[insert4:.*]] = llvm.insertvalue %[[one0:.*]], %[[insert3:.*]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[result_cast:.*]] = builtin.unrealized_conversion_cast %[[insert4:.*]] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> to memref + // CHECK: %[[result_cast:.*]] = builtin.unrealized_conversion_cast %[[insert4:.*]] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> to memref // CHECK: return %[[result_cast:.*]] : memref %1 = memref.reshape %arg0(%shape) : (memref, memref<1xindex>) -> memref @@ -344,31 +339,27 @@ func.func @memref.reshape_index(%arg0: memref, %shape: memref<1xindex>) // CHECK-LABEL: func.func @realloc_static( // CHECK-SAME: %[[arg0:.*]]: memref<2xi32>) -> memref<4xi32> { func.func @realloc_static(%in: memref<2xi32>) -> memref<4xi32>{ -// CHECK: %[[descriptor:.*]] = builtin.unrealized_conversion_cast %[[arg0]] : memref<2xi32> to !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> +// CHECK: %[[descriptor:.*]] = builtin.unrealized_conversion_cast %[[arg0]] : memref<2xi32> to !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // CHECK: %[[src_dim:.*]] = llvm.mlir.constant(2 : index) : i64 // CHECK: %[[dst_dim:.*]] = llvm.mlir.constant(4 : index) : i64 // CHECK: %[[cond:.*]] = llvm.icmp "ugt" %[[dst_dim]], %[[src_dim]] // CHECK: llvm.cond_br %[[cond]], ^bb1, ^bb2(%[[descriptor]] // CHECK: ^bb1: -// CHECK: %[[dst_null:.*]] = llvm.mlir.null : !llvm.ptr +// CHECK: %[[dst_null:.*]] = llvm.mlir.null : !llvm.ptr // CHECK: %[[dst_gep:.*]] = llvm.getelementptr %[[dst_null]][1] -// CHECK: %[[dst_es:.*]] = llvm.ptrtoint %[[dst_gep]] : !llvm.ptr to i64 +// CHECK: %[[dst_es:.*]] = llvm.ptrtoint %[[dst_gep]] : !llvm.ptr to i64 // CHECK: %[[dst_size:.*]] = llvm.mul %[[dst_dim]], %[[dst_es]] // CHECK: %[[src_size:.*]] = llvm.mul %[[src_dim]], %[[dst_es]] // CHECK: %[[new_buffer_raw:.*]] = llvm.call @malloc(%[[dst_size]]) -// CHECK: %[[new_buffer:.*]] = llvm.bitcast %[[new_buffer_raw]] : !llvm.ptr to !llvm.ptr // CHECK: %[[old_buffer_aligned:.*]] = llvm.extractvalue %[[descriptor]][1] // CHECK: %[[volatile:.*]] = llvm.mlir.constant(false) : i1 -// CHECK-DAG: %[[new_buffer_void:.*]] = llvm.bitcast %[[new_buffer]] : !llvm.ptr to !llvm.ptr -// CHECK-DAG: %[[old_buffer_void:.*]] = llvm.bitcast %[[old_buffer_aligned]] : !llvm.ptr to !llvm.ptr -// CHECK: "llvm.intr.memcpy"(%[[new_buffer_void]], %[[old_buffer_void]], %[[src_size]], %[[volatile]]) +// CHECK: "llvm.intr.memcpy"(%[[new_buffer_raw]], %[[old_buffer_aligned]], %[[src_size]], %[[volatile]]) // CHECK: %[[old_buffer_unaligned:.*]] = llvm.extractvalue %[[descriptor]][0] -// CHECK: %[[old_buffer_unaligned_void:.*]] = llvm.bitcast %[[old_buffer_unaligned]] : !llvm.ptr to !llvm.ptr -// CHECK: llvm.call @free(%[[old_buffer_unaligned_void]]) -// CHECK: %[[descriptor_update1:.*]] = llvm.insertvalue %[[new_buffer]], %[[descriptor]][0] -// CHECK: %[[descriptor_update2:.*]] = llvm.insertvalue %[[new_buffer]], %[[descriptor_update1]][1] -// CHECK: llvm.br ^bb2(%[[descriptor_update2]] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>) -// CHECK: ^bb2(%[[descriptor_update3:.*]]: !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>): +// CHECK: llvm.call @free(%[[old_buffer_unaligned]]) +// CHECK: %[[descriptor_update1:.*]] = llvm.insertvalue %[[new_buffer_raw]], %[[descriptor]][0] +// CHECK: %[[descriptor_update2:.*]] = llvm.insertvalue %[[new_buffer_raw]], %[[descriptor_update1]][1] +// CHECK: llvm.br ^bb2(%[[descriptor_update2]] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>) +// CHECK: ^bb2(%[[descriptor_update3:.*]]: !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>): // CHECK: %[[descriptor_update4:.*]] = llvm.insertvalue %[[dst_dim]], %[[descriptor_update3]][3, 0] // CHECK: %[[descriptor_update5:.*]] = builtin.unrealized_conversion_cast %[[descriptor_update4]] // CHECK: return %[[descriptor_update5]] : memref<4xi32> @@ -382,40 +373,36 @@ func.func @realloc_static(%in: memref<2xi32>) -> memref<4xi32>{ // CHECK-LABEL: func.func @realloc_static_alignment( // CHECK-SAME: %[[arg0:.*]]: memref<2xf32>) -> memref<4xf32> { func.func @realloc_static_alignment(%in: memref<2xf32>) -> memref<4xf32>{ -// CHECK: %[[descriptor:.*]] = builtin.unrealized_conversion_cast %[[arg0]] : memref<2xf32> to !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> +// CHECK: %[[descriptor:.*]] = builtin.unrealized_conversion_cast %[[arg0]] : memref<2xf32> to !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // CHECK: %[[src_dim:.*]] = llvm.mlir.constant(2 : index) : i64 // CHECK: %[[dst_dim:.*]] = llvm.mlir.constant(4 : index) : i64 // CHECK: %[[cond:.*]] = llvm.icmp "ugt" %[[dst_dim]], %[[src_dim]] : i64 // CHECK: llvm.cond_br %[[cond]], ^bb1, ^bb2(%[[descriptor]] // CHECK: ^bb1: -// CHECK: %[[dst_null:.*]] = llvm.mlir.null : !llvm.ptr +// CHECK: %[[dst_null:.*]] = llvm.mlir.null : !llvm.ptr // CHECK: %[[dst_gep:.*]] = llvm.getelementptr %[[dst_null]][1] -// CHECK: %[[dst_es:.*]] = llvm.ptrtoint %[[dst_gep]] : !llvm.ptr to i64 +// CHECK: %[[dst_es:.*]] = llvm.ptrtoint %[[dst_gep]] : !llvm.ptr to i64 // CHECK: %[[dst_size:.*]] = llvm.mul %[[dst_dim]], %[[dst_es]] // CHECK: %[[src_size:.*]] = llvm.mul %[[src_dim]], %[[dst_es]] // CHECK: %[[alignment:.*]] = llvm.mlir.constant(8 : index) : i64 // CHECK: %[[adjust_dst_size:.*]] = llvm.add %[[dst_size]], %[[alignment]] // CHECK: %[[new_buffer_raw:.*]] = llvm.call @malloc(%[[adjust_dst_size]]) -// CHECK: %[[new_buffer_unaligned:.*]] = llvm.bitcast %[[new_buffer_raw]] : !llvm.ptr to !llvm.ptr -// CHECK: %[[new_buffer_int:.*]] = llvm.ptrtoint %[[new_buffer_unaligned]] : !llvm.ptr +// CHECK: %[[new_buffer_int:.*]] = llvm.ptrtoint %[[new_buffer_raw]] : !llvm.ptr // CHECK: %[[const_1:.*]] = llvm.mlir.constant(1 : index) : i64 // CHECK: %[[alignment_m1:.*]] = llvm.sub %[[alignment]], %[[const_1]] // CHECK: %[[ptr_alignment_m1:.*]] = llvm.add %[[new_buffer_int]], %[[alignment_m1]] // CHECK: %[[padding:.*]] = llvm.urem %[[ptr_alignment_m1]], %[[alignment]] // CHECK: %[[new_buffer_aligned_int:.*]] = llvm.sub %[[ptr_alignment_m1]], %[[padding]] -// CHECK: %[[new_buffer_aligned:.*]] = llvm.inttoptr %[[new_buffer_aligned_int]] : i64 to !llvm.ptr +// CHECK: %[[new_buffer_aligned:.*]] = llvm.inttoptr %[[new_buffer_aligned_int]] : i64 to !llvm.ptr // CHECK: %[[old_buffer_aligned:.*]] = llvm.extractvalue %[[descriptor]][1] // CHECK: %[[volatile:.*]] = llvm.mlir.constant(false) : i1 -// CHECK-DAG: %[[new_buffer_void:.*]] = llvm.bitcast %[[new_buffer_aligned]] : !llvm.ptr to !llvm.ptr -// CHECK-DAG: %[[old_buffer_void:.*]] = llvm.bitcast %[[old_buffer_aligned]] : !llvm.ptr to !llvm.ptr -// CHECK: "llvm.intr.memcpy"(%[[new_buffer_void]], %[[old_buffer_void]], %[[src_size]], %[[volatile]]) +// CHECK: "llvm.intr.memcpy"(%[[new_buffer_aligned]], %[[old_buffer_aligned]], %[[src_size]], %[[volatile]]) // CHECK: %[[old_buffer_unaligned:.*]] = llvm.extractvalue %[[descriptor]][0] -// CHECK: %[[old_buffer_unaligned_void:.*]] = llvm.bitcast %[[old_buffer_unaligned]] : !llvm.ptr to !llvm.ptr -// CHECK: llvm.call @free(%[[old_buffer_unaligned_void]]) -// CHECK: %[[descriptor_update1:.*]] = llvm.insertvalue %[[new_buffer_unaligned]], %[[descriptor]][0] +// CHECK: llvm.call @free(%[[old_buffer_unaligned]]) +// CHECK: %[[descriptor_update1:.*]] = llvm.insertvalue %[[new_buffer_raw]], %[[descriptor]][0] // CHECK: %[[descriptor_update2:.*]] = llvm.insertvalue %[[new_buffer_aligned]], %[[descriptor_update1]][1] // CHECK: llvm.br ^bb2(%[[descriptor_update2]] -// CHECK: ^bb2(%[[descriptor_update3:.*]]: !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>): +// CHECK: ^bb2(%[[descriptor_update3:.*]]: !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>): // CHECK: %[[descriptor_update4:.*]] = llvm.insertvalue %[[dst_dim]], %[[descriptor_update3]][3, 0] // CHECK: %[[descriptor_update5:.*]] = builtin.unrealized_conversion_cast %[[descriptor_update4]] // CHECK: return %[[descriptor_update5]] : memref<4xf32> diff --git a/mlir/test/Conversion/MemRefToLLVM/expand-then-convert-to-llvm.mlir b/mlir/test/Conversion/MemRefToLLVM/expand-then-convert-to-llvm.mlir index f2be1b525696..7a21bf36b853 100644 --- a/mlir/test/Conversion/MemRefToLLVM/expand-then-convert-to-llvm.mlir +++ b/mlir/test/Conversion/MemRefToLLVM/expand-then-convert-to-llvm.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt -expand-strided-metadata -finalize-memref-to-llvm -lower-affine -convert-arith-to-llvm -cse %s -split-input-file | FileCheck %s +// RUN: mlir-opt -expand-strided-metadata -finalize-memref-to-llvm='use-opaque-pointers=1' -lower-affine -convert-arith-to-llvm -cse %s -split-input-file | FileCheck %s // // This test demonstrates a full "memref to llvm" pipeline where // we first expand some of the memref operations (using affine, @@ -56,8 +56,8 @@ func.func @subview(%0 : memref<64x4xf32, strided<[4, 1], offset: 0>>, %arg0 : in // CHECK-DAG: %[[ARG0:.*]] = builtin.unrealized_conversion_cast %[[ARG0f]] // CHECK-DAG: %[[ARG1:.*]] = builtin.unrealized_conversion_cast %[[ARG1f]] - // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64 - // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64 + // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64 + // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64 // CHECK: %[[STRIDE0:.*]] = llvm.mlir.constant(4 : index) : i64 // CHECK: %[[DESCSTRIDE0:.*]] = llvm.mul %[[ARG0]], %[[STRIDE0]] : i64 // CHECK: %[[TMP:.*]] = builtin.unrealized_conversion_cast %[[DESCSTRIDE0]] : i64 to index @@ -65,14 +65,14 @@ func.func @subview(%0 : memref<64x4xf32, strided<[4, 1], offset: 0>>, %arg0 : in // CHECK: %[[OFF2:.*]] = llvm.add %[[DESCSTRIDE0]], %[[ARG1]] : i64 // CHECK: %[[TMP:.*]] = builtin.unrealized_conversion_cast %[[OFF2]] : i64 to index // CHECK: %[[OFF2:.*]] = builtin.unrealized_conversion_cast %[[TMP]] : index to i64 - // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[ARG0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE0_V2]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[ARG0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE0_V2]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> %1 = memref.subview %0[%arg0, %arg1][%arg0, %arg1][%arg0, %arg1] : memref<64x4xf32, strided<[4, 1], offset: 0>> @@ -92,8 +92,8 @@ func.func @subview_non_zero_addrspace(%0 : memref<64x4xf32, strided<[4, 1], offs // CHECK-DAG: %[[ARG0:.*]] = builtin.unrealized_conversion_cast %[[ARG0f]] // CHECK-DAG: %[[ARG1:.*]] = builtin.unrealized_conversion_cast %[[ARG1f]] - // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[STRIDE0:.*]] = llvm.mlir.constant(4 : index) : i64 // CHECK: %[[DESCSTRIDE0:.*]] = llvm.mul %[[ARG0]], %[[STRIDE0]] : i64 // CHECK: %[[TMP:.*]] = builtin.unrealized_conversion_cast %[[DESCSTRIDE0]] : i64 to index @@ -101,14 +101,14 @@ func.func @subview_non_zero_addrspace(%0 : memref<64x4xf32, strided<[4, 1], offs // CHECK: %[[OFF2:.*]] = llvm.add %[[DESCSTRIDE0]], %[[ARG1]] : i64 // CHECK: %[[TMP:.*]] = builtin.unrealized_conversion_cast %[[OFF2]] : i64 to index // CHECK: %[[OFF2:.*]] = builtin.unrealized_conversion_cast %[[TMP]] : index to i64 - // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[ARG0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE0_V2]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[ARG0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE0_V2]], %[[DESC3]][4, 0] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)> %1 = memref.subview %0[%arg0, %arg1][%arg0, %arg1][%arg0, %arg1] : memref<64x4xf32, strided<[4, 1], offset: 0>, 3> @@ -128,8 +128,8 @@ func.func @subview_const_size(%0 : memref<64x4xf32, strided<[4, 1], offset: 0>>, // CHECK-DAG: %[[ARG0:.*]] = builtin.unrealized_conversion_cast %[[ARG0f]] // CHECK-DAG: %[[ARG1:.*]] = builtin.unrealized_conversion_cast %[[ARG1f]] - // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[C4:.*]] = llvm.mlir.constant(4 : index) : i64 // CHECK: %[[DESCSTRIDE0:.*]] = llvm.mul %[[ARG0]], %[[C4]] : i64 // CHECK: %[[TMP:.*]] = builtin.unrealized_conversion_cast %[[DESCSTRIDE0]] : i64 to index @@ -137,15 +137,15 @@ func.func @subview_const_size(%0 : memref<64x4xf32, strided<[4, 1], offset: 0>>, // CHECK: %[[OFF2:.*]] = llvm.add %[[DESCSTRIDE0]], %[[ARG1]] : i64 // CHECK: %[[TMP:.*]] = builtin.unrealized_conversion_cast %[[OFF2]] : i64 to index // CHECK: %[[OFF2:.*]] = builtin.unrealized_conversion_cast %[[TMP]] : index to i64 - // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C4]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE0_V2]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C4]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE0_V2]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[CST_SIZE1:.*]] = llvm.mlir.constant(2 : index) : i64 - // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[CST_SIZE1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[CST_SIZE1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> %1 = memref.subview %0[%arg0, %arg1][4, 2][%arg0, %arg1] : memref<64x4xf32, strided<[4, 1], offset: 0>> @@ -165,22 +165,22 @@ func.func @subview_const_stride(%0 : memref<64x4xf32, strided<[4, 1], offset: 0> // CHECK-DAG: %[[ARG0:.*]] = builtin.unrealized_conversion_cast %[[ARG0f]] // CHECK-DAG: %[[ARG1:.*]] = builtin.unrealized_conversion_cast %[[ARG1f]] - // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[C4:.*]] = llvm.mlir.constant(4 : index) : i64 // CHECK: %[[OFF0:.*]] = llvm.mul %[[ARG0]], %[[C4]] : i64 // CHECK: %[[OFF2:.*]] = llvm.add %[[OFF0]], %[[ARG1]] : i64 // CHECK: %[[TMP:.*]] = builtin.unrealized_conversion_cast %[[OFF2]] : i64 to index // CHECK: %[[OFF2:.*]] = builtin.unrealized_conversion_cast %[[TMP]] : index to i64 - // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[ARG0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[C4]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[ARG0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[C4]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[CST_STRIDE1:.*]] = llvm.mlir.constant(2 : index) : i64 - // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[CST_STRIDE1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[CST_STRIDE1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> %1 = memref.subview %0[%arg0, %arg1][%arg0, %arg1][1, 2] : memref<64x4xf32, strided<[4, 1], offset: 0>> @@ -196,21 +196,21 @@ func.func @subview_const_stride_and_offset(%0 : memref<64x4xf32, strided<[4, 1], // The last "insertvalue" that populates the memref descriptor from the function arguments. // CHECK: %[[MEMREF:.*]] = builtin.unrealized_conversion_cast %[[MEM]] - // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[CST_OFF:.*]] = llvm.mlir.constant(8 : index) : i64 - // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[CST_OFF]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[CST_OFF]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[CST_SIZE0:.*]] = llvm.mlir.constant(62 : index) : i64 - // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[CST_SIZE0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[CST_SIZE0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[CST_STRIDE0:.*]] = llvm.mlir.constant(4 : index) : i64 - // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[CST_STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[CST_STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[CST_SIZE1:.*]] = llvm.mlir.constant(3 : index) : i64 - // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[CST_SIZE1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[CST_SIZE1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[CST_STRIDE1:.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[CST_STRIDE1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[CST_STRIDE1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> %1 = memref.subview %0[0, 8][62, 3][1, 1] : memref<64x4xf32, strided<[4, 1], offset: 0>> @@ -231,8 +231,8 @@ func.func @subview_mixed_static_dynamic(%0 : memref<64x4xf32, strided<[4, 1], of // CHECK-DAG: %[[ARG1:.*]] = builtin.unrealized_conversion_cast %[[ARG1f]] // CHECK-DAG: %[[ARG2:.*]] = builtin.unrealized_conversion_cast %[[ARG2f]] - // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[STRIDE0:.*]] = llvm.mlir.constant(4 : index) : i64 // CHECK: %[[DESCSTRIDE0:.*]] = llvm.mul %[[ARG0]], %[[STRIDE0]] : i64 // CHECK: %[[TMP:.*]] = builtin.unrealized_conversion_cast %[[DESCSTRIDE0]] : i64 to index @@ -242,16 +242,16 @@ func.func @subview_mixed_static_dynamic(%0 : memref<64x4xf32, strided<[4, 1], of // CHECK: %[[OFF2:.*]] = llvm.add %[[OFF0]], %[[BASE_OFF]] : i64 // CHECK: %[[TMP:.*]] = builtin.unrealized_conversion_cast %[[OFF2]] : i64 to index // CHECK: %[[OFF2:.*]] = builtin.unrealized_conversion_cast %[[TMP]] : index to i64 - // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[CST_SIZE0:.*]] = llvm.mlir.constant(62 : index) : i64 - // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[CST_SIZE0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE0_V2]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[ARG2]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[CST_SIZE0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE0_V2]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[ARG2]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[CST_STRIDE1:.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[CST_STRIDE1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[CST_STRIDE1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> %1 = memref.subview %0[%arg1, 8][62, %arg2][%arg0, 1] : memref<64x4xf32, strided<[4, 1], offset: 0>> @@ -266,23 +266,23 @@ func.func @subview_mixed_static_dynamic(%0 : memref<64x4xf32, strided<[4, 1], of func.func @subview_leading_operands(%0 : memref<5x3xf32>, %1: memref<5x?xf32>) -> memref<3x3xf32, strided<[3, 1], offset: 6>> { // CHECK: %[[MEMREF:.*]] = builtin.unrealized_conversion_cast %[[MEM]] // Alloc ptr - // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64 + // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64 // Aligned ptr - // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64 - // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64 + // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // Offset // CHECK: %[[CST_OFF:.*]] = llvm.mlir.constant(6 : index) : i64 - // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[CST_OFF]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[CST_OFF]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // Sizes and strides @rank 0: both static extracted from type. // CHECK: %[[C3:.*]] = llvm.mlir.constant(3 : index) : i64 - // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C3]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[C3]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C3]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[C3]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // Sizes and strides @rank 1: both static. - // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[C3]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[C3]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[CST_STRIDE1:.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[CST_STRIDE1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[CST_STRIDE1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> %2 = memref.subview %0[2, 0][3, 3][1, 1]: memref<5x3xf32> to memref<3x3xf32, strided<[3, 1], offset: 6>> return %2 : memref<3x3xf32, strided<[3, 1], offset: 6>> @@ -294,30 +294,30 @@ func.func @subview_leading_operands(%0 : memref<5x3xf32>, %1: memref<5x?xf32>) - // CHECK: %[[MEM:[a-zA-Z0-9]*]]: memref func.func @subview_leading_operands_dynamic(%0 : memref<5x?xf32>) -> memref<3x?xf32, strided<[?, 1], offset: ?>> { // CHECK: %[[MEMREF:.*]] = builtin.unrealized_conversion_cast %[[MEM]] - // CHECK: %[[SIZE1:.*]] = llvm.extractvalue %[[MEMREF]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64 - // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64 + // CHECK: %[[SIZE1:.*]] = llvm.extractvalue %[[MEMREF]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64 + // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64 // Extract strides - // CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEMREF]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEMREF]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // Compute and insert offset from 2 + dynamic value. // CHECK: %[[CST_OFF0:.*]] = llvm.mlir.constant(2 : index) : i64 // CHECK: %[[OFF0:.*]] = llvm.mul %[[STRIDE0]], %[[CST_OFF0]] : i64 // CHECK: %[[TMP:.*]] = builtin.unrealized_conversion_cast %[[OFF0]] : i64 to index // CHECK: %[[OFF0:.*]] = builtin.unrealized_conversion_cast %[[TMP]] : index to i64 - // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // Alloc ptr - // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // Aligned ptr - // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // Sizes and strides @rank 0: both static. // CHECK: %[[CST_SIZE0:.*]] = llvm.mlir.constant(3 : index) : i64 - // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[CST_SIZE0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[CST_SIZE0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // Sizes and strides @rank 1: static stride 1, dynamic size unchanged from source memref. - // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[SIZE1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[SIZE1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[CST_STRIDE1:.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[CST_STRIDE1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[CST_STRIDE1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> %c0 = arith.constant 1 : index %d0 = memref.dim %0, %c0 : memref<5x?xf32> @@ -331,19 +331,19 @@ func.func @subview_leading_operands_dynamic(%0 : memref<5x?xf32>) -> memref<3x?x // CHECK: %[[MEM:.*]]: memref func.func @subview_rank_reducing_leading_operands(%0 : memref<5x3xf32>) -> memref<3xf32, strided<[1], offset: 3>> { // CHECK: %[[MEMREF:.*]] = builtin.unrealized_conversion_cast %[[MEM]] - // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64 - // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64 - // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64 + // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64 + // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // Alloc ptr - // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // Aligned ptr - // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // CHECK: %[[C3:.*]] = llvm.mlir.constant(3 : index) : i64 - // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[C3]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[C3]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // Sizes and strides @rank 0: both static. - // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C3]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C3]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // CHECK: %[[CST_STRIDE0:.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[CST_STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[CST_STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> %1 = memref.subview %0[1, 0][1, 3][1, 1]: memref<5x3xf32> to memref<3xf32, strided<[1], offset: 3>> @@ -356,18 +356,18 @@ func.func @subview_rank_reducing_leading_operands(%0 : memref<5x3xf32>) -> memre // CHECK-SAME: (%[[MEM:.*]]: memref<7xf32>) func.func @subview_negative_stride(%arg0 : memref<7xf32>) -> memref<7xf32, strided<[-1], offset: 6>> { // CHECK: %[[MEMREF:.*]] = builtin.unrealized_conversion_cast %[[MEM]] - // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64 - // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64 - // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64 + // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64 + // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // CHECK: %[[CST_OFF0:.*]] = llvm.mlir.constant(6 : index) : i64 - // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[CST_OFF0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[CST_OFF0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // CHECK: %[[CST_SIZE0:.*]] = llvm.mlir.constant(7 : index) : i64 - // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[CST_SIZE0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[CST_SIZE0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // CHECK: %[[CST_STRIDE0:.*]] = llvm.mlir.constant(-1 : index) : i64 - // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[CST_STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC4]] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> to memref<7xf32, strided<[-1], offset: 6>> + // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[CST_STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC4]] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> to memref<7xf32, strided<[-1], offset: 6>> // CHECK: return %[[RES]] : memref<7xf32, strided<[-1], offset: 6>> %0 = memref.subview %arg0[6] [7] [-1] : memref<7xf32> to memref<7xf32, strided<[-1], offset: 6>> @@ -383,26 +383,26 @@ func.func @collapse_shape_static(%arg0: memref<1x3x4x1x5xf32>) -> memref<3x4x5xf } // CHECK-LABEL: func @collapse_shape_static // CHECK-SAME: %[[ARG:.*]]: memref<1x3x4x1x5xf32>) -> memref<3x4x5xf32> { -// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<1x3x4x1x5xf32> to !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> -// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> -// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<1x3x4x1x5xf32> to !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64 -// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // CHECK: %[[C3:.*]] = llvm.mlir.constant(3 : index) : i64 -// CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C3]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C3]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // CHECK: %[[C20:.*]] = llvm.mlir.constant(20 : index) : i64 -// CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[C20]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[C20]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // CHECK: %[[C4:.*]] = llvm.mlir.constant(4 : index) : i64 -// CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[C4]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[C4]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // CHECK: %[[C5:.*]] = llvm.mlir.constant(5 : index) : i64 -// CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[C5]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC7:.*]] = llvm.insertvalue %[[C5]], %[[DESC6]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[C5]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC7:.*]] = llvm.insertvalue %[[C5]], %[[DESC6]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : index) : i64 -// CHECK: %[[DESC8:.*]] = llvm.insertvalue %[[C1]], %[[DESC7]][4, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC8]] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> to memref<3x4x5xf32> +// CHECK: %[[DESC8:.*]] = llvm.insertvalue %[[C1]], %[[DESC7]][4, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC8]] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> to memref<3x4x5xf32> // CHECK: return %[[RES]] : memref<3x4x5xf32> // CHECK: } @@ -418,29 +418,29 @@ func.func @collapse_shape_dynamic_with_non_identity_layout( } // CHECK-LABEL: func.func @collapse_shape_dynamic_with_non_identity_layout( // CHECK-SAME: %[[ARG:.*]]: memref<4x?x?xf32, strided<[?, 4, 1], offset: ?>>) -> memref<4x?xf32, strided<[?, ?], offset: ?>> { -// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<4x?x?xf32, strided<[?, 4, 1], offset: ?>> to !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, -// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, -// CHECK: %[[OFFSET:.*]] = llvm.extractvalue %[[MEM]][2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[SIZE1:.*]] = llvm.extractvalue %[[MEM]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[SIZE2:.*]] = llvm.extractvalue %[[MEM]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEM]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<4x?x?xf32, strided<[?, 4, 1], offset: ?>> to !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, +// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, +// CHECK: %[[OFFSET:.*]] = llvm.extractvalue %[[MEM]][2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[SIZE1:.*]] = llvm.extractvalue %[[MEM]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[SIZE2:.*]] = llvm.extractvalue %[[MEM]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEM]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // CHECK: %[[STRIDE0_TO_IDX:.*]] = builtin.unrealized_conversion_cast %[[STRIDE0]] : i64 to index // CHECK: %[[STRIDE0:.*]] = builtin.unrealized_conversion_cast %[[STRIDE0_TO_IDX]] : index to i64 // CHECK: %[[FINAL_SIZE1:.*]] = llvm.mul %[[SIZE1]], %[[SIZE2]] : i64 // CHECK: %[[SIZE1_TO_IDX:.*]] = builtin.unrealized_conversion_cast %[[FINAL_SIZE1]] : i64 to index // CHECK: %[[FINAL_SIZE1:.*]] = builtin.unrealized_conversion_cast %[[SIZE1_TO_IDX]] : index to i64 -// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFFSET]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFFSET]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[C4:.*]] = llvm.mlir.constant(4 : index) : i64 -// CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C4]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[FINAL_SIZE1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C4]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[FINAL_SIZE1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : index) : i64 -// CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[C1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC6]] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> to memref<4x?xf32, strided<[?, ?], offset: ?>> +// CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[C1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC6]] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> to memref<4x?xf32, strided<[?, ?], offset: ?>> // CHECK: return %[[RES]] : memref<4x?xf32, strided<[?, ?], offset: ?>> // CHECK: } // CHECK32-LABEL: func @collapse_shape_dynamic_with_non_identity_layout( @@ -459,31 +459,31 @@ func.func @expand_shape_static(%arg0: memref<3x4x5xf32>) -> memref<1x3x4x1x5xf32 } // CHECK-LABEL: func @expand_shape_static // CHECK-SAME: %[[ARG:.*]]: memref<3x4x5xf32>) -> memref<1x3x4x1x5xf32> { -// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<3x4x5xf32> to !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, -// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, +// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<3x4x5xf32> to !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, +// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64 -// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> -// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> -// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> -// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> // CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : index) : i64 -// CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C1]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C1]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> // CHECK: %[[C60:.*]] = llvm.mlir.constant(60 : index) : i64 -// CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[C60]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[C60]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> // CHECK: %[[C3:.*]] = llvm.mlir.constant(3 : index) : i64 -// CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[C3]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[C3]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> // CHECK: %[[C20:.*]] = llvm.mlir.constant(20 : index) : i64 -// CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[C20]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[C20]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> // CHECK: %[[C4:.*]] = llvm.mlir.constant(4 : index) : i64 -// CHECK: %[[DESC7:.*]] = llvm.insertvalue %[[C4]], %[[DESC6]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[DESC7:.*]] = llvm.insertvalue %[[C4]], %[[DESC6]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> // CHECK: %[[C5:.*]] = llvm.mlir.constant(5 : index) : i64 -// CHECK: %[[DESC8:.*]] = llvm.insertvalue %[[C5]], %[[DESC7]][4, 2] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> -// CHECK: %[[DESC9:.*]] = llvm.insertvalue %[[C1]], %[[DESC8]][3, 3] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> -// CHECK: %[[DESC10:.*]] = llvm.insertvalue %[[C5]], %[[DESC9]][4, 3] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> -// CHECK: %[[DESC11:.*]] = llvm.insertvalue %[[C5]], %[[DESC10]][3, 4] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> -// CHECK: %[[DESC12:.*]] = llvm.insertvalue %[[C1]], %[[DESC11]][4, 4] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> -// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC12]] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> to memref<1x3x4x1x5xf32> +// CHECK: %[[DESC8:.*]] = llvm.insertvalue %[[C5]], %[[DESC7]][4, 2] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[DESC9:.*]] = llvm.insertvalue %[[C1]], %[[DESC8]][3, 3] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[DESC10:.*]] = llvm.insertvalue %[[C5]], %[[DESC9]][4, 3] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[DESC11:.*]] = llvm.insertvalue %[[C5]], %[[DESC10]][3, 4] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[DESC12:.*]] = llvm.insertvalue %[[C1]], %[[DESC11]][4, 4] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC12]] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> to memref<1x3x4x1x5xf32> // CHECK: return %[[RES]] : memref<1x3x4x1x5xf32> // CHECK: } @@ -495,15 +495,15 @@ func.func @collapse_shape_fold_zero_dim(%arg0 : memref<1x1xf32>) -> memref } // CHECK-LABEL: func.func @collapse_shape_fold_zero_dim( // CHECK-SAME: %[[ARG:.*]]: memref<1x1xf32>) -> memref { -// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<1x1xf32> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, -// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, -// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64)> -// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64)> -// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64)> +// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<1x1xf32> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, +// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, +// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64)> +// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64)> +// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64)> // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64 -// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64)> -// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC2]] : !llvm.struct<(ptr, ptr, i64)> to memref +// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64)> +// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC2]] : !llvm.struct<(ptr, ptr, i64)> to memref // CHECK: return %[[RES]] : memref // CHECK: } @@ -516,20 +516,20 @@ func.func @expand_shape_zero_dim(%arg0 : memref) -> memref<1x1xf32> { // CHECK-LABEL: func.func @expand_shape_zero_dim( // CHECK-SAME: %[[ARG:.*]]: memref) -> memref<1x1xf32> { -// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref to !llvm.struct<(ptr, ptr, i64)> -// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64)> -// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64)> +// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref to !llvm.struct<(ptr, ptr, i64)> +// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64)> +// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64)> // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64 -// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : index) : i64 -// CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C1]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[C1]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[C1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[C1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC6]] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> to memref<1x1xf32> +// CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C1]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[C1]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[C1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[C1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC6]] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> to memref<1x1xf32> // CHECK: return %[[RES]] : memref<1x1xf32> // CHECK: } @@ -542,13 +542,13 @@ func.func @collapse_shape_dynamic(%arg0 : memref<1x2x?xf32>) -> memref<1x?xf32> // CHECK-LABEL: func.func @collapse_shape_dynamic( // CHECK-SAME: %[[ARG:.*]]: memref<1x2x?xf32>) -> memref<1x?xf32> { -// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<1x2x?xf32> to !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, -// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, +// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<1x2x?xf32> to !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, +// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64 -// CHECK: %[[SIZE2:.*]] = llvm.extractvalue %[[MEM]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEM]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEM]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[SIZE2:.*]] = llvm.extractvalue %[[MEM]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEM]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEM]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // CHECK: %[[C2:.*]] = llvm.mlir.constant(2 : index) : i64 // CHECK: %[[FINAL_SIZE1:.*]] = llvm.mul %[[SIZE2]], %[[C2]] : i64 // CHECK: %[[SIZE1_TO_IDX:.*]] = builtin.unrealized_conversion_cast %[[FINAL_SIZE1]] : i64 to index @@ -558,15 +558,15 @@ func.func @collapse_shape_dynamic(%arg0 : memref<1x2x?xf32>) -> memref<1x?xf32> // CHECK: %[[MIN_STRIDE1:.*]] = llvm.select %[[IS_MIN_STRIDE1]], %[[STRIDE1]], %[[C1]] : i1, i64 // CHECK: %[[MIN_STRIDE1_TO_IDX:.*]] = builtin.unrealized_conversion_cast %[[MIN_STRIDE1]] : i64 to index // CHECK: %[[MIN_STRIDE1:.*]] = builtin.unrealized_conversion_cast %[[MIN_STRIDE1_TO_IDX]] : index to i64 -// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C1]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[FINAL_SIZE1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[MIN_STRIDE1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC6]] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> to memref<1x?xf32> +// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C1]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[FINAL_SIZE1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[MIN_STRIDE1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC6]] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> to memref<1x?xf32> // CHECK: return %[[RES]] : memref<1x?xf32> // CHECK: } @@ -579,12 +579,12 @@ func.func @expand_shape_dynamic(%arg0 : memref<1x?xf32>) -> memref<1x2x?xf32> { // CHECK-LABEL: func.func @expand_shape_dynamic( // CHECK-SAME: %[[ARG:.*]]: memref<1x?xf32>) -> memref<1x2x?xf32> { -// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<1x?xf32> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, -// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, +// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<1x?xf32> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, +// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64 -// CHECK: %[[SIZE1:.*]] = llvm.extractvalue %[[MEM]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEM]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[SIZE1:.*]] = llvm.extractvalue %[[MEM]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEM]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[C2:.*]] = llvm.mlir.constant(2 : index) : i64 // CHECK: %[[CMINUS1:.*]] = llvm.mlir.constant(-1 : index) : i64 // CHECK: %[[IS_NEGATIVE_SIZE1:.*]] = llvm.icmp "slt" %[[SIZE1]], %[[C0]] : i64 @@ -595,20 +595,20 @@ func.func @expand_shape_dynamic(%arg0 : memref<1x?xf32>) -> memref<1x2x?xf32> { // CHECK: %[[FINAL_SIZE2:.*]] = llvm.select %[[IS_NEGATIVE_SIZE1]], %[[NEGATIVE_SIZE2]], %[[SIZE2]] : i1, i64 // CHECK: %[[SIZE2_TO_IDX:.*]] = builtin.unrealized_conversion_cast %[[FINAL_SIZE2]] : i64 to index // CHECK: %[[FINAL_SIZE2:.*]] = builtin.unrealized_conversion_cast %[[SIZE2_TO_IDX]] : index to i64 -// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : index) : i64 -// CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C1]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[C2]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C1]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[C2]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // In this example stride1 and size2 are the same. // Hence with CSE, we get the same SSA value. -// CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[FINAL_SIZE2]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC7:.*]] = llvm.insertvalue %[[FINAL_SIZE2]], %[[DESC6]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC8:.*]] = llvm.insertvalue %[[C1]], %[[DESC7]][4, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC8]] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> to memref<1x2x?xf32> +// CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[FINAL_SIZE2]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC7:.*]] = llvm.insertvalue %[[FINAL_SIZE2]], %[[DESC6]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC8:.*]] = llvm.insertvalue %[[C1]], %[[DESC7]][4, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC8]] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> to memref<1x2x?xf32> // CHECK: return %[[RES]] : memref<1x2x?xf32> // CHECK: } @@ -624,14 +624,14 @@ func.func @expand_shape_dynamic_with_non_identity_layout( } // CHECK-LABEL: func.func @expand_shape_dynamic_with_non_identity_layout( // CHECK-SAME: %[[ARG:.*]]: memref<1x?xf32, strided<[?, ?], offset: ?>>) -> memref<1x2x?xf32, strided<[?, ?, ?], offset: ?>> { -// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<1x?xf32, strided<[?, ?], offset: ?>> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, -// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, +// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<1x?xf32, strided<[?, ?], offset: ?>> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, +// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64 -// CHECK: %[[OFFSET:.*]] = llvm.extractvalue %[[MEM]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[SIZE1:.*]] = llvm.extractvalue %[[MEM]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEM]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEM]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[OFFSET:.*]] = llvm.extractvalue %[[MEM]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[SIZE1:.*]] = llvm.extractvalue %[[MEM]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEM]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEM]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[C2:.*]] = llvm.mlir.constant(2 : index) : i64 // CHECK: %[[CMINUS1:.*]] = llvm.mlir.constant(-1 : index) : i64 // CHECK: %[[IS_NEGATIVE_SIZE1:.*]] = llvm.icmp "slt" %[[SIZE1]], %[[C0]] : i64 @@ -645,18 +645,18 @@ func.func @expand_shape_dynamic_with_non_identity_layout( // CHECK: %[[FINAL_STRIDE1:.*]] = llvm.mul %[[TMP_SIZE2]], %[[STRIDE1]] // CHECK: %[[STRIDE1_TO_IDX:.*]] = builtin.unrealized_conversion_cast %[[FINAL_STRIDE1]] : i64 to index // CHECK: %[[FINAL_STRIDE1:.*]] = builtin.unrealized_conversion_cast %[[STRIDE1_TO_IDX]] : index to i64 -// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC1]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[OFFSET]], %[[DESC2]][2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC1]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[OFFSET]], %[[DESC2]][2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : index) : i64 -// CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[C1]], %[[DESC3]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[STRIDE0]], %[[DESC4]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[C2]], %[[DESC5]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC7:.*]] = llvm.insertvalue %[[FINAL_STRIDE1]], %[[DESC6]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC8:.*]] = llvm.insertvalue %[[FINAL_SIZE2]], %[[DESC7]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC9:.*]] = llvm.insertvalue %[[STRIDE1]], %[[DESC8]][4, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC9]] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> to memref<1x2x?xf32, strided<[?, ?, ?], offset: ?>> +// CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[C1]], %[[DESC3]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[STRIDE0]], %[[DESC4]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[C2]], %[[DESC5]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC7:.*]] = llvm.insertvalue %[[FINAL_STRIDE1]], %[[DESC6]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC8:.*]] = llvm.insertvalue %[[FINAL_SIZE2]], %[[DESC7]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC9:.*]] = llvm.insertvalue %[[STRIDE1]], %[[DESC8]][4, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC9]] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> to memref<1x2x?xf32, strided<[?, ?, ?], offset: ?>> // CHECK: return %[[RES]] : memref<1x2x?xf32, strided<[?, ?, ?], offset: ?>> // CHECK: } diff --git a/mlir/test/Conversion/MemRefToLLVM/generic-functions.mlir b/mlir/test/Conversion/MemRefToLLVM/generic-functions.mlir index 4de7fbe59581..d5ac0b0da979 100644 --- a/mlir/test/Conversion/MemRefToLLVM/generic-functions.mlir +++ b/mlir/test/Conversion/MemRefToLLVM/generic-functions.mlir @@ -1,13 +1,13 @@ -// RUN: mlir-opt -pass-pipeline="builtin.module(finalize-memref-to-llvm{use-generic-functions=1})" -split-input-file %s \ +// RUN: mlir-opt -pass-pipeline="builtin.module(finalize-memref-to-llvm{use-generic-functions=1 use-opaque-pointers=1})" -split-input-file %s \ // RUN: | FileCheck %s --check-prefix="CHECK-NOTALIGNED" -// RUN: mlir-opt -pass-pipeline="builtin.module(finalize-memref-to-llvm{use-generic-functions=1 use-aligned-alloc=1})" -split-input-file %s \ +// RUN: mlir-opt -pass-pipeline="builtin.module(finalize-memref-to-llvm{use-generic-functions=1 use-aligned-alloc=1 use-opaque-pointers=1})" -split-input-file %s \ // RUN: | FileCheck %s --check-prefix="CHECK-ALIGNED" // CHECK-LABEL: func @alloc() func.func @zero_d_alloc() -> memref { -// CHECK-NOTALIGNED: llvm.call @_mlir_memref_to_llvm_alloc(%{{.*}}) : (i64) -> !llvm.ptr -// CHECK-ALIGNED: llvm.call @_mlir_memref_to_llvm_aligned_alloc(%{{.*}}, %{{.*}}) : (i64, i64) -> !llvm.ptr +// CHECK-NOTALIGNED: llvm.call @_mlir_memref_to_llvm_alloc(%{{.*}}) : (i64) -> !llvm.ptr +// CHECK-ALIGNED: llvm.call @_mlir_memref_to_llvm_aligned_alloc(%{{.*}}, %{{.*}}) : (i64, i64) -> !llvm.ptr %0 = memref.alloc() : memref return %0 : memref } @@ -16,8 +16,8 @@ func.func @zero_d_alloc() -> memref { // CHECK-LABEL: func @dealloc() func.func @dealloc(%arg0: memref) { -// CHECK-NOTALIGNED: llvm.call @_mlir_memref_to_llvm_free(%{{.*}}) : (!llvm.ptr) -> () -// CHECK-ALIGNED: llvm.call @_mlir_memref_to_llvm_free(%{{.*}}) : (!llvm.ptr) -> () +// CHECK-NOTALIGNED: llvm.call @_mlir_memref_to_llvm_free(%{{.*}}) : (!llvm.ptr) -> () +// CHECK-ALIGNED: llvm.call @_mlir_memref_to_llvm_free(%{{.*}}) : (!llvm.ptr) -> () memref.dealloc %arg0 : memref return } diff --git a/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir b/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir index 3f61f6d78a7b..f6dc44cf4571 100644 --- a/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir +++ b/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir @@ -1,5 +1,5 @@ -// RUN: mlir-opt -finalize-memref-to-llvm %s -split-input-file | FileCheck %s -// RUN: mlir-opt -finalize-memref-to-llvm='index-bitwidth=32' %s -split-input-file | FileCheck --check-prefix=CHECK32 %s +// RUN: mlir-opt -finalize-memref-to-llvm='use-opaque-pointers=1' %s -split-input-file | FileCheck %s +// RUN: mlir-opt -finalize-memref-to-llvm='index-bitwidth=32 use-opaque-pointers=1' %s -split-input-file | FileCheck --check-prefix=CHECK32 %s // CHECK-LABEL: func @view( // CHECK: %[[ARG0F:.*]]: index, %[[ARG1F:.*]]: index, %[[ARG2F:.*]]: index @@ -8,80 +8,76 @@ func.func @view(%arg0 : index, %arg1 : index, %arg2 : index) { // CHECK: %[[ARG0:.*]] = builtin.unrealized_conversion_cast %[[ARG0F:.*]] // CHECK: %[[ARG1:.*]] = builtin.unrealized_conversion_cast %[[ARG1F:.*]] // CHECK: llvm.mlir.constant(2048 : index) : i64 - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> %0 = memref.alloc() : memref<2048xi8> // Test two dynamic sizes. - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[BASE_PTR:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[SHIFTED_BASE_PTR:.*]] = llvm.getelementptr %[[BASE_PTR]][%[[ARG2]]] : (!llvm.ptr, i64) -> !llvm.ptr - // CHECK: %[[CAST_SHIFTED_BASE_PTR:.*]] = llvm.bitcast %[[SHIFTED_BASE_PTR]] : !llvm.ptr to !llvm.ptr - // CHECK: llvm.insertvalue %[[CAST_SHIFTED_BASE_PTR]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE_PTR:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[SHIFTED_BASE_PTR:.*]] = llvm.getelementptr %[[BASE_PTR]][%[[ARG2]]] : (!llvm.ptr, i64) -> !llvm.ptr, i8 + // CHECK: llvm.insertvalue %[[SHIFTED_BASE_PTR]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64 - // CHECK: llvm.insertvalue %[[C0]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: llvm.insertvalue %[[ARG1]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[C0]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[ARG1]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(1 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: llvm.insertvalue %[[ARG0]], %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[ARG0]], %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mul %{{.*}}, %[[ARG1]] - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> %1 = memref.view %0[%arg2][%arg0, %arg1] : memref<2048xi8> to memref // Test one dynamic size. - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[BASE_PTR_2:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[SHIFTED_BASE_PTR_2:.*]] = llvm.getelementptr %[[BASE_PTR_2]][%[[ARG2]]] : (!llvm.ptr, i64) -> !llvm.ptr - // CHECK: %[[CAST_SHIFTED_BASE_PTR_2:.*]] = llvm.bitcast %[[SHIFTED_BASE_PTR_2]] : !llvm.ptr to !llvm.ptr - // CHECK: llvm.insertvalue %[[CAST_SHIFTED_BASE_PTR_2]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE_PTR_2:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[SHIFTED_BASE_PTR_2:.*]] = llvm.getelementptr %[[BASE_PTR_2]][%[[ARG2]]] : (!llvm.ptr, i64) -> !llvm.ptr, i8 + // CHECK: llvm.insertvalue %[[SHIFTED_BASE_PTR_2]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[C0_2:.*]] = llvm.mlir.constant(0 : index) : i64 - // CHECK: llvm.insertvalue %[[C0_2]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: llvm.insertvalue %[[ARG1]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[C0_2]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[ARG1]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(1 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(4 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mul %{{.*}}, %[[ARG1]] - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> %3 = memref.view %0[%arg2][%arg1] : memref<2048xi8> to memref<4x?xf32> // Test static sizes. - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[BASE_PTR_3:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[SHIFTED_BASE_PTR_3:.*]] = llvm.getelementptr %[[BASE_PTR_3]][%[[ARG2]]] : (!llvm.ptr, i64) -> !llvm.ptr - // CHECK: %[[CAST_SHIFTED_BASE_PTR_3:.*]] = llvm.bitcast %[[SHIFTED_BASE_PTR_3]] : !llvm.ptr to !llvm.ptr - // CHECK: llvm.insertvalue %[[CAST_SHIFTED_BASE_PTR_3]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE_PTR_3:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[SHIFTED_BASE_PTR_3:.*]] = llvm.getelementptr %[[BASE_PTR_3]][%[[ARG2]]] : (!llvm.ptr, i64) -> !llvm.ptr, i8 + // CHECK: llvm.insertvalue %[[SHIFTED_BASE_PTR_3]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[C0_3:.*]] = llvm.mlir.constant(0 : index) : i64 - // CHECK: llvm.insertvalue %[[C0_3]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[C0_3]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(4 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(1 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(64 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(4 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> %5 = memref.view %0[%arg2][] : memref<2048xi8> to memref<64x4xf32> // Test view memory space. // CHECK: llvm.mlir.constant(2048 : index) : i64 - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr<4>, ptr<4>, i64, array<1 x i64>, array<1 x i64>)> %6 = memref.alloc() : memref<2048xi8, 4> - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[BASE_PTR_4:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[SHIFTED_BASE_PTR_4:.*]] = llvm.getelementptr %[[BASE_PTR_4]][%[[ARG2]]] : (!llvm.ptr, i64) -> !llvm.ptr - // CHECK: %[[CAST_SHIFTED_BASE_PTR_4:.*]] = llvm.bitcast %[[SHIFTED_BASE_PTR_4]] : !llvm.ptr to !llvm.ptr - // CHECK: llvm.insertvalue %[[CAST_SHIFTED_BASE_PTR_4]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr<4>, ptr<4>, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE_PTR_4:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr<4>, ptr<4>, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[SHIFTED_BASE_PTR_4:.*]] = llvm.getelementptr %[[BASE_PTR_4]][%[[ARG2]]] : (!llvm.ptr<4>, i64) -> !llvm.ptr<4>, i8 + // CHECK: llvm.insertvalue %[[SHIFTED_BASE_PTR_4]], %{{.*}}[1] : !llvm.struct<(ptr<4>, ptr<4>, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[C0_4:.*]] = llvm.mlir.constant(0 : index) : i64 - // CHECK: llvm.insertvalue %[[C0_4]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[C0_4]], %{{.*}}[2] : !llvm.struct<(ptr<4>, ptr<4>, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(4 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 1] : !llvm.struct<(ptr<4>, ptr<4>, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(1 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr<4>, ptr<4>, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(64 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr<4>, ptr<4>, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(4 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr<4>, ptr<4>, i64, array<2 x i64>, array<2 x i64>)> %7 = memref.view %6[%arg2][] : memref<2048xi8, 4> to memref<64x4xf32, 4> return @@ -94,17 +90,17 @@ func.func @view(%arg0 : index, %arg1 : index, %arg2 : index) { // CHECK: %[[ARG1:.*]]: memref<0xi8>) func.func @view_empty_memref(%offset: index, %mem: memref<0xi8>) { - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(0 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(4 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(1 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(0 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(4 : index) : i64 - // CHECK: = llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: = llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> %0 = memref.view %mem[%offset][] : memref<0xi8> to memref<0x4xf32> return @@ -136,10 +132,10 @@ func.func @subview(%0 : memref<64x4xf32, strided<[4, 1], offset: 0>>, %arg0 : in // CHECK-LABEL: func @assume_alignment func.func @assume_alignment(%0 : memref<4x4xf16>) { - // CHECK: %[[PTR:.*]] = llvm.extractvalue %[[MEMREF:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[PTR:.*]] = llvm.extractvalue %[[MEMREF:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK-NEXT: %[[ZERO:.*]] = llvm.mlir.constant(0 : index) : i64 // CHECK-NEXT: %[[MASK:.*]] = llvm.mlir.constant(15 : index) : i64 - // CHECK-NEXT: %[[INT:.*]] = llvm.ptrtoint %[[PTR]] : !llvm.ptr to i64 + // CHECK-NEXT: %[[INT:.*]] = llvm.ptrtoint %[[PTR]] : !llvm.ptr to i64 // CHECK-NEXT: %[[MASKED_PTR:.*]] = llvm.and %[[INT]], %[[MASK:.*]] : i64 // CHECK-NEXT: %[[CONDITION:.*]] = llvm.icmp "eq" %[[MASKED_PTR]], %[[ZERO]] : i64 // CHECK-NEXT: "llvm.intr.assume"(%[[CONDITION]]) : (i1) -> () @@ -159,37 +155,32 @@ func.func @dim_of_unranked(%unranked: memref<*xi32>) -> index { // CHECK: %[[UNRANKED_DESC:.*]] = builtin.unrealized_conversion_cast // CHECK: %[[RANKED_DESC:.*]] = llvm.extractvalue %[[UNRANKED_DESC]][1] -// CHECK-SAME: : !llvm.struct<(i64, ptr)> +// CHECK-SAME: : !llvm.struct<(i64, ptr)> -// CHECK: %[[ZERO_D_DESC:.*]] = llvm.bitcast %[[RANKED_DESC]] -// CHECK-SAME: : !llvm.ptr to !llvm.ptr, ptr, i64)>> - -// CHECK: %[[OFFSET_PTR:.*]] = llvm.getelementptr %[[ZERO_D_DESC]]{{\[}} -// CHECK-SAME: 0, 2] : (!llvm.ptr, ptr, -// CHECK-SAME: i64)>>) -> !llvm.ptr +// CHECK: %[[OFFSET_PTR:.*]] = llvm.getelementptr %[[RANKED_DESC]]{{\[}} +// CHECK-SAME: 0, 2] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, ptr, i64)> // CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : index) : i64 // CHECK: %[[INDEX_INC:.*]] = llvm.add %[[C1]], %{{.*}} : i64 // CHECK: %[[SIZE_PTR:.*]] = llvm.getelementptr %[[OFFSET_PTR]]{{\[}} -// CHECK-SAME: %[[INDEX_INC]]] : (!llvm.ptr, i64) -> !llvm.ptr +// CHECK-SAME: %[[INDEX_INC]]] : (!llvm.ptr, i64) -> !llvm.ptr -// CHECK: %[[SIZE:.*]] = llvm.load %[[SIZE_PTR]] : !llvm.ptr +// CHECK: %[[SIZE:.*]] = llvm.load %[[SIZE_PTR]] : !llvm.ptr -> i64 -// CHECK32: %[[SIZE:.*]] = llvm.load %{{.*}} : !llvm.ptr +// CHECK32: %[[SIZE:.*]] = llvm.load %{{.*}} : !llvm.ptr -> i32 // ----- // CHECK-LABEL: func @address_space( func.func @address_space(%arg0 : memref<32xf32, affine_map<(d0) -> (d0)>, 7>) { // CHECK: %[[MEMORY:.*]] = llvm.call @malloc(%{{.*}}) - // CHECK: %[[CAST:.*]] = llvm.addrspacecast %[[MEMORY]] : !llvm.ptr to !llvm.ptr - // CHECK: %[[BCAST:.*]] = llvm.bitcast %[[CAST]] - // CHECK: llvm.insertvalue %[[BCAST]], %{{[[:alnum:]]+}}[0] - // CHECK: llvm.insertvalue %[[BCAST]], %{{[[:alnum:]]+}}[1] + // CHECK: %[[CAST:.*]] = llvm.addrspacecast %[[MEMORY]] : !llvm.ptr to !llvm.ptr<5> + // CHECK: llvm.insertvalue %[[CAST]], %{{[[:alnum:]]+}}[0] + // CHECK: llvm.insertvalue %[[CAST]], %{{[[:alnum:]]+}}[1] %0 = memref.alloc() : memref<32xf32, affine_map<(d0) -> (d0)>, 5> %1 = arith.constant 7 : index - // CHECK: llvm.load %{{.*}} : !llvm.ptr + // CHECK: llvm.load %{{.*}} : !llvm.ptr<5> -> f32 %2 = memref.load %0[%1] : memref<32xf32, affine_map<(d0) -> (d0)>, 5> func.return } @@ -197,16 +188,16 @@ func.func @address_space(%arg0 : memref<32xf32, affine_map<(d0) -> (d0)>, 7>) { // ----- // CHECK-LABEL: func @transpose -// CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: llvm.insertvalue {{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: llvm.insertvalue {{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: llvm.insertvalue {{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: llvm.extractvalue {{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: llvm.insertvalue {{.*}}[3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: llvm.extractvalue {{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: llvm.insertvalue {{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: llvm.extractvalue {{.*}}[3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: llvm.insertvalue {{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: llvm.insertvalue {{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: llvm.insertvalue {{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: llvm.insertvalue {{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: llvm.extractvalue {{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: llvm.insertvalue {{.*}}[3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: llvm.extractvalue {{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: llvm.insertvalue {{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: llvm.extractvalue {{.*}}[3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: llvm.insertvalue {{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> func.func @transpose(%arg0: memref>) { %0 = memref.transpose %arg0 (i, j, k) -> (k, i, j) : memref> to memref> return @@ -232,17 +223,17 @@ func.func @get_gv0_memref() { %0 = memref.get_global @gv0 : memref<2xf32> // CHECK: %[[DIM:.*]] = llvm.mlir.constant(2 : index) : i64 // CHECK: %[[STRIDE:.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK: %[[ADDR:.*]] = llvm.mlir.addressof @gv0 : !llvm.ptr> - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ADDR]][0, 0] : (!llvm.ptr>) -> !llvm.ptr + // CHECK: %[[ADDR:.*]] = llvm.mlir.addressof @gv0 : !llvm.ptr + // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ADDR]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.array<2 x f32> // CHECK: %[[DEADBEEF:.*]] = llvm.mlir.constant(3735928559 : index) : i64 - // CHECK: %[[DEADBEEFPTR:.*]] = llvm.inttoptr %[[DEADBEEF]] : i64 to !llvm.ptr - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: llvm.insertvalue %[[DEADBEEFPTR]], {{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: llvm.insertvalue %[[GEP]], {{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[DEADBEEFPTR:.*]] = llvm.inttoptr %[[DEADBEEF]] : i64 to !llvm.ptr + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.insertvalue %[[DEADBEEFPTR]], {{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.insertvalue %[[GEP]], {{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // CHECK: %[[OFFSET:.*]] = llvm.mlir.constant(0 : index) : i64 - // CHECK: llvm.insertvalue %[[OFFSET]], {{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: llvm.insertvalue %[[DIM]], {{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: llvm.insertvalue %[[STRIDE]], {{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.insertvalue %[[OFFSET]], {{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.insertvalue %[[DIM]], {{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.insertvalue %[[STRIDE]], {{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> return } @@ -252,19 +243,19 @@ func.func @get_gv2_memref() { // CHECK: %[[DIM0:.*]] = llvm.mlir.constant(2 : index) : i64 // CHECK: %[[DIM1:.*]] = llvm.mlir.constant(3 : index) : i64 // CHECK: %[[STRIDE1:.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK: %[[ADDR:.*]] = llvm.mlir.addressof @gv2 : !llvm.ptr>> - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ADDR]][0, 0, 0] : (!llvm.ptr>>) -> !llvm.ptr + // CHECK: %[[ADDR:.*]] = llvm.mlir.addressof @gv2 : !llvm.ptr + // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ADDR]][0, 0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.array<2 x array<3 x f32>> // CHECK: %[[DEADBEEF:.*]] = llvm.mlir.constant(3735928559 : index) : i64 - // CHECK: %[[DEADBEEFPTR:.*]] = llvm.inttoptr %[[DEADBEEF]] : i64 to !llvm.ptr - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: llvm.insertvalue %[[DEADBEEFPTR]], {{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: llvm.insertvalue %[[GEP]], {{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DEADBEEFPTR:.*]] = llvm.inttoptr %[[DEADBEEF]] : i64 to !llvm.ptr + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[DEADBEEFPTR]], {{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[GEP]], {{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[OFFSET:.*]] = llvm.mlir.constant(0 : index) : i64 - // CHECK: llvm.insertvalue %[[OFFSET]], {{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: llvm.insertvalue %[[DIM0]], {{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: llvm.insertvalue %[[DIM1]], {{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: llvm.insertvalue %[[DIM1]], {{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: llvm.insertvalue %[[STRIDE1]], {{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[OFFSET]], {{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[DIM0]], {{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[DIM1]], {{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[DIM1]], {{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[STRIDE1]], {{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> %0 = memref.get_global @gv2 : memref<2x3xf32> return @@ -276,15 +267,15 @@ memref.global @gv3 : memref = dense<1.0> // CHECK-LABEL: func @get_gv3_memref func.func @get_gv3_memref() { - // CHECK: %[[ADDR:.*]] = llvm.mlir.addressof @gv3 : !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ADDR]][0] : (!llvm.ptr) -> !llvm.ptr + // CHECK: %[[ADDR:.*]] = llvm.mlir.addressof @gv3 : !llvm.ptr + // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ADDR]][0] : (!llvm.ptr) -> !llvm.ptr, f32 // CHECK: %[[DEADBEEF:.*]] = llvm.mlir.constant(3735928559 : index) : i64 - // CHECK: %[[DEADBEEFPTR:.*]] = llvm.inttoptr %[[DEADBEEF]] : i64 to !llvm.ptr - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64)> - // CHECK: llvm.insertvalue %[[DEADBEEFPTR]], {{.*}}[0] : !llvm.struct<(ptr, ptr, i64)> - // CHECK: llvm.insertvalue %[[GEP]], {{.*}}[1] : !llvm.struct<(ptr, ptr, i64)> + // CHECK: %[[DEADBEEFPTR:.*]] = llvm.inttoptr %[[DEADBEEF]] : i64 to !llvm.ptr + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64)> + // CHECK: llvm.insertvalue %[[DEADBEEFPTR]], {{.*}}[0] : !llvm.struct<(ptr, ptr, i64)> + // CHECK: llvm.insertvalue %[[GEP]], {{.*}}[1] : !llvm.struct<(ptr, ptr, i64)> // CHECK: %[[OFFSET:.*]] = llvm.mlir.constant(0 : index) : i64 - // CHECK: llvm.insertvalue %[[OFFSET]], {{.*}}[2] : !llvm.struct<(ptr, ptr, i64)> + // CHECK: llvm.insertvalue %[[OFFSET]], {{.*}}[2] : !llvm.struct<(ptr, ptr, i64)> %0 = memref.get_global @gv3 : memref return } @@ -326,8 +317,8 @@ func.func @rank_of_unranked(%unranked: memref<*xi32>) { return } // CHECK: %[[UNRANKED_DESC:.*]] = builtin.unrealized_conversion_cast -// CHECK-NEXT: llvm.extractvalue %[[UNRANKED_DESC]][0] : !llvm.struct<(i64, ptr)> -// CHECK32: llvm.extractvalue %{{.*}}[0] : !llvm.struct<(i32, ptr)> +// CHECK-NEXT: llvm.extractvalue %[[UNRANKED_DESC]][0] : !llvm.struct<(i64, ptr)> +// CHECK32: llvm.extractvalue %{{.*}}[0] : !llvm.struct<(i32, ptr)> // CHECK-LABEL: func @rank_of_ranked // CHECK32-LABEL: func @rank_of_ranked @@ -371,11 +362,11 @@ func.func @generic_atomic_rmw(%I : memref<10xi32>, %i : index) { ^bb0(%old_value : i32): memref.atomic_yield %old_value : i32 } - // CHECK: [[init:%.*]] = llvm.load %{{.*}} : !llvm.ptr + // CHECK: [[init:%.*]] = llvm.load %{{.*}} : !llvm.ptr -> i32 // CHECK-NEXT: llvm.br ^bb1([[init]] : i32) // CHECK-NEXT: ^bb1([[loaded:%.*]]: i32): // CHECK-NEXT: [[pair:%.*]] = llvm.cmpxchg %{{.*}}, [[loaded]], [[loaded]] - // CHECK-SAME: acq_rel monotonic : !llvm.ptr, i32 + // CHECK-SAME: acq_rel monotonic : !llvm.ptr, i32 // CHECK-NEXT: [[new:%.*]] = llvm.extractvalue [[pair]][0] // CHECK-NEXT: [[ok:%.*]] = llvm.extractvalue [[pair]][1] // CHECK-NEXT: llvm.cond_br [[ok]], ^bb2, ^bb1([[new]] : i32) @@ -388,28 +379,28 @@ func.func @generic_atomic_rmw(%I : memref<10xi32>, %i : index) { func.func @memref_copy_ranked() { %0 = memref.alloc() : memref<2xf32> // CHECK: llvm.mlir.constant(2 : index) : i64 - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> %1 = memref.cast %0 : memref<2xf32> to memref %2 = memref.alloc() : memref<2xf32> // CHECK: llvm.mlir.constant(2 : index) : i64 - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> %3 = memref.cast %2 : memref<2xf32> to memref memref.copy %1, %3 : memref to memref // CHECK: [[ONE:%.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK: [[EXTRACT0:%.*]] = llvm.extractvalue {{%.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: [[EXTRACT0:%.*]] = llvm.extractvalue {{%.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // CHECK: [[MUL:%.*]] = llvm.mul [[ONE]], [[EXTRACT0]] : i64 - // CHECK: [[NULL:%.*]] = llvm.mlir.null : !llvm.ptr - // CHECK: [[GEP:%.*]] = llvm.getelementptr [[NULL]][1] : (!llvm.ptr) -> !llvm.ptr - // CHECK: [[PTRTOINT:%.*]] = llvm.ptrtoint [[GEP]] : !llvm.ptr to i64 + // CHECK: [[NULL:%.*]] = llvm.mlir.null : !llvm.ptr + // CHECK: [[GEP:%.*]] = llvm.getelementptr [[NULL]][1] : (!llvm.ptr) -> !llvm.ptr, f32 + // CHECK: [[PTRTOINT:%.*]] = llvm.ptrtoint [[GEP]] : !llvm.ptr to i64 // CHECK: [[SIZE:%.*]] = llvm.mul [[MUL]], [[PTRTOINT]] : i64 - // CHECK: [[EXTRACT1P:%.*]] = llvm.extractvalue {{%.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: [[EXTRACT1O:%.*]] = llvm.extractvalue {{%.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: [[GEP1:%.*]] = llvm.getelementptr [[EXTRACT1P]][[[EXTRACT1O]]] : (!llvm.ptr, i64) -> !llvm.ptr - // CHECK: [[EXTRACT2P:%.*]] = llvm.extractvalue {{%.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: [[EXTRACT2O:%.*]] = llvm.extractvalue {{%.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: [[GEP2:%.*]] = llvm.getelementptr [[EXTRACT2P]][[[EXTRACT2O]]] : (!llvm.ptr, i64) -> !llvm.ptr + // CHECK: [[EXTRACT1P:%.*]] = llvm.extractvalue {{%.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: [[EXTRACT1O:%.*]] = llvm.extractvalue {{%.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: [[GEP1:%.*]] = llvm.getelementptr [[EXTRACT1P]][[[EXTRACT1O]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 + // CHECK: [[EXTRACT2P:%.*]] = llvm.extractvalue {{%.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: [[EXTRACT2O:%.*]] = llvm.extractvalue {{%.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: [[GEP2:%.*]] = llvm.getelementptr [[EXTRACT2P]][[[EXTRACT2O]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 // CHECK: [[VOLATILE:%.*]] = llvm.mlir.constant(false) : i1 - // CHECK: "llvm.intr.memcpy"([[GEP2]], [[GEP1]], [[SIZE]], [[VOLATILE]]) : (!llvm.ptr, !llvm.ptr, i64, i1) -> () + // CHECK: "llvm.intr.memcpy"([[GEP2]], [[GEP1]], [[SIZE]], [[VOLATILE]]) : (!llvm.ptr, !llvm.ptr, i64, i1) -> () return } @@ -425,23 +416,23 @@ func.func @memref_copy_contiguous(%in: memref<16x2xi32>, %offset: index) { // CHECK: llvm.insertvalue {{%.*}}, {{%.*}}[4, 1] // Get the memref for the subview. // CHECK: %[[SUBVIEW:.*]] = memref.subview %{{.*}}[%{{.*}}, 0] [1, 2] [1, 1] : memref<16x2xi32> to memref<1x2xi32, strided<[2, 1], offset: ?>> - // CHECK: %[[DESC:.*]] = builtin.unrealized_conversion_cast %[[SUBVIEW]] : memref<1x2xi32, strided<[2, 1], offset: ?>> to !llvm.struct<(ptr - // CHECK: [[EXTRACT0:%.*]] = llvm.extractvalue %[[DESC]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC:.*]] = builtin.unrealized_conversion_cast %[[SUBVIEW]] : memref<1x2xi32, strided<[2, 1], offset: ?>> to !llvm.struct<(ptr + // CHECK: [[EXTRACT0:%.*]] = llvm.extractvalue %[[DESC]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: [[MUL1:%.*]] = llvm.mul {{.*}}, [[EXTRACT0]] : i64 - // CHECK: [[EXTRACT1:%.*]] = llvm.extractvalue %[[DESC]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: [[EXTRACT1:%.*]] = llvm.extractvalue %[[DESC]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: [[MUL2:%.*]] = llvm.mul [[MUL1]], [[EXTRACT1]] : i64 - // CHECK: [[NULL:%.*]] = llvm.mlir.null : !llvm.ptr - // CHECK: [[GEP:%.*]] = llvm.getelementptr [[NULL]][1] : (!llvm.ptr) -> !llvm.ptr - // CHECK: [[PTRTOINT:%.*]] = llvm.ptrtoint [[GEP]] : !llvm.ptr to i64 + // CHECK: [[NULL:%.*]] = llvm.mlir.null : !llvm.ptr + // CHECK: [[GEP:%.*]] = llvm.getelementptr [[NULL]][1] : (!llvm.ptr) -> !llvm.ptr, i32 + // CHECK: [[PTRTOINT:%.*]] = llvm.ptrtoint [[GEP]] : !llvm.ptr to i64 // CHECK: [[SIZE:%.*]] = llvm.mul [[MUL2]], [[PTRTOINT]] : i64 - // CHECK: [[EXTRACT1P:%.*]] = llvm.extractvalue {{%.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: [[EXTRACT1O:%.*]] = llvm.extractvalue {{%.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: [[GEP1:%.*]] = llvm.getelementptr [[EXTRACT1P]][[[EXTRACT1O]]] : (!llvm.ptr, i64) -> !llvm.ptr - // CHECK: [[EXTRACT2P:%.*]] = llvm.extractvalue {{%.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: [[EXTRACT2O:%.*]] = llvm.extractvalue {{%.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: [[GEP2:%.*]] = llvm.getelementptr [[EXTRACT2P]][[[EXTRACT2O]]] : (!llvm.ptr, i64) -> !llvm.ptr + // CHECK: [[EXTRACT1P:%.*]] = llvm.extractvalue {{%.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: [[EXTRACT1O:%.*]] = llvm.extractvalue {{%.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: [[GEP1:%.*]] = llvm.getelementptr [[EXTRACT1P]][[[EXTRACT1O]]] : (!llvm.ptr, i64) -> !llvm.ptr, i32 + // CHECK: [[EXTRACT2P:%.*]] = llvm.extractvalue {{%.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: [[EXTRACT2O:%.*]] = llvm.extractvalue {{%.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: [[GEP2:%.*]] = llvm.getelementptr [[EXTRACT2P]][[[EXTRACT2O]]] : (!llvm.ptr, i64) -> !llvm.ptr, i32 // CHECK: [[VOLATILE:%.*]] = llvm.mlir.constant(false) : i1 - // CHECK: "llvm.intr.memcpy"([[GEP2]], [[GEP1]], [[SIZE]], [[VOLATILE]]) : (!llvm.ptr, !llvm.ptr, i64, i1) -> () + // CHECK: "llvm.intr.memcpy"([[GEP2]], [[GEP1]], [[SIZE]], [[VOLATILE]]) : (!llvm.ptr, !llvm.ptr, i64, i1) -> () return } @@ -474,29 +465,28 @@ func.func @memref_copy_noncontiguous(%in: memref<16x2xi32>, %offset: index) { func.func @memref_copy_unranked() { %0 = memref.alloc() : memref<2xi1> // CHECK: llvm.mlir.constant(2 : index) : i64 - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> %1 = memref.cast %0 : memref<2xi1> to memref<*xi1> %2 = memref.alloc() : memref<2xi1> // CHECK: llvm.mlir.constant(2 : index) : i64 - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> %3 = memref.cast %2 : memref<2xi1> to memref<*xi1> memref.copy %1, %3 : memref<*xi1> to memref<*xi1> // CHECK: [[ONE:%.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK: [[ALLOCA:%.*]] = llvm.alloca %35 x !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> : (i64) -> !llvm.ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>> - // CHECK: llvm.store {{%.*}}, [[ALLOCA]] : !llvm.ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>> - // CHECK: [[BITCAST:%.*]] = llvm.bitcast [[ALLOCA]] : !llvm.ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>> to !llvm.ptr + // CHECK: [[ALLOCA:%.*]] = llvm.alloca [[ONE]] x !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> : (i64) -> !llvm.ptr + // CHECK: llvm.store {{%.*}}, [[ALLOCA]] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>, !llvm.ptr // CHECK: [[RANK:%.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK: [[UNDEF:%.*]] = llvm.mlir.undef : !llvm.struct<(i64, ptr)> - // CHECK: [[INSERT:%.*]] = llvm.insertvalue [[RANK]], [[UNDEF]][0] : !llvm.struct<(i64, ptr)> - // CHECK: [[INSERT2:%.*]] = llvm.insertvalue [[BITCAST]], [[INSERT]][1] : !llvm.struct<(i64, ptr)> - // CHECK: [[STACKSAVE:%.*]] = llvm.intr.stacksave : !llvm.ptr + // CHECK: [[UNDEF:%.*]] = llvm.mlir.undef : !llvm.struct<(i64, ptr)> + // CHECK: [[INSERT:%.*]] = llvm.insertvalue [[RANK]], [[UNDEF]][0] : !llvm.struct<(i64, ptr)> + // CHECK: [[INSERT2:%.*]] = llvm.insertvalue [[ALLOCA]], [[INSERT]][1] : !llvm.struct<(i64, ptr)> + // CHECK: [[STACKSAVE:%.*]] = llvm.intr.stacksave : !llvm.ptr // CHECK: [[RANK2:%.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK: [[ALLOCA2:%.*]] = llvm.alloca [[RANK2]] x !llvm.struct<(i64, ptr)> : (i64) -> !llvm.ptr)>> - // CHECK: llvm.store {{%.*}}, [[ALLOCA2]] : !llvm.ptr)>> - // CHECK: [[ALLOCA3:%.*]] = llvm.alloca [[RANK2]] x !llvm.struct<(i64, ptr)> : (i64) -> !llvm.ptr)>> - // CHECK: llvm.store [[INSERT2]], [[ALLOCA3]] : !llvm.ptr)>> + // CHECK: [[ALLOCA2:%.*]] = llvm.alloca [[RANK2]] x !llvm.struct<(i64, ptr)> : (i64) -> !llvm.ptr + // CHECK: llvm.store {{%.*}}, [[ALLOCA2]] : !llvm.struct<(i64, ptr)>, !llvm.ptr + // CHECK: [[ALLOCA3:%.*]] = llvm.alloca [[RANK2]] x !llvm.struct<(i64, ptr)> : (i64) -> !llvm.ptr + // CHECK: llvm.store [[INSERT2]], [[ALLOCA3]] : !llvm.struct<(i64, ptr)>, !llvm.ptr // CHECK: [[SIZE:%.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK: llvm.call @memrefCopy([[SIZE]], [[ALLOCA2]], [[ALLOCA3]]) : (i64, !llvm.ptr)>>, !llvm.ptr)>>) -> () + // CHECK: llvm.call @memrefCopy([[SIZE]], [[ALLOCA2]], [[ALLOCA3]]) : (i64, !llvm.ptr, !llvm.ptr) -> () // CHECK: llvm.intr.stackrestore [[STACKSAVE]] return } @@ -506,8 +496,8 @@ func.func @memref_copy_unranked() { // CHECK-LABEL: func @extract_aligned_pointer_as_index func.func @extract_aligned_pointer_as_index(%m: memref) -> index { %0 = memref.extract_aligned_pointer_as_index %m: memref -> index - // CHECK: %[[E:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[I64:.*]] = llvm.ptrtoint %[[E]] : !llvm.ptr to i64 + // CHECK: %[[E:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[I64:.*]] = llvm.ptrtoint %[[E]] : !llvm.ptr to i64 // CHECK: %[[R:.*]] = builtin.unrealized_conversion_cast %[[I64]] : i64 to index // CHECK: return %[[R:.*]] : index @@ -518,19 +508,19 @@ func.func @extract_aligned_pointer_as_index(%m: memref) -> index { // CHECK-LABEL: func @extract_strided_metadata( // CHECK-SAME: %[[ARG:.*]]: memref -// CHECK: %[[MEM_DESC:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEM_DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[ALIGNED_BASE:.*]] = llvm.extractvalue %[[MEM_DESC]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64)> -// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64)> -// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BASE]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64)> +// CHECK: %[[MEM_DESC:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEM_DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[ALIGNED_BASE:.*]] = llvm.extractvalue %[[MEM_DESC]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64)> +// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64)> +// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BASE]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64)> // CHECK: %[[OFF0:.*]] = llvm.mlir.constant(0 : index) : i64 -// CHECK: %[[BASE_BUFFER_DESC:.*]] = llvm.insertvalue %[[OFF0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64)> -// CHECK: %[[OFFSET:.*]] = llvm.extractvalue %[[MEM_DESC]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[SIZE0:.*]] = llvm.extractvalue %[[MEM_DESC]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[SIZE1:.*]] = llvm.extractvalue %[[MEM_DESC]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEM_DESC]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEM_DESC]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[BASE_BUFFER_DESC:.*]] = llvm.insertvalue %[[OFF0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64)> +// CHECK: %[[OFFSET:.*]] = llvm.extractvalue %[[MEM_DESC]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[SIZE0:.*]] = llvm.extractvalue %[[MEM_DESC]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[SIZE1:.*]] = llvm.extractvalue %[[MEM_DESC]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEM_DESC]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEM_DESC]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> func.func @extract_strided_metadata( %ref: memref>) { @@ -548,7 +538,7 @@ func.func @extract_strided_metadata( // CHECK-LABEL: func @load_non_temporal( func.func @load_non_temporal(%arg0 : memref<32xf32, affine_map<(d0) -> (d0)>>) { %1 = arith.constant 7 : index - // CHECK: llvm.load %{{.*}} {nontemporal} : !llvm.ptr + // CHECK: llvm.load %{{.*}} {nontemporal} : !llvm.ptr -> f32 %2 = memref.load %arg0[%1] {nontemporal = true} : memref<32xf32, affine_map<(d0) -> (d0)>> func.return } @@ -559,7 +549,7 @@ func.func @load_non_temporal(%arg0 : memref<32xf32, affine_map<(d0) -> (d0)>>) { func.func @store_non_temporal(%input : memref<32xf32, affine_map<(d0) -> (d0)>>, %output : memref<32xf32, affine_map<(d0) -> (d0)>>) { %1 = arith.constant 7 : index %2 = memref.load %input[%1] {nontemporal = true} : memref<32xf32, affine_map<(d0) -> (d0)>> - // CHECK: llvm.store %{{.*}}, %{{.*}} {nontemporal} : !llvm.ptr + // CHECK: llvm.store %{{.*}}, %{{.*}} {nontemporal} : f32, !llvm.ptr memref.store %2, %output[%1] {nontemporal = true} : memref<32xf32, affine_map<(d0) -> (d0)>> func.return } diff --git a/mlir/test/Conversion/MemRefToLLVM/typed-pointers.mlir b/mlir/test/Conversion/MemRefToLLVM/typed-pointers.mlir new file mode 100644 index 000000000000..e14f70df15b9 --- /dev/null +++ b/mlir/test/Conversion/MemRefToLLVM/typed-pointers.mlir @@ -0,0 +1,415 @@ +// RUN: mlir-opt -finalize-memref-to-llvm='use-opaque-pointers=0' %s -split-input-file | FileCheck %s +// RUN: mlir-opt -finalize-memref-to-llvm='index-bitwidth=32 use-opaque-pointers=0' -split-input-file %s | FileCheck --check-prefix=CHECK32 %s + +// CHECK-LABEL: func @view( +// CHECK: %[[ARG1F:.*]]: index, %[[ARG2F:.*]]: index +func.func @view(%arg1 : index, %arg2 : index) { + // CHECK: %[[ARG2:.*]] = builtin.unrealized_conversion_cast %[[ARG2F:.*]] + // CHECK: %[[ARG1:.*]] = builtin.unrealized_conversion_cast %[[ARG1F:.*]] + // CHECK: llvm.mlir.constant(2048 : index) : i64 + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + %0 = memref.alloc() : memref<2048xi8> + + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE_PTR_2:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[SHIFTED_BASE_PTR_2:.*]] = llvm.getelementptr %[[BASE_PTR_2]][%[[ARG2]]] : (!llvm.ptr, i64) -> !llvm.ptr + // CHECK: %[[CAST_SHIFTED_BASE_PTR_2:.*]] = llvm.bitcast %[[SHIFTED_BASE_PTR_2]] : !llvm.ptr to !llvm.ptr + // CHECK: llvm.insertvalue %[[CAST_SHIFTED_BASE_PTR_2]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[C0_2:.*]] = llvm.mlir.constant(0 : index) : i64 + // CHECK: llvm.insertvalue %[[C0_2]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[ARG1]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.mlir.constant(1 : index) : i64 + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.mlir.constant(4 : index) : i64 + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.mul %{{.*}}, %[[ARG1]] + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + %1 = memref.view %0[%arg2][%arg1] : memref<2048xi8> to memref<4x?xf32> + return +} + +// ----- + +// CHECK: llvm.mlir.global external @gv0() {addr_space = 0 : i32} : !llvm.array<2 x f32> { +// CHECK-NEXT: %0 = llvm.mlir.undef : !llvm.array<2 x f32> +// CHECK-NEXT: llvm.return %0 : !llvm.array<2 x f32> +// CHECK-NEXT: } +memref.global @gv0 : memref<2xf32> = uninitialized + +// CHECK-LABEL: func @get_gv0_memref +func.func @get_gv0_memref() { + %0 = memref.get_global @gv0 : memref<2xf32> + // CHECK: %[[DIM:.*]] = llvm.mlir.constant(2 : index) : i64 + // CHECK: %[[STRIDE:.*]] = llvm.mlir.constant(1 : index) : i64 + // CHECK: %[[ADDR:.*]] = llvm.mlir.addressof @gv0 : !llvm.ptr> + // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ADDR]][0, 0] : (!llvm.ptr>) -> !llvm.ptr + // CHECK: %[[DEADBEEF:.*]] = llvm.mlir.constant(3735928559 : index) : i64 + // CHECK: %[[DEADBEEFPTR:.*]] = llvm.inttoptr %[[DEADBEEF]] : i64 to !llvm.ptr + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.insertvalue %[[DEADBEEFPTR]], {{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.insertvalue %[[GEP]], {{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[OFFSET:.*]] = llvm.mlir.constant(0 : index) : i64 + // CHECK: llvm.insertvalue %[[OFFSET]], {{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.insertvalue %[[DIM]], {{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.insertvalue %[[STRIDE]], {{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + return +} + +// ----- + +// CHECK-LABEL: func @memref_copy_unranked +func.func @memref_copy_unranked() { + %0 = memref.alloc() : memref<2xi1> + // CHECK: llvm.mlir.constant(2 : index) : i64 + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + %1 = memref.cast %0 : memref<2xi1> to memref<*xi1> + %2 = memref.alloc() : memref<2xi1> + // CHECK: llvm.mlir.constant(2 : index) : i64 + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + %3 = memref.cast %2 : memref<2xi1> to memref<*xi1> + memref.copy %1, %3 : memref<*xi1> to memref<*xi1> + // CHECK: [[ONE:%.*]] = llvm.mlir.constant(1 : index) : i64 + // CHECK: [[ALLOCA:%.*]] = llvm.alloca %35 x !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> : (i64) -> !llvm.ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>> + // CHECK: llvm.store {{%.*}}, [[ALLOCA]] : !llvm.ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>> + // CHECK: [[BITCAST:%.*]] = llvm.bitcast [[ALLOCA]] : !llvm.ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>> to !llvm.ptr + // CHECK: [[RANK:%.*]] = llvm.mlir.constant(1 : index) : i64 + // CHECK: [[UNDEF:%.*]] = llvm.mlir.undef : !llvm.struct<(i64, ptr)> + // CHECK: [[INSERT:%.*]] = llvm.insertvalue [[RANK]], [[UNDEF]][0] : !llvm.struct<(i64, ptr)> + // CHECK: [[INSERT2:%.*]] = llvm.insertvalue [[BITCAST]], [[INSERT]][1] : !llvm.struct<(i64, ptr)> + // CHECK: [[STACKSAVE:%.*]] = llvm.intr.stacksave : !llvm.ptr + // CHECK: [[RANK2:%.*]] = llvm.mlir.constant(1 : index) : i64 + // CHECK: [[ALLOCA2:%.*]] = llvm.alloca [[RANK2]] x !llvm.struct<(i64, ptr)> : (i64) -> !llvm.ptr)>> + // CHECK: llvm.store {{%.*}}, [[ALLOCA2]] : !llvm.ptr)>> + // CHECK: [[ALLOCA3:%.*]] = llvm.alloca [[RANK2]] x !llvm.struct<(i64, ptr)> : (i64) -> !llvm.ptr)>> + // CHECK: llvm.store [[INSERT2]], [[ALLOCA3]] : !llvm.ptr)>> + // CHECK: [[SIZE:%.*]] = llvm.mlir.constant(1 : index) : i64 + // CHECK: llvm.call @memrefCopy([[SIZE]], [[ALLOCA2]], [[ALLOCA3]]) : (i64, !llvm.ptr)>>, !llvm.ptr)>>) -> () + // CHECK: llvm.intr.stackrestore [[STACKSAVE]] + return +} + +// ----- + +// CHECK-LABEL: func @mixed_alloc( +// CHECK: %[[Marg:.*]]: index, %[[Narg:.*]]: index) +func.func @mixed_alloc(%arg0: index, %arg1: index) -> memref { +// CHECK-DAG: %[[M:.*]] = builtin.unrealized_conversion_cast %[[Marg]] +// CHECK-DAG: %[[N:.*]] = builtin.unrealized_conversion_cast %[[Narg]] +// CHECK: %[[c42:.*]] = llvm.mlir.constant(42 : index) : i64 +// CHECK-NEXT: %[[one:.*]] = llvm.mlir.constant(1 : index) : i64 +// CHECK-NEXT: %[[st0:.*]] = llvm.mul %[[N]], %[[c42]] : i64 +// CHECK-NEXT: %[[sz:.*]] = llvm.mul %[[st0]], %[[M]] : i64 +// CHECK-NEXT: %[[null:.*]] = llvm.mlir.null : !llvm.ptr +// CHECK-NEXT: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[sz]]] : (!llvm.ptr, i64) -> !llvm.ptr +// CHECK-NEXT: %[[sz_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 +// CHECK-NEXT: llvm.call @malloc(%[[sz_bytes]]) : (i64) -> !llvm.ptr +// CHECK-NEXT: llvm.bitcast %{{.*}} : !llvm.ptr to !llvm.ptr +// CHECK-NEXT: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: %[[off:.*]] = llvm.mlir.constant(0 : index) : i64 +// CHECK-NEXT: llvm.insertvalue %[[off]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[M]], %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[c42]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[st0]], %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[one]], %{{.*}}[4, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + %0 = memref.alloc(%arg0, %arg1) : memref + return %0 : memref +} + +// ----- + +// CHECK-LABEL: func @mixed_dealloc +func.func @mixed_dealloc(%arg0: memref) { +// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: %[[ptri8:.*]] = llvm.bitcast %[[ptr]] : !llvm.ptr to !llvm.ptr +// CHECK-NEXT: llvm.call @free(%[[ptri8]]) : (!llvm.ptr) -> () + memref.dealloc %arg0 : memref + return +} + +// ----- + +// CHECK-LABEL: func @dynamic_alloc( +// CHECK: %[[Marg:.*]]: index, %[[Narg:.*]]: index) +func.func @dynamic_alloc(%arg0: index, %arg1: index) -> memref { +// CHECK-DAG: %[[M:.*]] = builtin.unrealized_conversion_cast %[[Marg]] +// CHECK-DAG: %[[N:.*]] = builtin.unrealized_conversion_cast %[[Narg]] +// CHECK-NEXT: %[[one:.*]] = llvm.mlir.constant(1 : index) : i64 +// CHECK-NEXT: %[[sz:.*]] = llvm.mul %[[N]], %[[M]] : i64 +// CHECK-NEXT: %[[null:.*]] = llvm.mlir.null : !llvm.ptr +// CHECK-NEXT: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[sz]]] : (!llvm.ptr, i64) -> !llvm.ptr +// CHECK-NEXT: %[[sz_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 +// CHECK-NEXT: llvm.call @malloc(%[[sz_bytes]]) : (i64) -> !llvm.ptr +// CHECK-NEXT: llvm.bitcast %{{.*}} : !llvm.ptr to !llvm.ptr +// CHECK-NEXT: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: %[[off:.*]] = llvm.mlir.constant(0 : index) : i64 +// CHECK-NEXT: llvm.insertvalue %[[off]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[M]], %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[one]], %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + %0 = memref.alloc(%arg0, %arg1) : memref + return %0 : memref +} + +// ----- + +// CHECK-LABEL: func @dynamic_dealloc +func.func @dynamic_dealloc(%arg0: memref) { +// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: %[[ptri8:.*]] = llvm.bitcast %[[ptr]] : !llvm.ptr to !llvm.ptr +// CHECK-NEXT: llvm.call @free(%[[ptri8]]) : (!llvm.ptr) -> () + memref.dealloc %arg0 : memref + return +} + +// ----- + +// CHECK-LABEL: func @memref_cast_static_to_dynamic +func.func @memref_cast_static_to_dynamic(%static : memref<10x42xf32>) { +// CHECK-NOT: llvm.bitcast + %0 = memref.cast %static : memref<10x42xf32> to memref + return +} + +// ----- + +// CHECK-LABEL: func @memref_cast_static_to_mixed +func.func @memref_cast_static_to_mixed(%static : memref<10x42xf32>) { +// CHECK-NOT: llvm.bitcast + %0 = memref.cast %static : memref<10x42xf32> to memref + return +} + +// ----- + +// CHECK-LABEL: func @memref_cast_dynamic_to_static +func.func @memref_cast_dynamic_to_static(%dynamic : memref) { +// CHECK-NOT: llvm.bitcast + %0 = memref.cast %dynamic : memref to memref<10x12xf32> + return +} + +// ----- + +// CHECK-LABEL: func @memref_cast_dynamic_to_mixed +func.func @memref_cast_dynamic_to_mixed(%dynamic : memref) { +// CHECK-NOT: llvm.bitcast + %0 = memref.cast %dynamic : memref to memref + return +} + +// ----- + +// CHECK-LABEL: func @memref_cast_mixed_to_dynamic +func.func @memref_cast_mixed_to_dynamic(%mixed : memref<42x?xf32>) { +// CHECK-NOT: llvm.bitcast + %0 = memref.cast %mixed : memref<42x?xf32> to memref + return +} + +// ----- + +// CHECK-LABEL: func @memref_cast_mixed_to_static +func.func @memref_cast_mixed_to_static(%mixed : memref<42x?xf32>) { +// CHECK-NOT: llvm.bitcast + %0 = memref.cast %mixed : memref<42x?xf32> to memref<42x1xf32> + return +} + +// ----- + +// CHECK-LABEL: func @memref_cast_mixed_to_mixed +func.func @memref_cast_mixed_to_mixed(%mixed : memref<42x?xf32>) { +// CHECK-NOT: llvm.bitcast + %0 = memref.cast %mixed : memref<42x?xf32> to memref + return +} + +// ----- + +// CHECK-LABEL: func @memref_cast_ranked_to_unranked +// CHECK32-LABEL: func @memref_cast_ranked_to_unranked +func.func @memref_cast_ranked_to_unranked(%arg : memref<42x2x?xf32>) { +// CHECK-DAG: %[[c:.*]] = llvm.mlir.constant(1 : index) : i64 +// CHECK-DAG: %[[p:.*]] = llvm.alloca %[[c]] x !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> : (i64) -> !llvm.ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>> +// CHECK-DAG: llvm.store %{{.*}}, %[[p]] : !llvm.ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>> +// CHECK-DAG: %[[p2:.*]] = llvm.bitcast %[[p]] : !llvm.ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>> to !llvm.ptr +// CHECK-DAG: %[[r:.*]] = llvm.mlir.constant(3 : index) : i64 +// CHECK : llvm.mlir.undef : !llvm.struct<(i64, ptr)> +// CHECK-DAG: llvm.insertvalue %[[r]], %{{.*}}[0] : !llvm.struct<(i64, ptr)> +// CHECK-DAG: llvm.insertvalue %[[p2]], %{{.*}}[1] : !llvm.struct<(i64, ptr)> +// CHECK32-DAG: %[[c:.*]] = llvm.mlir.constant(1 : index) : i64 +// CHECK32-DAG: %[[p:.*]] = llvm.alloca %[[c]] x !llvm.struct<(ptr, ptr, i32, array<3 x i32>, array<3 x i32>)> : (i64) -> !llvm.ptr, ptr, i32, array<3 x i32>, array<3 x i32>)>> +// CHECK32-DAG: llvm.store %{{.*}}, %[[p]] : !llvm.ptr, ptr, i32, array<3 x i32>, array<3 x i32>)>> +// CHECK32-DAG: %[[p2:.*]] = llvm.bitcast %[[p]] : !llvm.ptr, ptr, i32, array<3 x i32>, array<3 x i32>)>> to !llvm.ptr +// CHECK32-DAG: %[[r:.*]] = llvm.mlir.constant(3 : index) : i32 +// CHECK32 : llvm.mlir.undef : !llvm.struct<(i32, ptr)> +// CHECK32-DAG: llvm.insertvalue %[[r]], %{{.*}}[0] : !llvm.struct<(i32, ptr)> +// CHECK32-DAG: llvm.insertvalue %[[p2]], %{{.*}}[1] : !llvm.struct<(i32, ptr)> + %0 = memref.cast %arg : memref<42x2x?xf32> to memref<*xf32> + return +} + +// ----- + +// CHECK-LABEL: func @memref_cast_unranked_to_ranked +func.func @memref_cast_unranked_to_ranked(%arg : memref<*xf32>) { +// CHECK: %[[p:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(i64, ptr)> +// CHECK-NEXT: llvm.bitcast %[[p]] : !llvm.ptr to !llvm.ptr, ptr, i64, array<4 x i64>, array<4 x i64>)>> + %0 = memref.cast %arg : memref<*xf32> to memref + return +} + +// ----- + +// CHECK-LABEL: @memref_reinterpret_cast_unranked_to_dynamic_shape +func.func @memref_reinterpret_cast_unranked_to_dynamic_shape(%offset: index, + %size_0 : index, + %size_1 : index, + %stride_0 : index, + %stride_1 : index, + %input : memref<*xf32>) { + %output = memref.reinterpret_cast %input to + offset: [%offset], sizes: [%size_0, %size_1], + strides: [%stride_0, %stride_1] + : memref<*xf32> to memref> + return +} +// CHECK-SAME: ([[OFFSETarg:%[a-z,0-9]+]]: index, +// CHECK-SAME: [[SIZE_0arg:%[a-z,0-9]+]]: index, [[SIZE_1arg:%[a-z,0-9]+]]: index, +// CHECK-SAME: [[STRIDE_0arg:%[a-z,0-9]+]]: index, [[STRIDE_1arg:%[a-z,0-9]+]]: index, +// CHECK-DAG: [[OFFSET:%.*]] = builtin.unrealized_conversion_cast [[OFFSETarg]] +// CHECK-DAG: [[SIZE_0:%.*]] = builtin.unrealized_conversion_cast [[SIZE_0arg]] +// CHECK-DAG: [[SIZE_1:%.*]] = builtin.unrealized_conversion_cast [[SIZE_1arg]] +// CHECK-DAG: [[STRIDE_0:%.*]] = builtin.unrealized_conversion_cast [[STRIDE_0arg]] +// CHECK-DAG: [[STRIDE_1:%.*]] = builtin.unrealized_conversion_cast [[STRIDE_1arg]] +// CHECK-DAG: [[INPUT:%.*]] = builtin.unrealized_conversion_cast +// CHECK: [[OUT_0:%.*]] = llvm.mlir.undef : [[TY:!.*]] +// CHECK: [[DESCRIPTOR:%.*]] = llvm.extractvalue [[INPUT]][1] : !llvm.struct<(i64, ptr)> +// CHECK: [[BASE_PTR_PTR:%.*]] = llvm.bitcast [[DESCRIPTOR]] : !llvm.ptr to !llvm.ptr> +// CHECK: [[BASE_PTR:%.*]] = llvm.load [[BASE_PTR_PTR]] : !llvm.ptr> +// CHECK: [[BASE_PTR_PTR_:%.*]] = llvm.bitcast [[DESCRIPTOR]] : !llvm.ptr to !llvm.ptr> +// CHECK: [[ALIGNED_PTR_PTR:%.*]] = llvm.getelementptr [[BASE_PTR_PTR_]]{{\[}}1] +// CHECK-SAME: : (!llvm.ptr>) -> !llvm.ptr> +// CHECK: [[ALIGNED_PTR:%.*]] = llvm.load [[ALIGNED_PTR_PTR]] : !llvm.ptr> +// CHECK: [[OUT_1:%.*]] = llvm.insertvalue [[BASE_PTR]], [[OUT_0]][0] : [[TY]] +// CHECK: [[OUT_2:%.*]] = llvm.insertvalue [[ALIGNED_PTR]], [[OUT_1]][1] : [[TY]] +// CHECK: [[OUT_3:%.*]] = llvm.insertvalue [[OFFSET]], [[OUT_2]][2] : [[TY]] +// CHECK: [[OUT_4:%.*]] = llvm.insertvalue [[SIZE_0]], [[OUT_3]][3, 0] : [[TY]] +// CHECK: [[OUT_5:%.*]] = llvm.insertvalue [[STRIDE_0]], [[OUT_4]][4, 0] : [[TY]] +// CHECK: [[OUT_6:%.*]] = llvm.insertvalue [[SIZE_1]], [[OUT_5]][3, 1] : [[TY]] +// CHECK: [[OUT_7:%.*]] = llvm.insertvalue [[STRIDE_1]], [[OUT_6]][4, 1] : [[TY]] + +// ----- + +// CHECK-LABEL: @memref_reshape +func.func @memref_reshape(%input : memref<2x3xf32>, %shape : memref) { + %output = memref.reshape %input(%shape) + : (memref<2x3xf32>, memref) -> memref<*xf32> + return +} +// CHECK: [[INPUT:%.*]] = builtin.unrealized_conversion_cast %{{.*}} to [[INPUT_TY:!.*]] +// CHECK: [[SHAPE:%.*]] = builtin.unrealized_conversion_cast %{{.*}} to [[SHAPE_TY:!.*]] +// CHECK: [[RANK:%.*]] = llvm.extractvalue [[SHAPE]][3, 0] : [[SHAPE_TY]] +// CHECK: [[UNRANKED_OUT_O:%.*]] = llvm.mlir.undef : !llvm.struct<(i64, ptr)> +// CHECK: [[UNRANKED_OUT_1:%.*]] = llvm.insertvalue [[RANK]], [[UNRANKED_OUT_O]][0] : !llvm.struct<(i64, ptr)> + +// Compute size in bytes to allocate result ranked descriptor +// CHECK: [[C1:%.*]] = llvm.mlir.constant(1 : index) : i64 +// CHECK: [[C2:%.*]] = llvm.mlir.constant(2 : index) : i64 +// CHECK: [[PTR_SIZE:%.*]] = llvm.mlir.constant(8 : index) : i64 +// CHECK: [[INDEX_SIZE:%.*]] = llvm.mlir.constant(8 : index) : i64 +// CHECK: [[DOUBLE_PTR_SIZE:%.*]] = llvm.mul [[C2]], [[PTR_SIZE]] : i64 +// CHECK: [[DESC_ALLOC_SIZE:%.*]] = llvm.add [[DOUBLE_PTR_SIZE]], %{{.*}} +// CHECK: [[UNDERLYING_DESC:%.*]] = llvm.alloca [[DESC_ALLOC_SIZE]] x i8 +// CHECK: llvm.insertvalue [[UNDERLYING_DESC]], [[UNRANKED_OUT_1]][1] + +// Set allocated, aligned pointers and offset. +// CHECK: [[ALLOC_PTR:%.*]] = llvm.extractvalue [[INPUT]][0] : [[INPUT_TY]] +// CHECK: [[ALIGN_PTR:%.*]] = llvm.extractvalue [[INPUT]][1] : [[INPUT_TY]] +// CHECK: [[OFFSET:%.*]] = llvm.extractvalue [[INPUT]][2] : [[INPUT_TY]] +// CHECK: [[BASE_PTR_PTR:%.*]] = llvm.bitcast [[UNDERLYING_DESC]] +// CHECK-SAME: !llvm.ptr to !llvm.ptr> +// CHECK: llvm.store [[ALLOC_PTR]], [[BASE_PTR_PTR]] : !llvm.ptr> +// CHECK: [[BASE_PTR_PTR_:%.*]] = llvm.bitcast [[UNDERLYING_DESC]] : !llvm.ptr to !llvm.ptr> +// CHECK: [[ALIGNED_PTR_PTR:%.*]] = llvm.getelementptr [[BASE_PTR_PTR_]]{{\[}}1] +// CHECK: llvm.store [[ALIGN_PTR]], [[ALIGNED_PTR_PTR]] : !llvm.ptr> +// CHECK: [[BASE_PTR_PTR__:%.*]] = llvm.bitcast [[UNDERLYING_DESC]] : !llvm.ptr to !llvm.ptr> +// CHECK: [[OFFSET_PTR_:%.*]] = llvm.getelementptr [[BASE_PTR_PTR__]]{{\[}}2] +// CHECK: [[OFFSET_PTR:%.*]] = llvm.bitcast [[OFFSET_PTR_]] +// CHECK: llvm.store [[OFFSET]], [[OFFSET_PTR]] : !llvm.ptr + +// Iterate over shape operand in reverse order and set sizes and strides. +// CHECK: [[STRUCT_PTR:%.*]] = llvm.bitcast [[UNDERLYING_DESC]] +// CHECK-SAME: !llvm.ptr to !llvm.ptr, ptr, i64, i64)>> +// CHECK: [[SIZES_PTR:%.*]] = llvm.getelementptr [[STRUCT_PTR]]{{\[}}0, 3] +// CHECK: [[STRIDES_PTR:%.*]] = llvm.getelementptr [[SIZES_PTR]]{{\[}}[[RANK]]] +// CHECK: [[SHAPE_IN_PTR:%.*]] = llvm.extractvalue [[SHAPE]][1] : [[SHAPE_TY]] +// CHECK: [[C1_:%.*]] = llvm.mlir.constant(1 : index) : i64 +// CHECK: [[RANK_MIN_1:%.*]] = llvm.sub [[RANK]], [[C1_]] : i64 +// CHECK: llvm.br ^bb1([[RANK_MIN_1]], [[C1_]] : i64, i64) + +// CHECK: ^bb1([[DIM:%.*]]: i64, [[CUR_STRIDE:%.*]]: i64): +// CHECK: [[C0_:%.*]] = llvm.mlir.constant(0 : index) : i64 +// CHECK: [[COND:%.*]] = llvm.icmp "sge" [[DIM]], [[C0_]] : i64 +// CHECK: llvm.cond_br [[COND]], ^bb2, ^bb3 + +// CHECK: ^bb2: +// CHECK: [[SIZE_PTR:%.*]] = llvm.getelementptr [[SHAPE_IN_PTR]]{{\[}}[[DIM]]] +// CHECK: [[SIZE:%.*]] = llvm.load [[SIZE_PTR]] : !llvm.ptr +// CHECK: [[TARGET_SIZE_PTR:%.*]] = llvm.getelementptr [[SIZES_PTR]]{{\[}}[[DIM]]] +// CHECK: llvm.store [[SIZE]], [[TARGET_SIZE_PTR]] : !llvm.ptr +// CHECK: [[TARGET_STRIDE_PTR:%.*]] = llvm.getelementptr [[STRIDES_PTR]]{{\[}}[[DIM]]] +// CHECK: llvm.store [[CUR_STRIDE]], [[TARGET_STRIDE_PTR]] : !llvm.ptr +// CHECK: [[UPDATE_STRIDE:%.*]] = llvm.mul [[CUR_STRIDE]], [[SIZE]] : i64 +// CHECK: [[STRIDE_COND:%.*]] = llvm.sub [[DIM]], [[C1_]] : i64 +// CHECK: llvm.br ^bb1([[STRIDE_COND]], [[UPDATE_STRIDE]] : i64, i64) + +// CHECK: ^bb3: +// CHECK: return + +// ----- + +// CHECK-LABEL: func.func @realloc_dynamic( +// CHECK-SAME: %[[arg0:.*]]: memref, +// CHECK-SAME: %[[arg1:.*]]: index) -> memref { +func.func @realloc_dynamic(%in: memref, %d: index) -> memref{ +// CHECK: %[[descriptor:.*]] = builtin.unrealized_conversion_cast %[[arg0]] +// CHECK: %[[src_dim:.*]] = llvm.extractvalue %[[descriptor]][3, 0] +// CHECK: %[[dst_dim:.*]] = builtin.unrealized_conversion_cast %[[arg1]] : index to i64 +// CHECK: %[[cond:.*]] = llvm.icmp "ugt" %[[dst_dim]], %[[src_dim]] : i64 +// CHECK: llvm.cond_br %[[cond]], ^bb1, ^bb2(%[[descriptor]] +// CHECK: ^bb1: +// CHECK: %[[dst_null:.*]] = llvm.mlir.null : !llvm.ptr +// CHECK: %[[dst_gep:.*]] = llvm.getelementptr %[[dst_null]][1] +// CHECK: %[[dst_es:.*]] = llvm.ptrtoint %[[dst_gep]] : !llvm.ptr to i64 +// CHECK: %[[dst_size:.*]] = llvm.mul %[[dst_dim]], %[[dst_es]] +// CHECK: %[[src_size:.*]] = llvm.mul %[[src_dim]], %[[dst_es]] +// CHECK: %[[new_buffer_raw:.*]] = llvm.call @malloc(%[[dst_size]]) +// CHECK: %[[new_buffer:.*]] = llvm.bitcast %[[new_buffer_raw]] : !llvm.ptr to !llvm.ptr +// CHECK: %[[old_buffer_aligned:.*]] = llvm.extractvalue %[[descriptor]][1] +// CHECK: %[[volatile:.*]] = llvm.mlir.constant(false) : i1 +// CHECK-DAG: %[[new_buffer_void:.*]] = llvm.bitcast %[[new_buffer]] : !llvm.ptr to !llvm.ptr +// CHECK-DAG: %[[old_buffer_void:.*]] = llvm.bitcast %[[old_buffer_aligned]] : !llvm.ptr to !llvm.ptr +// CHECK: "llvm.intr.memcpy"(%[[new_buffer_void]], %[[old_buffer_void]], %[[src_size]], %[[volatile]]) +// CHECK: %[[old_buffer_unaligned:.*]] = llvm.extractvalue %[[descriptor]][0] +// CHECK: %[[old_buffer_unaligned_void:.*]] = llvm.bitcast %[[old_buffer_unaligned]] : !llvm.ptr to !llvm.ptr +// CHECK: llvm.call @free(%[[old_buffer_unaligned_void]]) +// CHECK: %[[descriptor_update1:.*]] = llvm.insertvalue %[[new_buffer]], %[[descriptor]][0] +// CHECK: %[[descriptor_update2:.*]] = llvm.insertvalue %[[new_buffer]], %[[descriptor_update1]][1] +// CHECK: llvm.br ^bb2(%[[descriptor_update2]] +// CHECK: ^bb2(%[[descriptor_update3:.*]]: !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>): +// CHECK: %[[descriptor_update4:.*]] = llvm.insertvalue %[[dst_dim]], %[[descriptor_update3]][3, 0] +// CHECK: %[[descriptor_update5:.*]] = builtin.unrealized_conversion_cast %[[descriptor_update4]] +// CHECK: return %[[descriptor_update5]] : memref + + %out = memref.realloc %in(%d) : memref to memref + return %out : memref +} + From d42d89395b95185466acb38c12ea7e50f0543086 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20B=C3=B6ck?= Date: Mon, 6 Mar 2023 11:11:59 +0100 Subject: [PATCH 085/147] [mlir][GPU] add required address space cast when lowering to LLVM The runtime functions `memset` and `memcpy` are lowered are declared with pointers to the defaul t address space (0) while their ops however are compatible with memrefs taking any address space. Such cases do not cause any issues with MLIRs LLVM Dialect due to `bitcast`s verifier being too lenient at the moment, but actual LLVM IR does not allow casting between address spaces using `bitca st`: https://godbolt.org/z/3a1z97rc9 This patch fixes the issue by inserting an address space cast before the bitcast, to first cast the pointer into the correct address space before doing the bitcast. Differential Revision: https://reviews.llvm.org/D143866 (cherry picked from commit 0aaf2e3bc057aa1d784455f8f4da66bc464733d6) --- .../GPUCommon/GPUToLLVMConversion.cpp | 35 ++++++++++++++----- .../lower-memcpy-to-gpu-runtime-calls.mlir | 4 ++- .../lower-memset-to-gpu-runtime-calls.mlir | 3 +- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp index 3f61be5471e7..4bb0e3ae028f 100644 --- a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp +++ b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp @@ -92,7 +92,7 @@ class ConvertOpToGpuRuntimeCallPattern : public ConvertOpToLLVMPattern { MLIRContext *context = &this->getTypeConverter()->getContext(); Type llvmVoidType = LLVM::LLVMVoidType::get(context); - Type llvmPointerType = + LLVM::LLVMPointerType llvmPointerType = LLVM::LLVMPointerType::get(IntegerType::get(context, 8)); Type llvmPointerPointerType = LLVM::LLVMPointerType::get(llvmPointerType); Type llvmInt8Type = IntegerType::get(context, 8); @@ -807,6 +807,22 @@ LogicalResult ConvertLaunchFuncOpToGpuRuntimeCallPattern::matchAndRewrite( return success(); } +static Value bitAndAddrspaceCast(Location loc, + ConversionPatternRewriter &rewriter, + LLVM::LLVMPointerType destinationType, + Value sourcePtr, + LLVMTypeConverter &typeConverter) { + auto sourceTy = sourcePtr.getType().cast(); + if (destinationType.getAddressSpace() != sourceTy.getAddressSpace()) + sourcePtr = rewriter.create( + loc, + typeConverter.getPointerType(sourceTy.getElementType(), + destinationType.getAddressSpace()), + sourcePtr); + + return rewriter.create(loc, destinationType, sourcePtr); +} + LogicalResult ConvertMemcpyOpToGpuRuntimeCallPattern::matchAndRewrite( gpu::MemcpyOp memcpyOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const { @@ -829,11 +845,13 @@ LogicalResult ConvertMemcpyOpToGpuRuntimeCallPattern::matchAndRewrite( auto sizeBytes = rewriter.create(loc, getIndexType(), gepPtr); - auto src = rewriter.create( - loc, llvmPointerType, srcDesc.alignedPtr(rewriter, loc)); - auto dst = rewriter.create( - loc, llvmPointerType, - MemRefDescriptor(adaptor.getDst()).alignedPtr(rewriter, loc)); + auto src = bitAndAddrspaceCast(loc, rewriter, llvmPointerType, + srcDesc.alignedPtr(rewriter, loc), + *getTypeConverter()); + auto dst = bitAndAddrspaceCast( + loc, rewriter, llvmPointerType, + MemRefDescriptor(adaptor.getDst()).alignedPtr(rewriter, loc), + *getTypeConverter()); auto stream = adaptor.getAsyncDependencies().front(); memcpyCallBuilder.create(loc, rewriter, {dst, src, sizeBytes, stream}); @@ -866,8 +884,9 @@ LogicalResult ConvertMemsetOpToGpuRuntimeCallPattern::matchAndRewrite( auto value = rewriter.create(loc, llvmInt32Type, adaptor.getValue()); - auto dst = rewriter.create( - loc, llvmPointerType, dstDesc.alignedPtr(rewriter, loc)); + auto dst = bitAndAddrspaceCast(loc, rewriter, llvmPointerType, + dstDesc.alignedPtr(rewriter, loc), + *getTypeConverter()); auto stream = adaptor.getAsyncDependencies().front(); memsetCallBuilder.create(loc, rewriter, {dst, value, numElements, stream}); diff --git a/mlir/test/Conversion/GPUCommon/lower-memcpy-to-gpu-runtime-calls.mlir b/mlir/test/Conversion/GPUCommon/lower-memcpy-to-gpu-runtime-calls.mlir index df10b317aa49..89c0268a0f72 100644 --- a/mlir/test/Conversion/GPUCommon/lower-memcpy-to-gpu-runtime-calls.mlir +++ b/mlir/test/Conversion/GPUCommon/lower-memcpy-to-gpu-runtime-calls.mlir @@ -7,8 +7,10 @@ module attributes {gpu.container_module} { // CHECK: %[[t0:.*]] = llvm.call @mgpuStreamCreate %t0 = gpu.wait async // CHECK: %[[size_bytes:.*]] = llvm.ptrtoint + // CHECK-NOT: llvm.addrspacecast // CHECK: %[[src:.*]] = llvm.bitcast - // CHECK: %[[dst:.*]] = llvm.bitcast + // CHECK: %[[addr_cast:.*]] = llvm.addrspacecast + // CHECK: %[[dst:.*]] = llvm.bitcast %[[addr_cast]] // CHECK: llvm.call @mgpuMemcpy(%[[dst]], %[[src]], %[[size_bytes]], %[[t0]]) %t1 = gpu.memcpy async [%t0] %dst, %src : memref<7xf32, 1>, memref<7xf32> // CHECK: llvm.call @mgpuStreamSynchronize(%[[t0]]) diff --git a/mlir/test/Conversion/GPUCommon/lower-memset-to-gpu-runtime-calls.mlir b/mlir/test/Conversion/GPUCommon/lower-memset-to-gpu-runtime-calls.mlir index ef5b6ef2c7bb..562c15583369 100644 --- a/mlir/test/Conversion/GPUCommon/lower-memset-to-gpu-runtime-calls.mlir +++ b/mlir/test/Conversion/GPUCommon/lower-memset-to-gpu-runtime-calls.mlir @@ -8,7 +8,8 @@ module attributes {gpu.container_module} { %t0 = gpu.wait async // CHECK: %[[size_bytes:.*]] = llvm.mlir.constant // CHECK: %[[value:.*]] = llvm.bitcast - // CHECK: %[[dst:.*]] = llvm.bitcast + // CHECK: %[[addr_cast:.*]] = llvm.addrspacecast + // CHECK: %[[dst:.*]] = llvm.bitcast %[[addr_cast]] // CHECK: llvm.call @mgpuMemset32(%[[dst]], %[[value]], %[[size_bytes]], %[[t0]]) %t1 = gpu.memset async [%t0] %dst, %value : memref<7xf32, 1>, f32 // CHECK: llvm.call @mgpuStreamSynchronize(%[[t0]]) From a12167c9aab8ce048ab07c47828edfbd5c67fba3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20B=C3=B6ck?= Date: Mon, 6 Mar 2023 11:12:39 +0100 Subject: [PATCH 086/147] [mlir][LLVM] Verify correct pointer casts with `llvm.bitcast` has so far not had a verifier and this allowed various bugs to sneak into the codebase (including within tests!) which could only be caught once translated to actual LLVM IR. This patch fixes those problematic cases by now verifying bitcasts on pointers are done correctly. Specifically, it verifies that if pointers are involved, that both result and source types are pointers, that this also applies to vector of pointers and that pointer casts are of the same address space. The only thing left unverified is the general case of "source type size does not match result type size". I think this case is less trivial and more prone to false positives, so I did not yet implement it. Differential Revision: https://reviews.llvm.org/D143868 (cherry picked from commit 009fc4ca3b4476826c89de05cb74d0afe2b5d609) --- mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td | 1 + mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 37 +++++++++++++++++- mlir/test/Dialect/LLVMIR/invalid.mlir | 43 +++++++++++++++++++++ 3 files changed, 80 insertions(+), 1 deletion(-) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td index 011231c15be5..db104e7dc790 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td @@ -446,6 +446,7 @@ class LLVM_CastOp { let hasFolder = 1; + let hasVerifier = 1; } def LLVM_AddrSpaceCastOp : LLVM_CastOp<"addrspacecast", "AddrSpaceCast", LLVM_ScalarOrVectorOf, diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp index 2dd6e40fd9db..2a86438eda44 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -2356,7 +2356,7 @@ LogicalResult FenceOp::verify() { } //===----------------------------------------------------------------------===// -// Folder for LLVM::BitcastOp +// Folder and verifier for LLVM::BitcastOp //===----------------------------------------------------------------------===// OpFoldResult LLVM::BitcastOp::fold(FoldAdaptor adaptor) { @@ -2370,6 +2370,41 @@ OpFoldResult LLVM::BitcastOp::fold(FoldAdaptor adaptor) { return {}; } +LogicalResult LLVM::BitcastOp::verify() { + auto resultType = extractVectorElementType(getResult().getType()) + .dyn_cast(); + auto sourceType = + extractVectorElementType(getArg().getType()).dyn_cast(); + + // If one of the types is a pointer (or vector of pointers), then + // both source and result type have to be pointers. + if (static_cast(resultType) != static_cast(sourceType)) + return emitOpError("can only cast pointers from and to pointers"); + + if (!resultType) + return success(); + + auto isVector = [](Type type) { + return type.isa(); + }; + + // Due to bitcast requiring both operands to be of the same size, it is not + // possible for only one of the two to be a pointer of vectors. + if (isVector(getResult().getType()) && !isVector(getArg().getType())) + return emitOpError("cannot cast pointer to vector of pointers"); + + if (!isVector(getResult().getType()) && isVector(getArg().getType())) + return emitOpError("cannot cast vector of pointers to pointer"); + + // Bitcast cannot cast between pointers of different address spaces. + // 'llvm.addrspacecast' must be used for this purpose instead. + if (resultType.getAddressSpace() != sourceType.getAddressSpace()) + return emitOpError("cannot cast pointers of different address spaces, " + "use 'llvm.addrspacecast' instead"); + + return success(); +} + //===----------------------------------------------------------------------===// // Folder for LLVM::AddrSpaceCastOp //===----------------------------------------------------------------------===// diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir index 18dfc4ce1ff0..256a50904925 100644 --- a/mlir/test/Dialect/LLVMIR/invalid.mlir +++ b/mlir/test/Dialect/LLVMIR/invalid.mlir @@ -1303,3 +1303,46 @@ func.func @extract_scalable_from_fixed_length_vector(%arg0 : vector<16xf32>) { #void = #llvm.di_void_result_type // expected-error@below {{expected subroutine to have non-void argument types}} #void_argument_type = #llvm.di_subroutine_type + +// ----- + +func.func @invalid_bitcast_ptr_to_i64(%arg : !llvm.ptr) { + // expected-error@+1 {{can only cast pointers from and to pointers}} + %1 = llvm.bitcast %arg : !llvm.ptr to i64 +} + +// ----- + +func.func @invalid_bitcast_i64_to_ptr() { + %0 = llvm.mlir.constant(2 : i64) : i64 + // expected-error@+1 {{can only cast pointers from and to pointers}} + %1 = llvm.bitcast %0 : i64 to !llvm.ptr +} + +// ----- + +func.func @invalid_bitcast_vec_to_ptr(%arg : !llvm.vec<4 x ptr>) { + // expected-error@+1 {{cannot cast vector of pointers to pointer}} + %0 = llvm.bitcast %arg : !llvm.vec<4 x ptr> to !llvm.ptr +} + +// ----- + +func.func @invalid_bitcast_ptr_to_vec(%arg : !llvm.ptr) { + // expected-error@+1 {{cannot cast pointer to vector of pointers}} + %0 = llvm.bitcast %arg : !llvm.ptr to !llvm.vec<4 x ptr> +} + +// ----- + +func.func @invalid_bitcast_addr_cast(%arg : !llvm.ptr<1>) { + // expected-error@+1 {{cannot cast pointers of different address spaces, use 'llvm.addrspacecast' instead}} + %0 = llvm.bitcast %arg : !llvm.ptr<1> to !llvm.ptr +} + +// ----- + +func.func @invalid_bitcast_addr_cast_vec(%arg : !llvm.vec<4 x ptr<1>>) { + // expected-error@+1 {{cannot cast pointers of different address spaces, use 'llvm.addrspacecast' instead}} + %0 = llvm.bitcast %arg : !llvm.vec<4 x ptr<1>> to !llvm.vec<4 x ptr> +} From 35c7bfcfcaa27d7e66648546f1c8618cf505fbca Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Mon, 6 Mar 2023 11:13:15 +0100 Subject: [PATCH 087/147] [mlir][llvm] Rename void debug type to null This commit renames the "di_void_result_type" to "di_null_type" as LLVM does use null not exclusively for void types. An added test demonstrates this for variadic function declarations, whose DISubroutine indicates the start of variadic types with `null`. Reviewed By: gysit Differential Revision: https://reviews.llvm.org/D144109 (cherry picked from commit 873ea45115a8e5ed1065d6aaad8f41fe6120cac8) --- .../mlir/Dialect/LLVMIR/LLVMAttrDefs.td | 9 ++++---- mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp | 21 ++----------------- mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 2 +- mlir/lib/Target/LLVMIR/DebugImporter.cpp | 10 ++++----- mlir/lib/Target/LLVMIR/DebugTranslation.cpp | 11 +++++----- mlir/lib/Target/LLVMIR/DebugTranslation.h | 2 +- mlir/test/Dialect/LLVMIR/debuginfo.mlir | 8 +++---- mlir/test/Dialect/LLVMIR/invalid.mlir | 6 ------ mlir/test/Target/LLVMIR/Import/debug-info.ll | 20 ++++++++++++++++-- mlir/test/Target/LLVMIR/llvmir-debug.mlir | 4 ++-- 10 files changed, 43 insertions(+), 50 deletions(-) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td index b150c71b37d4..2d4612947827 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td @@ -262,11 +262,11 @@ def LLVM_DITagParameter : LLVM_DIParameter< >; //===----------------------------------------------------------------------===// -// DIVoidResultTypeAttr +// DINullTypeAttr //===----------------------------------------------------------------------===// -def LLVM_DIVoidResultTypeAttr : LLVM_Attr<"DIVoidResultType", "di_void_result_type", - /*traits=*/[], "DITypeAttr"> { +def LLVM_DINullTypeAttr : LLVM_Attr<"DINullType", "di_null_type", + /*traits=*/[], "DITypeAttr"> { let parameters = (ins); } @@ -450,7 +450,7 @@ def LLVM_DISubprogramAttr : LLVM_Attr<"DISubprogram", "di_subprogram", [ SubElementAttrInterface ], "DIScopeAttr"> { let parameters = (ins - "DICompileUnitAttr":$compileUnit, + OptionalParameter<"DICompileUnitAttr">:$compileUnit, "DIScopeAttr":$scope, "StringAttr":$name, OptionalParameter<"StringAttr">:$linkageName, @@ -509,7 +509,6 @@ def LLVM_DISubroutineTypeAttr : LLVM_Attr<"DISubroutineType", "di_subroutine_typ }]> ]; let assemblyFormat = "`<` struct(params) `>`"; - let genVerifyDecl = 1; } //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp index 8e749259dad9..1bb0f6a9438d 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp @@ -42,7 +42,7 @@ void LLVMDialect::registerAttributes() { //===----------------------------------------------------------------------===// bool DINodeAttr::classof(Attribute attr) { - return llvm::isa(attr); } -//===----------------------------------------------------------------------===// -// DISubroutineTypeAttr -//===----------------------------------------------------------------------===// - -LogicalResult -DISubroutineTypeAttr::verify(function_ref emitError, - unsigned int callingConventions, - ArrayRef types) { - ArrayRef argumentTypes = - types.empty() ? types : types.drop_front(); - if (llvm::any_of(argumentTypes, [](DITypeAttr type) { - return type.isa(); - })) - return emitError() << "expected subroutine to have non-void argument types"; - return success(); -} - //===----------------------------------------------------------------------===// // MemoryEffectsAttr //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp index 2a86438eda44..7fa1dc872f6a 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -2759,7 +2759,7 @@ struct LLVMOpAsmDialectInterface : public OpAsmDialectInterface { AliasResult getAlias(Attribute attr, raw_ostream &os) const override { return TypeSwitch(attr) - .Case types; for (llvm::DIType *type : node->getTypeArray()) { if (!type) { - // A nullptr entry at the beginning of the subroutine types list models a - // void result type. Translate the nullptr to an explicit - // DIVoidResultTypeAttr since the attribute list cannot contain a nullptr - // entry. - types.push_back(DIVoidResultTypeAttr::get(context)); + // A nullptr entry may appear at the beginning or the end of the + // subroutine types list modeling either a void result type or the type of + // a variadic argument. Translate the nullptr to an explicit + // DINullTypeAttr since the attribute list cannot contain a nullptr entry. + types.push_back(DINullTypeAttr::get(context)); continue; } types.push_back(translate(type)); diff --git a/mlir/lib/Target/LLVMIR/DebugTranslation.cpp b/mlir/lib/Target/LLVMIR/DebugTranslation.cpp index fd80a773df8e..9b13e81ace67 100644 --- a/mlir/lib/Target/LLVMIR/DebugTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/DebugTranslation.cpp @@ -88,10 +88,11 @@ void DebugTranslation::translate(LLVMFuncOp func, llvm::Function &llvmFunc) { // Attributes //===----------------------------------------------------------------------===// -llvm::DIType *DebugTranslation::translateImpl(DIVoidResultTypeAttr attr) { - // A DIVoidResultTypeAttr at the beginning of the subroutine types list models - // a void result type. Translate the explicit DIVoidResultTypeAttr to a - // nullptr since LLVM IR metadata does not have an explicit void result type +llvm::DIType *DebugTranslation::translateImpl(DINullTypeAttr attr) { + // A DINullTypeAttr at the beginning of the subroutine types list models + // a void result type. If it is at the end, it models a variadic function. + // Translate the explicit DINullTypeAttr to a nullptr since LLVM IR metadata + // does not have an explicit void result type nor a variadic type // representation. return nullptr; } @@ -234,7 +235,7 @@ llvm::DINode *DebugTranslation::translate(DINodeAttr attr) { llvm::DINode *node = TypeSwitch(attr) - .Case( diff --git a/mlir/lib/Target/LLVMIR/DebugTranslation.h b/mlir/lib/Target/LLVMIR/DebugTranslation.h index b62bd2bab4ce..17bc35c8ab44 100644 --- a/mlir/lib/Target/LLVMIR/DebugTranslation.h +++ b/mlir/lib/Target/LLVMIR/DebugTranslation.h @@ -61,7 +61,7 @@ class DebugTranslation { llvm::DIFile *translateFile(StringRef fileName); /// Translate the given attribute to the corresponding llvm debug metadata. - llvm::DIType *translateImpl(DIVoidResultTypeAttr attr); + llvm::DIType *translateImpl(DINullTypeAttr attr); llvm::DIBasicType *translateImpl(DIBasicTypeAttr attr); llvm::DICompileUnit *translateImpl(DICompileUnitAttr attr); llvm::DICompositeType *translateImpl(DICompositeTypeAttr attr); diff --git a/mlir/test/Dialect/LLVMIR/debuginfo.mlir b/mlir/test/Dialect/LLVMIR/debuginfo.mlir index 4e13c5e01f2a..d7123d971a68 100644 --- a/mlir/test/Dialect/LLVMIR/debuginfo.mlir +++ b/mlir/test/Dialect/LLVMIR/debuginfo.mlir @@ -9,8 +9,8 @@ isOptimized = true, emissionKind = Full > -// CHECK-DAG: #[[VOID:.*]] = #llvm.di_void_result_type -#void = #llvm.di_void_result_type +// CHECK-DAG: #[[NULL:.*]] = #llvm.di_null_type +#null = #llvm.di_null_type // CHECK-DAG: #[[INT0:.*]] = #llvm.di_basic_type #int0 = #llvm.di_basic_type< @@ -56,9 +56,9 @@ flags = "TypePassByReference|NonTrivial" > -// CHECK-DAG: #[[SPTYPE0:.*]] = #llvm.di_subroutine_type +// CHECK-DAG: #[[SPTYPE0:.*]] = #llvm.di_subroutine_type #spType0 = #llvm.di_subroutine_type< - callingConvention = DW_CC_normal, types = #void, #int0, #ptr0, #ptr1, #comp0, #comp1, #comp2 + callingConvention = DW_CC_normal, types = #null, #int0, #ptr0, #ptr1, #comp0, #comp1, #comp2 > // CHECK-DAG: #[[SPTYPE1:.*]] = #llvm.di_subroutine_type diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir index 256a50904925..664a9136f69a 100644 --- a/mlir/test/Dialect/LLVMIR/invalid.mlir +++ b/mlir/test/Dialect/LLVMIR/invalid.mlir @@ -1300,12 +1300,6 @@ func.func @extract_scalable_from_fixed_length_vector(%arg0 : vector<16xf32>) { // ----- -#void = #llvm.di_void_result_type -// expected-error@below {{expected subroutine to have non-void argument types}} -#void_argument_type = #llvm.di_subroutine_type - -// ----- - func.func @invalid_bitcast_ptr_to_i64(%arg : !llvm.ptr) { // expected-error@+1 {{can only cast pointers from and to pointers}} %1 = llvm.bitcast %arg : !llvm.ptr to i64 diff --git a/mlir/test/Target/LLVMIR/Import/debug-info.ll b/mlir/test/Target/LLVMIR/Import/debug-info.ll index 588eea590de3..6dc5463006ae 100644 --- a/mlir/test/Target/LLVMIR/Import/debug-info.ll +++ b/mlir/test/Target/LLVMIR/Import/debug-info.ll @@ -112,10 +112,10 @@ define i32 @lexical_block_file(i32 %arg1) { ; // ----- -; CHECK-DAG: #[[VOID:.+]] = #llvm.di_void_result_type +; CHECK-DAG: #[[NULL:.+]] = #llvm.di_null_type ; CHECK-DAG: #[[INT1:.+]] = #llvm.di_basic_type ; CHECK-DAG: #[[INT2:.+]] = #llvm.di_basic_type -; CHECK-DAG: #llvm.di_subroutine_type +; CHECK-DAG: #llvm.di_subroutine_type define void @basic_type() !dbg !3 { ret void @@ -328,3 +328,19 @@ declare void @llvm.dbg.value(metadata, metadata, metadata) !7 = !DILocalVariable(scope: !8, name: "class_field", file: !2, type: !5); !8 = distinct !DISubprogram(name: "class_field", scope: !2, file: !2, spFlags: DISPFlagDefinition, unit: !1) !9 = !DILocation(line: 1, column: 2, scope: !8) + +; // ----- + +; CHECK-DAG: #[[NULL:.+]] = #llvm.di_null_type +; CHECK-DAG: #llvm.di_subroutine_type + +declare !dbg !3 void @variadic_func() + +!llvm.dbg.cu = !{!1} +!llvm.module.flags = !{!0} +!0 = !{i32 2, !"Debug Info Version", i32 3} +!1 = distinct !DICompileUnit(language: DW_LANG_C, file: !2) +!2 = !DIFile(filename: "debug-info.ll", directory: "/") +!3 = !DISubprogram(name: "variadic_func", scope: !2, file: !2, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, type: !4) +!4 = !DISubroutineType(types: !5) +!5 = !{null, null} diff --git a/mlir/test/Target/LLVMIR/llvmir-debug.mlir b/mlir/test/Target/LLVMIR/llvmir-debug.mlir index 50d18eb2dd5f..8b5a9f29864d 100644 --- a/mlir/test/Target/LLVMIR/llvmir-debug.mlir +++ b/mlir/test/Target/LLVMIR/llvmir-debug.mlir @@ -48,8 +48,8 @@ llvm.func @func_no_debug() { baseType = #si64, flags = Vector, elements = #llvm.di_subrange > -#void = #llvm.di_void_result_type -#spType0 = #llvm.di_subroutine_type +#null = #llvm.di_null_type +#spType0 = #llvm.di_subroutine_type #sp0 = #llvm.di_subprogram< compileUnit = #cu, scope = #file, name = "func_with_debug", linkageName = "func_with_debug", file = #file, line = 3, scopeLine = 3, subprogramFlags = "Definition|Optimized", type = #spType0 From fd71c2837f697f5749f1e3629912b3aca4edebd0 Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Mon, 6 Mar 2023 11:13:53 +0100 Subject: [PATCH 088/147] [mlir][llvm] Use before def debug intrinsic import This commit adds special handling for the debug intrinsic value handling. LLVM allows to relax the def before use property for debug intrinsics, so this property cannot be assumed for metadata values. Reviewed By: gysit Differential Revision: https://reviews.llvm.org/D144177 (cherry picked from commit 28542e99bb82b32dfc80ad372fcde3a016a8ca0b) --- .../mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td | 8 ++++-- .../include/mlir/Target/LLVMIR/ModuleImport.h | 5 ++++ mlir/lib/Target/LLVMIR/ModuleImport.cpp | 28 +++++++++++++++---- mlir/test/Target/LLVMIR/Import/debug-info.ll | 23 +++++++++++++++ 4 files changed, 57 insertions(+), 7 deletions(-) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td index c1b82b91c110..4f6ceea76e10 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td @@ -291,9 +291,13 @@ class LLVM_DbgIntrOp : LLVM_IntrOp argOperand = moduleImport.convertValue(llvmOperands[0]); + + FailureOr argOperand = moduleImport.convertMetadataValue(llvmOperands[0]); + // Drop the intrinsic when its operand could not be converted. This can + // happen for use before definition cases that are allowed for debug + // intrinsics. if (failed(argOperand)) - return failure(); + return success(); $_op = $_builder.create<$_qualCppClassName>($_location, *argOperand, $_var_attr($varInfo)); }]; diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h index aaf9b2cf55ef..be4f6e5717b1 100644 --- a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h +++ b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h @@ -119,6 +119,11 @@ class ModuleImport { /// LLVM values. FailureOr convertValue(llvm::Value *value); + /// Converts an LLVM metadata value to an MLIR value, or returns failure if + /// the conversion fails. Uses the `convertConstant` method to translate + /// constant LLVM values. + FailureOr convertMetadataValue(llvm::Value *value); + /// Converts a range of LLVM values to a range of MLIR values using the /// `convertValue` method, or returns failure if the conversion fails. FailureOr> convertValues(ArrayRef values); diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp index 94dd030ebc0f..fd4b69e63bab 100644 --- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp @@ -1127,11 +1127,8 @@ FailureOr ModuleImport::convertConstantExpr(llvm::Constant *constant) { } FailureOr ModuleImport::convertValue(llvm::Value *value) { - // A value may be wrapped as metadata, for example, when passed to a debug - // intrinsic. Unwrap these values before the conversion. - if (auto *nodeAsVal = dyn_cast(value)) - if (auto *node = dyn_cast(nodeAsVal->getMetadata())) - value = node->getValue(); + assert(!isa(value) && + "expected value to not be metadata"); // Return the mapped value if it has been converted before. if (valueMapping.count(value)) @@ -1147,6 +1144,27 @@ FailureOr ModuleImport::convertValue(llvm::Value *value) { return emitError(loc) << "unhandled value: " << diag(*value); } +FailureOr ModuleImport::convertMetadataValue(llvm::Value *value) { + // A value may be wrapped as metadata, for example, when passed to a debug + // intrinsic. Unwrap these values before the conversion. + auto *nodeAsVal = dyn_cast(value); + if (!nodeAsVal) + return failure(); + auto *node = dyn_cast(nodeAsVal->getMetadata()); + if (!node) + return failure(); + value = node->getValue(); + + // Return the mapped value if it has been converted before. + if (valueMapping.count(value)) + return lookupValue(value); + + // Convert constants such as immediate values that have no mapping yet. + if (auto *constant = dyn_cast(value)) + return convertConstantExpr(constant); + return failure(); +} + FailureOr> ModuleImport::convertValues(ArrayRef values) { SmallVector remapped; diff --git a/mlir/test/Target/LLVMIR/Import/debug-info.ll b/mlir/test/Target/LLVMIR/Import/debug-info.ll index 6dc5463006ae..896d42202c23 100644 --- a/mlir/test/Target/LLVMIR/Import/debug-info.ll +++ b/mlir/test/Target/LLVMIR/Import/debug-info.ll @@ -344,3 +344,26 @@ declare !dbg !3 void @variadic_func() !3 = !DISubprogram(name: "variadic_func", scope: !2, file: !2, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, type: !4) !4 = !DISubroutineType(types: !5) !5 = !{null, null} + +; // ----- + +define void @dbg_use_before_def(ptr %arg) { + call void @llvm.dbg.value(metadata ptr %dbg_arg, metadata !7, metadata !DIExpression()), !dbg !9 + %dbg_arg = getelementptr double, ptr %arg, i64 16 + ret void +} + +declare void @llvm.dbg.value(metadata, metadata, metadata) + +!llvm.dbg.cu = !{!1} +!llvm.module.flags = !{!0} +!0 = !{i32 2, !"Debug Info Version", i32 3} +!1 = distinct !DICompileUnit(language: DW_LANG_C, file: !2) +!2 = !DIFile(filename: "debug-info.ll", directory: "/") +!3 = !DICompositeType(tag: DW_TAG_class_type, name: "class_field", file: !2, line: 42, flags: DIFlagTypePassByReference | DIFlagNonTrivial, elements: !4) +!4 = !{!6} +!5 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !3, flags: DIFlagArtificial | DIFlagObjectPointer) +!6 = !DIDerivedType(tag: DW_TAG_member, name: "call_field", file: !2, baseType: !5) +!7 = !DILocalVariable(scope: !8, name: "var", file: !2, type: !5); +!8 = distinct !DISubprogram(name: "dbg_use_before_def", scope: !2, file: !2, spFlags: DISPFlagDefinition, unit: !1) +!9 = !DILocation(line: 1, column: 2, scope: !8) From 56656541a6a9df1929f459186405c1258af8f620 Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Mon, 6 Mar 2023 11:14:27 +0100 Subject: [PATCH 089/147] [mlir][llvm] Verify LLVM module before import This commit ensures that the importing of LLVM modules first verifies that the module is even valid. As many tests did not work with valid LLVM IR, they were fixed as part of this commit. Some error messages were only reachable with invalid input IR, thus they were replaced with a failures. Reviewed By: gysit Differential Revision: https://reviews.llvm.org/D144186 (cherry picked from commit bf91cd6ea9d6c03558667d417e496324468948da) --- mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp | 3 + .../LLVMIR/LLVMIRToLLVMTranslation.cpp | 20 +- .../Target/LLVMIR/LoopAnnotationImporter.cpp | 3 +- mlir/test/Target/LLVMIR/Import/basic.ll | 6 +- .../LLVMIR/Import/function-attributes.ll | 16 +- .../Target/LLVMIR/Import/import-failure.ll | 298 +----------------- .../Target/LLVMIR/Import/metadata-tbaa.ll | 25 +- 7 files changed, 45 insertions(+), 326 deletions(-) diff --git a/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp b/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp index 2e05f01119a0..bdfa6602915b 100644 --- a/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp +++ b/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp @@ -17,6 +17,7 @@ #include "mlir/Target/LLVMIR/Import.h" #include "mlir/Tools/mlir-translate/Translation.h" #include "llvm/IR/Module.h" +#include "llvm/IR/Verifier.h" #include "llvm/IRReader/IRReader.h" #include "llvm/Support/SourceMgr.h" @@ -40,6 +41,8 @@ void registerFromLLVMIRTranslation() { emitError(UnknownLoc::get(context)) << errStream.str(); return {}; } + if (llvm::verifyModule(*llvmModule, &llvm::errs())) + return nullptr; return translateLLVMIRToModule(std::move(llvmModule), context); }, [](DialectRegistry ®istry) { diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp index 88f0688180a3..ed9722bd8d9e 100644 --- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp @@ -107,30 +107,25 @@ struct AttributeSetter { static LogicalResult setProfilingAttr(OpBuilder &builder, llvm::MDNode *node, Operation *op, LLVM::ModuleImport &moduleImport) { - // Return success for empty metadata nodes since there is nothing to import. + // Return failure for empty metadata nodes since there is nothing to import. if (!node->getNumOperands()) - return op->emitWarning() << "expected non-empty profiling metadata node"; + return failure(); auto *name = dyn_cast(node->getOperand(0)); if (!name) - return op->emitWarning() - << "expected profiling metadata node to have a string identifier"; + return failure(); // Handle function entry count metadata. if (name->getString().equals("function_entry_count")) { - auto emitNodeWarning = [&]() { - return op->emitWarning() - << "expected function_entry_count to hold a single i64 value"; - }; // TODO support function entry count metadata with GUID fields. if (node->getNumOperands() != 2) - return emitNodeWarning(); + return failure(); llvm::ConstantInt *entryCount = llvm::mdconst::dyn_extract(node->getOperand(1)); if (!entryCount) - return emitNodeWarning(); + return failure(); if (auto funcOp = dyn_cast(op)) { funcOp.setFunctionEntryCount(entryCount->getZExtValue()); return success(); @@ -140,8 +135,7 @@ static LogicalResult setProfilingAttr(OpBuilder &builder, llvm::MDNode *node, } if (!name->getString().equals("branch_weights")) - return op->emitWarning() - << "unknown profiling metadata node " << name->getString(); + return failure(); // Handle branch weights metadata. SmallVector branchWeights; @@ -150,7 +144,7 @@ static LogicalResult setProfilingAttr(OpBuilder &builder, llvm::MDNode *node, llvm::ConstantInt *branchWeight = llvm::mdconst::dyn_extract(node->getOperand(i)); if (!branchWeight) - return op->emitWarning() << "expected branch weights to be integers"; + return failure(); branchWeights.push_back(branchWeight->getZExtValue()); } diff --git a/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp b/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp index 2986b64538c2..c086f4c318e7 100644 --- a/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp +++ b/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp @@ -463,8 +463,7 @@ LogicalResult LoopAnnotationImporter::translateAccessGroup( for (const llvm::MDOperand &operand : node->operands()) { auto *childNode = dyn_cast(operand); if (!childNode) - return emitWarning(loc) - << "expected access group operands to be metadata nodes"; + return failure(); accessGroups.push_back(cast(operand.get())); } diff --git a/mlir/test/Target/LLVMIR/Import/basic.ll b/mlir/test/Target/LLVMIR/Import/basic.ll index 3b9b1c2909c1..1ce3dedcc03a 100644 --- a/mlir/test/Target/LLVMIR/Import/basic.ll +++ b/mlir/test/Target/LLVMIR/Import/basic.ll @@ -69,11 +69,9 @@ define i32 @invokeLandingpad() personality ptr @__gxx_personality_v0 { invoke void @foo(ptr %1) to label %4 unwind label %2 ; CHECK: ^bb1: - ; CHECK: %{{[0-9]+}} = llvm.landingpad (catch %{{[0-9]+}} : !llvm.ptr) (catch %[[a1]] : !llvm.ptr) (filter %{{[0-9]+}} : !llvm.array<1 x i8>) : !llvm.struct<(ptr, i32)> + ; CHECK: %{{[0-9]+}} = llvm.landingpad (catch %{{[0-9]+}} : !llvm.ptr) (catch %[[a1]] : !llvm.ptr) (filter %{{[0-9]+}} : !llvm.array<1 x i1>) : !llvm.struct<(ptr, i32)> %3 = landingpad { ptr, i32 } catch ptr @_ZTIi catch ptr @_ZTIii - ; FIXME: Change filter to a constant array once they are handled. - ; Currently, even though it parses this, LLVM module is broken - filter [1 x i8] [i8 1] + filter [1 x i1] [i1 1] resume { ptr, i32 } %3 ; CHECK: ^bb2: diff --git a/mlir/test/Target/LLVMIR/Import/function-attributes.ll b/mlir/test/Target/LLVMIR/Import/function-attributes.ll index 1d21b708bd30..e70dc7622c76 100644 --- a/mlir/test/Target/LLVMIR/Import/function-attributes.ll +++ b/mlir/test/Target/LLVMIR/Import/function-attributes.ll @@ -29,8 +29,6 @@ attributes #0 = { readnone } ; CHECK-LABEL: @func_arg_attrs ; CHECK-SAME: !llvm.ptr {llvm.byval = i64} ; CHECK-SAME: !llvm.ptr {llvm.byref = i64} -; CHECK-SAME: !llvm.ptr {llvm.sret = i64} -; CHECK-SAME: !llvm.ptr {llvm.inalloca = i64} ; CHECK-SAME: !llvm.ptr {llvm.noalias} ; CHECK-SAME: !llvm.ptr {llvm.readonly} ; CHECK-SAME: !llvm.ptr {llvm.nest} @@ -50,8 +48,6 @@ attributes #0 = { readnone } define ptr @func_arg_attrs( ptr byval(i64) %arg0, ptr byref(i64) %arg1, - ptr sret(i64) %arg2, - ptr inalloca(i64) %arg3, ptr noalias %arg4, ptr readonly %arg5, ptr nest %arg6, @@ -71,6 +67,18 @@ define ptr @func_arg_attrs( ret ptr %arg17 } +; CHECK-LABEL: @sret +; CHECK-SAME: !llvm.ptr {llvm.sret = i64} +define void @sret(ptr sret(i64) %arg0) { + ret void +} + +; CHECK-LABEL: @inalloca +; CHECK-SAME: !llvm.ptr {llvm.inalloca = i64} +define void @inalloca(ptr inalloca(i64) %arg0) { + ret void +} + ; CHECK-LABEL: @allocator ; CHECK-SAME: i64 {llvm.allocalign} ; CHECK-SAME: ptr {llvm.allocptr} diff --git a/mlir/test/Target/LLVMIR/Import/import-failure.ll b/mlir/test/Target/LLVMIR/Import/import-failure.ll index 8454b511f2e6..5ef6e991d3c2 100644 --- a/mlir/test/Target/LLVMIR/Import/import-failure.ll +++ b/mlir/test/Target/LLVMIR/Import/import-failure.ll @@ -26,6 +26,7 @@ define i32 @unhandled_value(i32 %arg1) { ; CHECK: import-failure.ll ; CHECK-SAME: error: unhandled instruction: ret ptr blockaddress(@unhandled_constant, %bb1) define ptr @unhandled_constant() { + br label %bb1 bb1: ret ptr blockaddress(@unhandled_constant, %bb1) } @@ -39,6 +40,7 @@ bb1: @private = private global ptr blockaddress(@unhandled_global, %bb1) define void @unhandled_global() { + br label %bb1 bb1: ret void } @@ -48,37 +50,15 @@ bb1: declare void @llvm.gcroot(ptr %arg1, ptr %arg2) ; CHECK: import-failure.ll -; CHECK-SAME: error: unhandled intrinsic: call void @llvm.gcroot(ptr %arg1, ptr %arg2) -define void @unhandled_intrinsic(ptr %arg1, ptr %arg2) { - call void @llvm.gcroot(ptr %arg1, ptr %arg2) +; CHECK-SAME: error: unhandled intrinsic: call void @llvm.gcroot(ptr %arg1, ptr null) +define void @unhandled_intrinsic() gc "example" { + %arg1 = alloca ptr + call void @llvm.gcroot(ptr %arg1, ptr null) ret void } ; // ----- -; CHECK: warning: unhandled metadata: !0 = !{!"unknown metadata"} on br i1 %arg1, label %bb1, label %bb2, !prof !0 -define i64 @unhandled_metadata(i1 %arg1, i64 %arg2) { -entry: - br i1 %arg1, label %bb1, label %bb2, !prof !0 -bb1: - ret i64 %arg2 -bb2: - ret i64 %arg2 -} - -!0 = !{!"unknown metadata"} - -; // ----- - -; CHECK: warning: unhandled function metadata: !0 = !{!"unknown metadata"} on define void @unhandled_func_metadata(i1 %arg1, i64 %arg2) !prof !0 -define void @unhandled_func_metadata(i1 %arg1, i64 %arg2) !prof !0 { - ret void -} - -!0 = !{!"unknown metadata"} - -; // ----- - declare void @llvm.dbg.value(metadata, metadata, metadata) ; CHECK: import-failure.ll @@ -102,17 +82,6 @@ define void @dropped_instruction(i64 %arg1) { ; // ----- -; global_ctors requires the appending linkage type. -; CHECK: import-failure.ll -; CHECK-SAME: error: unhandled global variable: @llvm.global_ctors -@llvm.global_ctors = global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 0, ptr @foo, ptr null }] - -define void @foo() { - ret void -} - -; // ----- - ; global_dtors with non-null data fields cannot be represented in MLIR. ; CHECK: import-failure.ll ; CHECK-SAME: error: unhandled global variable: @llvm.global_dtors @@ -124,28 +93,6 @@ define void @foo() { ; // ----- -; global_ctors without a data field should not be imported. -; CHECK: import-failure.ll -; CHECK-SAME: error: unhandled global variable: @llvm.global_ctors -@llvm.global_ctors = appending global [1 x { i32, ptr }] [{ i32, ptr } { i32 0, ptr @foo }] - -define void @foo() { - ret void -} - -; // ----- - -; global_dtors with a wrong argument order should not be imported. -; CHECK: import-failure.ll -; CHECK-SAME: error: unhandled global variable: @llvm.global_dtors -@llvm.global_dtors = appending global [1 x { ptr, i32, ptr }] [{ ptr, i32, ptr } { ptr @foo, i32 0, ptr null }] - -define void @foo() { - ret void -} - -; // ----- - ; CHECK: import-failure.ll ; CHECK-SAME: error: TBAA root node must have non-empty identity: !2 = !{!""} define dso_local void @tbaa(ptr %0) { @@ -159,73 +106,6 @@ define dso_local void @tbaa(ptr %0) { ; // ----- -; CHECK: import-failure.ll -; CHECK-SAME: error: unsupported TBAA node format: !0 = !{!1, i64 0, i64 0} -define dso_local void @tbaa(ptr %0) { - store i8 1, ptr %0, align 4, !tbaa !2 - ret void -} - -!0 = !{!"Simple C/C++ TBAA"} -!1 = !{!"omnipotent char", !0, i64 0} -!2 = !{!1, i64 0, i64 0} - -; // ----- - -; CHECK: import-failure.ll -; CHECK-SAME: error: operand '1' must be MDNode: !1 = !{!"omnipotent char", i64 0, i64 0} -define dso_local void @tbaa(ptr %0) { - store i8 1, ptr %0, align 4, !tbaa !2 - ret void -} - -!0 = !{!"Simple C/C++ TBAA"} -!1 = !{!"omnipotent char", i64 0, i64 0} -!2 = !{!1, !1, i64 0} - -; // ----- - -; CHECK: import-failure.ll -; CHECK-SAME: error: missing member offset: !1 = !{!"agg_t", !2, i64 0, !2} -define dso_local void @tbaa(ptr %0) { - store i8 1, ptr %0, align 4, !tbaa !3 - ret void -} - -!0 = !{!"Simple C/C++ TBAA"} -!1 = !{!"omnipotent char", !0, i64 0} -!2 = !{!"agg_t", !1, i64 0, !1} -!3 = !{!2, !1, i64 0} - -; // ----- - -; CHECK: import-failure.ll -; CHECK-SAME: error: operand '4' must be ConstantInt: !1 = !{!"agg_t", !2, i64 0, !2, double 1.000000e+00} -define dso_local void @tbaa(ptr %0) { - store i8 1, ptr %0, align 4, !tbaa !3 - ret void -} - -!0 = !{!"Simple C/C++ TBAA"} -!1 = !{!"omnipotent char", !0, i64 0} -!2 = !{!"agg_t", !1, i64 0, !1, double 1.0} -!3 = !{!2, !1, i64 0} - -; // ----- - -; CHECK: import-failure.ll -; CHECK-SAME: error: operand '3' must be ConstantInt: !0 = !{!1, !1, i64 0, double 1.000000e+00} -define dso_local void @tbaa(ptr %0) { - store i8 1, ptr %0, align 4, !tbaa !2 - ret void -} - -!0 = !{!"Simple C/C++ TBAA"} -!1 = !{!"omnipotent char", !0, i64 0} -!2 = !{!1, !1, i64 0, double 1.0} - -; // ----- - ; CHECK: import-failure.ll ; CHECK-SAME: error: unsupported TBAA node format: !0 = !{!1, !1, i64 0, i64 4} define dso_local void @tbaa(ptr %0) { @@ -252,31 +132,6 @@ define void @access_group(ptr %arg1) { ; // ----- -; CHECK: import-failure.ll -; CHECK-SAME: warning: expected an access group node to be empty and distinct -; CHECK: error: unsupported access group node: !0 = !{!1} -define void @access_group(ptr %arg1) { - %1 = load i32, ptr %arg1, !llvm.access.group !0 - ret void -} - -!0 = !{!1} -!1 = distinct !{!"unsupported access group"} - -; // ----- - -; CHECK: import-failure.ll -; CHECK-SAME: warning: expected access group operands to be metadata nodes -; CHECK: error: unsupported access group node: !0 = !{i1 false} -define void @access_group(ptr %arg1) { - %1 = load i32, ptr %arg1, !llvm.access.group !0 - ret void -} - -!0 = !{i1 false} - -; // ----- - ; CHECK: import-failure.ll ; CHECK-SAME: warning: expected all loop properties to be either debug locations or metadata nodes ; CHECK: import-failure.ll @@ -454,70 +309,6 @@ end: ; // ----- -; CHECK: import-failure.ll -; CHECK-SAME: warning: expected non-empty profiling metadata node -; CHECK: warning: unhandled metadata: !0 = !{} -define void @cond_br(i1 %arg) { -entry: - br i1 %arg, label %bb1, label %bb2, !prof !0 -bb1: - ret void -bb2: - ret void -} - -!0 = !{} - -; // ----- - -; CHECK: import-failure.ll -; CHECK-SAME: warning: expected profiling metadata node to have a string identifier -; CHECK: import-failure.ll:{{.*}} warning: unhandled metadata: !0 = !{i32 64} -define void @cond_br(i1 %arg) { -entry: - br i1 %arg, label %bb1, label %bb2, !prof !0 -bb1: - ret void -bb2: - ret void -} - -!0 = !{i32 64} - -; // ----- - -; CHECK: import-failure.ll -; CHECK-SAME: warning: expected function_entry_count to hold a single i64 value -; CHECK: warning: unhandled function metadata: !0 = !{!"function_entry_count"} -define void @cond_br(i1 %arg) !prof !0 { -entry: - br i1 %arg, label %bb1, label %bb2 -bb1: - ret void -bb2: - ret void -} - -!0 = !{!"function_entry_count"} - -; // ----- - -; CHECK: import-failure.ll -; CHECK-SAME: warning: expected function_entry_count to hold a single i64 value -; CHECK: warning: unhandled function metadata: !0 = !{!"function_entry_count", !"string"} -define void @cond_br(i1 %arg) !prof !0 { -entry: - br i1 %arg, label %bb1, label %bb2 -bb1: - ret void -bb2: - ret void -} - -!0 = !{!"function_entry_count", !"string"} - -; // ----- - ; CHECK: import-failure.ll ; CHECK-SAME: warning: expected function_entry_count to be attached to a function ; CHECK: warning: unhandled metadata: !0 = !{!"function_entry_count", i64 42} @@ -531,80 +322,3 @@ bb2: } !0 = !{!"function_entry_count", i64 42} - -; // ----- - -; CHECK: import-failure.ll -; CHECK-SAME: warning: unknown profiling metadata node unknown_prof_type -; CHECK: warning: unhandled metadata: !0 = !{!"unknown_prof_type"} -define void @cond_br(i1 %arg) { -entry: - br i1 %arg, label %bb1, label %bb2, !prof !0 -bb1: - ret void -bb2: - ret void -} - -!0 = !{!"unknown_prof_type"} - -; // ----- - -; CHECK: import-failure.ll -; CHECK-SAME: warning: expected branch weights to be integers -; CHECK: warning: unhandled metadata: !0 = !{!"branch_weights", !"foo"} -define void @cond_br(i1 %arg) { -entry: - br i1 %arg, label %bb1, label %bb2, !prof !0 -bb1: - ret void -bb2: - ret void -} - -!0 = !{!"branch_weights", !"foo"} - -; // ----- - -; CHECK: import-failure.ll:{{.*}} warning: unhandled function metadata: !0 = !{!"branch_weights", i32 64} -define void @cond_br(i1 %arg) !prof !0 { - ret void -} - -!0 = !{!"branch_weights", i32 64} - -; // ----- - -; CHECK: import-failure.ll -; CHECK-SAME: error: unsupported alias scope node: ![[NODE:[0-9]+]] = distinct !{![[NODE]]} -define void @alias_scope(ptr %arg1) { - %1 = load i32, ptr %arg1, !alias.scope !0 - ret void -} - -!0 = !{!0} - -; // ----- - -; CHECK: import-failure.ll -; CHECK-SAME: error: unsupported alias scope node: ![[NODE:[0-9]+]] = distinct !{![[NODE]], !"The domain"} -define void @alias_scope(ptr %arg1) { - %1 = load i32, ptr %arg1, !alias.scope !1 - ret void -} - -!0 = distinct !{!0, !"The domain"} -!1 = !{!1, !0} - -; // ----- - -; CHECK: import-failure.ll -; CHECK-SAME: error: unsupported alias domain node: ![[NODE:[0-9]+]] = distinct !{![[NODE]], ![[NODE]]} -define void @alias_scope_domain(ptr %arg1) { - %1 = load i32, ptr %arg1, !alias.scope !2 - ret void -} - -!0 = distinct !{!0, !0} -!1 = !{!1, !0} -!2 = !{!1} diff --git a/mlir/test/Target/LLVMIR/Import/metadata-tbaa.ll b/mlir/test/Target/LLVMIR/Import/metadata-tbaa.ll index c0cbe7b9bdd0..9e3b7c2ca826 100644 --- a/mlir/test/Target/LLVMIR/Import/metadata-tbaa.ll +++ b/mlir/test/Target/LLVMIR/Import/metadata-tbaa.ll @@ -1,12 +1,12 @@ ; RUN: mlir-translate -import-llvm -split-input-file %s | FileCheck %s -// ----- - ; CHECK-LABEL: llvm.metadata @__llvm_global_metadata { -; CHECK-NEXT: llvm.tbaa_root @[[R0:tbaa_root_[0-9]+]] {id = "Simple C/C++ TBAA"} -; CHECK-NEXT: llvm.tbaa_tag @[[T0:tbaa_tag_[0-9]+]] {access_type = @[[R0]], base_type = @[[R0]], offset = 0 : i64} -; CHECK-NEXT: llvm.tbaa_root @[[R1:tbaa_root_[0-9]+]] {id = "Other language TBAA"} -; CHECK-NEXT: llvm.tbaa_tag @[[T1:tbaa_tag_[0-9]+]] {access_type = @[[R1]], base_type = @[[R1]], offset = 0 : i64} +; CHECK-DAG: llvm.tbaa_root @[[R0:tbaa_root_[0-9]+]] {id = "Simple C/C++ TBAA"} +; CHECK-DAG: llvm.tbaa_type_desc @[[D0:tbaa_type_desc_[0-9]+]] {id = "scalar type", members = {<@[[R0]], 0>}} +; CHECK-DAG: llvm.tbaa_tag @[[T0:tbaa_tag_[0-9]+]] {access_type = @[[D0]], base_type = @[[D0]], offset = 0 : i64} +; CHECK-DAG: llvm.tbaa_root @[[R1:tbaa_root_[0-9]+]] {id = "Other language TBAA"} +; CHECK-DAG: llvm.tbaa_type_desc @[[D1:tbaa_type_desc_[0-9]+]] {id = "other scalar type", members = {<@[[R1]], 0>}} +; CHECK-DAG: llvm.tbaa_tag @[[T1:tbaa_tag_[0-9]+]] {access_type = @[[D1]], base_type = @[[D1]], offset = 0 : i64} ; CHECK-NEXT: } ; CHECK: llvm.func @tbaa1 ; CHECK: llvm.store %{{.*}}, %{{.*}} { @@ -16,15 +16,18 @@ ; CHECK-SAME: tbaa = [@__llvm_global_metadata::@[[T1]]] ; CHECK-SAME: } : i8, !llvm.ptr define dso_local void @tbaa1(ptr %0, ptr %1) { - store i8 1, ptr %0, align 4, !tbaa !1 + store i8 1, ptr %0, align 4, !tbaa !0 store i8 1, ptr %1, align 4, !tbaa !3 ret void } -!0 = !{!"Simple C/C++ TBAA"} -!1 = !{!0, !0, i64 0} -!2 = !{!"Other language TBAA"} -!3 = !{!2, !2, i64 0} +!0 = !{!1, !1, i64 0} +!1 = !{!"scalar type", !2, i64 0} +!2 = !{!"Simple C/C++ TBAA"} + +!3 = !{!4, !4, i64 0} +!4 = !{!"other scalar type", !5, i64 0} +!5 = !{!"Other language TBAA"} // ----- From 6186345024af7f6b7f91fbe838dd57bdc033407a Mon Sep 17 00:00:00 2001 From: Alex Zinenko Date: Mon, 6 Mar 2023 11:15:02 +0100 Subject: [PATCH 090/147] [mlir] fix LLVM IR translation of vector<... x index> When the translation was written, `vector<... x index>` was not allowed at all. After it was added later, the translation was never adapted. It kept working in the most common case of index-typed attributes using 64-bit storage and being converted to 64-bit integers in LLVM IR, but not in the other cases that require truncation or extension, producing wrong results when using the raw data storage of the dense attrbute to construct the LLVM IR constant. When the storage size doesn't match, fall back to the per-element constant construction, which is slower but handles bitwidth differences correctly. Fixes #60614. Reviewed By: gysit Differential Revision: https://reviews.llvm.org/D143993 (cherry picked from commit 8c7cfa357280dd93d33b10bbba0fe33797e27d63) --- mlir/lib/Target/LLVMIR/ModuleTranslation.cpp | 10 ++++++++++ mlir/test/Target/LLVMIR/llvmir.mlir | 14 ++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp index b49fc383d73a..232b3d8b2216 100644 --- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp @@ -216,6 +216,16 @@ convertDenseElementsAttr(Location loc, DenseElementsAttr denseElementsAttr, if (type.getNumElements() == 0) return nullptr; + // Check that the raw data size matches what is expected for the scalar size. + // TODO: in theory, we could repack the data here to keep constructing from + // raw data. + // TODO: we may also need to consider endianness when cross-compiling to an + // architecture where it is different. + unsigned elementByteSize = denseElementsAttr.getRawData().size() / + denseElementsAttr.getNumElements(); + if (8 * elementByteSize != innermostLLVMType->getScalarSizeInBits()) + return nullptr; + // Compute the shape of all dimensions but the innermost. Note that the // innermost dimension may be that of the vector element type. bool hasVectorElementType = type.getElementType().isa(); diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir index 840b1f2caf04..9e208570776f 100644 --- a/mlir/test/Target/LLVMIR/llvmir.mlir +++ b/mlir/test/Target/LLVMIR/llvmir.mlir @@ -1251,6 +1251,20 @@ llvm.func @complexintconstantarray() -> !llvm.array<2 x !llvm.array<2 x !llvm.st llvm.return %1 : !llvm.array<2 x !llvm.array<2 x !llvm.struct<(i32, i32)>>> } +// CHECK-LABEL: @indexconstantsplat +llvm.func @indexconstantsplat() -> vector<3xi32> { + %1 = llvm.mlir.constant(dense<42> : vector<3xindex>) : vector<3xi32> + // CHECK: ret <3 x i32> + llvm.return %1 : vector<3xi32> +} + +// CHECK-LABEL: @indexconstantarray +llvm.func @indexconstantarray() -> vector<3xi32> { + %1 = llvm.mlir.constant(dense<[0, 1, 2]> : vector<3xindex>) : vector<3xi32> + // CHECK: ret <3 x i32> + llvm.return %1 : vector<3xi32> +} + llvm.func @noreach() { // CHECK: unreachable llvm.unreachable From 4adb556959398cd53d880b87baead7225cbfdd91 Mon Sep 17 00:00:00 2001 From: Tobias Gysi Date: Mon, 6 Mar 2023 11:15:46 +0100 Subject: [PATCH 091/147] [mlir][llvm] Add atomic support to the LoadOp. This revision adds atomic support to the LoadOp. It chooses to print the atomic keywords together with the syncscope and ordering arguments, which simplifies parsing and printing compared to the LLVM IR printer that puts the atomic keyword at the beginning. It uses the ordering attribute to check if the load is atomic. The revision also implements verifiers to ensure the constraints that apply to atomic load operations are checked. Reviewed By: Dinistro Differential Revision: https://reviews.llvm.org/D144112 (cherry picked from commit 6f4af64b7422942959ab128e89a24dffc83bc054) --- mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td | 61 +++++++++++---- mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 75 ++++++++++++++----- mlir/test/Dialect/LLVMIR/invalid.mlir | 35 +++++++++ mlir/test/Dialect/LLVMIR/roundtrip.mlir | 9 +++ .../test/Target/LLVMIR/Import/instructions.ll | 12 +++ mlir/test/Target/LLVMIR/llvmir.mlir | 14 +++- 6 files changed, 173 insertions(+), 33 deletions(-) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td index db104e7dc790..5fdc2cc59d49 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td @@ -187,6 +187,9 @@ class MemoryOpBase { inst->setSyncScopeID(llvmContext.getOrInsertSyncScopeID(*$syncscope)); } }]; + code setOrderingCode = [{ + inst->setAtomic(convertAtomicOrderingToLLVM($ordering)); + }]; code setNonTemporalMetadataCode = [{ if ($nontemporal) { llvm::MDNode *metadata = llvm::MDNode::get( @@ -350,17 +353,48 @@ def LLVM_LoadOp : LLVM_Op<"load">, MemoryOpBase { OptionalAttr:$alias_scopes, OptionalAttr:$noalias_scopes, OptionalAttr:$tbaa, - OptionalAttr:$alignment, UnitAttr:$volatile_, - UnitAttr:$nontemporal); + OptionalAttr:$alignment, + UnitAttr:$volatile_, + UnitAttr:$nontemporal, + DefaultValuedAttr:$ordering, + OptionalAttr:$syncscope); let results = (outs LLVM_LoadableType:$res); string llvmInstName = "Load"; + let description = [{ + The `load` operation is used to read from memory. A load may be marked as + atomic, volatile, and/or nontemporal, and takes a number of optional + attributes that specify aliasing information. + + An atomic load only supports a limited set of pointer, integer, and + floating point types, and requires an explicit alignment. + + Examples: + ```mlir + // A volatile load of a float variable. + %0 = llvm.load volatile %ptr : !llvm.ptr -> f32 + + // A nontemporal load of a float variable. + %0 = llvm.load %ptr {nontemporal} : !llvm.ptr -> f32 + + // An atomic load of an integer variable. + %0 = llvm.load %ptr atomic monotonic {alignment = 8 : i64} + : !llvm.ptr -> i64 + ``` + + See the following link for more details: + https://llvm.org/docs/LangRef.html#load-instruction + }]; let assemblyFormat = [{ - (`volatile` $volatile_^)? $addr attr-dict `:` - custom(type($addr), type($res)) + (`volatile` $volatile_^)? $addr + (`atomic` (`syncscope` `(` $syncscope^ `)`)? $ordering^)? + attr-dict `:` custom(type($addr), type($res)) }]; string llvmBuilder = [{ auto *inst = builder.CreateLoad($_resultType, $addr, $volatile_); - }] # setAlignmentCode + }] # setOrderingCode + # setSyncScopeCode + # setAlignmentCode # setNonTemporalMetadataCode # setAccessGroupsMetadataCode # setAliasScopeMetadataCode @@ -373,22 +407,19 @@ def LLVM_LoadOp : LLVM_Op<"load">, MemoryOpBase { unsigned alignment = loadInst->getAlign().value(); $res = $_builder.create($_location, $_resultType, $addr, alignment, loadInst->isVolatile(), - loadInst->hasMetadata(llvm::LLVMContext::MD_nontemporal)); + loadInst->hasMetadata(llvm::LLVMContext::MD_nontemporal), + convertAtomicOrderingFromLLVM(loadInst->getOrdering()), + getLLVMSyncScope(loadInst)); }]; let builders = [ OpBuilder<(ins "Value":$addr, CArg<"unsigned", "0">:$alignment, - CArg<"bool", "false">:$isVolatile, CArg<"bool", "false">:$isNonTemporal), - [{ - auto type = addr.getType().cast().getElementType(); - assert(type && "must provide explicit element type to the constructor " - "when the pointer type is opaque"); - build($_builder, $_state, type, addr, alignment, isVolatile, isNonTemporal); - }]>, + CArg<"bool", "false">:$isVolatile, CArg<"bool", "false">:$isNonTemporal)>, OpBuilder<(ins "Type":$type, "Value":$addr, CArg<"unsigned", "0">:$alignment, CArg<"bool", "false">:$isVolatile, - CArg<"bool", "false">:$isNonTemporal)> + CArg<"bool", "false">:$isNonTemporal, + CArg<"AtomicOrdering", "AtomicOrdering::not_atomic">:$ordering, + CArg<"StringRef", "StringRef()">:$syncscope)> ]; - let hasCustomAssemblyFormat = 1; let hasVerifier = 1; } diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp index 7fa1dc872f6a..0e31c18f41a8 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -804,15 +804,65 @@ LogicalResult verifyMemOpMetadata(OpTy memOp) { return success(); } -LogicalResult LoadOp::verify() { return verifyMemOpMetadata(*this); } +/// Returns true if the given type is supported by atomic operations. All +/// integer and float types with limited bit width are supported. Additionally, +/// depending on the operation pointers may be supported as well. +static bool isTypeCompatibleWithAtomicOp(Type type, bool isPointerTypeAllowed) { + if (type.isa()) + return isPointerTypeAllowed; + + std::optional bitWidth = std::nullopt; + if (auto floatType = type.dyn_cast()) { + if (!isCompatibleFloatingPointType(type)) + return false; + bitWidth = floatType.getWidth(); + } + if (auto integerType = type.dyn_cast()) + bitWidth = integerType.getWidth(); + // The type is neither an integer, float, or pointer type. + if (!bitWidth) + return false; + return *bitWidth == 8 || *bitWidth == 16 || *bitWidth == 32 || + *bitWidth == 64; +} + +LogicalResult LoadOp::verify() { + if (getOrdering() != AtomicOrdering::not_atomic) { + if (!isTypeCompatibleWithAtomicOp(getResult().getType(), + /*isPointerTypeAllowed=*/true)) + return emitOpError("unsupported type ") + << getResult().getType() << " for atomic access"; + if (getOrdering() == AtomicOrdering::release || + getOrdering() == AtomicOrdering::acq_rel) + return emitOpError("unsupported ordering '") + << stringifyAtomicOrdering(getOrdering()) << "'"; + if (!getAlignment()) + return emitOpError("expected alignment for atomic access"); + } else if (getSyncscope()) { + return emitOpError("expected syncscope to be null for non-atomic access"); + } + return verifyMemOpMetadata(*this); +} + +void LoadOp::build(OpBuilder &builder, OperationState &state, Value addr, + unsigned alignment, bool isVolatile, bool isNonTemporal) { + auto type = addr.getType().cast().getElementType(); + assert(type && "must provide explicit element type to the constructor " + "when the pointer type is opaque"); + build(builder, state, type, addr, alignment, isVolatile, isNonTemporal); +} void LoadOp::build(OpBuilder &builder, OperationState &state, Type type, Value addr, unsigned alignment, bool isVolatile, - bool isNonTemporal) { - build(builder, state, type, addr, /*access_groups=*/nullptr, - /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr, + bool isNonTemporal, AtomicOrdering ordering, + StringRef syncscope) { + build(builder, state, type, addr, + /*access_groups=*/nullptr, + /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, + /*tbaa=*/nullptr, alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile, - isNonTemporal); + isNonTemporal, ordering, + syncscope.empty() ? nullptr : builder.getStringAttr(syncscope)); } // Extract the pointee type from the LLVM pointer type wrapped in MLIR. Return @@ -2266,12 +2316,7 @@ LogicalResult AtomicRMWOp::verify() { if (!mlir::LLVM::isCompatibleFloatingPointType(valType)) return emitOpError("expected LLVM IR floating point type"); } else if (getBinOp() == AtomicBinOp::xchg) { - auto intType = valType.dyn_cast(); - unsigned intBitWidth = intType ? intType.getWidth() : 0; - if (intBitWidth != 8 && intBitWidth != 16 && intBitWidth != 32 && - intBitWidth != 64 && !valType.isa() && - !valType.isa() && !valType.isa() && - !valType.isa()) + if (!isTypeCompatibleWithAtomicOp(valType, /*isPointerTypeAllowed=*/false)) return emitOpError("unexpected LLVM IR type for 'xchg' bin_op"); } else { auto intType = valType.dyn_cast(); @@ -2320,12 +2365,8 @@ LogicalResult AtomicCmpXchgOp::verify() { if (!ptrType.isOpaque() && valType != ptrType.getElementType()) return emitOpError("expected LLVM IR element type for operand #0 to " "match type for all other operands"); - auto intType = valType.dyn_cast(); - unsigned intBitWidth = intType ? intType.getWidth() : 0; - if (!valType.isa() && intBitWidth != 8 && - intBitWidth != 16 && intBitWidth != 32 && intBitWidth != 64 && - !valType.isa() && !valType.isa() && - !valType.isa() && !valType.isa()) + if (!isTypeCompatibleWithAtomicOp(valType, + /*isPointerTypeAllowed=*/true)) return emitOpError("unexpected LLVM IR type"); if (getSuccessOrdering() < AtomicOrdering::monotonic || getFailureOrdering() < AtomicOrdering::monotonic) diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir index 664a9136f69a..cbcc633ae11b 100644 --- a/mlir/test/Dialect/LLVMIR/invalid.mlir +++ b/mlir/test/Dialect/LLVMIR/invalid.mlir @@ -153,6 +153,41 @@ func.func @load_non_ptr_type(%foo : f32) { // ----- +func.func @load_syncscope(%ptr : !llvm.ptr) { + // expected-error@below {{expected syncscope to be null for non-atomic access}} + %1 = "llvm.load"(%ptr) {syncscope = "singlethread"} : (!llvm.ptr) -> (f32) +} + +// ----- + +func.func @load_unsupported_ordering(%ptr : !llvm.ptr) { + // expected-error@below {{unsupported ordering 'release'}} + %1 = llvm.load %ptr atomic release {alignment = 4 : i64} : !llvm.ptr -> f32 +} + +// ----- + +func.func @load_unsupported_type(%ptr : !llvm.ptr) { + // expected-error@below {{unsupported type 'f80' for atomic access}} + %1 = llvm.load %ptr atomic monotonic {alignment = 16 : i64} : !llvm.ptr -> f80 +} + +// ----- + +func.func @load_unsupported_type(%ptr : !llvm.ptr) { + // expected-error@below {{unsupported type 'i1' for atomic access}} + %1 = llvm.load %ptr atomic monotonic {alignment = 16 : i64} : !llvm.ptr -> i1 +} + +// ----- + +func.func @load_unaligned_atomic(%ptr : !llvm.ptr) { + // expected-error@below {{expected alignment for atomic access}} + %1 = llvm.load %ptr atomic monotonic : !llvm.ptr -> f32 +} + +// ----- + func.func @store_non_llvm_type(%foo : memref, %bar : f32) { // expected-error@+1 {{expected LLVM pointer type}} llvm.store %bar, %foo : memref diff --git a/mlir/test/Dialect/LLVMIR/roundtrip.mlir b/mlir/test/Dialect/LLVMIR/roundtrip.mlir index e789978d713b..ebab8c35b90a 100644 --- a/mlir/test/Dialect/LLVMIR/roundtrip.mlir +++ b/mlir/test/Dialect/LLVMIR/roundtrip.mlir @@ -339,6 +339,15 @@ func.func @null() { llvm.return } +// CHECK-LABEL: @atomic_load +func.func @atomic_load(%ptr : !llvm.ptr) { + // CHECK: llvm.load %{{.*}} atomic monotonic {alignment = 4 : i64} : !llvm.ptr -> f32 + %0 = llvm.load %ptr atomic monotonic {alignment = 4 : i64} : !llvm.ptr -> f32 + // CHECK: llvm.load volatile %{{.*}} atomic syncscope("singlethread") monotonic {alignment = 16 : i64} : !llvm.ptr -> f32 + %1 = llvm.load volatile %ptr atomic syncscope("singlethread") monotonic {alignment = 16 : i64} : !llvm.ptr -> f32 + llvm.return +} + // CHECK-LABEL: @atomicrmw func.func @atomicrmw(%ptr : !llvm.ptr, %val : f32) { // CHECK: llvm.atomicrmw fadd %{{.*}}, %{{.*}} monotonic : !llvm.ptr, f32 diff --git a/mlir/test/Target/LLVMIR/Import/instructions.ll b/mlir/test/Target/LLVMIR/Import/instructions.ll index cd54411de4ef..f66d7b9cdf69 100644 --- a/mlir/test/Target/LLVMIR/Import/instructions.ll +++ b/mlir/test/Target/LLVMIR/Import/instructions.ll @@ -368,6 +368,18 @@ define void @load_store(ptr %ptr) { ; // ----- +; CHECK-LABEL: @atomic_load +; CHECK-SAME: %[[PTR:[a-zA-Z0-9]+]] +define void @atomic_load(ptr %ptr) { + ; CHECK: %[[V1:[0-9]+]] = llvm.load %[[PTR]] atomic acquire {alignment = 8 : i64} : !llvm.ptr -> f64 + ; CHECK: %[[V2:[0-9]+]] = llvm.load volatile %[[PTR]] atomic syncscope("singlethreaded") acquire {alignment = 16 : i64} : !llvm.ptr -> f64 + %1 = load atomic double, ptr %ptr acquire, align 8 + %2 = load atomic volatile double, ptr %ptr syncscope("singlethreaded") acquire, align 16 + ret void +} + +; // ----- + ; CHECK-LABEL: @atomic_rmw ; CHECK-SAME: %[[PTR1:[a-zA-Z0-9]+]] ; CHECK-SAME: %[[VAL1:[a-zA-Z0-9]+]] diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir index 9e208570776f..da7dc7f78ba5 100644 --- a/mlir/test/Target/LLVMIR/llvmir.mlir +++ b/mlir/test/Target/LLVMIR/llvmir.mlir @@ -1261,7 +1261,7 @@ llvm.func @indexconstantsplat() -> vector<3xi32> { // CHECK-LABEL: @indexconstantarray llvm.func @indexconstantarray() -> vector<3xi32> { %1 = llvm.mlir.constant(dense<[0, 1, 2]> : vector<3xindex>) : vector<3xi32> - // CHECK: ret <3 x i32> + // CHECK: ret <3 x i32> llvm.return %1 : vector<3xi32> } @@ -1780,6 +1780,18 @@ llvm.func @nontemporal_store_and_load() { // ----- +llvm.func @atomic_load(%ptr : !llvm.ptr) { + // CHECK: load atomic + // CHECK-SAME: monotonic, align 4 + %1 = llvm.load %ptr atomic monotonic {alignment = 4 : i64} : !llvm.ptr -> f32 + // CHECK: load atomic + // CHECK-SAME: syncscope("singlethread") monotonic, align 4 + %2 = llvm.load %ptr atomic syncscope("singlethread") monotonic {alignment = 4 : i64} : !llvm.ptr -> f32 + llvm.return +} + +// ----- + // Check that the translation does not crash in absence of a data layout. module { // CHECK: declare void @module_default_layout From cb3ba88b3a2b4a58b0e810dc0edf327451f5bd70 Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Mon, 6 Mar 2023 11:16:23 +0100 Subject: [PATCH 092/147] [mlir][llvm] Make DI param optional to match LLVM This commit makes DIDerivedTypeAttr's baseType parameter optional to ensure imported IR doesn't break the printer. Reviewed By: gysit Differential Revision: https://reviews.llvm.org/D144247 (cherry picked from commit 276358fd3af5e01b4e41e442703e835d26d36deb) --- mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td | 2 +- mlir/test/Dialect/LLVMIR/debuginfo.mlir | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td index 2d4612947827..1fa17b12d695 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td @@ -344,7 +344,7 @@ def LLVM_DIDerivedTypeAttr : LLVM_Attr<"DIDerivedType", "di_derived_type", [ let parameters = (ins LLVM_DITagParameter:$tag, OptionalParameter<"StringAttr">:$name, - "DITypeAttr":$baseType, + OptionalParameter<"DITypeAttr">:$baseType, OptionalParameter<"uint64_t">:$sizeInBits, OptionalParameter<"uint32_t">:$alignInBits, OptionalParameter<"uint64_t">:$offsetInBits diff --git a/mlir/test/Dialect/LLVMIR/debuginfo.mlir b/mlir/test/Dialect/LLVMIR/debuginfo.mlir index d7123d971a68..7004d57762b9 100644 --- a/mlir/test/Dialect/LLVMIR/debuginfo.mlir +++ b/mlir/test/Dialect/LLVMIR/debuginfo.mlir @@ -30,10 +30,10 @@ sizeInBits = 64, alignInBits = 32, offsetInBits = 4 > -// CHECK-DAG: #[[PTR1:.*]] = #llvm.di_derived_type +// CHECK-DAG: #[[PTR1:.*]] = #llvm.di_derived_type #ptr1 = #llvm.di_derived_type< // Specify the name parameter. - tag = DW_TAG_pointer_type, name = "ptr1", baseType = #int0 + tag = DW_TAG_pointer_type, name = "ptr1" > // CHECK-DAG: #[[COMP0:.*]] = #llvm.di_composite_type From 4e6f95eaf44bee14a28df4b682087e377e79bcfb Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Mon, 6 Mar 2023 11:17:01 +0100 Subject: [PATCH 093/147] [mlir][llvm] Add DINamespace attribute This commit introduces the DINamespaceAttr to model LLVM's DINamespace metadata. Reviewed By: gysit Differential Revision: https://reviews.llvm.org/D144243 (cherry picked from commit e133cb9c6daf5f6c9fb2c1bdb138e0a02b00fadb) --- .../mlir/Dialect/LLVMIR/LLVMAttrDefs.td | 15 +++++++++++++ mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp | 10 ++++----- mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 22 +++++++++---------- mlir/lib/Target/LLVMIR/DebugImporter.cpp | 8 +++++++ mlir/lib/Target/LLVMIR/DebugImporter.h | 1 + mlir/lib/Target/LLVMIR/DebugTranslation.cpp | 14 ++++++++---- mlir/lib/Target/LLVMIR/DebugTranslation.h | 1 + mlir/test/Dialect/LLVMIR/debuginfo.mlir | 14 ++++++++++-- mlir/test/Target/LLVMIR/Import/debug-info.ll | 22 +++++++++++++++++++ mlir/test/Target/LLVMIR/llvmir-debug.mlir | 12 ++++++++-- 10 files changed, 95 insertions(+), 24 deletions(-) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td index 1fa17b12d695..d31524a29ac9 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td @@ -477,6 +477,21 @@ def LLVM_DISubprogramAttr : LLVM_Attr<"DISubprogram", "di_subprogram", [ let assemblyFormat = "`<` struct(params) `>`"; } +//===----------------------------------------------------------------------===// +// DINamespaceAttr +//===----------------------------------------------------------------------===// + +def LLVM_DINamespaceAttr : LLVM_Attr<"DINamespace", "di_namespace", + /*traits=*/[], "DIScopeAttr"> { + let parameters = (ins + "StringAttr":$name, + OptionalParameter<"DIScopeAttr">:$scope, + "bool":$exportSymbols + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + //===----------------------------------------------------------------------===// // DISubrangeAttr //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp index 1bb0f6a9438d..808428937988 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp @@ -42,10 +42,10 @@ void LLVMDialect::registerAttributes() { //===----------------------------------------------------------------------===// bool DINodeAttr::classof(Attribute attr) { - return llvm::isa(attr); } @@ -63,7 +63,7 @@ bool DIScopeAttr::classof(Attribute attr) { //===----------------------------------------------------------------------===// bool DILocalScopeAttr::classof(Attribute attr) { - return llvm::isa(attr); + return llvm::isa(attr); } //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp index 0e31c18f41a8..de6fa64769ab 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -2800,17 +2800,17 @@ struct LLVMOpAsmDialectInterface : public OpAsmDialectInterface { AliasResult getAlias(Attribute attr, raw_ostream &os) const override { return TypeSwitch(attr) - .Case( - [&](auto attr) { - os << decltype(attr)::getMnemonic(); - return AliasResult::OverridableAlias; - }) + .Case([&](auto attr) { + os << decltype(attr)::getMnemonic(); + return AliasResult::OverridableAlias; + }) .Default([](Attribute) { return AliasResult::NoAlias; }); } }; diff --git a/mlir/lib/Target/LLVMIR/DebugImporter.cpp b/mlir/lib/Target/LLVMIR/DebugImporter.cpp index d6ddd6efd149..20cd5da30ae5 100644 --- a/mlir/lib/Target/LLVMIR/DebugImporter.cpp +++ b/mlir/lib/Target/LLVMIR/DebugImporter.cpp @@ -113,6 +113,12 @@ DIScopeAttr DebugImporter::translateImpl(llvm::DIScope *node) { return cast(translate(static_cast(node))); } +DINamespaceAttr DebugImporter::translateImpl(llvm::DINamespace *node) { + return DINamespaceAttr::get( + context, StringAttr::get(context, node->getName()), + translate(node->getScope()), node->getExportSymbols()); +} + DISubprogramAttr DebugImporter::translateImpl(llvm::DISubprogram *node) { std::optional subprogramFlags = symbolizeDISubprogramFlags(node->getSubprogram()->getSPFlags()); @@ -205,6 +211,8 @@ DINodeAttr DebugImporter::translate(llvm::DINode *node) { return translateImpl(casted); if (auto *casted = dyn_cast(node)) return translateImpl(casted); + if (auto *casted = dyn_cast(node)) + return translateImpl(casted); if (auto *casted = dyn_cast(node)) return translateImpl(casted); if (auto *casted = dyn_cast(node)) diff --git a/mlir/lib/Target/LLVMIR/DebugImporter.h b/mlir/lib/Target/LLVMIR/DebugImporter.h index 7ae633df6e7b..2ce19bfbe0be 100644 --- a/mlir/lib/Target/LLVMIR/DebugImporter.h +++ b/mlir/lib/Target/LLVMIR/DebugImporter.h @@ -62,6 +62,7 @@ class DebugImporter { DILocalVariableAttr translateImpl(llvm::DILocalVariable *node); DIScopeAttr translateImpl(llvm::DIScope *node); DISubprogramAttr translateImpl(llvm::DISubprogram *node); + DINamespaceAttr translateImpl(llvm::DINamespace *node); DISubrangeAttr translateImpl(llvm::DISubrange *node); DISubroutineTypeAttr translateImpl(llvm::DISubroutineType *node); DITypeAttr translateImpl(llvm::DIType *node); diff --git a/mlir/lib/Target/LLVMIR/DebugTranslation.cpp b/mlir/lib/Target/LLVMIR/DebugTranslation.cpp index 9b13e81ace67..478216df312a 100644 --- a/mlir/lib/Target/LLVMIR/DebugTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/DebugTranslation.cpp @@ -198,6 +198,11 @@ llvm::DISubprogram *DebugTranslation::translateImpl(DISubprogramAttr attr) { translate(attr.getCompileUnit())); } +llvm::DINamespace *DebugTranslation::translateImpl(DINamespaceAttr attr) { + return llvm::DINamespace::get(llvmCtx, translate(attr.getScope()), + attr.getName(), attr.getExportSymbols()); +} + llvm::DISubrange *DebugTranslation::translateImpl(DISubrangeAttr attr) { auto getMetadataOrNull = [&](IntegerAttr attr) -> llvm::Metadata * { if (!attr) @@ -235,10 +240,11 @@ llvm::DINode *DebugTranslation::translate(DINodeAttr attr) { llvm::DINode *node = TypeSwitch(attr) - .Case( + .Case( [&](auto attr) { return translateImpl(attr); }); attrToNode.insert({attr, node}); return node; diff --git a/mlir/lib/Target/LLVMIR/DebugTranslation.h b/mlir/lib/Target/LLVMIR/DebugTranslation.h index 17bc35c8ab44..4160ca09a1c9 100644 --- a/mlir/lib/Target/LLVMIR/DebugTranslation.h +++ b/mlir/lib/Target/LLVMIR/DebugTranslation.h @@ -73,6 +73,7 @@ class DebugTranslation { llvm::DILocalVariable *translateImpl(DILocalVariableAttr attr); llvm::DIScope *translateImpl(DIScopeAttr attr); llvm::DISubprogram *translateImpl(DISubprogramAttr attr); + llvm::DINamespace *translateImpl(DINamespaceAttr attr); llvm::DISubrange *translateImpl(DISubrangeAttr attr); llvm::DISubroutineType *translateImpl(DISubroutineTypeAttr attr); llvm::DIType *translateImpl(DITypeAttr attr); diff --git a/mlir/test/Dialect/LLVMIR/debuginfo.mlir b/mlir/test/Dialect/LLVMIR/debuginfo.mlir index 7004d57762b9..1c3c553f1340 100644 --- a/mlir/test/Dialect/LLVMIR/debuginfo.mlir +++ b/mlir/test/Dialect/LLVMIR/debuginfo.mlir @@ -50,9 +50,19 @@ elements = #llvm.di_subrange > -// CHECK-DAG: #[[COMP2:.*]] = #llvm.di_composite_type +// CHECK-DAG: #[[TOPLEVEL:.*]] = #llvm.di_namespace +#toplevel_namespace = #llvm.di_namespace< + name = "toplevel", exportSymbols = true +> + +// CHECK-DAG: #[[NESTED:.*]] = #llvm.di_namespace +#nested_namespace = #llvm.di_namespace< + name = "nested", scope = #toplevel_namespace, exportSymbols = false +> + +// CHECK-DAG: #[[COMP2:.*]] = #llvm.di_composite_type #comp2 = #llvm.di_composite_type< - tag = DW_TAG_class_type, name = "class_name", file = #file, scope = #file, + tag = DW_TAG_class_type, name = "class_name", file = #file, scope = #nested_namespace, flags = "TypePassByReference|NonTrivial" > diff --git a/mlir/test/Target/LLVMIR/Import/debug-info.ll b/mlir/test/Target/LLVMIR/Import/debug-info.ll index 896d42202c23..fa1dccb47179 100644 --- a/mlir/test/Target/LLVMIR/Import/debug-info.ll +++ b/mlir/test/Target/LLVMIR/Import/debug-info.ll @@ -367,3 +367,25 @@ declare void @llvm.dbg.value(metadata, metadata, metadata) !7 = !DILocalVariable(scope: !8, name: "var", file: !2, type: !5); !8 = distinct !DISubprogram(name: "dbg_use_before_def", scope: !2, file: !2, spFlags: DISPFlagDefinition, unit: !1) !9 = !DILocation(line: 1, column: 2, scope: !8) + +; // ----- + +; CHECK-DAG: #[[NAMESPACE:.+]] = #llvm.di_namespace +; CHECK-DAG: #[[SUBPROGRAM:.+]] = #llvm.di_subprogram #null = #llvm.di_null_type #spType0 = #llvm.di_subroutine_type +#toplevel_namespace = #llvm.di_namespace< + name = "toplevel", exportSymbols = true +> +#nested_namespace = #llvm.di_namespace< + name = "nested", scope = #toplevel_namespace, exportSymbols = false +> #sp0 = #llvm.di_subprogram< - compileUnit = #cu, scope = #file, name = "func_with_debug", linkageName = "func_with_debug", + compileUnit = #cu, scope = #nested_namespace, name = "func_with_debug", linkageName = "func_with_debug", file = #file, line = 3, scopeLine = 3, subprogramFlags = "Definition|Optimized", type = #spType0 > #calleeType = #llvm.di_subroutine_type< @@ -113,7 +119,9 @@ llvm.func @empty_types() { // CHECK: ![[CU_LOC:.*]] = distinct !DICompileUnit(language: DW_LANG_C, file: ![[CU_FILE_LOC:.*]], producer: "MLIR", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug) // CHECK: ![[CU_FILE_LOC]] = !DIFile(filename: "foo.mlir", directory: "/test/") -// CHECK: ![[FUNC_LOC]] = distinct !DISubprogram(name: "func_with_debug", linkageName: "func_with_debug", scope: ![[CU_FILE_LOC]], file: ![[CU_FILE_LOC]], line: 3, type: ![[FUNC_TYPE:.*]], scopeLine: 3, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: ![[CU_LOC]]) +// CHECK: ![[FUNC_LOC]] = distinct !DISubprogram(name: "func_with_debug", linkageName: "func_with_debug", scope: ![[NESTED_NAMESPACE:.*]], file: ![[CU_FILE_LOC]], line: 3, type: ![[FUNC_TYPE:.*]], scopeLine: 3, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: ![[CU_LOC]]) +// CHECK: ![[NESTED_NAMESPACE]] = !DINamespace(name: "nested", scope: ![[TOPLEVEL_NAMESPACE:.*]]) +// CHECK: ![[TOPLEVEL_NAMESPACE]] = !DINamespace(name: "toplevel", scope: null, exportSymbols: true) // CHECK: ![[FUNC_TYPE]] = !DISubroutineType(cc: DW_CC_normal, types: ![[FUNC_ARGS:.*]]) // CHECK: ![[FUNC_ARGS]] = !{null, ![[ARG_TYPE:.*]], ![[PTR_TYPE:.*]], ![[NAMED_TYPE:.*]], ![[COMPOSITE_TYPE:.*]], ![[VECTOR_TYPE:.*]]} // CHECK: ![[ARG_TYPE]] = !DIBasicType(name: "si64") From ba467c2b18119692c618c49743acea8b97e77890 Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Mon, 6 Mar 2023 11:17:36 +0100 Subject: [PATCH 094/147] [mlir][llvm] Fuse access_group & loop export (NFC) This commit moves the access group translation into the LoopAnnotationTranslation class as these two metadata kinds only appear together. Drops the access group cleanup from `ModuleTranslation::forgetMapping` as this is only used on function regions. Access groups only appear in the region of a global metadata operation and will thus not be cleaned here. Analogous to https://reviews.llvm.org/D143577 Reviewed By: gysit Differential Revision: https://reviews.llvm.org/D144253 (cherry picked from commit 87a0479538fe4fad1cbbf729ee6e1ee35326f093) --- .../mlir/Target/LLVMIR/ModuleTranslation.h | 10 ---- .../LLVMIR/LoopAnnotationTranslation.cpp | 60 +++++++++++++++---- .../Target/LLVMIR/LoopAnnotationTranslation.h | 32 ++++++++-- mlir/lib/Target/LLVMIR/ModuleTranslation.cpp | 47 +++------------ 4 files changed, 83 insertions(+), 66 deletions(-) diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h index 2b08d96b680b..faca8fc5e4fb 100644 --- a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h +++ b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h @@ -120,11 +120,6 @@ class ModuleTranslation { /// in these blocks. void forgetMapping(Region ®ion); - /// Returns the LLVM metadata corresponding to a symbol reference to an mlir - /// LLVM dialect access group operation. - llvm::MDNode *getAccessGroup(Operation *op, - SymbolRefAttr accessGroupRef) const; - /// Returns the LLVM metadata corresponding to a symbol reference to an mlir /// LLVM dialect alias scope operation llvm::MDNode *getAliasScope(Operation *op, SymbolRefAttr aliasScopeRef) const; @@ -332,11 +327,6 @@ class ModuleTranslation { /// values after all operations are converted. DenseMap branchMapping; - /// Mapping from an access group metadata operation to its LLVM metadata. - /// This map is populated on module entry and is used to annotate loops (as - /// identified via their branches) and contained memory accesses. - DenseMap accessGroupMetadataMapping; - /// Mapping from an alias scope metadata operation to its LLVM metadata. /// This map is populated on module entry. DenseMap aliasScopeMetadataMapping; diff --git a/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp index 5f27f97f2fd4..4a535dcc3a8d 100644 --- a/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp @@ -15,12 +15,11 @@ using namespace mlir::LLVM::detail; namespace { /// Helper class that keeps the state of one attribute to metadata conversion. struct LoopAnnotationConversion { - LoopAnnotationConversion(LoopAnnotationAttr attr, - ModuleTranslation &moduleTranslation, Operation *op, - LoopAnnotationTranslation &loopAnnotationTranslation) - : attr(attr), moduleTranslation(moduleTranslation), op(op), - loopAnnotationTranslation(loopAnnotationTranslation), - ctx(moduleTranslation.getLLVMContext()) {} + LoopAnnotationConversion(LoopAnnotationAttr attr, Operation *op, + LoopAnnotationTranslation &loopAnnotationTranslation, + llvm::LLVMContext &ctx) + : attr(attr), op(op), + loopAnnotationTranslation(loopAnnotationTranslation), ctx(ctx) {} /// Converts this struct's loop annotation into a corresponding LLVMIR /// metadata representation. @@ -46,7 +45,6 @@ struct LoopAnnotationConversion { void convertLoopOptions(LoopUnswitchAttr options); LoopAnnotationAttr attr; - ModuleTranslation &moduleTranslation; Operation *op; LoopAnnotationTranslation &loopAnnotationTranslation; llvm::LLVMContext &ctx; @@ -95,7 +93,8 @@ void LoopAnnotationConversion::convertFollowupNode(StringRef name, if (!attr) return; - llvm::MDNode *node = loopAnnotationTranslation.translate(attr, op); + llvm::MDNode *node = + loopAnnotationTranslation.translateLoopAnnotation(attr, op); metadataNodes.push_back( llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name), node})); @@ -225,7 +224,7 @@ llvm::MDNode *LoopAnnotationConversion::convert() { llvm::MDString::get(ctx, "llvm.loop.parallel_accesses")); for (SymbolRefAttr accessGroupRef : parallelAccessGroups) parallelAccess.push_back( - moduleTranslation.getAccessGroup(op, accessGroupRef)); + loopAnnotationTranslation.getAccessGroup(op, accessGroupRef)); metadataNodes.push_back(llvm::MDNode::get(ctx, parallelAccess)); } @@ -236,7 +235,8 @@ llvm::MDNode *LoopAnnotationConversion::convert() { return loopMD; } -llvm::MDNode *LoopAnnotationTranslation::translate(LoopAnnotationAttr attr, +llvm::MDNode * +LoopAnnotationTranslation::translateLoopAnnotation(LoopAnnotationAttr attr, Operation *op) { if (!attr) return nullptr; @@ -246,9 +246,47 @@ llvm::MDNode *LoopAnnotationTranslation::translate(LoopAnnotationAttr attr, return loopMD; loopMD = - LoopAnnotationConversion(attr, moduleTranslation, op, *this).convert(); + LoopAnnotationConversion(attr, op, *this, this->llvmModule.getContext()) + .convert(); // Store a map from this Attribute to the LLVM metadata in case we // encounter it again. mapLoopMetadata(attr, loopMD); return loopMD; } + +LogicalResult LoopAnnotationTranslation::createAccessGroupMetadata() { + mlirModule->walk([&](LLVM::MetadataOp metadatas) { + metadatas.walk([&](LLVM::AccessGroupMetadataOp op) { + llvm::MDNode *accessGroup = + llvm::MDNode::getDistinct(llvmModule.getContext(), {}); + accessGroupMetadataMapping.insert({op, accessGroup}); + }); + }); + return success(); +} + +llvm::MDNode * +LoopAnnotationTranslation::getAccessGroup(Operation *op, + SymbolRefAttr accessGroupRef) const { + auto metadataName = accessGroupRef.getRootReference(); + auto accessGroupName = accessGroupRef.getLeafReference(); + auto metadataOp = SymbolTable::lookupNearestSymbolFrom( + op->getParentOp(), metadataName); + auto *accessGroupOp = + SymbolTable::lookupNearestSymbolFrom(metadataOp, accessGroupName); + return accessGroupMetadataMapping.lookup(accessGroupOp); +} + +llvm::MDNode * +LoopAnnotationTranslation::getAccessGroups(Operation *op, + ArrayAttr accessGroupRefs) const { + if (!accessGroupRefs || accessGroupRefs.empty()) + return nullptr; + + SmallVector groupMDs; + for (SymbolRefAttr groupRef : accessGroupRefs.getAsRange()) + groupMDs.push_back(getAccessGroup(op, groupRef)); + if (groupMDs.size() == 1) + return llvm::cast(groupMDs.front()); + return llvm::MDNode::get(llvmModule.getContext(), groupMDs); +} diff --git a/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.h b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.h index 0bbd5442510f..4de54f399830 100644 --- a/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.h +++ b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.h @@ -21,14 +21,28 @@ namespace mlir { namespace LLVM { namespace detail { -/// A helper class that converts a LoopAnnotationAttr into a corresponding -/// llvm::MDNode. +/// A helper class that converts LoopAnnotationAttrs and AccessGroupMetadataOps +/// into a corresponding llvm::MDNodes. class LoopAnnotationTranslation { public: - LoopAnnotationTranslation(LLVM::ModuleTranslation &moduleTranslation) - : moduleTranslation(moduleTranslation) {} + LoopAnnotationTranslation(Operation *mlirModule, llvm::Module &llvmModule) + : mlirModule(mlirModule), llvmModule(llvmModule) {} - llvm::MDNode *translate(LoopAnnotationAttr attr, Operation *op); + llvm::MDNode *translateLoopAnnotation(LoopAnnotationAttr attr, Operation *op); + + /// Traverses the global access group metadata operation in the `mlirModule` + /// and creates corresponding LLVM metadata nodes. + LogicalResult createAccessGroupMetadata(); + + /// Returns the LLVM metadata corresponding to a symbol reference to an mlir + /// LLVM dialect access group operation. + llvm::MDNode *getAccessGroup(Operation *op, + SymbolRefAttr accessGroupRef) const; + + /// Returns the LLVM metadata corresponding to a list of symbol reference to + /// an mlir LLVM dialect access group operation. Returns nullptr if + /// `accessGroupRefs` is null or empty. + llvm::MDNode *getAccessGroups(Operation *op, ArrayAttr accessGroupRefs) const; private: /// Returns the LLVM metadata corresponding to a llvm loop metadata attribute. @@ -47,7 +61,13 @@ class LoopAnnotationTranslation { /// The metadata is attached to Latch block branches with this attribute. DenseMap loopMetadataMapping; - LLVM::ModuleTranslation &moduleTranslation; + /// Mapping from an access group metadata operation to its LLVM metadata. + /// This map is populated on module entry and is used to annotate loops (as + /// identified via their branches) and contained memory accesses. + DenseMap accessGroupMetadataMapping; + + Operation *mlirModule; + llvm::Module &llvmModule; }; } // namespace detail diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp index 232b3d8b2216..04eddde310cf 100644 --- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp @@ -421,8 +421,8 @@ ModuleTranslation::ModuleTranslation(Operation *module, : mlirModule(module), llvmModule(std::move(llvmModule)), debugTranslation( std::make_unique(module, *this->llvmModule)), - loopAnnotationTranslation( - std::make_unique(*this)), + loopAnnotationTranslation(std::make_unique( + module, *this->llvmModule)), typeTranslator(this->llvmModule->getContext()), iface(module->getContext()) { assert(satisfiesLLVMModule(mlirModule) && @@ -449,7 +449,6 @@ void ModuleTranslation::forgetMapping(Region ®ion) { branchMapping.erase(&op); if (isa(op)) globalsMapping.erase(&op); - accessGroupMetadataMapping.erase(&op); llvm::append_range( toProcess, llvm::map_range(op.getRegions(), [](Region &r) { return &r; })); @@ -994,47 +993,16 @@ LogicalResult ModuleTranslation::convertFunctions() { return success(); } -llvm::MDNode * -ModuleTranslation::getAccessGroup(Operation *op, - SymbolRefAttr accessGroupRef) const { - auto metadataName = accessGroupRef.getRootReference(); - auto accessGroupName = accessGroupRef.getLeafReference(); - auto metadataOp = SymbolTable::lookupNearestSymbolFrom( - op->getParentOp(), metadataName); - auto *accessGroupOp = - SymbolTable::lookupNearestSymbolFrom(metadataOp, accessGroupName); - return accessGroupMetadataMapping.lookup(accessGroupOp); -} - LogicalResult ModuleTranslation::createAccessGroupMetadata() { - mlirModule->walk([&](LLVM::MetadataOp metadatas) { - metadatas.walk([&](LLVM::AccessGroupMetadataOp op) { - llvm::LLVMContext &ctx = llvmModule->getContext(); - llvm::MDNode *accessGroup = llvm::MDNode::getDistinct(ctx, {}); - accessGroupMetadataMapping.insert({op, accessGroup}); - }); - }); - return success(); + return loopAnnotationTranslation->createAccessGroupMetadata(); } void ModuleTranslation::setAccessGroupsMetadata(Operation *op, llvm::Instruction *inst) { auto populateGroupsMetadata = [&](ArrayAttr groupRefs) { - if (!groupRefs || groupRefs.empty()) - return; - - llvm::Module *module = inst->getModule(); - SmallVector groupMDs; - for (SymbolRefAttr groupRef : groupRefs.getAsRange()) - groupMDs.push_back(getAccessGroup(op, groupRef)); - - llvm::MDNode *node = nullptr; - if (groupMDs.size() == 1) - node = llvm::cast(groupMDs.front()); - else if (groupMDs.size() >= 2) - node = llvm::MDNode::get(module->getContext(), groupMDs); - - inst->setMetadata(llvm::LLVMContext::MD_access_group, node); + if (llvm::MDNode *node = + loopAnnotationTranslation->getAccessGroups(op, groupRefs)) + inst->setMetadata(llvm::LLVMContext::MD_access_group, node); }; auto groupRefs = @@ -1250,7 +1218,8 @@ void ModuleTranslation::setLoopMetadata(Operation *op, [](auto branchOp) { return branchOp.getLoopAnnotationAttr(); }); if (!attr) return; - llvm::MDNode *loopMD = loopAnnotationTranslation->translate(attr, op); + llvm::MDNode *loopMD = + loopAnnotationTranslation->translateLoopAnnotation(attr, op); inst->setMetadata(llvm::LLVMContext::MD_loop, loopMD); } From 44863706474a365a7946f62396bcc97c9a99dc83 Mon Sep 17 00:00:00 2001 From: Tobias Gysi Date: Mon, 6 Mar 2023 11:18:15 +0100 Subject: [PATCH 095/147] [mlir][llvm] Add atomic support to the StoreOp. This revision adds atomic support to the StoreOp. It chooses to print the atomic keywords together with the syncscope and ordering arguments. The revision also implements verifiers to ensure the constraints that apply to atomic store operations are checked. Depends on D144112 Reviewed By: Dinistro Differential Revision: https://reviews.llvm.org/D144200 (cherry picked from commit 9185896a36457f8e6858c9a4c789975fc067228a) --- mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td | 49 +++++++++++++--- mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 56 +++++++++++++------ mlir/test/Dialect/LLVMIR/invalid.mlir | 35 ++++++++++++ mlir/test/Dialect/LLVMIR/roundtrip.mlir | 9 +++ .../test/Target/LLVMIR/Import/instructions.ll | 9 ++- mlir/test/Target/LLVMIR/llvmir.mlir | 17 ++++-- 6 files changed, 145 insertions(+), 30 deletions(-) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td index 5fdc2cc59d49..bfacb18f3a0e 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td @@ -430,16 +430,47 @@ def LLVM_StoreOp : LLVM_Op<"store">, MemoryOpBase { OptionalAttr:$alias_scopes, OptionalAttr:$noalias_scopes, OptionalAttr:$tbaa, - OptionalAttr:$alignment, UnitAttr:$volatile_, - UnitAttr:$nontemporal); + OptionalAttr:$alignment, + UnitAttr:$volatile_, + UnitAttr:$nontemporal, + DefaultValuedAttr:$ordering, + OptionalAttr:$syncscope); string llvmInstName = "Store"; + let description = [{ + The `store` operation is used to write to memory. A store may be marked as + atomic, volatile, and/or nontemporal, and takes a number of optional + attributes that specify aliasing information. + + An atomic store only supports a limited set of pointer, integer, and + floating point types, and requires an explicit alignment. + + Examples: + ```mlir + // A volatile store of a float variable. + llvm.store volatile %val, %ptr : f32, !llvm.ptr + + // A nontemporal store of a float variable. + llvm.store %val, %ptr {nontemporal} : f32, !llvm.ptr + + // An atomic store of an integer variable. + llvm.store %val, %ptr atomic monotonic {alignment = 8 : i64} + : i64, !llvm.ptr + ``` + + See the following link for more details: + https://llvm.org/docs/LangRef.html#store-instruction + }]; let assemblyFormat = [{ - (`volatile` $volatile_^)? $value `,` $addr attr-dict `:` - custom(type($value), type($addr)) + (`volatile` $volatile_^)? $value `,` $addr + (`atomic` (`syncscope` `(` $syncscope^ `)`)? $ordering^)? + attr-dict `:` custom(type($value), type($addr)) }]; string llvmBuilder = [{ auto *inst = builder.CreateStore($value, $addr, $volatile_); - }] # setAlignmentCode + }] # setOrderingCode + # setSyncScopeCode + # setAlignmentCode # setNonTemporalMetadataCode # setAccessGroupsMetadataCode # setAliasScopeMetadataCode @@ -449,12 +480,16 @@ def LLVM_StoreOp : LLVM_Op<"store">, MemoryOpBase { unsigned alignment = storeInst->getAlign().value(); $_op = $_builder.create($_location, $value, $addr, alignment, storeInst->isVolatile(), - storeInst->hasMetadata(llvm::LLVMContext::MD_nontemporal)); + storeInst->hasMetadata(llvm::LLVMContext::MD_nontemporal), + convertAtomicOrderingFromLLVM(storeInst->getOrdering()), + getLLVMSyncScope(storeInst)); }]; let builders = [ OpBuilder<(ins "Value":$value, "Value":$addr, CArg<"unsigned", "0">:$alignment, CArg<"bool", "false">:$isVolatile, - CArg<"bool", "false">:$isNonTemporal)> + CArg<"bool", "false">:$isNonTemporal, + CArg<"AtomicOrdering", "AtomicOrdering::not_atomic">:$ordering, + CArg<"StringRef", "StringRef()">:$syncscope)> ]; let hasVerifier = 1; } diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp index de6fa64769ab..510abda7a676 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -826,21 +826,35 @@ static bool isTypeCompatibleWithAtomicOp(Type type, bool isPointerTypeAllowed) { *bitWidth == 64; } -LogicalResult LoadOp::verify() { - if (getOrdering() != AtomicOrdering::not_atomic) { - if (!isTypeCompatibleWithAtomicOp(getResult().getType(), +/// Verifies the attributes and the type of atomic memory access operations. +template +LogicalResult verifyAtomicMemOp(OpTy memOp, Type valueType, + ArrayRef unsupportedOrderings) { + if (memOp.getOrdering() != AtomicOrdering::not_atomic) { + if (!isTypeCompatibleWithAtomicOp(valueType, /*isPointerTypeAllowed=*/true)) - return emitOpError("unsupported type ") - << getResult().getType() << " for atomic access"; - if (getOrdering() == AtomicOrdering::release || - getOrdering() == AtomicOrdering::acq_rel) - return emitOpError("unsupported ordering '") - << stringifyAtomicOrdering(getOrdering()) << "'"; - if (!getAlignment()) - return emitOpError("expected alignment for atomic access"); - } else if (getSyncscope()) { - return emitOpError("expected syncscope to be null for non-atomic access"); + return memOp.emitOpError("unsupported type ") + << valueType << " for atomic access"; + if (llvm::is_contained(unsupportedOrderings, memOp.getOrdering())) + return memOp.emitOpError("unsupported ordering '") + << stringifyAtomicOrdering(memOp.getOrdering()) << "'"; + if (!memOp.getAlignment()) + return memOp.emitOpError("expected alignment for atomic access"); + return success(); } + if (memOp.getSyncscope()) + return memOp.emitOpError( + "expected syncscope to be null for non-atomic access"); + return success(); +} + +LogicalResult LoadOp::verify() { + Type valueType = getResult().getType(); + if (failed(verifyAtomicMemOp( + *this, valueType, + {AtomicOrdering::release, AtomicOrdering::acq_rel}))) + return failure(); + return verifyMemOpMetadata(*this); } @@ -914,15 +928,25 @@ static void printLoadType(OpAsmPrinter &printer, Operation *op, Type type, // StoreOp //===----------------------------------------------------------------------===// -LogicalResult StoreOp::verify() { return verifyMemOpMetadata(*this); } +LogicalResult StoreOp::verify() { + Type valueType = getValue().getType(); + if (failed(verifyAtomicMemOp( + *this, valueType, + {AtomicOrdering::acquire, AtomicOrdering::acq_rel}))) + return failure(); + + return verifyMemOpMetadata(*this); +} void StoreOp::build(OpBuilder &builder, OperationState &state, Value value, Value addr, unsigned alignment, bool isVolatile, - bool isNonTemporal) { + bool isNonTemporal, AtomicOrdering ordering, + StringRef syncscope) { build(builder, state, value, addr, /*access_groups=*/nullptr, /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr, alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile, - isNonTemporal); + isNonTemporal, ordering, + syncscope.empty() ? nullptr : builder.getStringAttr(syncscope)); } /// Parses the StoreOp type either using the typed or opaque pointer format. diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir index cbcc633ae11b..642639b18dee 100644 --- a/mlir/test/Dialect/LLVMIR/invalid.mlir +++ b/mlir/test/Dialect/LLVMIR/invalid.mlir @@ -209,6 +209,41 @@ func.func @store_malformed_elem_type(%foo: !llvm.ptr, %bar: f32) { // ----- +func.func @store_syncscope(%val : f32, %ptr : !llvm.ptr) { + // expected-error@below {{expected syncscope to be null for non-atomic access}} + "llvm.store"(%val, %ptr) {syncscope = "singlethread"} : (f32, !llvm.ptr) -> () +} + +// ----- + +func.func @store_unsupported_ordering(%val : f32, %ptr : !llvm.ptr) { + // expected-error@below {{unsupported ordering 'acquire'}} + llvm.store %val, %ptr atomic acquire {alignment = 4 : i64} : f32, !llvm.ptr +} + +// ----- + +func.func @store_unsupported_type(%val : f80, %ptr : !llvm.ptr) { + // expected-error@below {{unsupported type 'f80' for atomic access}} + llvm.store %val, %ptr atomic monotonic {alignment = 16 : i64} : f80, !llvm.ptr +} + +// ----- + +func.func @store_unsupported_type(%val : i1, %ptr : !llvm.ptr) { + // expected-error@below {{unsupported type 'i1' for atomic access}} + llvm.store %val, %ptr atomic monotonic {alignment = 16 : i64} : i1, !llvm.ptr +} + +// ----- + +func.func @store_unaligned_atomic(%val : f32, %ptr : !llvm.ptr) { + // expected-error@below {{expected alignment for atomic access}} + llvm.store %val, %ptr atomic monotonic : f32, !llvm.ptr +} + +// ----- + func.func @invalid_call() { // expected-error@+1 {{'llvm.call' op must have either a `callee` attribute or at least an operand}} "llvm.call"() : () -> () diff --git a/mlir/test/Dialect/LLVMIR/roundtrip.mlir b/mlir/test/Dialect/LLVMIR/roundtrip.mlir index ebab8c35b90a..1e49b4e3124e 100644 --- a/mlir/test/Dialect/LLVMIR/roundtrip.mlir +++ b/mlir/test/Dialect/LLVMIR/roundtrip.mlir @@ -348,6 +348,15 @@ func.func @atomic_load(%ptr : !llvm.ptr) { llvm.return } +// CHECK-LABEL: @atomic_store +func.func @atomic_store(%val : f32, %ptr : !llvm.ptr) { + // CHECK: llvm.store %{{.*}}, %{{.*}} atomic monotonic {alignment = 4 : i64} : f32, !llvm.ptr + llvm.store %val, %ptr atomic monotonic {alignment = 4 : i64} : f32, !llvm.ptr + // CHECK: llvm.store volatile %{{.*}}, %{{.*}} atomic syncscope("singlethread") monotonic {alignment = 16 : i64} : f32, !llvm.ptr + llvm.store volatile %val, %ptr atomic syncscope("singlethread") monotonic {alignment = 16 : i64} : f32, !llvm.ptr + llvm.return +} + // CHECK-LABEL: @atomicrmw func.func @atomicrmw(%ptr : !llvm.ptr, %val : f32) { // CHECK: llvm.atomicrmw fadd %{{.*}}, %{{.*}} monotonic : !llvm.ptr, f32 diff --git a/mlir/test/Target/LLVMIR/Import/instructions.ll b/mlir/test/Target/LLVMIR/Import/instructions.ll index f66d7b9cdf69..47076a7abb01 100644 --- a/mlir/test/Target/LLVMIR/Import/instructions.ll +++ b/mlir/test/Target/LLVMIR/Import/instructions.ll @@ -368,13 +368,18 @@ define void @load_store(ptr %ptr) { ; // ----- -; CHECK-LABEL: @atomic_load +; CHECK-LABEL: @atomic_load_store ; CHECK-SAME: %[[PTR:[a-zA-Z0-9]+]] -define void @atomic_load(ptr %ptr) { +define void @atomic_load_store(ptr %ptr) { ; CHECK: %[[V1:[0-9]+]] = llvm.load %[[PTR]] atomic acquire {alignment = 8 : i64} : !llvm.ptr -> f64 ; CHECK: %[[V2:[0-9]+]] = llvm.load volatile %[[PTR]] atomic syncscope("singlethreaded") acquire {alignment = 16 : i64} : !llvm.ptr -> f64 %1 = load atomic double, ptr %ptr acquire, align 8 %2 = load atomic volatile double, ptr %ptr syncscope("singlethreaded") acquire, align 16 + + ; CHECK: llvm.store %[[V1]], %[[PTR]] atomic release {alignment = 8 : i64} : f64, !llvm.ptr + ; CHECK: llvm.store volatile %[[V2]], %[[PTR]] atomic syncscope("singlethreaded") release {alignment = 16 : i64} : f64, !llvm.ptr + store atomic double %1, ptr %ptr release, align 8 + store atomic volatile double %2, ptr %ptr syncscope("singlethreaded") release, align 16 ret void } diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir index da7dc7f78ba5..a58dfde99b05 100644 --- a/mlir/test/Target/LLVMIR/llvmir.mlir +++ b/mlir/test/Target/LLVMIR/llvmir.mlir @@ -1780,13 +1780,20 @@ llvm.func @nontemporal_store_and_load() { // ----- -llvm.func @atomic_load(%ptr : !llvm.ptr) { +llvm.func @atomic_store_and_load(%ptr : !llvm.ptr) { // CHECK: load atomic - // CHECK-SAME: monotonic, align 4 - %1 = llvm.load %ptr atomic monotonic {alignment = 4 : i64} : !llvm.ptr -> f32 + // CHECK-SAME: acquire, align 4 + %1 = llvm.load %ptr atomic acquire {alignment = 4 : i64} : !llvm.ptr -> f32 // CHECK: load atomic - // CHECK-SAME: syncscope("singlethread") monotonic, align 4 - %2 = llvm.load %ptr atomic syncscope("singlethread") monotonic {alignment = 4 : i64} : !llvm.ptr -> f32 + // CHECK-SAME: syncscope("singlethread") acquire, align 4 + %2 = llvm.load %ptr atomic syncscope("singlethread") acquire {alignment = 4 : i64} : !llvm.ptr -> f32 + + // CHECK: store atomic + // CHECK-SAME: release, align 4 + llvm.store %1, %ptr atomic release {alignment = 4 : i64} : f32, !llvm.ptr + // CHECK: store atomic + // CHECK-SAME: syncscope("singlethread") release, align 4 + llvm.store %2, %ptr atomic syncscope("singlethread") release {alignment = 4 : i64} : f32, !llvm.ptr llvm.return } From 9858b1979bba9fe986daf8feaa3b4da3d763a60d Mon Sep 17 00:00:00 2001 From: Tobias Gysi Date: Mon, 6 Mar 2023 11:18:55 +0100 Subject: [PATCH 096/147] [mlir][llvm] Add FastmathFlagsInterface only once (NFC). Previously, the FastmathFlagsInterface has been added twice to the LLVM::PowIOp. Once directly and once indirectly using the requiresFastmath flag. This revision only uses the flag and drops the redundant interface. Reviewed By: definelicht Differential Revision: https://reviews.llvm.org/D144702 (cherry picked from commit a52332df5d1f9de05d9cd5b4a02b0888e28aa5cf) --- mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td index 4f6ceea76e10..a90b5f64b2cb 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td @@ -109,8 +109,7 @@ def LLVM_FTruncOp : LLVM_UnaryIntrOpF<"trunc">; def LLVM_SqrtOp : LLVM_UnaryIntrOpF<"sqrt">; def LLVM_PowOp : LLVM_BinarySameArgsIntrOpF<"pow">; def LLVM_PowIOp : LLVM_OneResultIntrOp<"powi", [], [0,1], - [DeclareOpInterfaceMethods, Pure], - /*requiresFastmath=*/1> { + [Pure], /*requiresFastmath=*/1> { let arguments = (ins LLVM_ScalarOrVectorOf:$val, AnySignlessInteger:$power, From d9b6d70d8c6bef59c13175c9786d16e74992d1e0 Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Mon, 6 Mar 2023 11:19:40 +0100 Subject: [PATCH 097/147] [mlir][llvm] Builders dont access null attr (NFC) Reviewed By: gysit Differential Revision: https://reviews.llvm.org/D144267 (cherry picked from commit ddd1d1c5657989b75bdf4b1ae7c64366a34fbb35) --- mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td index d31524a29ac9..ae4f26439750 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td @@ -384,7 +384,7 @@ def LLVM_DILexicalBlockAttr : LLVM_Attr<"DILexicalBlock", "di_lexical_block", [ "DIScopeAttr":$scope, "DIFileAttr":$file, "unsigned":$line, "unsigned":$column ), [{ - return $_get(file.getContext(), scope, file, line, column); + return $_get(scope.getContext(), scope, file, line, column); }]> ]; let assemblyFormat = "`<` struct(params) `>`"; @@ -406,7 +406,7 @@ def LLVM_DILexicalBlockFile : LLVM_Attr<"DILexicalBlockFile", "di_lexical_block_ AttrBuilderWithInferredContext<(ins "DIScopeAttr":$scope, "DIFileAttr":$file, "unsigned":$discriminator ), [{ - return $_get(file.getContext(), scope, file, discriminator); + return $_get(scope.getContext(), scope, file, discriminator); }]> ]; let assemblyFormat = "`<` struct(params) `>`"; @@ -434,7 +434,7 @@ def LLVM_DILocalVariableAttr : LLVM_Attr<"DILocalVariable", "di_local_variable", "unsigned":$line, "unsigned":$arg, "unsigned":$alignInBits, "DITypeAttr":$type ), [{ - MLIRContext *ctx = file.getContext(); + MLIRContext *ctx = scope.getContext(); return $_get(ctx, scope, StringAttr::get(ctx, name), file, line, arg, alignInBits, type); }]> From 48d3773f917636ce1a874303d2fada06d5b60809 Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Mon, 6 Mar 2023 11:20:17 +0100 Subject: [PATCH 098/147] [mlir][llvm] Stop exporting empty debug MD strings This commit ensures that no empty debug metadata strings are exported as these are not legal names. Additionally, this commit ensures that non-existing strings are not accidentially imported as empty strings. Reviewed By: gysit Differential Revision: https://reviews.llvm.org/D144263 (cherry picked from commit d9391a37a9f8eb222a951606e0ca5d6a5b6b2199) --- .../mlir/Dialect/LLVMIR/LLVMAttrDefs.td | 6 +-- mlir/lib/Target/LLVMIR/DebugImporter.cpp | 35 +++++++++------- mlir/lib/Target/LLVMIR/DebugImporter.h | 4 ++ mlir/lib/Target/LLVMIR/DebugTranslation.cpp | 42 +++++++++++-------- mlir/lib/Target/LLVMIR/DebugTranslation.h | 4 ++ mlir/test/Target/LLVMIR/Import/debug-info.ll | 23 +++++++++- mlir/test/Target/LLVMIR/llvmir-debug.mlir | 4 ++ 7 files changed, 80 insertions(+), 38 deletions(-) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td index ae4f26439750..26a469dc79da 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td @@ -305,7 +305,7 @@ def LLVM_DICompileUnitAttr : LLVM_Attr<"DICompileUnit", "di_compile_unit", [ let parameters = (ins LLVM_DILanguageParameter:$sourceLanguage, "DIFileAttr":$file, - "StringAttr":$producer, + OptionalParameter<"StringAttr">:$producer, "bool":$isOptimized, "DIEmissionKind":$emissionKind ); @@ -321,7 +321,7 @@ def LLVM_DICompositeTypeAttr : LLVM_Attr<"DICompositeType", "di_composite_type", ], "DITypeAttr"> { let parameters = (ins LLVM_DITagParameter:$tag, - "StringAttr":$name, + OptionalParameter<"StringAttr">:$name, OptionalParameter<"DIFileAttr">:$file, OptionalParameter<"uint32_t">:$line, OptionalParameter<"DIScopeAttr">:$scope, @@ -421,7 +421,7 @@ def LLVM_DILocalVariableAttr : LLVM_Attr<"DILocalVariable", "di_local_variable", ], "DINodeAttr"> { let parameters = (ins "DIScopeAttr":$scope, - "StringAttr":$name, + OptionalParameter<"StringAttr">:$name, OptionalParameter<"DIFileAttr">:$file, OptionalParameter<"unsigned">:$line, OptionalParameter<"unsigned">:$arg, diff --git a/mlir/lib/Target/LLVMIR/DebugImporter.cpp b/mlir/lib/Target/LLVMIR/DebugImporter.cpp index 20cd5da30ae5..cff8357457cb 100644 --- a/mlir/lib/Target/LLVMIR/DebugImporter.cpp +++ b/mlir/lib/Target/LLVMIR/DebugImporter.cpp @@ -47,7 +47,7 @@ DICompileUnitAttr DebugImporter::translateImpl(llvm::DICompileUnit *node) { symbolizeDIEmissionKind(node->getEmissionKind()); return DICompileUnitAttr::get(context, node->getSourceLanguage(), translate(node->getFile()), - StringAttr::get(context, node->getProducer()), + getStringAttrOrNull(node->getRawProducer()), node->isOptimized(), emissionKind.value()); } @@ -66,7 +66,7 @@ DICompositeTypeAttr DebugImporter::translateImpl(llvm::DICompositeType *node) { if (llvm::is_contained(elements, nullptr)) elements.clear(); return DICompositeTypeAttr::get( - context, node->getTag(), StringAttr::get(context, node->getName()), + context, node->getTag(), getStringAttrOrNull(node->getRawName()), translate(node->getFile()), node->getLine(), translate(node->getScope()), translate(node->getBaseType()), flags.value_or(DIFlags::Zero), node->getSizeInBits(), node->getAlignInBits(), elements); @@ -78,8 +78,7 @@ DIDerivedTypeAttr DebugImporter::translateImpl(llvm::DIDerivedType *node) { if (node->getBaseType() && !baseType) return nullptr; return DIDerivedTypeAttr::get( - context, node->getTag(), - node->getRawName() ? StringAttr::get(context, node->getName()) : nullptr, + context, node->getTag(), getStringAttrOrNull(node->getRawName()), baseType, node->getSizeInBits(), node->getAlignInBits(), node->getOffsetInBits()); } @@ -103,7 +102,7 @@ DebugImporter::translateImpl(llvm::DILexicalBlockFile *node) { DILocalVariableAttr DebugImporter::translateImpl(llvm::DILocalVariable *node) { return DILocalVariableAttr::get(context, translate(node->getScope()), - StringAttr::get(context, node->getName()), + getStringAttrOrNull(node->getRawName()), translate(node->getFile()), node->getLine(), node->getArg(), node->getAlignInBits(), translate(node->getType())); @@ -114,9 +113,9 @@ DIScopeAttr DebugImporter::translateImpl(llvm::DIScope *node) { } DINamespaceAttr DebugImporter::translateImpl(llvm::DINamespace *node) { - return DINamespaceAttr::get( - context, StringAttr::get(context, node->getName()), - translate(node->getScope()), node->getExportSymbols()); + return DINamespaceAttr::get(context, getStringAttrOrNull(node->getRawName()), + translate(node->getScope()), + node->getExportSymbols()); } DISubprogramAttr DebugImporter::translateImpl(llvm::DISubprogram *node) { @@ -129,14 +128,12 @@ DISubprogramAttr DebugImporter::translateImpl(llvm::DISubprogram *node) { DISubroutineTypeAttr type = translate(node->getType()); if (node->getType() && !type) return nullptr; - return DISubprogramAttr::get( - context, translate(node->getUnit()), scope, - StringAttr::get(context, node->getName()), - node->getRawLinkageName() - ? StringAttr::get(context, node->getLinkageName()) - : nullptr, - translate(node->getFile()), node->getLine(), node->getScopeLine(), - subprogramFlags.value(), type); + return DISubprogramAttr::get(context, translate(node->getUnit()), scope, + getStringAttrOrNull(node->getRawName()), + getStringAttrOrNull(node->getRawLinkageName()), + translate(node->getFile()), node->getLine(), + node->getScopeLine(), subprogramFlags.value(), + type); } DISubrangeAttr DebugImporter::translateImpl(llvm::DISubrange *node) { @@ -248,3 +245,9 @@ Location DebugImporter::translateLoc(llvm::DILocation *loc) { context); return result; } + +StringAttr DebugImporter::getStringAttrOrNull(llvm::MDString *stringNode) { + if (!stringNode) + return StringAttr(); + return StringAttr::get(context, stringNode->getString()); +} diff --git a/mlir/lib/Target/LLVMIR/DebugImporter.h b/mlir/lib/Target/LLVMIR/DebugImporter.h index 2ce19bfbe0be..e7f2332ff436 100644 --- a/mlir/lib/Target/LLVMIR/DebugImporter.h +++ b/mlir/lib/Target/LLVMIR/DebugImporter.h @@ -67,6 +67,10 @@ class DebugImporter { DISubroutineTypeAttr translateImpl(llvm::DISubroutineType *node); DITypeAttr translateImpl(llvm::DIType *node); + /// Constructs a StringAttr from the MDString if it is non-null. Returns a + /// null attribute otherwise. + StringAttr getStringAttrOrNull(llvm::MDString *stringNode); + /// A mapping between LLVM debug metadata and the corresponding attribute. DenseMap nodeToAttr; diff --git a/mlir/lib/Target/LLVMIR/DebugTranslation.cpp b/mlir/lib/Target/LLVMIR/DebugTranslation.cpp index 478216df312a..dae9930ea309 100644 --- a/mlir/lib/Target/LLVMIR/DebugTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/DebugTranslation.cpp @@ -97,17 +97,26 @@ llvm::DIType *DebugTranslation::translateImpl(DINullTypeAttr attr) { return nullptr; } +llvm::MDString *DebugTranslation::getMDStringOrNull(StringAttr stringAttr) { + if (!stringAttr || stringAttr.getValue().empty()) + return nullptr; + return llvm::MDString::get(llvmCtx, stringAttr); +} + llvm::DIBasicType *DebugTranslation::translateImpl(DIBasicTypeAttr attr) { return llvm::DIBasicType::get( - llvmCtx, attr.getTag(), attr.getName(), attr.getSizeInBits(), + llvmCtx, attr.getTag(), getMDStringOrNull(attr.getName()), + attr.getSizeInBits(), /*AlignInBits=*/0, attr.getEncoding(), llvm::DINode::FlagZero); } llvm::DICompileUnit *DebugTranslation::translateImpl(DICompileUnitAttr attr) { llvm::DIBuilder builder(llvmModule); return builder.createCompileUnit( - attr.getSourceLanguage(), translate(attr.getFile()), attr.getProducer(), - attr.getIsOptimized(), /*Flags=*/"", /*RV=*/0); + attr.getSourceLanguage(), translate(attr.getFile()), + attr.getProducer() ? attr.getProducer().getValue() : "", + attr.getIsOptimized(), + /*Flags=*/"", /*RV=*/0); } llvm::DICompositeType * @@ -116,9 +125,10 @@ DebugTranslation::translateImpl(DICompositeTypeAttr attr) { for (auto member : attr.getElements()) elements.push_back(translate(member)); return llvm::DICompositeType::get( - llvmCtx, attr.getTag(), attr.getName(), translate(attr.getFile()), - attr.getLine(), translate(attr.getScope()), translate(attr.getBaseType()), - attr.getSizeInBits(), attr.getAlignInBits(), + llvmCtx, attr.getTag(), getMDStringOrNull(attr.getName()), + translate(attr.getFile()), attr.getLine(), translate(attr.getScope()), + translate(attr.getBaseType()), attr.getSizeInBits(), + attr.getAlignInBits(), /*OffsetInBits=*/0, /*Flags=*/static_cast(attr.getFlags()), llvm::MDNode::get(llvmCtx, elements), @@ -126,9 +136,6 @@ DebugTranslation::translateImpl(DICompositeTypeAttr attr) { } llvm::DIDerivedType *DebugTranslation::translateImpl(DIDerivedTypeAttr attr) { - auto getMDStringOrNull = [&](StringAttr attr) -> llvm::MDString * { - return attr ? llvm::MDString::get(llvmCtx, attr) : nullptr; - }; return llvm::DIDerivedType::get( llvmCtx, attr.getTag(), getMDStringOrNull(attr.getName()), /*File=*/nullptr, /*Line=*/0, @@ -138,7 +145,8 @@ llvm::DIDerivedType *DebugTranslation::translateImpl(DIDerivedTypeAttr attr) { } llvm::DIFile *DebugTranslation::translateImpl(DIFileAttr attr) { - return llvm::DIFile::get(llvmCtx, attr.getName(), attr.getDirectory()); + return llvm::DIFile::get(llvmCtx, getMDStringOrNull(attr.getName()), + getMDStringOrNull(attr.getDirectory())); } llvm::DILexicalBlock *DebugTranslation::translateImpl(DILexicalBlockAttr attr) { @@ -161,9 +169,9 @@ llvm::DILocalScope *DebugTranslation::translateImpl(DILocalScopeAttr attr) { llvm::DILocalVariable * DebugTranslation::translateImpl(DILocalVariableAttr attr) { return llvm::DILocalVariable::get( - llvmCtx, translate(attr.getScope()), - llvm::MDString::get(llvmCtx, attr.getName()), translate(attr.getFile()), - attr.getLine(), translate(attr.getType()), attr.getArg(), + llvmCtx, translate(attr.getScope()), getMDStringOrNull(attr.getName()), + translate(attr.getFile()), attr.getLine(), translate(attr.getType()), + attr.getArg(), /*Flags=*/llvm::DINode::FlagZero, attr.getAlignInBits(), /*Annotations=*/nullptr); } @@ -184,12 +192,9 @@ static llvm::DISubprogram *getSubprogram(bool isDistinct, Ts &&...args) { llvm::DISubprogram *DebugTranslation::translateImpl(DISubprogramAttr attr) { bool isDefinition = static_cast(attr.getSubprogramFlags() & LLVM::DISubprogramFlags::Definition); - auto getMDStringOrNull = [&](StringAttr attr) -> llvm::MDString * { - return attr ? llvm::MDString::get(llvmCtx, attr) : nullptr; - }; return getSubprogram( isDefinition, llvmCtx, translate(attr.getScope()), - llvm::MDString::get(llvmCtx, attr.getName()), + getMDStringOrNull(attr.getName()), getMDStringOrNull(attr.getLinkageName()), translate(attr.getFile()), attr.getLine(), translate(attr.getType()), attr.getScopeLine(), /*ContainingType=*/nullptr, /*VirtualIndex=*/0, @@ -200,7 +205,8 @@ llvm::DISubprogram *DebugTranslation::translateImpl(DISubprogramAttr attr) { llvm::DINamespace *DebugTranslation::translateImpl(DINamespaceAttr attr) { return llvm::DINamespace::get(llvmCtx, translate(attr.getScope()), - attr.getName(), attr.getExportSymbols()); + getMDStringOrNull(attr.getName()), + attr.getExportSymbols()); } llvm::DISubrange *DebugTranslation::translateImpl(DISubrangeAttr attr) { diff --git a/mlir/lib/Target/LLVMIR/DebugTranslation.h b/mlir/lib/Target/LLVMIR/DebugTranslation.h index 4160ca09a1c9..3ea077a80bfa 100644 --- a/mlir/lib/Target/LLVMIR/DebugTranslation.h +++ b/mlir/lib/Target/LLVMIR/DebugTranslation.h @@ -78,6 +78,10 @@ class DebugTranslation { llvm::DISubroutineType *translateImpl(DISubroutineTypeAttr attr); llvm::DIType *translateImpl(DITypeAttr attr); + /// Constructs a string metadata node from the string attribute. Returns + /// nullptr if `stringAttr` is null or contains and empty string. + llvm::MDString *getMDStringOrNull(StringAttr stringAttr); + /// A mapping between mlir location+scope and the corresponding llvm debug /// metadata. DenseMap, diff --git a/mlir/test/Target/LLVMIR/Import/debug-info.ll b/mlir/test/Target/LLVMIR/Import/debug-info.ll index fa1dccb47179..a8aae4573e9a 100644 --- a/mlir/test/Target/LLVMIR/Import/debug-info.ll +++ b/mlir/test/Target/LLVMIR/Import/debug-info.ll @@ -190,7 +190,7 @@ define void @composite_type() !dbg !3 { ; // ----- ; CHECK-DAG: #[[FILE:.+]] = #llvm.di_file<"debug-info.ll" in "/"> -; CHECK-DAG: #[[CU:.+]] = #llvm.di_compile_unit +; CHECK-DAG: #[[CU:.+]] = #llvm.di_compile_unit ; Verify an empty subroutine types list is supported. ; CHECK-DAG: #[[SP_TYPE:.+]] = #llvm.di_subroutine_type ; CHECK-DAG: #[[SP:.+]] = #llvm.di_subprogram @@ -389,3 +389,24 @@ declare void @llvm.dbg.value(metadata, metadata, metadata) !8 = distinct !DISubprogram(name: "namespace", scope: !10, file: !2, unit: !1); !9 = !DILocation(line: 1, column: 2, scope: !8) !10 = !DINamespace(name: "std", scope: null) + +; // ----- + +; CHECK-DAG: #[[SUBPROGRAM:.+]] = #llvm.di_subprogram + +define void @noname_variable(ptr %arg) { + call void @llvm.dbg.value(metadata ptr %arg, metadata !7, metadata !DIExpression()), !dbg !9 + ret void +} + +declare void @llvm.dbg.value(metadata, metadata, metadata) + +!llvm.dbg.cu = !{!1} +!llvm.module.flags = !{!0} +!0 = !{i32 2, !"Debug Info Version", i32 3} +!1 = distinct !DICompileUnit(language: DW_LANG_C, file: !2) +!2 = !DIFile(filename: "debug-info.ll", directory: "/") +!7 = !DILocalVariable(scope: !8) +!8 = distinct !DISubprogram(name: "noname_variable", scope: !2, file: !2, unit: !1); +!9 = !DILocation(line: 1, column: 2, scope: !8) diff --git a/mlir/test/Target/LLVMIR/llvmir-debug.mlir b/mlir/test/Target/LLVMIR/llvmir-debug.mlir index 6e07b685abd9..a628349951e6 100644 --- a/mlir/test/Target/LLVMIR/llvmir-debug.mlir +++ b/mlir/test/Target/LLVMIR/llvmir-debug.mlir @@ -72,6 +72,7 @@ llvm.func @func_no_debug() { #blockScope = #llvm.di_lexical_block #variable = #llvm.di_local_variable #variableAddr = #llvm.di_local_variable +#noNameVariable = #llvm.di_local_variable #spType1 = #llvm.di_subroutine_type #sp1 = #llvm.di_subprogram< @@ -89,9 +90,11 @@ llvm.func @func_with_debug(%arg: i64) { // CHECK: call void @llvm.dbg.value(metadata i64 %[[ARG]], metadata ![[VAR_LOC:[0-9]+]], metadata !DIExpression()) // CHECK: call void @llvm.dbg.addr(metadata ptr %[[ALLOC]], metadata ![[ADDR_LOC:[0-9]+]], metadata !DIExpression()) // CHECK: call void @llvm.dbg.declare(metadata ptr %[[ALLOC]], metadata ![[ADDR_LOC]], metadata !DIExpression()) + // CHECK: call void @llvm.dbg.value(metadata i64 %[[ARG]], metadata ![[NO_NAME_VAR:[0-9]+]], metadata !DIExpression()) llvm.intr.dbg.value #variable = %arg : i64 llvm.intr.dbg.addr #variableAddr = %alloc : !llvm.ptr llvm.intr.dbg.declare #variableAddr = %alloc : !llvm.ptr + llvm.intr.dbg.value #noNameVariable= %arg : i64 // CHECK: call void @func_no_debug(), !dbg ![[CALLSITE_LOC:[0-9]+]] llvm.call @func_no_debug() : () -> () loc(callsite("mysource.cc":3:4 at "mysource.cc":5:6)) @@ -139,6 +142,7 @@ llvm.func @empty_types() { // CHECK: ![[VAR_SCOPE]] = distinct !DILexicalBlockFile(scope: ![[FUNC_LOC]], file: ![[CU_FILE_LOC]], discriminator: 0) // CHECK: ![[ADDR_LOC]] = !DILocalVariable(name: "alloc", scope: ![[BLOCK_LOC:.*]]) // CHECK: ![[BLOCK_LOC]] = distinct !DILexicalBlock(scope: ![[FUNC_LOC]]) +// CHECK: ![[NO_NAME_VAR]] = !DILocalVariable(scope: ![[BLOCK_LOC]]) // CHECK-DAG: ![[CALLSITE_LOC]] = !DILocation(line: 3, column: 4, // CHECK-DAG: ![[FILE_LOC]] = !DILocation(line: 1, column: 2, From cdfbd1274cce41a85c8447e618d1bf822c443ca9 Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Thu, 13 Jul 2023 14:54:43 +0200 Subject: [PATCH 099/147] [mlir][llvm] Add missing bit manipulation intrs This commit adds three missing bit manipulation intrinsics. LangRef: https://llvm.org/docs/LangRef.html#bit-manipulation-intrinsics Reviewed By: gysit Differential Revision: https://reviews.llvm.org/D147711 (cherry picked from commit 5b7881b0b3f0d385d3b6b6cb98cc245bd72821df) --- .../mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td | 29 +++++++++++++---- mlir/test/Target/LLVMIR/Import/intrinsic.ll | 32 +++++++++++++++++++ .../test/Target/LLVMIR/llvmir-intrinsics.mlir | 27 ++++++++++++++++ mlir/test/Target/LLVMIR/llvmir-invalid.mlir | 2 +- 4 files changed, 82 insertions(+), 8 deletions(-) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td index a90b5f64b2cb..1620c7f6b670 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td @@ -58,19 +58,31 @@ class LLVM_BinarySameArgsIntrOpF traits = []> : let arguments = !con(commonArgs, fmfArg); } -class LLVM_TernarySameArgsIntrOpF traits = []> : +class LLVM_TernarySameArgsIntrOpBase traits = [], bit requiresFastmath = 0> : LLVM_OneResultIntrOp { - let arguments = (ins LLVM_ScalarOrVectorOf:$a, - LLVM_ScalarOrVectorOf:$b, - LLVM_ScalarOrVectorOf:$c, - DefaultValuedAttr:$fastmathFlags); + requiresFastmath> { + dag commonArgs = (ins LLVM_ScalarOrVectorOf:$a, + LLVM_ScalarOrVectorOf:$b, + LLVM_ScalarOrVectorOf:$c); let assemblyFormat = "`(` operands `)` custom(attr-dict) `:` " "functional-type(operands, results)"; } +class LLVM_TernarySameArgsIntrOpI traits = []> : + LLVM_TernarySameArgsIntrOpBase { + let arguments = commonArgs; +} + +class LLVM_TernarySameArgsIntrOpF traits = []> : + LLVM_TernarySameArgsIntrOpBase { + dag fmfArg = ( + ins DefaultValuedAttr:$fastmathFlags); + let arguments = !con(commonArgs, fmfArg); +} + class LLVM_CountZerosIntrOp traits = []> : LLVM_OneResultIntrOp { @@ -118,9 +130,12 @@ def LLVM_PowIOp : LLVM_OneResultIntrOp<"powi", [], [0,1], "functional-type(operands, results)"; } def LLVM_BitReverseOp : LLVM_UnaryIntrOpI<"bitreverse">; +def LLVM_BitSwapOp : LLVM_UnaryIntrOpI<"bswap">; def LLVM_CountLeadingZerosOp : LLVM_CountZerosIntrOp<"ctlz">; def LLVM_CountTrailingZerosOp : LLVM_CountZerosIntrOp<"cttz">; def LLVM_CtPopOp : LLVM_UnaryIntrOpI<"ctpop">; +def LLVM_FshlOp : LLVM_TernarySameArgsIntrOpI<"fshl">; +def LLVM_FshrOp : LLVM_TernarySameArgsIntrOpI<"fshr">; def LLVM_MaxNumOp : LLVM_BinarySameArgsIntrOpF<"maxnum">; def LLVM_MinNumOp : LLVM_BinarySameArgsIntrOpF<"minnum">; def LLVM_MaximumOp : LLVM_BinarySameArgsIntrOpF<"maximum">; diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll index 1bcafb7d5c23..12adf83ff730 100644 --- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll +++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll @@ -125,6 +125,14 @@ define void @bitreverse_test(i32 %0, <8 x i32> %1) { %4 = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %1) ret void } +; CHECK-LABEL: llvm.func @bitswap_test +define void @bitswap_test(i32 %0, <8 x i32> %1) { + ; CHECK: llvm.intr.bswap(%{{.*}}) : (i32) -> i32 + %3 = call i32 @llvm.bswap.i32(i32 %0) + ; CHECK: llvm.intr.bswap(%{{.*}}) : (vector<8xi32>) -> vector<8xi32> + %4 = call <8 x i32> @llvm.bswap.v8i32(<8 x i32> %1) + ret void +} ; CHECK-LABEL: llvm.func @ctlz_test define void @ctlz_test(i32 %0, <8 x i32> %1) { ; CHECK: %[[FALSE:.+]] = llvm.mlir.constant(false) : i1 @@ -153,6 +161,24 @@ define void @ctpop_test(i32 %0, <8 x i32> %1) { ret void } +; CHECK-LABEL: llvm.func @fshl_test +define void @fshl_test(i32 %0, i32 %1, i32 %2, <8 x i32> %3, <8 x i32> %4, <8 x i32> %5) { + ; CHECK: llvm.intr.fshl(%{{.*}}, %{{.*}}, %{{.*}}) : (i32, i32, i32) -> i32 + %7 = call i32 @llvm.fshl.i32(i32 %0, i32 %1, i32 %2) + ; CHECK: llvm.intr.fshl(%{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi32>, vector<8xi32>, vector<8xi32>) -> vector<8xi32> + %8 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %3, <8 x i32> %4, <8 x i32> %5) + ret void +} + +; CHECK-LABEL: llvm.func @fshr_test +define void @fshr_test(i32 %0, i32 %1, i32 %2, <8 x i32> %3, <8 x i32> %4, <8 x i32> %5) { + ; CHECK: llvm.intr.fshr(%{{.*}}, %{{.*}}, %{{.*}}) : (i32, i32, i32) -> i32 + %7 = call i32 @llvm.fshr.i32(i32 %0, i32 %1, i32 %2) + ; CHECK: llvm.intr.fshr(%{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi32>, vector<8xi32>, vector<8xi32>) -> vector<8xi32> + %8 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %3, <8 x i32> %4, <8 x i32> %5) + ret void +} + ; CHECK-LABEL: llvm.func @maximum_test define void @maximum_test(float %0, float %1, <8 x float> %2, <8 x float> %3) { ; CHECK: llvm.intr.maximum(%{{.*}}, %{{.*}}) : (f32, f32) -> f32 @@ -663,12 +689,18 @@ declare float @llvm.pow.f32(float, float) declare <8 x float> @llvm.pow.v8f32(<8 x float>, <8 x float>) declare i32 @llvm.bitreverse.i32(i32) declare <8 x i32> @llvm.bitreverse.v8i32(<8 x i32>) +declare i32 @llvm.bswap.i32(i32) +declare <8 x i32> @llvm.bswap.v8i32(<8 x i32>) declare i32 @llvm.ctlz.i32(i32, i1 immarg) declare <8 x i32> @llvm.ctlz.v8i32(<8 x i32>, i1 immarg) declare i32 @llvm.cttz.i32(i32, i1 immarg) declare <8 x i32> @llvm.cttz.v8i32(<8 x i32>, i1 immarg) declare i32 @llvm.ctpop.i32(i32) declare <8 x i32> @llvm.ctpop.v8i32(<8 x i32>) +declare i32 @llvm.fshl.i32(i32, i32, i32) +declare <8 x i32> @llvm.fshl.v8i32(<8 x i32>, <8 x i32>, <8 x i32>) +declare i32 @llvm.fshr.i32(i32, i32, i32) +declare <8 x i32> @llvm.fshr.v8i32(<8 x i32>, <8 x i32>, <8 x i32>) declare float @llvm.maximum.f32(float, float) declare <8 x float> @llvm.maximum.v8f32(<8 x float>, <8 x float>) declare float @llvm.minimum.f32(float, float) diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir index e6ff83f15ba7..2db5b0bdae34 100644 --- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir +++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir @@ -143,6 +143,15 @@ llvm.func @bitreverse_test(%arg0: i32, %arg1: vector<8xi32>) { llvm.return } +// CHECK-LABEL: @bitswap_test +llvm.func @bitswap_test(%arg0: i32, %arg1: vector<8xi32>) { + // CHECK: call i32 @llvm.bswap.i32 + "llvm.intr.bswap"(%arg0) : (i32) -> i32 + // CHECK: call <8 x i32> @llvm.bswap.v8i32 + "llvm.intr.bswap"(%arg1) : (vector<8xi32>) -> vector<8xi32> + llvm.return +} + // CHECK-LABEL: @ctlz_test llvm.func @ctlz_test(%arg0: i32, %arg1: vector<8xi32>) { %i1 = llvm.mlir.constant(false) : i1 @@ -172,6 +181,24 @@ llvm.func @ctpop_test(%arg0: i32, %arg1: vector<8xi32>) { llvm.return } +// CHECK-LABEL: @fshl_test +llvm.func @fshl_test(%arg0: i32, %arg1: i32, %arg2: i32, %arg3: vector<8xi32>, %arg4: vector<8xi32>, %arg5: vector<8xi32>) { + // CHECK: call i32 @llvm.fshl.i32 + "llvm.intr.fshl"(%arg0, %arg1, %arg2) : (i32, i32, i32) -> i32 + // CHECK: call <8 x i32> @llvm.fshl.v8i32 + "llvm.intr.fshl"(%arg3, %arg4, %arg5) : (vector<8xi32>, vector<8xi32>, vector<8xi32>) -> vector<8xi32> + llvm.return +} + +// CHECK-LABEL: @fshr_test +llvm.func @fshr_test(%arg0: i32, %arg1: i32, %arg2: i32, %arg3: vector<8xi32>, %arg4: vector<8xi32>, %arg5: vector<8xi32>) { + // CHECK: call i32 @llvm.fshr.i32 + "llvm.intr.fshr"(%arg0, %arg1, %arg2) : (i32, i32, i32) -> i32 + // CHECK: call <8 x i32> @llvm.fshr.v8i32 + "llvm.intr.fshr"(%arg3, %arg4, %arg5) : (vector<8xi32>, vector<8xi32>, vector<8xi32>) -> vector<8xi32> + llvm.return +} + // CHECK-LABEL: @maximum_test llvm.func @maximum_test(%arg0: f32, %arg1: f32, %arg2: vector<8xf32>, %arg3: vector<8xf32>) { // CHECK: call float @llvm.maximum.f32 diff --git a/mlir/test/Target/LLVMIR/llvmir-invalid.mlir b/mlir/test/Target/LLVMIR/llvmir-invalid.mlir index 19e0b1501e09..bc06841040fc 100644 --- a/mlir/test/Target/LLVMIR/llvmir-invalid.mlir +++ b/mlir/test/Target/LLVMIR/llvmir-invalid.mlir @@ -91,7 +91,7 @@ llvm.func @binary_int_intr_wrong_type(%arg0 : i32, %arg1 : f32) -> f32 { // ----- llvm.func @ternary_float_intr_wrong_type(%arg0 : f32, %arg1 : f32, %arg2 : i32) -> f32 { - // expected-error @below{{op operand #2 must be floating-point or LLVM dialect-compatible vector of floating-point}} + // expected-error @below{{op operand #2 must be floating point LLVM type or LLVM dialect-compatible vector of floating point LLVM type}} %0 = "llvm.intr.fma"(%arg0, %arg1, %arg2) : (f32, f32, i32) -> f32 llvm.return %0 : f32 } From b526bba4fa74cfa2582168713b0c1b5fc09ef471 Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Thu, 13 Jul 2023 14:57:21 +0200 Subject: [PATCH 100/147] [mlir][llvm] Fix bswap naming (NFC) As pointed out in https://reviews.llvm.org/D147711#inline-1427665, the bswap intrinsic is called byteswap, not bitswap. (cherry picked from commit 3f4d16018d51c9deff8e7deb01cefcf52e966cd3) --- mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td | 2 +- mlir/test/Target/LLVMIR/Import/intrinsic.ll | 4 ++-- mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td index 1620c7f6b670..3e9fb426006d 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td @@ -130,7 +130,7 @@ def LLVM_PowIOp : LLVM_OneResultIntrOp<"powi", [], [0,1], "functional-type(operands, results)"; } def LLVM_BitReverseOp : LLVM_UnaryIntrOpI<"bitreverse">; -def LLVM_BitSwapOp : LLVM_UnaryIntrOpI<"bswap">; +def LLVM_ByteSwapOp : LLVM_UnaryIntrOpI<"bswap">; def LLVM_CountLeadingZerosOp : LLVM_CountZerosIntrOp<"ctlz">; def LLVM_CountTrailingZerosOp : LLVM_CountZerosIntrOp<"cttz">; def LLVM_CtPopOp : LLVM_UnaryIntrOpI<"ctpop">; diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll index 12adf83ff730..be0c28d98b42 100644 --- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll +++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll @@ -125,8 +125,8 @@ define void @bitreverse_test(i32 %0, <8 x i32> %1) { %4 = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %1) ret void } -; CHECK-LABEL: llvm.func @bitswap_test -define void @bitswap_test(i32 %0, <8 x i32> %1) { +; CHECK-LABEL: llvm.func @byteswap_test +define void @byteswap_test(i32 %0, <8 x i32> %1) { ; CHECK: llvm.intr.bswap(%{{.*}}) : (i32) -> i32 %3 = call i32 @llvm.bswap.i32(i32 %0) ; CHECK: llvm.intr.bswap(%{{.*}}) : (vector<8xi32>) -> vector<8xi32> diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir index 2db5b0bdae34..b2de923764d2 100644 --- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir +++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir @@ -143,8 +143,8 @@ llvm.func @bitreverse_test(%arg0: i32, %arg1: vector<8xi32>) { llvm.return } -// CHECK-LABEL: @bitswap_test -llvm.func @bitswap_test(%arg0: i32, %arg1: vector<8xi32>) { +// CHECK-LABEL: @byteswap_test +llvm.func @byteswap_test(%arg0: i32, %arg1: vector<8xi32>) { // CHECK: call i32 @llvm.bswap.i32 "llvm.intr.bswap"(%arg0) : (i32) -> i32 // CHECK: call <8 x i32> @llvm.bswap.v8i32 From f53c2fe251e65ae45cd3d64b8135075ff6323591 Mon Sep 17 00:00:00 2001 From: Victor Perez Date: Thu, 13 Jul 2023 15:12:15 +0200 Subject: [PATCH 101/147] [mlir][llvm] Add trap intrinsics to the dialect Define llvm.trap, llvm.debugtrap, and llvm.ubsantrap intrinsics in the llvm dialect. Signed-off-by: Victor Perez Differential Revision: https://reviews.llvm.org/D149574 (cherry picked from commit b8064c1a039ef819a2f2ce0ca79990666b23366f) --- .../mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td | 20 +++++++++++++++++++ mlir/test/Target/LLVMIR/Import/intrinsic.ll | 14 +++++++++++++ .../test/Target/LLVMIR/llvmir-intrinsics.mlir | 14 +++++++++++++ 3 files changed, 48 insertions(+) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td index 3e9fb426006d..413516e1a771 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td @@ -664,6 +664,26 @@ def LLVM_masked_compressstore let arguments = (ins LLVM_AnyVector, LLVM_AnyPointer, LLVM_VectorOf); } +// +// Trap intrinsics. +// + +def LLVM_Trap : LLVM_ZeroResultIntrOp<"trap">; + +def LLVM_DebugTrap : LLVM_ZeroResultIntrOp<"debugtrap">; + +def LLVM_UBSanTrap : LLVM_ZeroResultIntrOp<"ubsantrap"> { + let arguments = (ins I8Attr:$failureKind); + string llvmBuilder = [{ + builder.CreateIntrinsic( + llvm::Intrinsic::ubsantrap, {}, {builder.getInt8($failureKind)}); + }]; + string mlirBuilder = [{ + $_op = + $_builder.create($_location, $_int_attr($failureKind)); + }]; +} + /// Create a call to vscale intrinsic. def LLVM_vscale : LLVM_IntrOp<"vscale", [0], [], [], 1>; diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll index be0c28d98b42..2115a1d9bc81 100644 --- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll +++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll @@ -355,6 +355,17 @@ define void @masked_expand_compress_intrinsics(ptr %0, <7 x i1> %1, <7 x float> ret void } +; CHECK-LABEL: llvm.func @trap_intrinsics +define void @trap_intrinsics() { + ; CHECK: "llvm.intr.trap"() : () -> () + call void @llvm.trap() + ; CHECK: "llvm.intr.debugtrap"() : () -> () + call void @llvm.debugtrap() + ; CHECK: "llvm.intr.ubsantrap"() {failureKind = 1 : i8} : () -> () + call void @llvm.ubsantrap(i8 1) + ret void +} + ; CHECK-LABEL: llvm.func @memcpy_test define void @memcpy_test(i32 %0, ptr %1, ptr %2) { ; CHECK: %[[FALSE:.+]] = llvm.mlir.constant(false) : i1 @@ -741,6 +752,9 @@ declare <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr>, i32 immarg, <7 x i declare void @llvm.masked.scatter.v7f32.v7p0(<7 x float>, <7 x ptr>, i32 immarg, <7 x i1>) declare <7 x float> @llvm.masked.expandload.v7f32(ptr, <7 x i1>, <7 x float>) declare void @llvm.masked.compressstore.v7f32(<7 x float>, ptr, <7 x i1>) +declare void @llvm.trap() +declare void @llvm.debugtrap() +declare void @llvm.ubsantrap(i8 immarg) declare void @llvm.memcpy.p0.p0.i32(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i32, i1 immarg) declare void @llvm.memcpy.inline.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64 immarg, i1 immarg) declare void @llvm.memmove.p0.p0.i32(ptr nocapture writeonly, ptr nocapture readonly, i32, i1 immarg) diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir index b2de923764d2..a841a9d6c687 100644 --- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir +++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir @@ -374,6 +374,17 @@ llvm.func @masked_expand_compress_intrinsics(%ptr: !llvm.ptr, %mask: vector llvm.return } +// CHECK-LABEL: @trap_intrinsics +llvm.func @trap_intrinsics() { + // CHECK: call void @llvm.trap() + "llvm.intr.trap"() : () -> () + // CHECK: call void @llvm.debugtrap() + "llvm.intr.debugtrap"() : () -> () + // CHECK: call void @llvm.ubsantrap(i8 1) + "llvm.intr.ubsantrap"() {failureKind = 1 : i8} : () -> () + llvm.return +} + // CHECK-LABEL: @memcpy_test llvm.func @memcpy_test(%arg0: i32, %arg2: !llvm.ptr, %arg3: !llvm.ptr) { %i1 = llvm.mlir.constant(false) : i1 @@ -785,6 +796,9 @@ llvm.func @lifetime(%p: !llvm.ptr) { // CHECK-DAG: declare void @llvm.masked.scatter.v7f32.v7p0(<7 x float>, <7 x ptr>, i32 immarg, <7 x i1>) // CHECK-DAG: declare <7 x float> @llvm.masked.expandload.v7f32(ptr nocapture, <7 x i1>, <7 x float>) // CHECK-DAG: declare void @llvm.masked.compressstore.v7f32(<7 x float>, ptr nocapture, <7 x i1>) +// CHECK-DAG: declare void @llvm.trap() +// CHECK-DAG: declare void @llvm.debugtrap() +// CHECK-DAG: declare void @llvm.ubsantrap(i8 immarg) // CHECK-DAG: declare void @llvm.memcpy.p0.p0.i32(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i32, i1 immarg) // CHECK-DAG: declare void @llvm.memcpy.inline.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64 immarg, i1 immarg) // CHECK-DAG: declare { i32, i1 } @llvm.sadd.with.overflow.i32(i32, i32) From 35ae20839e152993ed420a517c472149d8eaf9da Mon Sep 17 00:00:00 2001 From: Tobias Gysi Date: Tue, 18 Jul 2023 10:11:07 +0200 Subject: [PATCH 102/147] [mlir][llvm] Saturation arithmetic intrinsics. Add the saturation arithmetic intrinsics to the LLVM dialect. Reviewed By: Dinistro Differential Revision: https://reviews.llvm.org/D150676 (cherry picked from commit c04cf58dfc5430f0c82c8ef42c3a8cb43f84020b) --- .../mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td | 11 ++++ mlir/test/Target/LLVMIR/Import/intrinsic.ll | 66 +++++++++++++++++++ .../test/Target/LLVMIR/llvmir-intrinsics.mlir | 66 +++++++++++++++++++ 3 files changed, 143 insertions(+) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td index 413516e1a771..dbb565dcd1c9 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td @@ -276,6 +276,17 @@ def LLVM_CoroResumeOp : LLVM_IntrOp<"coro.resume", [], [], [], 0> { let assemblyFormat = "$handle attr-dict"; } +// +// Saturation Arithmetic Intrinsics. +// + +def LLVM_SAddSat : LLVM_BinarySameArgsIntrOpI<"sadd.sat">; +def LLVM_UAddSat : LLVM_BinarySameArgsIntrOpI<"uadd.sat">; +def LLVM_SSubSat : LLVM_BinarySameArgsIntrOpI<"ssub.sat">; +def LLVM_USubSat : LLVM_BinarySameArgsIntrOpI<"usub.sat">; +def LLVM_SSHLSat : LLVM_BinarySameArgsIntrOpI<"sshl.sat">; +def LLVM_USHLSat : LLVM_BinarySameArgsIntrOpI<"ushl.sat">; + // // Debug function intrinsics. // diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll index 2115a1d9bc81..9bceb000b104 100644 --- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll +++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll @@ -447,6 +447,60 @@ define void @umul_with_overflow_test(i32 %0, i32 %1, <8 x i32> %2, <8 x i32> %3) ret void } +; CHECK-LABEL: llvm.func @sadd_sat_test +define void @sadd_sat_test(i32 %0, i32 %1, <8 x i32> %2, <8 x i32> %3) { + ; CHECK: llvm.intr.sadd.sat(%{{.*}}, %{{.*}}) : (i32, i32) -> i32 + %5 = call i32 @llvm.sadd.sat.i32(i32 %0, i32 %1) + ; CHECK: llvm.intr.sadd.sat(%{{.*}}, %{{.*}}) : (vector<8xi32>, vector<8xi32>) -> vector<8xi32> + %6 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> %2, <8 x i32> %3) + ret void +} + +; CHECK-LABEL: llvm.func @uadd_sat_test +define void @uadd_sat_test(i32 %0, i32 %1, <8 x i32> %2, <8 x i32> %3) { + ; CHECK: llvm.intr.uadd.sat(%{{.*}}, %{{.*}}) : (i32, i32) -> i32 + %5 = call i32 @llvm.uadd.sat.i32(i32 %0, i32 %1) + ; CHECK: llvm.intr.uadd.sat(%{{.*}}, %{{.*}}) : (vector<8xi32>, vector<8xi32>) -> vector<8xi32> + %6 = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> %2, <8 x i32> %3) + ret void +} + +; CHECK-LABEL: llvm.func @ssub_sat_test +define void @ssub_sat_test(i32 %0, i32 %1, <8 x i32> %2, <8 x i32> %3) { + ; CHECK: llvm.intr.ssub.sat(%{{.*}}, %{{.*}}) : (i32, i32) -> i32 + %5 = call i32 @llvm.ssub.sat.i32(i32 %0, i32 %1) + ; CHECK: llvm.intr.ssub.sat(%{{.*}}, %{{.*}}) : (vector<8xi32>, vector<8xi32>) -> vector<8xi32> + %6 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> %2, <8 x i32> %3) + ret void +} + +; CHECK-LABEL: llvm.func @usub_sat_test +define void @usub_sat_test(i32 %0, i32 %1, <8 x i32> %2, <8 x i32> %3) { + ; CHECK: llvm.intr.usub.sat(%{{.*}}, %{{.*}}) : (i32, i32) -> i32 + %5 = call i32 @llvm.usub.sat.i32(i32 %0, i32 %1) + ; CHECK: llvm.intr.usub.sat(%{{.*}}, %{{.*}}) : (vector<8xi32>, vector<8xi32>) -> vector<8xi32> + %6 = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> %2, <8 x i32> %3) + ret void +} + +; CHECK-LABEL: llvm.func @sshl_sat_test +define void @sshl_sat_test(i32 %0, i32 %1, <8 x i32> %2, <8 x i32> %3) { + ; CHECK: llvm.intr.sshl.sat(%{{.*}}, %{{.*}}) : (i32, i32) -> i32 + %5 = call i32 @llvm.sshl.sat.i32(i32 %0, i32 %1) + ; CHECK: llvm.intr.sshl.sat(%{{.*}}, %{{.*}}) : (vector<8xi32>, vector<8xi32>) -> vector<8xi32> + %6 = call <8 x i32> @llvm.sshl.sat.v8i32(<8 x i32> %2, <8 x i32> %3) + ret void +} + +; CHECK-LABEL: llvm.func @ushl_sat_test +define void @ushl_sat_test(i32 %0, i32 %1, <8 x i32> %2, <8 x i32> %3) { + ; CHECK: llvm.intr.ushl.sat(%{{.*}}, %{{.*}}) : (i32, i32) -> i32 + %5 = call i32 @llvm.ushl.sat.i32(i32 %0, i32 %1) + ; CHECK: llvm.intr.ushl.sat(%{{.*}}, %{{.*}}) : (vector<8xi32>, vector<8xi32>) -> vector<8xi32> + %6 = call <8 x i32> @llvm.ushl.sat.v8i32(<8 x i32> %2, <8 x i32> %3) + ret void +} + ; CHECK-LABEL: llvm.func @va_intrinsics_test define void @va_intrinsics_test(ptr %0, ptr %1) { ; CHECK: llvm.intr.vastart %{{.*}} @@ -771,6 +825,18 @@ declare { i32, i1 } @llvm.smul.with.overflow.i32(i32, i32) declare { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32>, <8 x i32>) declare { i32, i1 } @llvm.umul.with.overflow.i32(i32, i32) declare { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32>, <8 x i32>) +declare i32 @llvm.sadd.sat.i32(i32, i32) +declare <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32>, <8 x i32>) +declare i32 @llvm.uadd.sat.i32(i32, i32) +declare <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32>, <8 x i32>) +declare i32 @llvm.ssub.sat.i32(i32, i32) +declare <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32>, <8 x i32>) +declare i32 @llvm.usub.sat.i32(i32, i32) +declare <8 x i32> @llvm.usub.sat.v8i32(<8 x i32>, <8 x i32>) +declare i32 @llvm.sshl.sat.i32(i32, i32) +declare <8 x i32> @llvm.sshl.sat.v8i32(<8 x i32>, <8 x i32>) +declare i32 @llvm.ushl.sat.i32(i32, i32) +declare <8 x i32> @llvm.ushl.sat.v8i32(<8 x i32>, <8 x i32>) declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) declare ptr @llvm.coro.begin(token, ptr writeonly) declare i64 @llvm.coro.size.i64() diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir index a841a9d6c687..1dc46159dcce 100644 --- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir +++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir @@ -466,6 +466,60 @@ llvm.func @umul_with_overflow_test(%arg0: i32, %arg1: i32, %arg2: vector<8xi32>, llvm.return } +// CHECK-LABEL: @sadd_sat_test +llvm.func @sadd_sat_test(%arg0: i32, %arg1: i32, %arg2: vector<8xi32>, %arg3: vector<8xi32>) { + // CHECK: call i32 @llvm.sadd.sat.i32 + "llvm.intr.sadd.sat"(%arg0, %arg1) : (i32, i32) -> i32 + // CHECK: call <8 x i32> @llvm.sadd.sat.v8i32 + "llvm.intr.sadd.sat"(%arg2, %arg3) : (vector<8xi32>, vector<8xi32>) -> vector<8xi32> + llvm.return +} + +// CHECK-LABEL: @uadd_sat_test +llvm.func @uadd_sat_test(%arg0: i32, %arg1: i32, %arg2: vector<8xi32>, %arg3: vector<8xi32>) { + // CHECK: call i32 @llvm.uadd.sat.i32 + "llvm.intr.uadd.sat"(%arg0, %arg1) : (i32, i32) -> i32 + // CHECK: call <8 x i32> @llvm.uadd.sat.v8i32 + "llvm.intr.uadd.sat"(%arg2, %arg3) : (vector<8xi32>, vector<8xi32>) -> vector<8xi32> + llvm.return +} + +// CHECK-LABEL: @ssub_sat_test +llvm.func @ssub_sat_test(%arg0: i32, %arg1: i32, %arg2: vector<8xi32>, %arg3: vector<8xi32>) { + // CHECK: call i32 @llvm.ssub.sat.i32 + "llvm.intr.ssub.sat"(%arg0, %arg1) : (i32, i32) -> i32 + // CHECK: call <8 x i32> @llvm.ssub.sat.v8i32 + "llvm.intr.ssub.sat"(%arg2, %arg3) : (vector<8xi32>, vector<8xi32>) -> vector<8xi32> + llvm.return +} + +// CHECK-LABEL: @usub_sat_test +llvm.func @usub_sat_test(%arg0: i32, %arg1: i32, %arg2: vector<8xi32>, %arg3: vector<8xi32>) { + // CHECK: call i32 @llvm.usub.sat.i32 + "llvm.intr.usub.sat"(%arg0, %arg1) : (i32, i32) -> i32 + // CHECK: call <8 x i32> @llvm.usub.sat.v8i32 + "llvm.intr.usub.sat"(%arg2, %arg3) : (vector<8xi32>, vector<8xi32>) -> vector<8xi32> + llvm.return +} + +// CHECK-LABEL: @sshl_sat_test +llvm.func @sshl_sat_test(%arg0: i32, %arg1: i32, %arg2: vector<8xi32>, %arg3: vector<8xi32>) { + // CHECK: call i32 @llvm.sshl.sat.i32 + "llvm.intr.sshl.sat"(%arg0, %arg1) : (i32, i32) -> i32 + // CHECK: call <8 x i32> @llvm.sshl.sat.v8i32 + "llvm.intr.sshl.sat"(%arg2, %arg3) : (vector<8xi32>, vector<8xi32>) -> vector<8xi32> + llvm.return +} + +// CHECK-LABEL: @ushl_sat_test +llvm.func @ushl_sat_test(%arg0: i32, %arg1: i32, %arg2: vector<8xi32>, %arg3: vector<8xi32>) { + // CHECK: call i32 @llvm.ushl.sat.i32 + "llvm.intr.ushl.sat"(%arg0, %arg1) : (i32, i32) -> i32 + // CHECK: call <8 x i32> @llvm.ushl.sat.v8i32 + "llvm.intr.ushl.sat"(%arg2, %arg3) : (vector<8xi32>, vector<8xi32>) -> vector<8xi32> + llvm.return +} + // CHECK-LABEL: @coro_id llvm.func @coro_id(%arg0: i32, %arg1: !llvm.ptr) { // CHECK: call token @llvm.coro.id @@ -811,6 +865,18 @@ llvm.func @lifetime(%p: !llvm.ptr) { // CHECK-DAG: declare { <8 x i32>, <8 x i1> } @llvm.usub.with.overflow.v8i32(<8 x i32>, <8 x i32>) #0 // CHECK-DAG: declare { i32, i1 } @llvm.umul.with.overflow.i32(i32, i32) // CHECK-DAG: declare { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32>, <8 x i32>) #0 +// CHECK-DAG: declare i32 @llvm.sadd.sat.i32(i32, i32) +// CHECK-DAG: declare <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32>, <8 x i32>) +// CHECK-DAG: declare i32 @llvm.uadd.sat.i32(i32, i32) +// CHECK-DAG: declare <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32>, <8 x i32>) +// CHECK-DAG: declare i32 @llvm.ssub.sat.i32(i32, i32) +// CHECK-DAG: declare <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32>, <8 x i32>) +// CHECK-DAG: declare i32 @llvm.usub.sat.i32(i32, i32) +// CHECK-DAG: declare <8 x i32> @llvm.usub.sat.v8i32(<8 x i32>, <8 x i32>) +// CHECK-DAG: declare i32 @llvm.sshl.sat.i32(i32, i32) +// CHECK-DAG: declare <8 x i32> @llvm.sshl.sat.v8i32(<8 x i32>, <8 x i32>) +// CHECK-DAG: declare i32 @llvm.ushl.sat.i32(i32, i32) +// CHECK-DAG: declare <8 x i32> @llvm.ushl.sat.v8i32(<8 x i32>, <8 x i32>) // CHECK-DAG: declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) // CHECK-DAG: declare ptr @llvm.coro.begin(token, ptr writeonly) // CHECK-DAG: declare i64 @llvm.coro.size.i64() From 5cdd836ddfd0973be16d3520a01dfda76d0b8edc Mon Sep 17 00:00:00 2001 From: Andrea Gussoni Date: Mon, 8 Jan 2024 14:49:33 +0100 Subject: [PATCH 103/147] GenericDomTreeConstruction: add `constexpr` Refactor the code and add `constexpr`esness in order to avoid and instantion of `llvm::Inverse` `GraphTraits` when using only a dominator, and not a postdominator. This is useful in order to compute a `DominatorTreeBase` class on a `GenericGraph` built with simple `ForwardNode`s. --- .../llvm/Support/GenericDomTreeConstruction.h | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/llvm/include/llvm/Support/GenericDomTreeConstruction.h b/llvm/include/llvm/Support/GenericDomTreeConstruction.h index 4f695eccb4cb..f48a1b56bb8e 100644 --- a/llvm/include/llvm/Support/GenericDomTreeConstruction.h +++ b/llvm/include/llvm/Support/GenericDomTreeConstruction.h @@ -347,19 +347,10 @@ struct SemiNCAInfoOnView { return GraphTraits::getEntryNode(DT.Parent); } - // Finds all roots without relaying on the set of roots already stored in the - // tree. - // We define roots to be some non-redundant set of the CFG nodes - static RootsT FindRoots(const DomTreeT &DT, BatchUpdatePtr BUI) { - assert(DT.Parent && "Parent pointer is not set"); + // We factor the code of `FindRoots` for postdominators in this static + // function so we can constexpr away it when instantiating it on a dominator. + static RootsT FindRootsPostDom(const DomTreeT &DT, BatchUpdatePtr BUI) { RootsT Roots; - - // For dominators, function entry CFG node is always a tree root node. - if (!IsPostDom) { - Roots.push_back(GetEntryNode(DT)); - return Roots; - } - SemiNCAInfoOnView SNCA(BUI); // PostDominatorTree always has a virtual root. @@ -498,6 +489,24 @@ struct SemiNCAInfoOnView { return Roots; } + // Finds all roots without relaying on the set of roots already stored in the + // tree. + // We define roots to be some non-redundant set of the CFG nodes + static RootsT FindRoots(const DomTreeT &DT, BatchUpdatePtr BUI) { + assert(DT.Parent && "Parent pointer is not set"); + + // We avoid compiling the code handling postdominators, when instantiating a + // dominator. + if constexpr (IsPostDom) { + return FindRootsPostDom(DT, BUI); + } + + // For dominators, function entry CFG node is always a tree root node. + RootsT Roots; + Roots.push_back(GetEntryNode(DT)); + return Roots; + } + // This function only makes sense for postdominators. // We define roots to be some set of CFG nodes where (reverse) DFS walks have // to start in order to visit all the CFG nodes (including the From fb136e71d27ee4b8f4fa4ddbd3b86232db89a166 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20B=C3=B6ck?= Date: Sat, 26 Aug 2023 16:10:52 +0200 Subject: [PATCH 104/147] [mlir] Fix infinite recursion in alias initializer The alias initializer keeps a list of child indices around. When an alias is then marked as non-deferrable, all children are also marked non-deferrable. This is currently done naively which leads to an infinite recursion if using mutable types or attributes containing a cycle. This patch fixes this by adding an early return if the alias is already marked non-deferrable. Since this function is the only way to mark an alias as non-deferrable, it is guaranteed that if it is marked non-deferrable, all its children are as well, and it is not required to walk all the children. This incidentally makes the non-deferrable marking also `O(n)` instead of `O(n^2)` (although not performance sensitive obviously). Differential Revision: https://reviews.llvm.org/D158932 Partial cherry-pick of commit de3f7e2f0fb4363c17eec73ce79ca30e221ea844. This commit was cherry-picked partially to avoid having to cherry-pick all the previous commits on which it depends. When upgrading, discarding this commit in favour of the full original should be fine. --- mlir/lib/IR/AsmPrinter.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mlir/lib/IR/AsmPrinter.cpp b/mlir/lib/IR/AsmPrinter.cpp index 0981b3b1cbe2..6c1cc671ab56 100644 --- a/mlir/lib/IR/AsmPrinter.cpp +++ b/mlir/lib/IR/AsmPrinter.cpp @@ -1028,6 +1028,12 @@ std::pair AliasInitializer::visitImpl( void AliasInitializer::markAliasNonDeferrable(size_t aliasIndex) { auto it = std::next(aliases.begin(), aliasIndex); + + // If already marked non-deferrable stop the recursion. + // All children should already be marked non-deferrable as well. + if (!it->second.canBeDeferred) + return; + it->second.canBeDeferred = false; // Propagate the non-deferrable flag to any child aliases. From a1da679897a38859105953ea6d3858469a75200b Mon Sep 17 00:00:00 2001 From: Giacomo Vercesi Date: Wed, 11 Sep 2024 13:07:19 +0200 Subject: [PATCH 105/147] ErrorHandling: reserve memory for OOM At statup reserve some memory via mmap to be later released when `report_bad_alloc_error` is called to prevent the stacktrace handler from running out of memory. --- llvm/lib/Support/ErrorHandling.cpp | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Support/ErrorHandling.cpp b/llvm/lib/Support/ErrorHandling.cpp index b8b3b7424ac6..7b9ecaf4f75a 100644 --- a/llvm/lib/Support/ErrorHandling.cpp +++ b/llvm/lib/Support/ErrorHandling.cpp @@ -19,6 +19,7 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/Errc.h" #include "llvm/Support/Error.h" +#include "llvm/Support/Memory.h" #include "llvm/Support/Process.h" #include "llvm/Support/Signals.h" #include "llvm/Support/Threading.h" @@ -39,12 +40,37 @@ using namespace llvm; +class MemoryReserver { +private: + std::error_code EC; + sys::MemoryBlock Ptr; + +public: + MemoryReserver(size_t Pages): Ptr() { + using sys::Memory; + unsigned int PSize = sys::Process::getPageSizeEstimate(); + Ptr = Memory::allocateMappedMemory(PSize * Pages, nullptr, Memory::MF_READ | Memory::MF_WRITE, EC); + assert(!EC); + } + + ~MemoryReserver() { + release(); + } + + void release() { + EC = sys::Memory::releaseMappedMemory(Ptr); + assert(!EC); + } +}; + static fatal_error_handler_t ErrorHandler = nullptr; static void *ErrorHandlerUserData = nullptr; static fatal_error_handler_t BadAllocErrorHandler = nullptr; static void *BadAllocErrorHandlerUserData = nullptr; +static MemoryReserver OOMReservedMemory(1024); + #if LLVM_ENABLE_THREADS == 1 // Mutexes to synchronize installing error handlers and calling error handlers. // Do not use ManagedStatic, or that may allocate memory while attempting to @@ -152,6 +178,7 @@ void llvm::report_bad_alloc_error(const char *Reason, bool GenCrashDiag) { #if LLVM_ENABLE_THREADS == 1 std::lock_guard Lock(BadAllocErrorHandlerMutex); #endif + OOMReservedMemory.release(); Handler = BadAllocErrorHandler; HandlerData = BadAllocErrorHandlerUserData; } @@ -172,7 +199,8 @@ void llvm::report_bad_alloc_error(const char *Reason, bool GenCrashDiag) { (void)!::write(2, OOMMessage, strlen(OOMMessage)); (void)!::write(2, Reason, strlen(Reason)); (void)!::write(2, Newline, strlen(Newline)); - abort(); + sys::PrintStackTrace(llvm::errs()); + ::_Exit(235); #endif } From f0700622a1967d89f0b7b536090eca15f14efef6 Mon Sep 17 00:00:00 2001 From: Giacomo Vercesi Date: Mon, 16 Sep 2024 14:45:10 +0200 Subject: [PATCH 106/147] ErrorHandling: do not print stack twice on OOM Add a boolean to ErrorHandling which avoids PrintStackTrace being called twice. This avoids situations where the `report_bad_alloc_error` is called recursively due to memory exhaustion. --- llvm/lib/Support/ErrorHandling.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Support/ErrorHandling.cpp b/llvm/lib/Support/ErrorHandling.cpp index 7b9ecaf4f75a..1959966858c9 100644 --- a/llvm/lib/Support/ErrorHandling.cpp +++ b/llvm/lib/Support/ErrorHandling.cpp @@ -25,6 +25,7 @@ #include "llvm/Support/Threading.h" #include "llvm/Support/WindowsError.h" #include "llvm/Support/raw_ostream.h" +#include #include #include #include @@ -70,6 +71,7 @@ static fatal_error_handler_t BadAllocErrorHandler = nullptr; static void *BadAllocErrorHandlerUserData = nullptr; static MemoryReserver OOMReservedMemory(1024); +static std::atomic_bool BadAllocCalled(false); #if LLVM_ENABLE_THREADS == 1 // Mutexes to synchronize installing error handlers and calling error handlers. @@ -172,6 +174,7 @@ void llvm::remove_bad_alloc_error_handler() { void llvm::report_bad_alloc_error(const char *Reason, bool GenCrashDiag) { fatal_error_handler_t Handler = nullptr; void *HandlerData = nullptr; + bool BadAllocCalledPrev = BadAllocCalled.exchange(true); { // Only acquire the mutex while reading the handler, so as not to invoke a // user-supplied callback under a lock. @@ -199,7 +202,9 @@ void llvm::report_bad_alloc_error(const char *Reason, bool GenCrashDiag) { (void)!::write(2, OOMMessage, strlen(OOMMessage)); (void)!::write(2, Reason, strlen(Reason)); (void)!::write(2, Newline, strlen(Newline)); - sys::PrintStackTrace(llvm::errs()); + if (not BadAllocCalledPrev) { + sys::PrintStackTrace(llvm::errs()); + } ::_Exit(235); #endif } From 3c90f1f64a5d04927434a578c103b535dfa8e8cc Mon Sep 17 00:00:00 2001 From: Lauri Vasama Date: Wed, 30 Oct 2024 09:37:46 +0200 Subject: [PATCH 107/147] [STLExtras] Convert friend == and != to member See upstream commit 77f2ccb. This solves an ambiguity introduced by C++20 operator rewriting rules. The upstream commit was supposed to solve this as well, but did not seem to solve the problem when cherry-picked. It may be that it depends on other changes in LLVM. This commit can likely be discarded when LLVM is upgraded and the upstream commit is introduced instead. --- llvm/include/llvm/ADT/STLExtras.h | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h index c38864b5147e..a64ef504bfd5 100644 --- a/llvm/include/llvm/ADT/STLExtras.h +++ b/llvm/include/llvm/ADT/STLExtras.h @@ -1315,15 +1315,11 @@ class indexed_accessor_range_base { } /// Compare this range with another. - template - friend bool operator==(const indexed_accessor_range_base &lhs, - const OtherT &rhs) { - return std::equal(lhs.begin(), lhs.end(), rhs.begin(), rhs.end()); - } - template - friend bool operator!=(const indexed_accessor_range_base &lhs, - const OtherT &rhs) { - return !(lhs == rhs); + template bool operator==(const OtherT &rhs) const { + return std::equal(begin(), end(), rhs.begin(), rhs.end()); + } + template bool operator!=(const OtherT &rhs) const { + return !(*this == rhs); } /// Return the size of this range. From 22aff49c723a2bb100dc46a44dbde3e8e736b1fd Mon Sep 17 00:00:00 2001 From: Giacomo Vercesi Date: Thu, 14 Nov 2024 15:30:38 +0100 Subject: [PATCH 108/147] Add zstd support to llvm::parseIR Augment the `llvm::parseIR` to also work with .ll/.bc compressed with zstd. --- llvm/include/llvm/Support/Compression.h | 9 +++ llvm/lib/IRReader/IRReader.cpp | 38 +++++++++++- llvm/lib/Support/Compression.cpp | 81 +++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 3 deletions(-) diff --git a/llvm/include/llvm/Support/Compression.h b/llvm/include/llvm/Support/Compression.h index c3ba3274d6ed..e56be721a6aa 100644 --- a/llvm/include/llvm/Support/Compression.h +++ b/llvm/include/llvm/Support/Compression.h @@ -61,6 +61,9 @@ constexpr int BestSizeCompression = 12; bool isAvailable(); +// Will return true if the buffer begins with the zstd magic sequence +bool isZstd(ArrayRef Input); + void compress(ArrayRef Input, SmallVectorImpl &CompressedBuffer, int Level = DefaultCompression); @@ -71,6 +74,12 @@ Error decompress(ArrayRef Input, uint8_t *Output, Error decompress(ArrayRef Input, SmallVectorImpl &Output, size_t UncompressedSize); +// This version of decompress does not require knowing the uncompressed size +// ahead of time, this makes it more flexible than the two versions above but +// also more dangerous since it makes the program more susceptible to running +// out of memory. +Error decompress(ArrayRef Input, SmallVectorImpl &Output); + } // End of namespace zstd enum class Format { diff --git a/llvm/lib/IRReader/IRReader.cpp b/llvm/lib/IRReader/IRReader.cpp index 7885c36a7987..ed9569383d5d 100644 --- a/llvm/lib/IRReader/IRReader.cpp +++ b/llvm/lib/IRReader/IRReader.cpp @@ -12,6 +12,7 @@ #include "llvm/Bitcode/BitcodeReader.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" +#include "llvm/Support/Compression.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/SourceMgr.h" #include "llvm/Support/Timer.h" @@ -66,9 +67,10 @@ std::unique_ptr llvm::getLazyIRFileModule(StringRef Filename, ShouldLazyLoadMetadata); } -std::unique_ptr llvm::parseIR(MemoryBufferRef Buffer, SMDiagnostic &Err, - LLVMContext &Context, - ParserCallbacks Callbacks) { +static std::unique_ptr parseIRImpl(MemoryBufferRef Buffer, + SMDiagnostic &Err, + LLVMContext &Context, + ParserCallbacks Callbacks) { NamedRegionTimer T(TimeIRParsingName, TimeIRParsingDescription, TimeIRParsingGroupName, TimeIRParsingGroupDescription, TimePassesIsEnabled); @@ -91,6 +93,36 @@ std::unique_ptr llvm::parseIR(MemoryBufferRef Buffer, SMDiagnostic &Err, [](StringRef, StringRef) { return std::nullopt; })); } +std::unique_ptr llvm::parseIR(MemoryBufferRef Buffer, SMDiagnostic &Err, + LLVMContext &Context, + ParserCallbacks Callbacks) { + llvm::ArrayRef BufferRef( + reinterpret_cast(Buffer.getBufferStart()), + reinterpret_cast(Buffer.getBufferEnd())); + if (compression::zstd::isAvailable() && + compression::zstd::isZstd(BufferRef)) { + SmallVector DecompressedBuffer; + Error Error = compression::zstd::decompress(BufferRef, DecompressedBuffer); + if (Error) { + handleAllErrors(std::move(Error), [&](ErrorInfoBase &EIB) { + Err = SMDiagnostic(Buffer.getBufferIdentifier(), SourceMgr::DK_Error, + EIB.message()); + }); + return nullptr; + } + + std::string Identifier = + Buffer.getBufferIdentifier().str() + "-decompressed"; + MemoryBufferRef DecompressedBufferRef( + {reinterpret_cast(DecompressedBuffer.data()), + DecompressedBuffer.size()}, + Identifier); + return parseIRImpl(DecompressedBufferRef, Err, Context, Callbacks); + } else { + return parseIRImpl(Buffer, Err, Context, Callbacks); + } +} + std::unique_ptr llvm::parseIRFile(StringRef Filename, SMDiagnostic &Err, LLVMContext &Context, ParserCallbacks Callbacks) { diff --git a/llvm/lib/Support/Compression.cpp b/llvm/lib/Support/Compression.cpp index 8e57ba798f52..9f148dbbf627 100644 --- a/llvm/lib/Support/Compression.cpp +++ b/llvm/lib/Support/Compression.cpp @@ -159,6 +159,11 @@ Error zlib::decompress(ArrayRef Input, } #endif +bool zstd::isZstd(ArrayRef Input) { + static uint8_t Magic[] = {0x28, 0xb5, 0x2f, 0xfd}; + return ::memcmp(Input.data(), Magic, 4) == 0; +} + #if LLVM_ENABLE_ZSTD bool zstd::isAvailable() { return true; } @@ -202,6 +207,78 @@ Error zstd::decompress(ArrayRef Input, return E; } +static void zstdContextFree(ZSTD_DCtx *Ctx) { + size_t RC = ::ZSTD_freeDCtx(Ctx); + assert(::ZSTD_isError(RC) == 0); +} + +static Expected +zstdDecompressRead(::ZSTD_DCtx *Ctx, ::ZSTD_inBuffer &ZSTDInput, + SmallVector &Buffer, + SmallVectorImpl &Decompressed) { + ::ZSTD_outBuffer ZSTDOutput = { + .dst = Buffer.data(), .size = Buffer.size(), .pos = 0}; + size_t Res = ::ZSTD_decompressStream(Ctx, &ZSTDOutput, &ZSTDInput); + if (::ZSTD_isError(Res)) { + return make_error(::ZSTD_getErrorName(Res), + inconvertibleErrorCode()); + } + + // Check if the buffer was filled, this might not happen in all iterations + // because the data read might still reside in the internal state of the + // decompressor + if (ZSTDOutput.pos > 0) { + Decompressed.append(Buffer.begin(), Buffer.begin() + ZSTDOutput.pos); + return true; + } + return false; +} + +Error zstd::decompress(ArrayRef Input, + SmallVectorImpl &Output) { + auto &Decompressed = Output; + std::unique_ptr<::ZSTD_DCtx, void (*)(::ZSTD_DCtx *)> Ctx(::ZSTD_createDCtx(), + zstdContextFree); + size_t Res = ::ZSTD_initDStream(&*Ctx); + if (::ZSTD_isError(Res)) + return make_error(::ZSTD_getErrorName(Res), + inconvertibleErrorCode()); + + // Create a buffer, this will be filled up repeatedly by + // ZSTD_decompressStream and appended to Output + SmallVector Buffer; + Buffer.resize_for_overwrite(4096); + + ::ZSTD_inBuffer ZSTDInput = { + .src = Input.data(), .size = Input.size(), .pos = 0}; + + // First loop, read the entirety of `InputBuffer`. Stop when + // ZSTD_decompressedStream reports pos to be equal to size. This signals that + // all the data has been read and is at least in the internal state of the + // decompressor. + while (ZSTDInput.pos < ZSTDInput.size) { + Expected Err = + zstdDecompressRead(&*Ctx, ZSTDInput, Buffer, Decompressed); + if (!Err) + return Err.takeError(); + } + + // Second loop, in this one there isn't any data to read, but there still + // might be data to be outputted from the decompressor internal state + ZSTDInput = {.src = nullptr, .size = 0, .pos = 0}; + while (true) { + Expected MaybeRes = + zstdDecompressRead(&*Ctx, ZSTDInput, Buffer, Decompressed); + if (!MaybeRes) + return MaybeRes.takeError(); + // If this if is taken it means that the decompress function did not write + // anything, which means that there isn't more data to decompress + if (!MaybeRes.get()) + break; + } + return Error::success(); +} + #else bool zstd::isAvailable() { return false; } void zstd::compress(ArrayRef Input, @@ -217,4 +294,8 @@ Error zstd::decompress(ArrayRef Input, size_t UncompressedSize) { llvm_unreachable("zstd::decompress is unavailable"); } +Error zstd::decompress(ArrayRef Input, + SmallVectorImpl &Output) { + llvm_unreachable("zstd::decompress is unavailable"); +} #endif From d1de186c8881e29ff9e219f7057aaa5c72c1d8c5 Mon Sep 17 00:00:00 2001 From: Andrea Gussoni Date: Fri, 31 Jan 2025 11:04:14 +0100 Subject: [PATCH 109/147] ADT::GenericCycleInfo: Introduce GraphView param The `GenericCycleInfo` analysis, and all the classes used to perform the computation, now accept and additional `GraphView` template parameter, which can be used to specify a view used as the graph on which the analysis is computed. This means that we cannot use the `successors` and `predecessors` helper functions to iterate over the graph, but instead make use of `llvm::children` and `llvm::children>`. --- llvm/include/llvm/ADT/GenericCycleImpl.h | 88 +++++++++++++----------- llvm/include/llvm/ADT/GenericCycleInfo.h | 42 ++++++----- 2 files changed, 74 insertions(+), 56 deletions(-) diff --git a/llvm/include/llvm/ADT/GenericCycleImpl.h b/llvm/include/llvm/ADT/GenericCycleImpl.h index 07ac1768ea27..582ad6fcbfc8 100644 --- a/llvm/include/llvm/ADT/GenericCycleImpl.h +++ b/llvm/include/llvm/ADT/GenericCycleImpl.h @@ -31,8 +31,8 @@ namespace llvm { -template -bool GenericCycle::contains(const GenericCycle *C) const { +template +bool GenericCycle::contains(const GenericCycle *C) const { if (!C) return false; @@ -43,14 +43,15 @@ bool GenericCycle::contains(const GenericCycle *C) const { return this == C; } -template -void GenericCycle::getExitBlocks( +template +void GenericCycle::getExitBlocks( SmallVectorImpl &TmpStorage) const { TmpStorage.clear(); size_t NumExitBlocks = 0; for (BlockT *Block : blocks()) { - llvm::append_range(TmpStorage, successors(Block)); + auto BlockSuccessors = llvm::children(Block); + llvm::append_range(TmpStorage, BlockSuccessors); for (size_t Idx = NumExitBlocks, End = TmpStorage.size(); Idx < End; ++Idx) { @@ -66,8 +67,8 @@ void GenericCycle::getExitBlocks( } } -template -auto GenericCycle::getCyclePreheader() const -> BlockT * { +template +auto GenericCycle::getCyclePreheader() const -> BlockT * { BlockT *Predecessor = getCyclePredecessor(); if (!Predecessor) return nullptr; @@ -84,8 +85,9 @@ auto GenericCycle::getCyclePreheader() const -> BlockT * { return Predecessor; } -template -auto GenericCycle::getCyclePredecessor() const -> BlockT * { +template +auto GenericCycle::getCyclePredecessor() const + -> BlockT * { if (!isReducible()) return nullptr; @@ -93,7 +95,8 @@ auto GenericCycle::getCyclePredecessor() const -> BlockT * { // Loop over the predecessors of the header node... BlockT *Header = getHeader(); - for (const auto Pred : predecessors(Header)) { + auto BlockPredecessors = llvm::children>(Header); + for (const auto Pred : BlockPredecessors) { if (!contains(Pred)) { if (Out && Out != Pred) return nullptr; @@ -105,9 +108,9 @@ auto GenericCycle::getCyclePredecessor() const -> BlockT * { } /// \brief Helper class for computing cycle information. -template class GenericCycleInfoCompute { +template class GenericCycleInfoCompute { using BlockT = typename ContextT::BlockT; - using CycleInfoT = GenericCycleInfo; + using CycleInfoT = GenericCycleInfo; using CycleT = typename CycleInfoT::CycleT; CycleInfoT &Info; @@ -143,9 +146,9 @@ template class GenericCycleInfoCompute { void dfs(BlockT *EntryBlock); }; -template -auto GenericCycleInfo::getTopLevelParentCycle(BlockT *Block) - -> CycleT * { +template +auto GenericCycleInfo::getTopLevelParentCycle( + BlockT *Block) -> CycleT * { auto Cycle = BlockMapTopLevel.find(Block); if (Cycle != BlockMapTopLevel.end()) return Cycle->second; @@ -161,9 +164,9 @@ auto GenericCycleInfo::getTopLevelParentCycle(BlockT *Block) return C; } -template -void GenericCycleInfo::moveTopLevelCycleToNewParent(CycleT *NewParent, - CycleT *Child) { +template +void GenericCycleInfo::moveTopLevelCycleToNewParent( + CycleT *NewParent, CycleT *Child) { assert((!Child->ParentCycle && !NewParent->ParentCycle) && "NewParent and Child must be both top level cycle!\n"); auto &CurrentContainer = @@ -186,8 +189,8 @@ void GenericCycleInfo::moveTopLevelCycleToNewParent(CycleT *NewParent, } /// \brief Main function of the cycle info computations. -template -void GenericCycleInfoCompute::run(BlockT *EntryBlock) { +template +void GenericCycleInfoCompute::run(BlockT *EntryBlock) { LLVM_DEBUG(errs() << "Entry block: " << Info.Context.print(EntryBlock) << "\n"); dfs(EntryBlock); @@ -197,7 +200,9 @@ void GenericCycleInfoCompute::run(BlockT *EntryBlock) { for (BlockT *HeaderCandidate : llvm::reverse(BlockPreorder)) { const DFSInfo CandidateInfo = BlockDFSInfo.lookup(HeaderCandidate); - for (BlockT *Pred : predecessors(HeaderCandidate)) { + auto BlockPredecessors = + llvm::children>(HeaderCandidate); + for (BlockT *Pred : BlockPredecessors) { const DFSInfo PredDFSInfo = BlockDFSInfo.lookup(Pred); if (CandidateInfo.isAncestorOf(PredDFSInfo)) Worklist.push_back(Pred); @@ -221,7 +226,8 @@ void GenericCycleInfoCompute::run(BlockT *EntryBlock) { LLVM_DEBUG(errs() << " block " << Info.Context.print(Block) << ": "); bool IsEntry = false; - for (BlockT *Pred : predecessors(Block)) { + auto BlockPredecessors = llvm::children>(Block); + for (BlockT *Pred : BlockPredecessors) { const DFSInfo PredDFSInfo = BlockDFSInfo.lookup(Pred); if (CandidateInfo.isAncestorOf(PredDFSInfo)) { Worklist.push_back(Pred); @@ -286,8 +292,9 @@ void GenericCycleInfoCompute::run(BlockT *EntryBlock) { } /// \brief Recompute depth values of \p SubTree and all descendants. -template -void GenericCycleInfoCompute::updateDepth(CycleT *SubTree) { +template +void GenericCycleInfoCompute::updateDepth( + CycleT *SubTree) { for (CycleT *Cycle : depth_first(SubTree)) Cycle->Depth = Cycle->ParentCycle ? Cycle->ParentCycle->Depth + 1 : 1; } @@ -295,8 +302,8 @@ void GenericCycleInfoCompute::updateDepth(CycleT *SubTree) { /// \brief Compute a DFS of basic blocks starting at the function entry. /// /// Fills BlockDFSInfo with start/end counters and BlockPreorder. -template -void GenericCycleInfoCompute::dfs(BlockT *EntryBlock) { +template +void GenericCycleInfoCompute::dfs(BlockT *EntryBlock) { SmallVector DFSTreeStack; SmallVector TraverseStack; unsigned Counter = 0; @@ -315,7 +322,8 @@ void GenericCycleInfoCompute::dfs(BlockT *EntryBlock) { << TraverseStack.size() << "\n"); DFSTreeStack.emplace_back(TraverseStack.size()); - llvm::append_range(TraverseStack, successors(Block)); + auto BlockSuccessors = llvm::children(Block); + llvm::append_range(TraverseStack, BlockSuccessors); bool Added = BlockDFSInfo.try_emplace(Block, ++Counter).second; (void)Added; @@ -345,16 +353,17 @@ void GenericCycleInfoCompute::dfs(BlockT *EntryBlock) { } /// \brief Reset the object to its initial state. -template void GenericCycleInfo::clear() { +template +void GenericCycleInfo::clear() { TopLevelCycles.clear(); BlockMap.clear(); BlockMapTopLevel.clear(); } /// \brief Compute the cycle info for a function. -template -void GenericCycleInfo::compute(FunctionT &F) { - GenericCycleInfoCompute Compute(*this); +template +void GenericCycleInfo::compute(FunctionT &F) { + GenericCycleInfoCompute Compute(*this); Context.setFunction(F); LLVM_DEBUG(errs() << "Computing cycles for function: " << F.getName() @@ -368,8 +377,8 @@ void GenericCycleInfo::compute(FunctionT &F) { /// /// \returns the innermost cycle containing \p Block or nullptr if /// it is not contained in any cycle. -template -auto GenericCycleInfo::getCycle(const BlockT *Block) const +template +auto GenericCycleInfo::getCycle(const BlockT *Block) const -> CycleT * { auto MapIt = BlockMap.find(Block); if (MapIt != BlockMap.end()) @@ -381,8 +390,9 @@ auto GenericCycleInfo::getCycle(const BlockT *Block) const /// /// \returns the depth for the innermost cycle containing \p Block or 0 if it is /// not contained in any cycle. -template -unsigned GenericCycleInfo::getCycleDepth(const BlockT *Block) const { +template +unsigned GenericCycleInfo::getCycleDepth( + const BlockT *Block) const { CycleT *Cycle = getCycle(Block); if (!Cycle) return 0; @@ -394,8 +404,8 @@ unsigned GenericCycleInfo::getCycleDepth(const BlockT *Block) const { /// /// Note that this does \em not check that cycles are really cycles in the CFG, /// or that the right set of cycles in the CFG were found. -template -bool GenericCycleInfo::validateTree() const { +template +bool GenericCycleInfo::validateTree() const { DenseSet Blocks; DenseSet Entries; @@ -458,8 +468,8 @@ bool GenericCycleInfo::validateTree() const { #endif /// \brief Print the cycle info. -template -void GenericCycleInfo::print(raw_ostream &Out) const { +template +void GenericCycleInfo::print(raw_ostream &Out) const { for (const auto *TLC : toplevel_cycles()) { for (const CycleT *Cycle : depth_first(TLC)) { for (unsigned I = 0; I < Cycle->Depth; ++I) diff --git a/llvm/include/llvm/ADT/GenericCycleInfo.h b/llvm/include/llvm/ADT/GenericCycleInfo.h index 63db9eb9a601..9cc182f32a7a 100644 --- a/llvm/include/llvm/ADT/GenericCycleInfo.h +++ b/llvm/include/llvm/ADT/GenericCycleInfo.h @@ -34,6 +34,7 @@ #include "llvm/ADT/GraphTraits.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/iterator.h" +#include "llvm/IR/BasicBlock.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Printable.h" #include "llvm/Support/raw_ostream.h" @@ -41,16 +42,19 @@ namespace llvm { -template class GenericCycleInfo; -template class GenericCycleInfoCompute; +template +class GenericCycleInfo; +template +class GenericCycleInfoCompute; /// A possibly irreducible generalization of a \ref Loop. -template class GenericCycle { +template +class GenericCycle { public: using BlockT = typename ContextT::BlockT; using FunctionT = typename ContextT::FunctionT; - template friend class GenericCycleInfo; - template friend class GenericCycleInfoCompute; + template friend class GenericCycleInfo; + template friend class GenericCycleInfoCompute; private: /// The parent cycle. Is null for the root "cycle". Top-level cycles point @@ -223,13 +227,15 @@ template class GenericCycle { }; /// \brief Cycle information for a function. -template class GenericCycleInfo { +/// \tparam GraphView A graph view which is used, through `GraphTraits`, as the +/// the graph on which `GenericCycleInfo` is computed. +template class GenericCycleInfo { public: using BlockT = typename ContextT::BlockT; - using CycleT = GenericCycle; + using CycleT = GenericCycle; using FunctionT = typename ContextT::FunctionT; - template friend class GenericCycle; - template friend class GenericCycleInfoCompute; + template friend class GenericCycle; + template friend class GenericCycleInfoCompute; private: ContextT Context; @@ -343,14 +349,16 @@ template struct CycleGraphTraits { // Return total number of nodes in the graph }; -template -struct GraphTraits *> - : CycleGraphTraits *, - typename GenericCycle::const_child_iterator> {}; -template -struct GraphTraits *> - : CycleGraphTraits *, - typename GenericCycle::const_child_iterator> {}; +template +struct GraphTraits *> + : CycleGraphTraits< + const GenericCycle *, + typename GenericCycle::const_child_iterator> {}; +template +struct GraphTraits *> + : CycleGraphTraits< + GenericCycle *, + typename GenericCycle::const_child_iterator> {}; } // namespace llvm From 64ca0ffdeb58a2f5ce5ccf1a17553b7b6c8680ac Mon Sep 17 00:00:00 2001 From: Pietro Fezzardi Date: Mon, 7 Apr 2025 17:00:45 +0200 Subject: [PATCH 110/147] Add SROANoArrays cl::opt This option prevents SROA from splitting a single memory access that spans many elements of an array into many smaller accesses and GEPs. It is disabled by default. --- llvm/include/llvm/Transforms/Scalar/SROA.h | 3 +++ llvm/lib/Transforms/Scalar/SROA.cpp | 20 ++++++++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/llvm/include/llvm/Transforms/Scalar/SROA.h b/llvm/include/llvm/Transforms/Scalar/SROA.h index 26348da22021..a50b9bc51161 100644 --- a/llvm/include/llvm/Transforms/Scalar/SROA.h +++ b/llvm/include/llvm/Transforms/Scalar/SROA.h @@ -21,8 +21,11 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/ValueHandle.h" +#include "llvm/Support/CommandLine.h" #include +extern llvm::cl::opt SROANoArrays; + namespace llvm { class AllocaInst; diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp index 8339981e1bdc..50818a5209b7 100644 --- a/llvm/lib/Transforms/Scalar/SROA.cpp +++ b/llvm/lib/Transforms/Scalar/SROA.cpp @@ -118,6 +118,12 @@ STATISTIC(NumVectorized, "Number of vectorized aggregates"); /// GEPs. static cl::opt SROAStrictInbounds("sroa-strict-inbounds", cl::init(false), cl::Hidden); + +cl::opt +SROANoArrays("sroa-no-arrays", + cl::desc("Disables SROA for array aggregates"), + cl::init(false)); + namespace { /// Find linked dbg.assign and generate a new one with the correct /// FragmentInfo. Link Inst to the new dbg.assign. If Value is nullptr the @@ -882,7 +888,9 @@ class AllocaSlices::SliceBuilder : public PtrUseVisitor { } void visitLoadInst(LoadInst &LI) { - assert((!LI.isSimple() || LI.getType()->isSingleValueType()) && + assert((!LI.isSimple() || + LI.getType()->isSingleValueType() || + (isa(LI.getType()) && SROANoArrays)) && "All simple FCA loads should have been pre-split"); if (!IsOffsetKnown) @@ -923,7 +931,9 @@ class AllocaSlices::SliceBuilder : public PtrUseVisitor { return markAsDead(SI); } - assert((!SI.isSimple() || ValOp->getType()->isSingleValueType()) && + assert((!SI.isSimple() || + ValOp->getType()->isSingleValueType() || + (isa(ValOp->getType()) && SROANoArrays)) && "All simple FCA stores should have been pre-split"); handleLoadOrStore(ValOp->getType(), SI, Offset, Size, SI.isVolatile()); } @@ -3722,6 +3732,9 @@ class AggLoadStoreRewriter : public InstVisitor { if (!LI.isSimple() || LI.getType()->isSingleValueType()) return false; + if (isa(LI.getType()) and SROANoArrays) + return false; + // We have an aggregate being loaded, split it apart. LLVM_DEBUG(dbgs() << " original: " << LI << "\n"); LoadOpSplitter Splitter(&LI, *U, LI.getType(), LI.getAAMetadata(), @@ -3793,6 +3806,9 @@ class AggLoadStoreRewriter : public InstVisitor { if (V->getType()->isSingleValueType()) return false; + if (isa(V->getType()) and SROANoArrays) + return false; + // We have an aggregate being stored, split it apart. LLVM_DEBUG(dbgs() << " original: " << SI << "\n"); StoreOpSplitter Splitter(&SI, *U, V->getType(), SI.getAAMetadata(), &SI, From 1197e64532b67b1934614a3289c65b02a9fc2f15 Mon Sep 17 00:00:00 2001 From: Pietro Fezzardi Date: Mon, 7 Apr 2025 17:02:45 +0200 Subject: [PATCH 111/147] Make InstCombine MaxArraySize cl::opt public --- .../Transforms/InstCombine/InstCombiner.h | 2 ++ .../InstCombineLoadStoreAlloca.cpp | 27 ++++++++++--------- .../InstCombine/InstructionCombining.cpp | 8 +++--- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h b/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h index a876385581e7..d49c9039b0e8 100644 --- a/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h +++ b/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h @@ -30,6 +30,8 @@ #define DEBUG_TYPE "instcombine" #include "llvm/Transforms/Utils/InstructionWorklist.h" +extern llvm::cl::opt MaxArraySize; + namespace llvm { class AAResults; diff --git a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp index 41bc65620ff6..56f82ab39efe 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp @@ -768,14 +768,7 @@ static Instruction *unpackLoadToAggregate(InstCombinerImpl &IC, LoadInst &LI) { } if (auto *AT = dyn_cast(T)) { - auto *ET = AT->getElementType(); auto NumElements = AT->getNumElements(); - if (NumElements == 1) { - LoadInst *NewLoad = IC.combineLoadToNewType(LI, ET, ".unpack"); - NewLoad->setAAMetadata(LI.getAAMetadata()); - return IC.replaceInstUsesWith(LI, IC.Builder.CreateInsertValue( - PoisonValue::get(T), NewLoad, 0, Name)); - } // Bail out if the array is too large. Ideally we would like to optimize // arrays of arbitrary size but this has a terrible impact on compile time. @@ -784,6 +777,14 @@ static Instruction *unpackLoadToAggregate(InstCombinerImpl &IC, LoadInst &LI) { if (NumElements > IC.MaxArraySizeForCombine) return nullptr; + auto *ET = AT->getElementType(); + if (NumElements == 1) { + LoadInst *NewLoad = IC.combineLoadToNewType(LI, ET, ".unpack"); + NewLoad->setAAMetadata(LI.getAAMetadata()); + return IC.replaceInstUsesWith(LI, IC.Builder.CreateInsertValue( + PoisonValue::get(T), NewLoad, 0, Name)); + } + const DataLayout &DL = IC.getDataLayout(); auto EltSize = DL.getTypeAllocSize(ET); const auto Align = LI.getAlign(); @@ -1291,13 +1292,7 @@ static bool unpackStoreToAggregate(InstCombinerImpl &IC, StoreInst &SI) { } if (auto *AT = dyn_cast(T)) { - // If the array only have one element, we unpack. auto NumElements = AT->getNumElements(); - if (NumElements == 1) { - V = IC.Builder.CreateExtractValue(V, 0); - combineStoreToNewValue(IC, SI, V); - return true; - } // Bail out if the array is too large. Ideally we would like to optimize // arrays of arbitrary size but this has a terrible impact on compile time. @@ -1306,6 +1301,12 @@ static bool unpackStoreToAggregate(InstCombinerImpl &IC, StoreInst &SI) { if (NumElements > IC.MaxArraySizeForCombine) return false; + // If the array only have one element, we unpack. + if (NumElements == 1) { + V = IC.Builder.CreateExtractValue(V, 0); + combineStoreToNewValue(IC, SI, V); + return true; + } const DataLayout &DL = IC.getDataLayout(); auto EltSize = DL.getTypeAllocSize(AT->getElementType()); const auto Align = SI.getAlign(); diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp index 9c6a57650ed5..8dff0117f898 100644 --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -135,6 +135,7 @@ static constexpr unsigned InstCombineDefaultInfiniteLoopThreshold = 100; #else static constexpr unsigned InstCombineDefaultInfiniteLoopThreshold = 1000; #endif +static constexpr unsigned InstCombineDefaultMaxArraySize = 1024; static cl::opt EnableCodeSinking("instcombine-code-sinking", cl::desc("Enable code sinking"), @@ -155,9 +156,10 @@ static cl::opt InfiniteLoopDetectionThreshold( "infinite loop"), cl::init(InstCombineDefaultInfiniteLoopThreshold), cl::Hidden); -static cl::opt -MaxArraySize("instcombine-maxarray-size", cl::init(1024), - cl::desc("Maximum array size considered when doing a combine")); +cl::opt MaxArraySize( + "instcombine-maxarray-size", + cl::desc("Maximum array size considered when doing a combine"), + cl::init(InstCombineDefaultMaxArraySize)); // FIXME: Remove this flag when it is no longer necessary to convert // llvm.dbg.declare to avoid inaccurate debug info. Setting this to false From eaf4788611d3b816636e1d6892eec9d0135ae455 Mon Sep 17 00:00:00 2001 From: Lauri Vasama Date: Mon, 28 Apr 2025 07:23:19 +0300 Subject: [PATCH 112/147] [mlir] Print aliases for recursive types Based on upstream commit: 8773bd0 --- mlir/lib/IR/AsmPrinter.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/mlir/lib/IR/AsmPrinter.cpp b/mlir/lib/IR/AsmPrinter.cpp index 6c1cc671ab56..2c0bbd33e948 100644 --- a/mlir/lib/IR/AsmPrinter.cpp +++ b/mlir/lib/IR/AsmPrinter.cpp @@ -478,6 +478,10 @@ class SymbolAlias { bool isType : 1; /// A flag indicating whether this alias may be deferred or not. bool isDeferrable : 1; + +public: + /// Used to avoid printing incomplete aliases for recursive types. + bool isPrinted = false; }; /// This class represents a utility that initializes the set of attribute and @@ -1133,6 +1137,8 @@ LogicalResult AliasState::getAlias(Type ty, raw_ostream &os) const { auto it = attrTypeToAlias.find(ty.getAsOpaquePointer()); if (it == attrTypeToAlias.end()) return failure(); + if (!it->second.isPrinted) + return failure(); it->second.print(os); return success(); @@ -1151,10 +1157,8 @@ void AliasState::printAliases(AsmPrinter::Impl &p, NewLineCounter &newLine, if (alias.isTypeAlias()) { // TODO: Support nested aliases in mutable types. Type type = Type::getFromOpaquePointer(opaqueSymbol); - if (type.hasTrait()) - p.getStream() << type; - else - p.printTypeImpl(type); + p.printTypeImpl(type); + alias.isPrinted = true; } else { // TODO: Support nested aliases in mutable attributes. Attribute attr = Attribute::getFromOpaquePointer(opaqueSymbol); From c09d2536b4cfec671e169380847779d1f0825d1f Mon Sep 17 00:00:00 2001 From: Lauri Vasama Date: Mon, 28 Apr 2025 07:26:36 +0300 Subject: [PATCH 113/147] Add AsmPrinter::getCurrentTypeOrAttrAlias It is unclear at this time if this change will be needed after LLVM is upgraded. Using the first class support for cyclic printing and parsing may solve all the issues we've had in another way. --- mlir/include/mlir/IR/OpImplementation.h | 2 ++ mlir/lib/IR/AsmPrinter.cpp | 28 +++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/mlir/include/mlir/IR/OpImplementation.h b/mlir/include/mlir/IR/OpImplementation.h index 92f4c610a903..e73434a9f080 100644 --- a/mlir/include/mlir/IR/OpImplementation.h +++ b/mlir/include/mlir/IR/OpImplementation.h @@ -213,6 +213,8 @@ class AsmPrinter { printArrowTypeList(results); } + virtual const void *getCurrentTypeOrAttrAlias() const; + protected: /// Initialize the printer with no internal implementation. In this case, all /// virtual methods of this class must be overriden. diff --git a/mlir/lib/IR/AsmPrinter.cpp b/mlir/lib/IR/AsmPrinter.cpp index 2c0bbd33e948..cfc729d47eda 100644 --- a/mlir/lib/IR/AsmPrinter.cpp +++ b/mlir/lib/IR/AsmPrinter.cpp @@ -377,6 +377,9 @@ class AsmPrinter::Impl { /// dialect. void printResourceHandle(const AsmDialectResourceHandle &resource); + const void *getCurrentTypeOrAttrAlias() const; + void setCurrentTypeOrAttrAlias(const void *opaque); + void printAffineMap(AffineMap map); void printAffineExpr(AffineExpr expr, @@ -757,6 +760,8 @@ class DummyAliasOperationPrinter : private OpAsmPrinter { void printSuccessorAndUseList(Block *, ValueRange) override {} void shadowRegionArgs(Region &, ValueRange) override {} + const void *getCurrentTypeOrAttrAlias() const override { return nullptr; } + /// The printer flags to use when determining potential aliases. const OpPrintingFlags &printerFlags; @@ -894,6 +899,8 @@ class DummyAliasDialectAsmPrinter : public DialectAsmPrinter { void printSymbolName(StringRef) override {} void printResourceHandle(const AsmDialectResourceHandle &) override {} + const void *getCurrentTypeOrAttrAlias() const override { return nullptr; } + /// The initializer to use when identifying aliases. AliasInitializer &initializer; @@ -1115,6 +1122,9 @@ class AliasState { /// An allocator used for alias names. llvm::BumpPtrAllocator aliasAllocator; + +public: + const void *currentTypeOrAttrAlias = nullptr; }; } // namespace @@ -1154,6 +1164,11 @@ void AliasState::printAliases(AsmPrinter::Impl &p, NewLineCounter &newLine, alias.print(p.getStream()); p.getStream() << " = "; + currentTypeOrAttrAlias = opaqueSymbol; + auto guard = llvm::make_scope_exit([&]() { + currentTypeOrAttrAlias = nullptr; + }); + if (alias.isTypeAlias()) { // TODO: Support nested aliases in mutable types. Type type = Type::getFromOpaquePointer(opaqueSymbol); @@ -2620,6 +2635,14 @@ void AsmPrinter::Impl::printHexString(ArrayRef data) { printHexString(StringRef(data.data(), data.size())); } +const void *AsmPrinter::Impl::getCurrentTypeOrAttrAlias() const { + return state.getAliasState().currentTypeOrAttrAlias; +} + +void AsmPrinter::Impl::setCurrentTypeOrAttrAlias(const void *opaque) { + state.getAliasState().currentTypeOrAttrAlias = opaque; +} + //===--------------------------------------------------------------------===// // AsmPrinter //===--------------------------------------------------------------------===// @@ -2678,6 +2701,11 @@ void AsmPrinter::printResourceHandle(const AsmDialectResourceHandle &resource) { impl->printResourceHandle(resource); } +const void *AsmPrinter::getCurrentTypeOrAttrAlias() const { + assert(impl && "expected AsmPrinter::getCurrentTypeOrAttrAlias to be overriden"); + return impl->getCurrentTypeOrAttrAlias(); +} + //===----------------------------------------------------------------------===// // Affine expressions and maps //===----------------------------------------------------------------------===// From 3815ef1d29653de7ec37a0584103ef4e96e0ba9f Mon Sep 17 00:00:00 2001 From: Pietro Fezzardi Date: Mon, 9 Jun 2025 18:25:47 +0200 Subject: [PATCH 114/147] InstructionSimplify: respect NullPointerIsValid --- llvm/lib/Analysis/InstructionSimplify.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp index c83eb96bbc69..ff535811b566 100644 --- a/llvm/lib/Analysis/InstructionSimplify.cpp +++ b/llvm/lib/Analysis/InstructionSimplify.cpp @@ -6515,12 +6515,21 @@ Value *llvm::simplifyCall(CallBase *Call, const SimplifyQuery &Q) { if (Call->isMustTailCall()) return nullptr; - // call undef -> poison - // call null -> poison Value *Callee = Call->getCalledOperand(); - if (isa(Callee) || isa(Callee)) + + // call undef -> poison + if (isa(Callee)) return PoisonValue::get(Call->getType()); + // call null -> poison, but only if Call is in a function where null pointer + // is undefined + if (isa(Callee)) { + if (!Call->getParent() || !Call->getFunction() || + !NullPointerIsDefined(Call->getFunction())) { + return PoisonValue::get(Call->getType()); + } + } + if (Value *V = tryConstantFoldCall(Call, Q)) return V; From 4c67a2075601ab56170ad905daf86a50da154823 Mon Sep 17 00:00:00 2001 From: Lauri Vasama Date: Mon, 28 Apr 2025 07:24:43 +0300 Subject: [PATCH 115/147] Add DialectBytecodeReader::getContext This commit adds simple getters to expose the context from a DialectBytecodeReader. The same functions are already available in upstream, but cherry-picking the commit introducing them was not feasible. See the commit bff6a42 in upstream. --- mlir/include/mlir/Bytecode/BytecodeImplementation.h | 2 ++ mlir/lib/Bytecode/Reader/BytecodeReader.cpp | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/mlir/include/mlir/Bytecode/BytecodeImplementation.h b/mlir/include/mlir/Bytecode/BytecodeImplementation.h index 1ae839d5f40e..ed40be6a6f43 100644 --- a/mlir/include/mlir/Bytecode/BytecodeImplementation.h +++ b/mlir/include/mlir/Bytecode/BytecodeImplementation.h @@ -39,6 +39,8 @@ class DialectBytecodeReader { /// Emit an error to the reader. virtual InFlightDiagnostic emitError(const Twine &msg = {}) = 0; + virtual MLIRContext *getContext() const = 0; + /// Read out a list of elements, invoking the provided callback for each /// element. The callback function may be in any of the following forms: /// * LogicalResult(T &) diff --git a/mlir/lib/Bytecode/Reader/BytecodeReader.cpp b/mlir/lib/Bytecode/Reader/BytecodeReader.cpp index 53e55470fcd3..1223a6c9da5a 100644 --- a/mlir/lib/Bytecode/Reader/BytecodeReader.cpp +++ b/mlir/lib/Bytecode/Reader/BytecodeReader.cpp @@ -268,6 +268,8 @@ class EncodingReader { return parseBytes(static_cast(length), sectionData); } + Location getLoc() const { return fileLoc; } + private: /// Parse a variable length encoded integer from the byte stream. This method /// is a fallback when the number of bytes used to encode the value is greater @@ -835,6 +837,10 @@ class DialectReader : public DialectBytecodeReader { return reader.emitError(msg); } + MLIRContext *getContext() const override { return getLoc().getContext(); } + + Location getLoc() const { return reader.getLoc(); } + //===--------------------------------------------------------------------===// // IR //===--------------------------------------------------------------------===// From bb023b7bfa9352f348d7a34fae701b63f5a8cbab Mon Sep 17 00:00:00 2001 From: Shan Huang <52285902006@stu.ecnu.edu.cn> Date: Fri, 5 Jul 2024 14:10:04 +0800 Subject: [PATCH 116/147] [DebugInfo][SimplifyCFGPass] Fix the missing debug location update for the new br instruction (#97389) Fix #97388 . --- .../lib/Transforms/Scalar/SimplifyCFGPass.cpp | 4 +- .../SimplifyCFG/preserving-debugloc-br.ll | 72 +++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 llvm/test/Transforms/SimplifyCFG/preserving-debugloc-br.ll diff --git a/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp b/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp index e014f5d1eb04..5a9cf5176e59 100644 --- a/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp +++ b/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp @@ -142,8 +142,10 @@ performBlockTailMerging(Function &F, ArrayRef BBs, // And turn BB into a block that just unconditionally branches // to the canonical block. + Instruction *BI = BranchInst::Create(CanonicalBB, BB); + BI->setDebugLoc(Term->getDebugLoc()); Term->eraseFromParent(); - BranchInst::Create(CanonicalBB, BB); + if (Updates) Updates->push_back({DominatorTree::Insert, BB, CanonicalBB}); } diff --git a/llvm/test/Transforms/SimplifyCFG/preserving-debugloc-br.ll b/llvm/test/Transforms/SimplifyCFG/preserving-debugloc-br.ll new file mode 100644 index 000000000000..298701ff61b4 --- /dev/null +++ b/llvm/test/Transforms/SimplifyCFG/preserving-debugloc-br.ll @@ -0,0 +1,72 @@ +; RUN: opt -S -passes=simplifycfg < %s | FileCheck %s + +; Check that SimplifyCFGPass's performBlockTailMerging() propagates +; the debug location of the old block terminator to the new branch +; instruction. + +define i32 @foo(i64 %x, i64 %y) !dbg !5 { +; CHECK-LABEL: define i32 @foo( +; CHECK: a: +; CHECK: br label %[[COMMON_RET:.*]], !dbg [[DBG14:![0-9]+]] +; CHECK: b: +; CHECK: br label %[[COMMON_RET]], !dbg [[DBG17:![0-9]+]] +; +entry: + %eq = icmp eq i64 %x, %y, !dbg !8 + br i1 %eq, label %b, label %switch, !dbg !9 + +switch: ; preds = %entry + %lt = icmp slt i64 %x, %y, !dbg !10 + %qux = select i1 %lt, i32 0, i32 2, !dbg !11 + switch i32 %qux, label %bees [ + i32 0, label %a + i32 1, label %b + i32 2, label %b + ], !dbg !12 + +a: ; preds = %switch + tail call void @bees.a(), !dbg !13 + ret i32 1, !dbg !14 + +b: ; preds = %switch, %switch, %entry + %retval = phi i32 [ 0, %switch ], [ 0, %switch ], [ 2, %entry ], !dbg !15 + tail call void @bees.b(), !dbg !16 + ret i32 %retval, !dbg !17 + +bees: ; preds = %switch + tail call void @llvm.trap(), !dbg !18 + unreachable, !dbg !19 +} + + +declare void @llvm.trap() +declare void @bees.a() +declare void @bees.b() + +!llvm.dbg.cu = !{!0} +!llvm.debugify = !{!2, !3} +!llvm.module.flags = !{!4} + +; CHECK: [[DBG14]] = !DILocation(line: 7, +; CHECK: [[DBG17]] = !DILocation(line: 10, + +!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug) +!1 = !DIFile(filename: "main.ll", directory: "/") +!2 = !{i32 12} +!3 = !{i32 0} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!5 = distinct !DISubprogram(name: "foo", linkageName: "foo", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0) +!6 = !DISubroutineType(types: !7) +!7 = !{} +!8 = !DILocation(line: 1, column: 1, scope: !5) +!9 = !DILocation(line: 2, column: 1, scope: !5) +!10 = !DILocation(line: 3, column: 1, scope: !5) +!11 = !DILocation(line: 4, column: 1, scope: !5) +!12 = !DILocation(line: 5, column: 1, scope: !5) +!13 = !DILocation(line: 6, column: 1, scope: !5) +!14 = !DILocation(line: 7, column: 1, scope: !5) +!15 = !DILocation(line: 8, column: 1, scope: !5) +!16 = !DILocation(line: 9, column: 1, scope: !5) +!17 = !DILocation(line: 10, column: 1, scope: !5) +!18 = !DILocation(line: 11, column: 1, scope: !5) +!19 = !DILocation(line: 12, column: 1, scope: !5) From 4d36a0c3cf7323e0a0159cb26f666b3c6956cf46 Mon Sep 17 00:00:00 2001 From: OCHyams Date: Thu, 27 Apr 2023 13:44:12 +0100 Subject: [PATCH 117/147] [DebugInfo][InstCombine] Fix missing source and variable locations after foldOpIntoPhi Reviewed By: fdeazeve Differential Revision: https://reviews.llvm.org/D149335 --- .../InstCombine/InstructionCombining.cpp | 5 ++ .../X86/instcombine-fold-cast-into-phi.ll | 48 +++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 llvm/test/DebugInfo/X86/instcombine-fold-cast-into-phi.ll diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp index 8dff0117f898..5a5d6a9cab5b 100644 --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -1265,6 +1265,7 @@ Instruction *InstCombinerImpl::foldOpIntoPhi(Instruction &I, PHINode *PN) { PHINode *NewPN = PHINode::Create(I.getType(), PN->getNumIncomingValues()); InsertNewInstBefore(NewPN, *PN); NewPN->takeName(PN); + NewPN->setDebugLoc(PN->getDebugLoc()); // If we are going to have to insert a new computation, do so right before the // predecessor's terminator. @@ -1293,6 +1294,10 @@ Instruction *InstCombinerImpl::foldOpIntoPhi(Instruction &I, PHINode *PN) { replaceInstUsesWith(*User, NewPN); eraseInstFromFunction(*User); } + + replaceAllDbgUsesWith(const_cast(*PN), + const_cast(*NewPN), + const_cast(*PN), DT); return replaceInstUsesWith(I, NewPN); } diff --git a/llvm/test/DebugInfo/X86/instcombine-fold-cast-into-phi.ll b/llvm/test/DebugInfo/X86/instcombine-fold-cast-into-phi.ll new file mode 100644 index 000000000000..b8259dcfab85 --- /dev/null +++ b/llvm/test/DebugInfo/X86/instcombine-fold-cast-into-phi.ll @@ -0,0 +1,48 @@ +; RUN: opt -passes=instcombine -S %s | FileCheck %s + +;; Instcombine folds the trunc %x into the phi. Check it updates the phi's dbg +;; user (otherwise the dbg use becomes poison after the original phi is +;; deleted). Check the new phi inherits the DebugLoc. + +; CHECK: %[[phi:.*]] = phi i8 [ 1, %{{.*}} ], [ 0, %{{.*}} ], !dbg ![[dbg:[0-9]+]] +; CHECK: call void @llvm.dbg.value(metadata i8 %[[phi]], metadata ![[#]], metadata !DIExpression(DW_OP_LLVM_convert, 8, DW_ATE_signed, DW_OP_LLVM_convert, 32, DW_ATE_signed, DW_OP_stack_value) +; CHECK: ![[dbg]] = !DILocation(line: 123, + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define linkonce_odr float @f(i1 %cond) { +entry: + br i1 %cond, label %if.then, label %if.end + +if.then: ; preds = entry + br label %if.end + +if.end: ; preds = %if.then, %entry + %p.0 = phi i32 [ 1, %if.then ], [ 0, %entry ], !dbg !13 + call void @llvm.dbg.value(metadata i32 %p.0, metadata !4, metadata !DIExpression()), !dbg !13 + %x = trunc i32 %p.0 to i8 + %callff = call float @ff(i8 %x) + ret float %callff +} + +declare void @llvm.dbg.value(metadata, metadata, metadata) +declare float @ff(float) + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3} + +!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_11, file: !1, producer: "clang version 17.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, retainedTypes: !2, splitDebugInlining: false, nameTableKind: None) +!1 = !DIFile(filename: "btConvexTriangleMeshShape.cpp", directory: "/") +!2 = !{} +!3 = !{i32 2, !"Debug Info Version", i32 3} +!4 = !DILocalVariable(name: "p", scope: !5, file: !6, line: 2057, type: !12) +!5 = distinct !DILexicalBlock(scope: !7, file: !6, line: 2056, column: 4) +!6 = !DIFile(filename: "reduce.cpp", directory: "/") +!7 = distinct !DISubprogram(name: "diagonalize", linkageName: "_ZN11btMatrix3x311diagonalizeERS_fi", scope: !8, file: !6, line: 2054, type: !9, scopeLine: 2055, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, declaration: !11, retainedNodes: !2) +!8 = distinct !DICompositeType(tag: DW_TAG_class_type, name: "btMatrix3x3", file: !6, line: 2050, size: 24, flags: DIFlagTypePassByValue, elements: !2, identifier: "_ZTS11btMatrix3x3") +!9 = !DISubroutineType(types: !10) +!10 = !{null} +!11 = !DISubprogram(name: "diagonalize", linkageName: "_ZN11btMatrix3x311diagonalizeERS_fi", scope: !8, file: !6, line: 2054, type: !9, scopeLine: 2054, flags: DIFlagPublic | DIFlagPrototyped, spFlags: DISPFlagOptimized) +!12 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!13 = !DILocation(line: 123, scope: !5) From 7028be8c1af90a89c7a64d42e59710522a6c9136 Mon Sep 17 00:00:00 2001 From: Ivan Krysak Date: Mon, 14 Jul 2025 17:30:44 +0300 Subject: [PATCH 118/147] BasicBlockUtils: preserve `split` phi debug information --- llvm/lib/Transforms/Utils/BasicBlockUtils.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp index 58a226fc601c..d35c35cd1359 100644 --- a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp +++ b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp @@ -856,6 +856,7 @@ void llvm::createPHIsForSplitLoopExit(ArrayRef Preds, SplitBB->isLandingPad() ? &SplitBB->front() : SplitBB->getTerminator()); for (BasicBlock *BB : Preds) NewPN->addIncoming(V, BB); + NewPN->setDebugLoc(PN.getDebugLoc()); // Update the original PHI. PN.setIncomingValue(Idx, NewPN); From cff29a2f2017b94268b9d05b14bec809c48176d0 Mon Sep 17 00:00:00 2001 From: Ivan Krysak Date: Mon, 14 Jul 2025 17:35:56 +0300 Subject: [PATCH 119/147] BasicBlockUtils: preserve `.ph` phi debug information --- llvm/lib/Transforms/Utils/BasicBlockUtils.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp index d35c35cd1359..a58282651199 100644 --- a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp +++ b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp @@ -1157,6 +1157,7 @@ static void UpdatePHINodes(BasicBlock *OrigBB, BasicBlock *NewBB, // Create the new PHI node, insert it into NewBB at the end of the block PHINode *NewPHI = PHINode::Create(PN->getType(), Preds.size(), PN->getName() + ".ph", BI); + NewPHI->setDebugLoc(PN->getDebugLoc()); // NOTE! This loop walks backwards for a reason! First off, this minimizes // the cost of removal if we end up removing a large number of values, and From cb2caeea0997c10cf3039abe35540ee406838319 Mon Sep 17 00:00:00 2001 From: Ivan Krysak Date: Mon, 14 Jul 2025 17:36:09 +0300 Subject: [PATCH 120/147] LoopSimplify: preserve `split` phi debug information --- llvm/lib/Transforms/Utils/LoopSimplify.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Transforms/Utils/LoopSimplify.cpp b/llvm/lib/Transforms/Utils/LoopSimplify.cpp index 87a0e54e2704..19473c08b82a 100644 --- a/llvm/lib/Transforms/Utils/LoopSimplify.cpp +++ b/llvm/lib/Transforms/Utils/LoopSimplify.cpp @@ -400,6 +400,7 @@ static BasicBlock *insertUniqueBackedgeBlock(Loop *L, BasicBlock *Preheader, PHINode *PN = cast(I); PHINode *NewPN = PHINode::Create(PN->getType(), BackedgeBlocks.size(), PN->getName()+".be", BETerminator); + NewPN->setDebugLoc(PN->getDebugLoc()); // Loop over the PHI node, moving all entries except the one for the // preheader over to the new PHI node. From 5d0c83bdf28131ea3b4f682f656f1472a85feaf3 Mon Sep 17 00:00:00 2001 From: Ivan Krysak Date: Mon, 14 Jul 2025 17:36:52 +0300 Subject: [PATCH 121/147] ScalarEvolutionExpander: preserve `indvar` phi debug information --- llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp index 24f1966edd37..df95b4d38729 100644 --- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp +++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp @@ -21,6 +21,7 @@ #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/DataLayout.h" +#include "llvm/IR/DebugLoc.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/PatternMatch.h" @@ -1600,8 +1601,11 @@ Value *SCEVExpander::visitAddRecExpr(const SCEVAddRecExpr *S) { // specified loop. BasicBlock *Header = L->getHeader(); pred_iterator HPB = pred_begin(Header), HPE = pred_end(Header); + + DebugLoc DebugLocation = Header->front().getDebugLoc(); CanonicalIV = PHINode::Create(Ty, std::distance(HPB, HPE), "indvar", &Header->front()); + CanonicalIV->setDebugLoc(DebugLocation); rememberInstruction(CanonicalIV); SmallSet PredSeen; From 8077941ef918d480b60c70aff03963f6f18ea6a8 Mon Sep 17 00:00:00 2001 From: Ivan Krysak Date: Mon, 14 Jul 2025 17:37:43 +0300 Subject: [PATCH 122/147] SimplifyCFG: preserve `.sink` phi debug information --- llvm/lib/Transforms/Utils/SimplifyCFG.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp index 9e0483966d3e..7e21d2e0cd45 100644 --- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp @@ -1970,6 +1970,12 @@ static bool sinkLastInstruction(ArrayRef Blocks) { for (auto *I : Insts) PN->addIncoming(I->getOperand(O), I->getParent()); NewOperands.push_back(PN); + + // Merge all the incoming debug locations together. + PN->setDebugLoc(Insts.front()->getDebugLoc()); + for (Instruction *I : Insts) + if (I->getDebugLoc()) + PN->applyMergedLocation(PN->getDebugLoc(), I->getDebugLoc()); } // Arbitrarily use I0 as the new "common" instruction; remap its operands From 7f38b8eac2fed585171ae1c4c1e8326b7b880e6f Mon Sep 17 00:00:00 2001 From: Ivan Krysak Date: Mon, 14 Jul 2025 17:38:27 +0300 Subject: [PATCH 123/147] LICM: preserve `.lcssa` phi debug information --- llvm/lib/Transforms/Scalar/LICM.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp index 2865dece8723..73e480cb9b0b 100644 --- a/llvm/lib/Transforms/Scalar/LICM.cpp +++ b/llvm/lib/Transforms/Scalar/LICM.cpp @@ -1465,6 +1465,7 @@ static Instruction *cloneInstructionInExitBlock( PHINode *OpPN = PHINode::Create(OInst->getType(), PN.getNumIncomingValues(), OInst->getName() + ".lcssa", &ExitBlock.front()); + OpPN->setDebugLoc(OInst->getDebugLoc()); for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i) OpPN->addIncoming(OInst, PN.getIncomingBlock(i)); Op = OpPN; From 4630c471b46ac36777c75462cd40812cf9b7a4d2 Mon Sep 17 00:00:00 2001 From: Ivan Krysak Date: Mon, 14 Jul 2025 17:38:59 +0300 Subject: [PATCH 124/147] SimplifyCFGPass: preserve `common.` basic block debug information --- llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp b/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp index 5a9cf5176e59..6fd4624ea564 100644 --- a/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp +++ b/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp @@ -32,6 +32,7 @@ #include "llvm/IR/Attributes.h" #include "llvm/IR/CFG.h" #include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/DebugLoc.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" @@ -108,6 +109,7 @@ performBlockTailMerging(Function &F, ArrayRef BBs, std::get<1>(I) = PHINode::Create(std::get<0>(I)->getType(), /*NumReservedValues=*/BBs.size(), CanonicalBB->getName() + ".op"); + std::get<1>(I)->setDebugLoc(Term->getDebugLoc()); std::get<1>(I)->insertInto(CanonicalBB, CanonicalBB->end()); } // Make it so that this canonical block actually has the right @@ -117,6 +119,8 @@ performBlockTailMerging(Function &F, ArrayRef BBs, // If the canonical terminator has operands, rewrite it to take PHI's. for (auto I : zip(NewOps, CanonicalTerm->operands())) std::get<1>(I) = std::get<0>(I); + + CanonicalTerm->setDebugLoc(Term->getDebugLoc()); } // Now, go through each block (with the current terminator type) From 9e26069cc13cdb17e7ac2786ccbd3ca3d97c9972 Mon Sep 17 00:00:00 2001 From: Ivan Krysak Date: Mon, 14 Jul 2025 17:41:28 +0300 Subject: [PATCH 125/147] PointerReplacer: preserve phi debug information --- llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp index 56f82ab39efe..0fc30d643e83 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp @@ -374,6 +374,7 @@ void PointerReplacer::replace(Instruction *I) { Type *NewTy = getReplacement(PHI->getIncomingValue(0))->getType(); auto *NewPHI = PHINode::Create(NewTy, PHI->getNumIncomingValues(), PHI->getName(), PHI); + NewPHI->setDebugLoc(PHI->getDebugLoc()); for (unsigned int I = 0; I < PHI->getNumIncomingValues(); ++I) NewPHI->addIncoming(getReplacement(PHI->getIncomingValue(I)), PHI->getIncomingBlock(I)); From e659f8a8b50bdaf9d1b2486d48c5f318972a22de Mon Sep 17 00:00:00 2001 From: Ivan Krysak Date: Mon, 14 Jul 2025 17:41:51 +0300 Subject: [PATCH 126/147] InstCombinePHI: preserve `.pn` phi debug information --- llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp index 7f59729f0085..5c6a980aaa72 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp @@ -450,6 +450,7 @@ Instruction *InstCombinerImpl::foldPHIArgBinOpIntoPHI(PHINode &PN) { NewLHS = PHINode::Create(LHSType, PN.getNumIncomingValues(), FirstInst->getOperand(0)->getName() + ".pn"); NewLHS->addIncoming(InLHS, PN.getIncomingBlock(0)); + NewLHS->setDebugLoc(PN.getDebugLoc()); InsertNewInstBefore(NewLHS, PN); LHSVal = NewLHS; } @@ -458,6 +459,7 @@ Instruction *InstCombinerImpl::foldPHIArgBinOpIntoPHI(PHINode &PN) { NewRHS = PHINode::Create(RHSType, PN.getNumIncomingValues(), FirstInst->getOperand(1)->getName() + ".pn"); NewRHS->addIncoming(InRHS, PN.getIncomingBlock(0)); + NewRHS->setDebugLoc(PN.getDebugLoc()); InsertNewInstBefore(NewRHS, PN); RHSVal = NewRHS; } From 11001378c4627e5d03aba4da3cfb0c52441ec0ec Mon Sep 17 00:00:00 2001 From: Ivan Krysak Date: Mon, 14 Jul 2025 17:41:58 +0300 Subject: [PATCH 127/147] InstCombinePHI: preserve `.in` phi debug information --- llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp index 5c6a980aaa72..304b39d484d8 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp @@ -730,6 +730,7 @@ Instruction *InstCombinerImpl::foldPHIArgLoadIntoPHI(PHINode &PN) { PHINode *NewPN = PHINode::Create(FirstLI->getOperand(0)->getType(), PN.getNumIncomingValues(), PN.getName()+".in"); + NewPN->setDebugLoc(PN.getDebugLoc()); Value *InVal = FirstLI->getOperand(0); NewPN->addIncoming(InVal, PN.getIncomingBlock(0)); @@ -923,6 +924,7 @@ Instruction *InstCombinerImpl::foldPHIArgOpIntoPHI(PHINode &PN) { PHINode *NewPN = PHINode::Create(FirstInst->getOperand(0)->getType(), PN.getNumIncomingValues(), PN.getName()+".in"); + NewPN->setDebugLoc(PN.getDebugLoc()); Value *InVal = FirstInst->getOperand(0); NewPN->addIncoming(InVal, PN.getIncomingBlock(0)); From 0d5434a6f32c0924bbd633e6b0f6d94560915f14 Mon Sep 17 00:00:00 2001 From: Ivan Krysak Date: Mon, 14 Jul 2025 17:42:08 +0300 Subject: [PATCH 128/147] InstCombinePHI: preserve `.shrunk` phi debug information --- llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp index 304b39d484d8..b3cb2d4167ed 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp @@ -854,6 +854,7 @@ Instruction *InstCombinerImpl::foldPHIArgZextsIntoPHI(PHINode &Phi) { Phi.getName() + ".shrunk"); for (unsigned I = 0; I != NumIncomingValues; ++I) NewPhi->addIncoming(NewIncoming[I], Phi.getIncomingBlock(I)); + NewPhi->setDebugLoc(Phi.getDebugLoc()); InsertNewInstBefore(NewPhi, Phi); return CastInst::CreateZExtOrBitCast(NewPhi, Phi.getType()); From a04c466ac778b49c195b06a38e768f2d99abef32 Mon Sep 17 00:00:00 2001 From: Ivan Krysak Date: Mon, 11 Aug 2025 10:21:31 +0300 Subject: [PATCH 129/147] Introduce a debug information preservation style It's guarded behind the `EnableStrictDebugInformationPreservationStyle` option. --- .../llvm/Support/DebugInfoPreservation.h | 28 +++++++++++++++++++ llvm/lib/Support/CMakeLists.txt | 1 + llvm/lib/Support/DebugInfoPreservation.cpp | 24 ++++++++++++++++ 3 files changed, 53 insertions(+) create mode 100644 llvm/include/llvm/Support/DebugInfoPreservation.h create mode 100644 llvm/lib/Support/DebugInfoPreservation.cpp diff --git a/llvm/include/llvm/Support/DebugInfoPreservation.h b/llvm/include/llvm/Support/DebugInfoPreservation.h new file mode 100644 index 000000000000..fbf82ab40c63 --- /dev/null +++ b/llvm/include/llvm/Support/DebugInfoPreservation.h @@ -0,0 +1,28 @@ +//===- llvm/Support/CommandLine.h - Command line handler --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides an extra command line option that changes debug +// information preservation logic accross multiple transformation utilities. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_DEBUGINFOPRESERVATION_H +#define LLVM_SUPPORT_DEBUGINFOPRESERVATION_H + +#include "llvm/Support/CommandLine.h" + +namespace llvm { + +// This enables stricter debug information preservation. +// In particular, when this is set, debug information should never be dropped, +// even if it leads to worse debugger (gdb, lldb, etc) experience. +extern llvm::cl::opt EnableStrictDebugInformationPreservationStyle; + +} // end namespace llvm + +#endif // LLVM_SUPPORT_DEBUGINFOPRESERVATION_H diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt index 1058f4848601..57b7debfc078 100644 --- a/llvm/lib/Support/CMakeLists.txt +++ b/llvm/lib/Support/CMakeLists.txt @@ -154,6 +154,7 @@ add_llvm_component_library(LLVMSupport DataExtractor.cpp Debug.cpp DebugCounter.cpp + DebugInfoPreservation.cpp DeltaAlgorithm.cpp DivisionByConstantInfo.cpp DAGDeltaAlgorithm.cpp diff --git a/llvm/lib/Support/DebugInfoPreservation.cpp b/llvm/lib/Support/DebugInfoPreservation.cpp new file mode 100644 index 000000000000..ff667ea3c029 --- /dev/null +++ b/llvm/lib/Support/DebugInfoPreservation.cpp @@ -0,0 +1,24 @@ +//===-- CommandLine.cpp - Command line parser implementation --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides an extra command line option that changes debug +// information preservation logic accross multiple transformation utilities. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/DebugInfoPreservation.h" + +static constexpr llvm::StringRef Description = "Enable stricter debug " + "information preservation. In " + "particular avoid dropping it " + "even if it means conventional " + "debugger (gdb, lldb, etc) " + "experience degrades."; +llvm::cl::opt llvm::EnableStrictDebugInformationPreservationStyle( + "enable-strict-debug-information-preservation-style", + llvm::cl::desc(Description)); From 60bc843729dda298f69463042c70c118585ea81f Mon Sep 17 00:00:00 2001 From: Ivan Krysak Date: Mon, 14 Jul 2025 17:43:01 +0300 Subject: [PATCH 130/147] LICM: do not discard debug information --- llvm/lib/Transforms/Scalar/LICM.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp index 73e480cb9b0b..fe73f418f782 100644 --- a/llvm/lib/Transforms/Scalar/LICM.cpp +++ b/llvm/lib/Transforms/Scalar/LICM.cpp @@ -75,6 +75,7 @@ #include "llvm/InitializePasses.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/DebugInfoPreservation.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetOptions.h" #include "llvm/Transforms/Scalar.h" @@ -1739,7 +1740,8 @@ static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop, // Move the new node to the destination block, before its terminator. moveInstructionBefore(I, *Dest->getTerminator(), *SafetyInfo, MSSAU, SE); - I.updateLocationAfterHoist(); + if (not EnableStrictDebugInformationPreservationStyle) + I.updateLocationAfterHoist(); if (isa(I)) ++NumMovedLoads; From fd5eb0520afbc41f8e8a1c9c94b7aed7e583b765 Mon Sep 17 00:00:00 2001 From: Ivan Krysak Date: Mon, 14 Jul 2025 17:43:22 +0300 Subject: [PATCH 131/147] PromoteMem2Reg: do not discard debug information --- .../Transforms/Utils/PromoteMemoryToRegister.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp index 75ea9dc5dfc0..1a51a3ba3e44 100644 --- a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp +++ b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp @@ -43,6 +43,7 @@ #include "llvm/IR/Type.h" #include "llvm/IR/User.h" #include "llvm/Support/Casting.h" +#include "llvm/Support/DebugInfoPreservation.h" #include "llvm/Transforms/Utils/Local.h" #include "llvm/Transforms/Utils/PromoteMemToReg.h" #include @@ -959,6 +960,13 @@ bool PromoteMem2Reg::QueuePhiNode(BasicBlock *BB, unsigned AllocaNo, PN = PHINode::Create(Allocas[AllocaNo]->getAllocatedType(), getNumPreds(BB), Allocas[AllocaNo]->getName() + "." + Twine(Version++), &BB->front()); + + if (EnableStrictDebugInformationPreservationStyle) { + // Keep the debug information from the original alloca, to avoid discarding + // debug information. + PN->setDebugLoc(Allocas[AllocaNo]->getDebugLoc()); + } + ++NumPHIInsert; PhiToAllocaMap[PN] = AllocaNo; return true; @@ -968,6 +976,11 @@ bool PromoteMem2Reg::QueuePhiNode(BasicBlock *BB, unsigned AllocaNo, /// create a merged location incorporating \p DL, or to set \p DL directly. static void updateForIncomingValueLocation(PHINode *PN, DebugLoc DL, bool ApplyMergedLoc) { + if (EnableStrictDebugInformationPreservationStyle and not DL) { + // Do not override valid debug information with an empty location. + return; + } + if (ApplyMergedLoc) PN->applyMergedLocation(PN->getDebugLoc(), DL); else From 04d1d9e319a8246589a2477d8c1a0f3d401232c8 Mon Sep 17 00:00:00 2001 From: Ivan Krysak Date: Mon, 14 Jul 2025 17:43:34 +0300 Subject: [PATCH 132/147] SimplifyCFG: do not discard debug information --- llvm/lib/Transforms/Utils/SimplifyCFG.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp index 7e21d2e0cd45..d212a065abed 100644 --- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp @@ -67,6 +67,7 @@ #include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/DebugInfoPreservation.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/KnownBits.h" #include "llvm/Support/MathExtras.h" @@ -1105,12 +1106,14 @@ static void CloneInstructionsIntoPredecessorBlockAndUpdateSSAUses( Instruction *NewBonusInst = BonusInst.clone(); - if (PTI->getDebugLoc() != NewBonusInst->getDebugLoc()) { - // Unless the instruction has the same !dbg location as the original - // branch, drop it. When we fold the bonus instructions we want to make - // sure we reset their debug locations in order to avoid stepping on - // dead code caused by folding dead branches. - NewBonusInst->setDebugLoc(DebugLoc()); + if (not EnableStrictDebugInformationPreservationStyle) { + if (PTI->getDebugLoc() != NewBonusInst->getDebugLoc()) { + // Unless the instruction has the same !dbg location as the original + // branch, drop it. When we fold the bonus instructions we want to make + // sure we reset their debug locations in order to avoid stepping on + // dead code caused by folding dead branches. + NewBonusInst->setDebugLoc(DebugLoc()); + } } RemapInstruction(NewBonusInst, VMap, From 35001d7053f1307e17fb2eef28f7f091e12a13b8 Mon Sep 17 00:00:00 2001 From: Ivan Krysak Date: Mon, 14 Jul 2025 17:44:12 +0300 Subject: [PATCH 133/147] SSAUpdater: attach basic debug information to new Phis --- llvm/lib/Transforms/Utils/SSAUpdater.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/llvm/lib/Transforms/Utils/SSAUpdater.cpp b/llvm/lib/Transforms/Utils/SSAUpdater.cpp index 2520aa5d9db0..2453ea24699c 100644 --- a/llvm/lib/Transforms/Utils/SSAUpdater.cpp +++ b/llvm/lib/Transforms/Utils/SSAUpdater.cpp @@ -27,6 +27,7 @@ #include "llvm/IR/Value.h" #include "llvm/Support/Casting.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/DebugInfoPreservation.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Utils/SSAUpdaterImpl.h" #include @@ -269,6 +270,13 @@ class SSAUpdaterTraits { SSAUpdater *Updater) { PHINode *PHI = PHINode::Create(Updater->ProtoType, NumPreds, Updater->ProtoName, &BB->front()); + + if (EnableStrictDebugInformationPreservationStyle) { + // This is not the best, the proper solution would be to accept the debug + // location from the callee, but this will do for now. + PHI->setDebugLoc(BB->getTerminator()->getDebugLoc()); + } + return PHI; } From d8814bea6a4852f9d698ebece1d47683a373d3da Mon Sep 17 00:00:00 2001 From: Ivan Krysak Date: Mon, 21 Jul 2025 11:50:01 +0300 Subject: [PATCH 134/147] Error: print error messages in non-debug builds --- llvm/include/llvm/Support/Error.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/llvm/include/llvm/Support/Error.h b/llvm/include/llvm/Support/Error.h index ece888f247f6..714e1de198b2 100644 --- a/llvm/include/llvm/Support/Error.h +++ b/llvm/include/llvm/Support/Error.h @@ -746,12 +746,10 @@ inline void cantFail(Error Err, const char *Msg = nullptr) { if (Err) { if (!Msg) Msg = "Failure value returned from cantFail wrapped call"; -#ifndef NDEBUG std::string Str; raw_string_ostream OS(Str); OS << Msg << "\n" << Err; Msg = OS.str().c_str(); -#endif llvm_unreachable(Msg); } } @@ -776,13 +774,11 @@ T cantFail(Expected ValOrErr, const char *Msg = nullptr) { else { if (!Msg) Msg = "Failure value returned from cantFail wrapped call"; -#ifndef NDEBUG std::string Str; raw_string_ostream OS(Str); auto E = ValOrErr.takeError(); OS << Msg << "\n" << E; Msg = OS.str().c_str(); -#endif llvm_unreachable(Msg); } } @@ -807,13 +803,11 @@ T& cantFail(Expected ValOrErr, const char *Msg = nullptr) { else { if (!Msg) Msg = "Failure value returned from cantFail wrapped call"; -#ifndef NDEBUG std::string Str; raw_string_ostream OS(Str); auto E = ValOrErr.takeError(); OS << Msg << "\n" << E; Msg = OS.str().c_str(); -#endif llvm_unreachable(Msg); } } From f25067668935b26e0b16fe64c8d1a45867251f4d Mon Sep 17 00:00:00 2001 From: Alessandro Di Federico Date: Wed, 12 Nov 2025 13:47:57 +0100 Subject: [PATCH 135/147] InstCombine: prevent usage of illegal types --- llvm/lib/Transforms/Utils/Local.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp index b2ed95b05e04..581264152aa3 100644 --- a/llvm/lib/Transforms/Utils/Local.cpp +++ b/llvm/lib/Transforms/Utils/Local.cpp @@ -3302,7 +3302,12 @@ bool llvm::recognizeBSwapOrBitReverseIdiom( BitProvenance = BitProvenance.drop_back(); if (BitProvenance.empty()) return false; // TODO - handle null value? - DemandedTy = Type::getIntNTy(I->getContext(), BitProvenance.size()); + + unsigned Size = BitProvenance.size(); + if (!I->getModule()->getDataLayout().isLegalInteger(Size)) + return false; + + DemandedTy = Type::getIntNTy(I->getContext(), Size); if (auto *IVecTy = dyn_cast(ITy)) DemandedTy = VectorType::get(DemandedTy, IVecTy); } From f6c18566d2c601a19efe26920c49879f57681340 Mon Sep 17 00:00:00 2001 From: Alessandro Di Federico Date: Thu, 27 Nov 2025 10:19:32 +0100 Subject: [PATCH 136/147] CodeExtractor: fix debug message --- llvm/lib/Transforms/Utils/CodeExtractor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp index c1fe10504e45..b2df50f26594 100644 --- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp +++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp @@ -231,8 +231,8 @@ buildExtractionBlockSet(ArrayRef BBs, DominatorTree *DT, if (!Result.count(PBB)) { LLVM_DEBUG(dbgs() << "No blocks in this region may have entries from " "outside the region except for the first block!\n" - << "Problematic source BB: " << BB->getName() << "\n" - << "Problematic destination BB: " << PBB->getName() + << "Problematic source BB: " << PBB->getName() << "\n" + << "Problematic destination BB: " << BB->getName() << "\n"); return {}; } From ba502e0cb74929d0dac1a4a23d439d0737997ef6 Mon Sep 17 00:00:00 2001 From: Alessandro Di Federico Date: Thu, 27 Nov 2025 10:19:56 +0100 Subject: [PATCH 137/147] AsmWriter: introduce --emit-hex-constant-literals-from --- llvm/lib/IR/AsmWriter.cpp | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp index 6108ce09c289..ed7fe2cf07dd 100644 --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -88,6 +88,14 @@ using namespace llvm; +static cl::opt +EmitHexConstantLiteralsFrom("emit-hex-constant-literals-from", + cl::desc("Emit constant literals greater than or" + " equal to a certain unsigned value in" + " hexadecimal in LLVM IR."), + cl::init(std::numeric_limits::max())); + + // Make virtual table appear in this compilation unit. AssemblyAnnotationWriter::~AssemblyAnnotationWriter() = default; @@ -1353,7 +1361,16 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV, Out << (CI->getZExtValue() ? "true" : "false"); return; } - Out << CI->getValue(); + + if (EmitHexConstantLiteralsFrom != std::numeric_limits::max() + && CI->getValue().uge(EmitHexConstantLiteralsFrom)) { + std::string Value = llvm::toString(CI->getValue(), 16, false, false); + llvm::transform(Value, Value.begin(), tolower); + Out << "u0x" << Value; + } else { + Out << CI->getValue(); + } + return; } From 14d95e6a7fe991e252b8c45b54dd0318a808906e Mon Sep 17 00:00:00 2001 From: Giacomo Vercesi Date: Tue, 27 Jan 2026 15:42:48 +0100 Subject: [PATCH 138/147] cloneModule: allow omitting values Augment the `llvm::CloneModule` filter function to allow some `GlobalValue`s to be omitted altogether from the cloned module. --- llvm/include/llvm/Transforms/Utils/Cloning.h | 15 +++++++ llvm/lib/Transforms/Utils/CloneModule.cpp | 44 ++++++++++++++++++-- 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/llvm/include/llvm/Transforms/Utils/Cloning.h b/llvm/include/llvm/Transforms/Utils/Cloning.h index 77050391746b..e1a14a7a5f8c 100644 --- a/llvm/include/llvm/Transforms/Utils/Cloning.h +++ b/llvm/include/llvm/Transforms/Utils/Cloning.h @@ -57,6 +57,21 @@ std::unique_ptr CloneModule(const Module &M, ValueToValueMapTy &VMap, function_ref ShouldCloneDefinition); +enum class CloneAction : uint8_t { + Omit, + MakeDeclaration, + Clone, +}; + +/// This version works similarly to the one above with `ShouldCloneDefinition`, +/// however it allows specifying that a GlobalValue is to be completely skipped +/// for cloning via `CloneAction::Omit`. +/// NOTE: passing `Omit` is dangerous, if a GV is omitted and it's used by e.g. +/// a function then it will lead to an assertion when cloning. +std::unique_ptr +CloneModule(const Module &M, ValueToValueMapTy &VMap, + function_ref Action); + /// This struct can be used to capture information about code /// being cloned, while it is being cloned. struct ClonedCodeInfo { diff --git a/llvm/lib/Transforms/Utils/CloneModule.cpp b/llvm/lib/Transforms/Utils/CloneModule.cpp index 55e051298a9a..f0f6de4c4b9f 100644 --- a/llvm/lib/Transforms/Utils/CloneModule.cpp +++ b/llvm/lib/Transforms/Utils/CloneModule.cpp @@ -49,6 +49,15 @@ std::unique_ptr llvm::CloneModule(const Module &M, std::unique_ptr llvm::CloneModule( const Module &M, ValueToValueMapTy &VMap, function_ref ShouldCloneDefinition) { + return CloneModule(M, VMap, [&ShouldCloneDefinition](const GlobalValue *GV) { + return ShouldCloneDefinition(GV) ? CloneAction::Clone + : CloneAction::MakeDeclaration; + }); +} + +std::unique_ptr +llvm::CloneModule(const Module &M, ValueToValueMapTy &VMap, + function_ref Action) { // First off, we need to create the new module. std::unique_ptr New = std::make_unique(M.getModuleIdentifier(), M.getContext()); @@ -57,11 +66,16 @@ std::unique_ptr llvm::CloneModule( New->setTargetTriple(M.getTargetTriple()); New->setModuleInlineAsm(M.getModuleInlineAsm()); + DenseMap Actions; // Loop over all of the global variables, making corresponding globals in the // new module. Here we add them to the VMap and to the new Module. We // don't worry about attributes or initializers, they will come later. // for (const GlobalVariable &I : M.globals()) { + Actions[&I] = Action(&I); + if (Actions[&I] == CloneAction::Omit) + continue; + GlobalVariable *NewGV = new GlobalVariable( *New, I.getValueType(), I.isConstant(), I.getLinkage(), (Constant *)nullptr, I.getName(), (GlobalVariable *)nullptr, @@ -72,6 +86,10 @@ std::unique_ptr llvm::CloneModule( // Loop over the functions in the module, making external functions as before for (const Function &I : M) { + Actions[&I] = Action(&I); + if (Actions[&I] == CloneAction::Omit) + continue; + Function *NF = Function::Create(cast(I.getValueType()), I.getLinkage(), I.getAddressSpace(), I.getName(), New.get()); @@ -81,7 +99,11 @@ std::unique_ptr llvm::CloneModule( // Loop over the aliases in the module for (const GlobalAlias &I : M.aliases()) { - if (!ShouldCloneDefinition(&I)) { + Actions[&I] = Action(&I); + if (Actions[&I] == CloneAction::Omit) + continue; + + if (Actions[&I] == CloneAction::MakeDeclaration) { // An alias cannot act as an external reference, so we need to create // either a function or a global variable depending on the value type. // FIXME: Once pointee types are gone we can probably pick one or the @@ -110,6 +132,10 @@ std::unique_ptr llvm::CloneModule( } for (const GlobalIFunc &I : M.ifuncs()) { + Actions[&I] = Action(&I); + if (Actions[&I] == CloneAction::Omit) + continue; + // Defer setting the resolver function until after functions are cloned. auto *GI = GlobalIFunc::create(I.getValueType(), I.getAddressSpace(), @@ -123,6 +149,9 @@ std::unique_ptr llvm::CloneModule( // over... We also set the attributes on the global now. // for (const GlobalVariable &G : M.globals()) { + if (Actions[&G] == CloneAction::Omit) + continue; + GlobalVariable *GV = cast(VMap[&G]); SmallVector, 1> MDs; @@ -133,7 +162,7 @@ std::unique_ptr llvm::CloneModule( if (G.isDeclaration()) continue; - if (!ShouldCloneDefinition(&G)) { + if (Actions[&G] == CloneAction::MakeDeclaration) { // Skip after setting the correct linkage for an external reference. GV->setLinkage(GlobalValue::ExternalLinkage); continue; @@ -147,6 +176,9 @@ std::unique_ptr llvm::CloneModule( // Similarly, copy over function bodies now... // for (const Function &I : M) { + if (Actions[&I] == CloneAction::Omit) + continue; + Function *F = cast(VMap[&I]); if (I.isDeclaration()) { @@ -159,7 +191,7 @@ std::unique_ptr llvm::CloneModule( continue; } - if (!ShouldCloneDefinition(&I)) { + if (Actions[&I] == CloneAction::MakeDeclaration) { // Skip after setting the correct linkage for an external reference. F->setLinkage(GlobalValue::ExternalLinkage); // Personality function is not valid on a declaration. @@ -186,14 +218,18 @@ std::unique_ptr llvm::CloneModule( // And aliases for (const GlobalAlias &I : M.aliases()) { // We already dealt with undefined aliases above. - if (!ShouldCloneDefinition(&I)) + if (Actions[&I] != CloneAction::Clone) continue; + GlobalAlias *GA = cast(VMap[&I]); if (const Constant *C = I.getAliasee()) GA->setAliasee(MapValue(C, VMap)); } for (const GlobalIFunc &I : M.ifuncs()) { + if (Actions[&I] == CloneAction::Omit) + continue; + GlobalIFunc *GI = cast(VMap[&I]); if (const Constant *Resolver = I.getResolver()) GI->setResolver(MapValue(Resolver, VMap)); From 7c3510881414eabe27f55e90269600bd709655b6 Mon Sep 17 00:00:00 2001 From: Jacques Pienaar Date: Wed, 30 Aug 2023 15:29:53 -0700 Subject: [PATCH 139/147] [mlir] Relax requirement on memory buffer creation. parseSourceString does not require null-terminated string, hence requirement on memory buffer can be relaxed. Differential Revision: https://reviews.llvm.org/D159214 --- mlir/lib/Parser/Parser.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mlir/lib/Parser/Parser.cpp b/mlir/lib/Parser/Parser.cpp index 57dd3eeb2714..b2fa463d94d1 100644 --- a/mlir/lib/Parser/Parser.cpp +++ b/mlir/lib/Parser/Parser.cpp @@ -90,7 +90,9 @@ LogicalResult mlir::parseSourceFile( LogicalResult mlir::parseSourceString(llvm::StringRef sourceStr, Block *block, const ParserConfig &config, LocationAttr *sourceFileLoc) { - auto memBuffer = llvm::MemoryBuffer::getMemBuffer(sourceStr); + auto memBuffer = + llvm::MemoryBuffer::getMemBuffer(sourceStr, "", + /*RequiresNullTerminator=*/false); if (!memBuffer) return failure(); From 272ff29c874b568df875d941f2e630ecfadf3bc1 Mon Sep 17 00:00:00 2001 From: Balazs Benics Date: Mon, 3 Jul 2023 09:29:37 +0200 Subject: [PATCH 140/147] [ADT] Add deduction guide for iterator_range This commit also enhances the container overload constructor so that it now considers the `begin` and `end` free functions, and performs ADL lookups to find the best match. To make this possible, I had to split-off the ADL helper functions from the STLExtras header so that the iterator_range still remains pretty cheap to include. Differential Revision: https://reviews.llvm.org/D152891 --- llvm/include/llvm/ADT/ADL.h | 103 +++++++++++++++++++++++++ llvm/include/llvm/ADT/STLExtras.h | 58 +------------- llvm/include/llvm/ADT/iterator_range.h | 25 ++++-- 3 files changed, 124 insertions(+), 62 deletions(-) create mode 100644 llvm/include/llvm/ADT/ADL.h diff --git a/llvm/include/llvm/ADT/ADL.h b/llvm/include/llvm/ADT/ADL.h new file mode 100644 index 000000000000..ab1f28ff6b9c --- /dev/null +++ b/llvm/include/llvm/ADT/ADL.h @@ -0,0 +1,103 @@ +//===- llvm/ADT/ADL.h - Argument dependent lookup utilities -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ADT_ADL_H +#define LLVM_ADT_ADL_H + +#include +#include +#include + +namespace llvm { + +// Only used by compiler if both template types are the same. Useful when +// using SFINAE to test for the existence of member functions. +template struct SameType; + +namespace adl_detail { + +using std::begin; + +template +constexpr auto begin_impl(RangeT &&range) + -> decltype(begin(std::forward(range))) { + return begin(std::forward(range)); +} + +using std::end; + +template +constexpr auto end_impl(RangeT &&range) + -> decltype(end(std::forward(range))) { + return end(std::forward(range)); +} + +using std::swap; + +template +constexpr void swap_impl(T &&lhs, + T &&rhs) noexcept(noexcept(swap(std::declval(), + std::declval()))) { + swap(std::forward(lhs), std::forward(rhs)); +} + +using std::size; + +template +constexpr auto size_impl(RangeT &&range) + -> decltype(size(std::forward(range))) { + return size(std::forward(range)); +} + +} // end namespace adl_detail + +/// Returns the begin iterator to \p range using `std::begin` and +/// function found through Argument-Dependent Lookup (ADL). +template +constexpr auto adl_begin(RangeT &&range) + -> decltype(adl_detail::begin_impl(std::forward(range))) { + return adl_detail::begin_impl(std::forward(range)); +} + +/// Returns the end iterator to \p range using `std::end` and +/// functions found through Argument-Dependent Lookup (ADL). +template +constexpr auto adl_end(RangeT &&range) + -> decltype(adl_detail::end_impl(std::forward(range))) { + return adl_detail::end_impl(std::forward(range)); +} + +/// Swaps \p lhs with \p rhs using `std::swap` and functions found through +/// Argument-Dependent Lookup (ADL). +template +constexpr void adl_swap(T &&lhs, T &&rhs) noexcept( + noexcept(adl_detail::swap_impl(std::declval(), std::declval()))) { + adl_detail::swap_impl(std::forward(lhs), std::forward(rhs)); +} + +/// Returns the size of \p range using `std::size` and functions found through +/// Argument-Dependent Lookup (ADL). +template +constexpr auto adl_size(RangeT &&range) + -> decltype(adl_detail::size_impl(std::forward(range))) { + return adl_detail::size_impl(std::forward(range)); +} + +namespace detail { + +template +using IterOfRange = decltype(adl_begin(std::declval())); + +template +using ValueOfRange = + std::remove_reference_t()))>; + +} // namespace detail +} // namespace llvm + +#endif // LLVM_ADT_ADL_H diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h index a64ef504bfd5..0fc41d432bfb 100644 --- a/llvm/include/llvm/ADT/STLExtras.h +++ b/llvm/include/llvm/ADT/STLExtras.h @@ -17,6 +17,7 @@ #ifndef LLVM_ADT_STLEXTRAS_H #define LLVM_ADT_STLEXTRAS_H +#include "llvm/ADT/ADL.h" #include "llvm/ADT/Hashing.h" #include "llvm/ADT/STLForwardCompat.h" #include "llvm/ADT/STLFunctionalExtras.h" @@ -46,21 +47,6 @@ namespace llvm { -// Only used by compiler if both template types are the same. Useful when -// using SFINAE to test for the existence of member functions. -template struct SameType; - -namespace detail { - -template -using IterOfRange = decltype(std::begin(std::declval())); - -template -using ValueOfRange = - std::remove_reference_t()))>; - -} // end namespace detail - //===----------------------------------------------------------------------===// // Extra additions to //===----------------------------------------------------------------------===// @@ -333,48 +319,6 @@ template class Callable { } // namespace callable_detail -namespace adl_detail { - -using std::begin; - -template -decltype(auto) adl_begin(ContainerTy &&container) { - return begin(std::forward(container)); -} - -using std::end; - -template -decltype(auto) adl_end(ContainerTy &&container) { - return end(std::forward(container)); -} - -using std::swap; - -template -void adl_swap(T &&lhs, T &&rhs) noexcept(noexcept(swap(std::declval(), - std::declval()))) { - swap(std::forward(lhs), std::forward(rhs)); -} - -} // end namespace adl_detail - -template -decltype(auto) adl_begin(ContainerTy &&container) { - return adl_detail::adl_begin(std::forward(container)); -} - -template -decltype(auto) adl_end(ContainerTy &&container) { - return adl_detail::adl_end(std::forward(container)); -} - -template -void adl_swap(T &&lhs, T &&rhs) noexcept( - noexcept(adl_detail::adl_swap(std::declval(), std::declval()))) { - adl_detail::adl_swap(std::forward(lhs), std::forward(rhs)); -} - /// Returns true if the given container only contains a single element. template bool hasSingleElement(ContainerTy &&C) { auto B = std::begin(C), E = std::end(C); diff --git a/llvm/include/llvm/ADT/iterator_range.h b/llvm/include/llvm/ADT/iterator_range.h index a9b46a3aa45b..4f33a27adadd 100644 --- a/llvm/include/llvm/ADT/iterator_range.h +++ b/llvm/include/llvm/ADT/iterator_range.h @@ -18,10 +18,22 @@ #ifndef LLVM_ADT_ITERATOR_RANGE_H #define LLVM_ADT_ITERATOR_RANGE_H +#include "llvm/ADT/ADL.h" +#include #include namespace llvm { +template +struct explicitly_convertable : std::false_type {}; + +template +struct explicitly_convertable< + From, To, + std::void_t( + std::declval>()))>> : std::true_type { +}; + /// A range adaptor for a pair of iterators. /// /// This just wraps two iterators into a range-compatible interface. Nothing @@ -31,12 +43,12 @@ class iterator_range { IteratorT begin_iterator, end_iterator; public: - //TODO: Add SFINAE to test that the Container's iterators match the range's - // iterators. - template + template , IteratorT>::value> * = nullptr> iterator_range(Container &&c) - //TODO: Consider ADL/non-member begin/end calls. - : begin_iterator(c.begin()), end_iterator(c.end()) {} + : begin_iterator(adl_begin(std::forward(c))), + end_iterator(adl_end(std::forward(c))) {} iterator_range(IteratorT begin_iterator, IteratorT end_iterator) : begin_iterator(std::move(begin_iterator)), end_iterator(std::move(end_iterator)) {} @@ -46,6 +58,9 @@ class iterator_range { bool empty() const { return begin_iterator == end_iterator; } }; +template +iterator_range(Container &&) -> iterator_range>; + /// Convenience function for iterating over sub-ranges. /// /// This provides a bit of syntactic sugar to make using sub-ranges From 098e77fe7d07be413e25c8e8262c80950c1f1c60 Mon Sep 17 00:00:00 2001 From: Rahul Joshi Date: Thu, 22 Aug 2024 14:44:35 -0700 Subject: [PATCH 141/147] [NFC][ADT] Add reverse iterators and `value_type` to StringRef (#105579) - Add reverse iterators and `value_type` to StringRef. - Add unit test for all 4 iterator flavors. - This prepares StringRef to be used with `SequenceToOffsetTable`. --- llvm/include/llvm/ADT/StringRef.h | 12 ++++++++++++ llvm/unittests/ADT/StringRefTest.cpp | 14 +++++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/llvm/include/llvm/ADT/StringRef.h b/llvm/include/llvm/ADT/StringRef.h index 2d2f0bedfe1f..8d34c31aae39 100644 --- a/llvm/include/llvm/ADT/StringRef.h +++ b/llvm/include/llvm/ADT/StringRef.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -54,6 +55,9 @@ namespace llvm { using iterator = const char *; using const_iterator = const char *; using size_type = size_t; + using value_type = char; + using reverse_iterator = std::reverse_iterator; + using const_reverse_iterator = std::reverse_iterator; private: /// The start of the string, in an external buffer. @@ -112,6 +116,14 @@ namespace llvm { iterator end() const { return Data + Length; } + reverse_iterator rbegin() const { + return std::make_reverse_iterator(end()); + } + + reverse_iterator rend() const { + return std::make_reverse_iterator(begin()); + } + const unsigned char *bytes_begin() const { return reinterpret_cast(begin()); } diff --git a/llvm/unittests/ADT/StringRefTest.cpp b/llvm/unittests/ADT/StringRefTest.cpp index cad1974dd126..9e6eee368c0c 100644 --- a/llvm/unittests/ADT/StringRefTest.cpp +++ b/llvm/unittests/ADT/StringRefTest.cpp @@ -71,9 +71,17 @@ TEST(StringRefTest, EmptyInitializerList) { TEST(StringRefTest, Iteration) { StringRef S("hello"); - const char *p = "hello"; - for (const char *it = S.begin(), *ie = S.end(); it != ie; ++it, ++p) - EXPECT_EQ(*it, *p); + constexpr StringLiteral CS("hello"); + + // Note: Cannot use literal strings in equal() as iteration over a literal + // string includes the null terminator. + const std::string_view RefFwd("hello"); + const std::string_view RefRev("olleh"); + + EXPECT_TRUE(equal(S, RefFwd)); + EXPECT_TRUE(equal(CS, RefFwd)); + EXPECT_TRUE(equal(make_range(S.rbegin(), S.rend()), RefRev)); + EXPECT_TRUE(equal(make_range(CS.rbegin(), CS.rend()), RefRev)); } TEST(StringRefTest, StringOps) { From 86c41665f8c96582e6d01cf45edfc91ce571d553 Mon Sep 17 00:00:00 2001 From: Jakub Kuderski Date: Tue, 31 Mar 2026 10:53:13 +0200 Subject: [PATCH 142/147] [ADT] Allow reverse to find free rbegin/rend functions (#87840) Lift the requirement that rbegin/rend must be member functions. Also allow the rbegin/rend to be found through Argument Dependent Lookup (ADL) and add `adl_rbegin`/`adl_rend` to STLExtras. --- llvm/include/llvm/ADT/ADL.h | 32 ++++++++++++++++++++++++ llvm/include/llvm/ADT/STLExtras.h | 33 +++++++++---------------- llvm/unittests/ADT/IteratorTest.cpp | 17 +++++++++++++ llvm/unittests/ADT/RangeAdapterTest.cpp | 4 --- llvm/unittests/ADT/STLExtrasTest.cpp | 10 ++++++++ 5 files changed, 71 insertions(+), 25 deletions(-) diff --git a/llvm/include/llvm/ADT/ADL.h b/llvm/include/llvm/ADT/ADL.h index ab1f28ff6b9c..812d9a4b52d8 100644 --- a/llvm/include/llvm/ADT/ADL.h +++ b/llvm/include/llvm/ADT/ADL.h @@ -37,6 +37,22 @@ constexpr auto end_impl(RangeT &&range) return end(std::forward(range)); } +using std::rbegin; + +template +constexpr auto rbegin_impl(RangeT &&range) + -> decltype(rbegin(std::forward(range))) { + return rbegin(std::forward(range)); +} + +using std::rend; + +template +constexpr auto rend_impl(RangeT &&range) + -> decltype(rend(std::forward(range))) { + return rend(std::forward(range)); +} + using std::swap; template @@ -72,6 +88,22 @@ constexpr auto adl_end(RangeT &&range) return adl_detail::end_impl(std::forward(range)); } +/// Returns the reverse-begin iterator to \p range using `std::rbegin` and +/// function found through Argument-Dependent Lookup (ADL). +template +constexpr auto adl_rbegin(RangeT &&range) + -> decltype(adl_detail::rbegin_impl(std::forward(range))) { + return adl_detail::rbegin_impl(std::forward(range)); +} + +/// Returns the reverse-end iterator to \p range using `std::rend` and +/// functions found through Argument-Dependent Lookup (ADL). +template +constexpr auto adl_rend(RangeT &&range) + -> decltype(adl_detail::rend_impl(std::forward(range))) { + return adl_detail::rend_impl(std::forward(range)); +} + /// Swaps \p lhs with \p rhs using `std::swap` and functions found through /// Argument-Dependent Lookup (ADL). template diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h index 0fc41d432bfb..a378564fc77d 100644 --- a/llvm/include/llvm/ADT/STLExtras.h +++ b/llvm/include/llvm/ADT/STLExtras.h @@ -405,32 +405,23 @@ class mapped_iterator_base } }; -/// Helper to determine if type T has a member called rbegin(). -template class has_rbegin_impl { - using yes = char[1]; - using no = char[2]; - - template - static yes& test(Inner *I, decltype(I->rbegin()) * = nullptr); - - template - static no& test(...); - -public: - static const bool value = sizeof(test(nullptr)) == sizeof(yes); -}; +namespace detail { +template +using check_has_free_function_rbegin = + decltype(adl_rbegin(std::declval())); -/// Metafunction to determine if T& or T has a member called rbegin(). -template -struct has_rbegin : has_rbegin_impl> {}; +template +static constexpr bool HasFreeFunctionRBegin = + is_detected::value; +} // namespace detail // Returns an iterator_range over the given container which iterates in reverse. template auto reverse(ContainerTy &&C) { - if constexpr (has_rbegin::value) - return make_range(C.rbegin(), C.rend()); + if constexpr (detail::HasFreeFunctionRBegin) + return make_range(adl_rbegin(C), adl_rend(C)); else - return make_range(std::make_reverse_iterator(std::end(C)), - std::make_reverse_iterator(std::begin(C))); + return make_range(std::make_reverse_iterator(adl_end(C)), + std::make_reverse_iterator(adl_begin(C))); } /// An iterator adaptor that filters the elements of given inner iterators. diff --git a/llvm/unittests/ADT/IteratorTest.cpp b/llvm/unittests/ADT/IteratorTest.cpp index 641ff4fcc7f5..67504c7fbf5f 100644 --- a/llvm/unittests/ADT/IteratorTest.cpp +++ b/llvm/unittests/ADT/IteratorTest.cpp @@ -11,10 +11,12 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" +#include "gmock/gmock.h" #include "gtest/gtest.h" #include using namespace llvm; +using testing::ElementsAre; namespace { @@ -391,6 +393,21 @@ TEST(PointerIterator, Range) { EXPECT_EQ(A + I++, P); } +namespace rbegin_detail { +struct WithFreeRBegin { + int data[3] = {42, 43, 44}; +}; + +auto rbegin(const WithFreeRBegin &X) { return std::rbegin(X.data); } +auto rend(const WithFreeRBegin &X) { return std::rend(X.data); } +} // namespace rbegin_detail + +TEST(ReverseTest, ADL) { + // Check that we can find the rbegin/rend functions via ADL. + rbegin_detail::WithFreeRBegin Foo; + EXPECT_THAT(reverse(Foo), ElementsAre(44, 43, 42)); +} + TEST(ZipIteratorTest, Basic) { using namespace std; const SmallVector pi{3, 1, 4, 1, 5, 9}; diff --git a/llvm/unittests/ADT/RangeAdapterTest.cpp b/llvm/unittests/ADT/RangeAdapterTest.cpp index 3c68103e1745..5b8a99dd42c8 100644 --- a/llvm/unittests/ADT/RangeAdapterTest.cpp +++ b/llvm/unittests/ADT/RangeAdapterTest.cpp @@ -151,10 +151,6 @@ TYPED_TEST(RangeAdapterRValueTest, TrivialOperation) { TestRev(reverse(TypeParam({0, 1, 2, 3}))); } -TYPED_TEST(RangeAdapterRValueTest, HasRbegin) { - static_assert(has_rbegin::value, "rbegin() should be defined"); -} - TYPED_TEST(RangeAdapterRValueTest, RangeType) { static_assert( std::is_same< diff --git a/llvm/unittests/ADT/STLExtrasTest.cpp b/llvm/unittests/ADT/STLExtrasTest.cpp index 4a9c745f5328..ad2c9bdb0c8a 100644 --- a/llvm/unittests/ADT/STLExtrasTest.cpp +++ b/llvm/unittests/ADT/STLExtrasTest.cpp @@ -375,6 +375,14 @@ std::vector::const_iterator end(const some_struct &s) { return s.data.end(); } +std::vector::const_reverse_iterator rbegin(const some_struct &s) { + return s.data.rbegin(); +} + +std::vector::const_reverse_iterator rend(const some_struct &s) { + return s.data.rend(); +} + void swap(some_struct &lhs, some_struct &rhs) { // make swap visible as non-adl swap would even seem to // work with std::swap which defaults to moving @@ -389,6 +397,8 @@ TEST(STLExtrasTest, ADLTest) { EXPECT_EQ(*adl_begin(s), 1); EXPECT_EQ(*(adl_end(s) - 1), 5); + EXPECT_EQ(*adl_rbegin(s), 5); + EXPECT_EQ(*(adl_rend(s) - 1), 1); adl_swap(s, s2); EXPECT_EQ(s.swap_val, "lhs"); From 7ee11986ce609f9a18dc771d3313e957c6e78778 Mon Sep 17 00:00:00 2001 From: Alessandro Di Federico Date: Tue, 31 Mar 2026 10:53:33 +0200 Subject: [PATCH 143/147] [ADT] Add llvm::mismatch and llvm::range_size range wrappers Add range-based wrappers for std::mismatch and std::distance, needed by RadixTree. --- llvm/include/llvm/ADT/STLExtras.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h index a378564fc77d..8c0421daf2da 100644 --- a/llvm/include/llvm/ADT/STLExtras.h +++ b/llvm/include/llvm/ADT/STLExtras.h @@ -1879,6 +1879,19 @@ auto upper_bound(R &&Range, T &&Value, Compare C) { std::forward(Value), C); } +/// Wrapper function around std::mismatch to find the first mismatching pair +/// in two ranges. +template +auto mismatch(R1 &&Range1, R2 &&Range2) { + return std::mismatch(adl_begin(Range1), adl_end(Range1), adl_begin(Range2), + adl_end(Range2)); +} + +/// Returns the size of a range, i.e., the number of elements. +template auto range_size(R &&Range) { + return std::distance(adl_begin(Range), adl_end(Range)); +} + template void stable_sort(R &&Range) { std::stable_sort(adl_begin(Range), adl_end(Range)); From f82a01844f56b8319d5f74ff4e83ffe19346a49d Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Tue, 31 Mar 2026 10:53:57 +0200 Subject: [PATCH 144/147] [NFC][ADT] Add RadixTree (#164524) --- llvm/docs/ProgrammersManual.rst | 10 + llvm/include/llvm/ADT/RadixTree.h | 350 ++++++++++++++++++++++++ llvm/unittests/ADT/CMakeLists.txt | 1 + llvm/unittests/ADT/RadixTreeTest.cpp | 385 +++++++++++++++++++++++++++ 4 files changed, 746 insertions(+) create mode 100644 llvm/include/llvm/ADT/RadixTree.h create mode 100644 llvm/unittests/ADT/RadixTreeTest.cpp diff --git a/llvm/docs/ProgrammersManual.rst b/llvm/docs/ProgrammersManual.rst index d1b3744e22da..99391b43de87 100644 --- a/llvm/docs/ProgrammersManual.rst +++ b/llvm/docs/ProgrammersManual.rst @@ -2040,6 +2040,16 @@ that are not simple pointers (use :ref:`SmallPtrSet ` for pointers). Note that DenseSet has the same requirements for the value type that :ref:`DenseMap ` has. +.. _dss_radixtree: + +llvm/ADT/RadixTree.h +^^^^^^^^^^^^^^^^^^^^ + +``RadixTree`` is a trie-based data structure that stores range-like keys and +their associated values. It is particularly efficient for storing keys that +share common prefixes, as it can compress these prefixes to save memory. It +supports efficient search of matching prefixes. + .. _dss_sparseset: llvm/ADT/SparseSet.h diff --git a/llvm/include/llvm/ADT/RadixTree.h b/llvm/include/llvm/ADT/RadixTree.h new file mode 100644 index 000000000000..d3c44e4e6345 --- /dev/null +++ b/llvm/include/llvm/ADT/RadixTree.h @@ -0,0 +1,350 @@ +//===-- llvm/ADT/RadixTree.h - Radix Tree implementation --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +//===----------------------------------------------------------------------===// +// +// This file implements a Radix Tree. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ADT_RADIXTREE_H +#define LLVM_ADT_RADIXTREE_H + +#include "llvm/ADT/ADL.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/iterator.h" +#include "llvm/ADT/iterator_range.h" +#include +#include +#include +#include +#include +#include + +namespace llvm { + +/// \brief A Radix Tree implementation. +/// +/// A Radix Tree (also known as a compact prefix tree or radix trie) is a +/// data structure that stores a dynamic set or associative array where keys +/// are strings and values are associated with these keys. Unlike a regular +/// trie, the edges of a radix tree can be labeled with sequences of characters +/// as well as single characters. This makes radix trees more efficient for +/// storing sparse data sets, where many nodes in a regular trie would have +/// only one child. +/// +/// This implementation supports arbitrary key types that can be iterated over +/// (e.g., `std::string`, `std::vector`, `ArrayRef`). The key type +/// must provide `begin()` and `end()` for iteration. +/// +/// The tree stores `std::pair` as its value type. +/// +/// Example usage: +/// \code +/// llvm::RadixTree Tree; +/// Tree.emplace("apple", 1); +/// Tree.emplace("grapefruit", 2); +/// Tree.emplace("grape", 3); +/// +/// // Find prefixes +/// for (const auto &[Key, Value] : Tree.find_prefixes("grapefruit juice")) { +/// // pair will be {"grape", 3} +/// // pair will be {"grapefruit", 2} +/// llvm::outs() << Key << ": " << Value << "\n"; +/// } +/// +/// // Iterate over all elements +/// for (const auto &[Key, Value] : Tree) +/// llvm::outs() << Key << ": " << Value << "\n"; +/// \endcode +/// +/// \note +/// The `RadixTree` takes ownership of the `KeyType` and `T` objects +/// inserted into it. When an element is removed or the tree is destroyed, +/// these objects will be destructed. +/// However, if `KeyType` is a reference-like type, e.g., StringRef or range, +/// the user must guarantee that the referenced data has a lifetime longer than +/// the tree. +template class RadixTree { +public: + using key_type = KeyType; + using mapped_type = T; + using value_type = std::pair; + +private: + using KeyConstIteratorType = + decltype(adl_begin(std::declval())); + using KeyConstIteratorRangeType = iterator_range; + using KeyValueType = + remove_cvref_t()))>; + using ContainerType = std::list; + + /// Represents an internal node in the Radix Tree. + struct Node { + KeyConstIteratorRangeType Key{KeyConstIteratorType{}, + KeyConstIteratorType{}}; + std::vector Children; + + /// An iterator to the value associated with this node. + /// + /// If this node does not have a value (i.e., it's an internal node that + /// only serves as a path to other values), this iterator will be equal + /// to default constructed `ContainerType::iterator()`. + typename ContainerType::iterator Value; + + /// The first character of the Key. Used for fast child lookup. + KeyValueType KeyFront; + + Node() = default; + Node(const KeyConstIteratorRangeType &Key) + : Key(Key), KeyFront(*Key.begin()) { + assert(!Key.empty()); + } + + Node(Node &&) = default; + Node &operator=(Node &&) = default; + + Node(const Node &) = delete; + Node &operator=(const Node &) = delete; + + const Node *findChild(const KeyConstIteratorRangeType &Key) const { + if (Key.empty()) + return nullptr; + for (const Node &Child : Children) { + assert(!Child.Key.empty()); // Only root can be empty. + if (Child.KeyFront == *Key.begin()) + return &Child; + } + return nullptr; + } + + Node *findChild(const KeyConstIteratorRangeType &Query) { + const Node *This = this; + return const_cast(This->findChild(Query)); + } + + size_t countNodes() const { + size_t R = 1; + for (const Node &C : Children) + R += C.countNodes(); + return R; + } + + /// + /// Splits the current node into two. + /// + /// This function is used when a new key needs to be inserted that shares + /// a common prefix with the current node's key, but then diverges. + /// The current `Key` is truncated to the common prefix, and a new child + /// node is created for the remainder of the original node's `Key`. + /// + /// \param SplitPoint An iterator pointing to the character in the current + /// `Key` where the split should occur. + void split(KeyConstIteratorType SplitPoint) { + Node Child(make_range(SplitPoint, Key.end())); + Key = make_range(Key.begin(), SplitPoint); + + Children.swap(Child.Children); + std::swap(Value, Child.Value); + + Children.emplace_back(std::move(Child)); + } + }; + + /// Root always corresponds to the empty key, which is the shortest possible + /// prefix for everything. + Node Root; + ContainerType KeyValuePairs; + + /// Finds or creates a new tail or leaf node corresponding to the `Key`. + Node &findOrCreate(KeyConstIteratorRangeType Key) { + Node *Curr = &Root; + if (Key.empty()) + return *Curr; + + for (;;) { + auto [I1, I2] = llvm::mismatch(Key, Curr->Key); + Key = make_range(I1, Key.end()); + + if (I2 != Curr->Key.end()) { + // Match is partial. Either query is too short, or there is mismatching + // character. Split either way, and put new node in between of the + // current and its children. + Curr->split(I2); + + // Split was caused by mismatch, so `findChild` would fail. + break; + } + + Node *Child = Curr->findChild(Key); + if (!Child) + break; + + // Move to child with the same first character. + Curr = Child; + } + + if (Key.empty()) { + // The current node completely matches the key, return it. + return *Curr; + } + + // `Key` is a suffix of original `Key` unmatched by path from the `Root` to + // the `Curr`, and we have no candidate in the children to match more. + // Create a new one. + return Curr->Children.emplace_back(Key); + } + + /// + /// An iterator for traversing prefixes search results. + /// + /// This iterator is used by `find_prefixes` to traverse the tree and find + /// elements that are prefixes to the given key. It's a forward iterator. + /// + /// \tparam MappedType The type of the value pointed to by the iterator. + /// This will be `value_type` for non-const iterators + /// and `const value_type` for const iterators. + template + class IteratorImpl + : public iterator_facade_base, + std::forward_iterator_tag, MappedType> { + const Node *Curr = nullptr; + KeyConstIteratorRangeType Query{KeyConstIteratorType{}, + KeyConstIteratorType{}}; + + void findNextValid() { + while (Curr && Curr->Value == typename ContainerType::iterator()) + advance(); + } + + void advance() { + assert(Curr); + if (Query.empty()) { + Curr = nullptr; + return; + } + + Curr = Curr->findChild(Query); + if (!Curr) { + Curr = nullptr; + return; + } + + auto [I1, I2] = llvm::mismatch(Query, Curr->Key); + if (I2 != Curr->Key.end()) { + Curr = nullptr; + return; + } + Query = make_range(I1, Query.end()); + } + + friend class RadixTree; + IteratorImpl(const Node *C, const KeyConstIteratorRangeType &Q) + : Curr(C), Query(Q) { + findNextValid(); + } + + public: + IteratorImpl() = default; + + MappedType &operator*() const { return *Curr->Value; } + + IteratorImpl &operator++() { + advance(); + findNextValid(); + return *this; + } + + bool operator==(const IteratorImpl &Other) const { + return Curr == Other.Curr; + } + }; + +public: + RadixTree() = default; + RadixTree(RadixTree &&) = default; + RadixTree &operator=(RadixTree &&) = default; + + using prefix_iterator = IteratorImpl; + using const_prefix_iterator = IteratorImpl; + + using iterator = typename ContainerType::iterator; + using const_iterator = typename ContainerType::const_iterator; + + /// Returns true if the tree is empty. + bool empty() const { return KeyValuePairs.empty(); } + + /// Returns the number of elements in the tree. + size_t size() const { return KeyValuePairs.size(); } + + /// Returns the number of nodes in the tree. + /// + /// This function counts all internal nodes in the tree. It can be useful for + /// understanding the memory footprint or complexity of the tree structure. + size_t countNodes() const { return Root.countNodes(); } + + /// Returns an iterator to the first element. + iterator begin() { return KeyValuePairs.begin(); } + const_iterator begin() const { return KeyValuePairs.begin(); } + + /// Returns an iterator to the end of the tree. + iterator end() { return KeyValuePairs.end(); } + const_iterator end() const { return KeyValuePairs.end(); } + + /// Constructs and inserts a new element into the tree. + /// + /// This function constructs an element in place within the tree. If an + /// element with the same key already exists, the insertion fails and the + /// function returns an iterator to the existing element along with `false`. + /// Otherwise, the new element is inserted and the function returns an + /// iterator to the new element along with `true`. + /// + /// \param Key The key of the element to construct. + /// \param Args Arguments to forward to the constructor of the mapped_type. + /// \return A pair consisting of an iterator to the inserted element (or to + /// the element that prevented insertion) and a boolean value + /// indicating whether the insertion took place. + template + std::pair emplace(key_type &&Key, Ts &&...Args) { + // We want to make new `Node` to refer key in the container, not the one + // from the argument. + // FIXME: Determine that we need a new node, before expanding + // `KeyValuePairs`. + const value_type &NewValue = KeyValuePairs.emplace_front( + std::move(Key), T(std::forward(Args)...)); + Node &Node = findOrCreate(NewValue.first); + bool HasValue = Node.Value != typename ContainerType::iterator(); + if (!HasValue) + Node.Value = KeyValuePairs.begin(); + else + KeyValuePairs.pop_front(); + return {Node.Value, !HasValue}; + } + + /// + /// Finds all elements whose keys are prefixes of the given `Key`. + /// + /// This function returns an iterator range over all elements in the tree + /// whose keys are prefixes of the provided `Key`. For example, if the tree + /// contains "abcde", "abc", "abcdefgh", and `Key` is "abcde", this function + /// would return iterators to "abcde" and "abc". + /// + /// \param Key The key to search for prefixes of. + /// \return An `iterator_range` of `const_prefix_iterator`s, allowing + /// iteration over the found prefix elements. + /// \note The returned iterators reference the `Key` provided by the caller. + /// The caller must ensure that `Key` remains valid for the lifetime + /// of the iterators. + iterator_range + find_prefixes(const key_type &Key) const { + return iterator_range{ + const_prefix_iterator(&Root, KeyConstIteratorRangeType(Key)), + const_prefix_iterator{}}; + } +}; + +} // namespace llvm + +#endif // LLVM_ADT_RADIXTREE_H diff --git a/llvm/unittests/ADT/CMakeLists.txt b/llvm/unittests/ADT/CMakeLists.txt index 900294d4216e..8f0a3c501b52 100644 --- a/llvm/unittests/ADT/CMakeLists.txt +++ b/llvm/unittests/ADT/CMakeLists.txt @@ -56,6 +56,7 @@ add_llvm_unittest(ADTTests PointerUnionTest.cpp PostOrderIteratorTest.cpp PriorityWorklistTest.cpp + RadixTreeTest.cpp RangeAdapterTest.cpp SCCIteratorTest.cpp STLExtrasTest.cpp diff --git a/llvm/unittests/ADT/RadixTreeTest.cpp b/llvm/unittests/ADT/RadixTreeTest.cpp new file mode 100644 index 000000000000..54f4697c2f3e --- /dev/null +++ b/llvm/unittests/ADT/RadixTreeTest.cpp @@ -0,0 +1,385 @@ +//===- llvm/unittest/ADT/RadixTreeTest.cpp --------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/RadixTree.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringRef.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include +#include +#include + +using namespace llvm; +namespace { + +using ::testing::ElementsAre; +using ::testing::ElementsAreArray; +using ::testing::Pair; +using ::testing::UnorderedElementsAre; + +// Wrapper for ElementsAreArray that works with ranges lacking ::value_type +// (e.g., iterator_range). +template auto RangeElementsAre(const Range &R) { + return ElementsAreArray(adl_begin(R), adl_end(R)); +} + +// Test with StringRef. + +TEST(RadixTreeTest, Empty) { + RadixTree T; + EXPECT_TRUE(T.empty()); + EXPECT_EQ(T.size(), 0u); + + EXPECT_TRUE(T.find_prefixes("").empty()); + EXPECT_TRUE(T.find_prefixes("A").empty()); + + EXPECT_EQ(T.countNodes(), 1u); +} + +TEST(RadixTreeTest, InsertEmpty) { + RadixTree T; + auto [It, IsNew] = T.emplace("", 4); + EXPECT_TRUE(!T.empty()); + EXPECT_EQ(T.size(), 1u); + EXPECT_TRUE(IsNew); + const auto &[K, V] = *It; + EXPECT_TRUE(K.empty()); + EXPECT_EQ(4, V); + + EXPECT_THAT(T, ElementsAre(Pair("", 4))); + + EXPECT_THAT(T.find_prefixes(""), ElementsAre(Pair("", 4))); + + EXPECT_THAT(T.find_prefixes("a"), ElementsAre(Pair("", 4))); + + EXPECT_EQ(T.countNodes(), 1u); +} + +TEST(RadixTreeTest, Complex) { + RadixTree T; + T.emplace("abcd", 1); + EXPECT_EQ(T.countNodes(), 2u); + T.emplace("abklm", 2); + EXPECT_EQ(T.countNodes(), 4u); + T.emplace("123abklm", 3); + EXPECT_EQ(T.countNodes(), 5u); + T.emplace("123abklm", 4); + EXPECT_EQ(T.countNodes(), 5u); + T.emplace("ab", 5); + EXPECT_EQ(T.countNodes(), 5u); + T.emplace("1234567", 6); + EXPECT_EQ(T.countNodes(), 7u); + T.emplace("123456", 7); + EXPECT_EQ(T.countNodes(), 8u); + T.emplace("123456789", 8); + EXPECT_EQ(T.countNodes(), 9u); + + EXPECT_THAT(T, UnorderedElementsAre(Pair("abcd", 1), Pair("abklm", 2), + Pair("123abklm", 3), Pair("ab", 5), + Pair("1234567", 6), Pair("123456", 7), + Pair("123456789", 8))); + + EXPECT_THAT(T.find_prefixes("1234567890"), + UnorderedElementsAre(Pair("1234567", 6), Pair("123456", 7), + Pair("123456789", 8))); + + EXPECT_THAT(T.find_prefixes("123abklm"), + UnorderedElementsAre(Pair("123abklm", 3))); + + EXPECT_THAT(T.find_prefixes("abcdefg"), + UnorderedElementsAre(Pair("abcd", 1), Pair("ab", 5))); + + EXPECT_EQ(T.countNodes(), 9u); +} + +TEST(RadixTreeTest, ValueWith2Parameters) { + RadixTree> T; + T.emplace("abcd", "a", 3); + + EXPECT_THAT(T, UnorderedElementsAre(Pair("abcd", Pair("a", 3)))); +} + +// Test different types, less readable. + +template struct TestData { + static const T Data1[]; + static const T Data2[]; +}; + +template <> const char TestData::Data1[] = "abcdedcba"; +template <> const char TestData::Data2[] = "abCDEDCba"; + +template <> const int TestData::Data1[] = {1, 2, 3, 4, 5, 4, 3, 2, 1}; +template <> const int TestData::Data2[] = {1, 2, 4, 8, 16, 8, 4, 2, 1}; + +template class RadixTreeTypeTest : public ::testing::Test { +public: + using IteratorType = decltype(adl_begin(std::declval())); + using CharType = remove_cvref_t()))>; + + T make(const CharType *Data, size_t N) { return T(StringRef(Data, N)); } + + T make1(size_t N) { return make(TestData::Data1, N); } + T make2(size_t N) { return make(TestData::Data2, N); } +}; + +template <> +iterator_range +RadixTreeTypeTest>::make( + const char *Data, size_t N) { + return StringRef(Data).take_front(N); +} + +template <> +iterator_range +RadixTreeTypeTest>::make( + const char *Data, size_t N) { + return reverse(StringRef(Data).take_back(N)); +} + +template <> +ArrayRef RadixTreeTypeTest>::make(const int *Data, + size_t N) { + return ArrayRef(Data, Data + N); +} + +template <> +std::vector RadixTreeTypeTest>::make(const int *Data, + size_t N) { + return std::vector(Data, Data + N); +} + +template <> +std::list RadixTreeTypeTest>::make(const int *Data, + size_t N) { + return std::list(Data, Data + N); +} + +class TypeNameGenerator { +public: + template static std::string GetName(int) { + if (std::is_same_v) + return "StringRef"; + if (std::is_same_v) + return "string"; + if (std::is_same_v>) + return "iterator_range"; + if (std::is_same_v>) + return "reverse_iterator_range"; + if (std::is_same_v>) + return "ArrayRef"; + if (std::is_same_v>) + return "vector"; + if (std::is_same_v>) + return "list"; + return "Unknown"; + } +}; + +using TestTypes = + ::testing::Types, + iterator_range, + ArrayRef, std::vector, std::list>; + +TYPED_TEST_SUITE(RadixTreeTypeTest, TestTypes, TypeNameGenerator); + +TYPED_TEST(RadixTreeTypeTest, Helpers) { + for (size_t i = 0; i < 9; ++i) { + auto R1 = this->make1(i); + auto R2 = this->make2(i); + EXPECT_EQ(llvm::range_size(R1), i); + EXPECT_EQ(llvm::range_size(R2), i); + auto [I1, I2] = llvm::mismatch(R1, R2); + // Exactly 2 first elements of Data1 and Data2 must match. + EXPECT_EQ(std::distance(R1.begin(), I1), std::min(2, i)); + } +} + +TYPED_TEST(RadixTreeTypeTest, Empty) { + RadixTree T; + EXPECT_TRUE(T.empty()); + EXPECT_EQ(T.size(), 0u); + + EXPECT_TRUE(T.find_prefixes(this->make1(0)).empty()); + EXPECT_TRUE(T.find_prefixes(this->make2(1)).empty()); + + EXPECT_EQ(T.countNodes(), 1u); +} + +TYPED_TEST(RadixTreeTypeTest, InsertEmpty) { + using TreeType = RadixTree; + TreeType T; + auto [It, IsNew] = T.emplace(this->make1(0), 5); + EXPECT_TRUE(!T.empty()); + EXPECT_EQ(T.size(), 1u); + EXPECT_TRUE(IsNew); + const auto &[K, V] = *It; + EXPECT_TRUE(K.empty()); + EXPECT_EQ(5, V); + + EXPECT_THAT(T.find_prefixes(this->make1(0)), + ElementsAre(Pair(ElementsAre(), 5))); + + EXPECT_THAT(T.find_prefixes(this->make2(1)), + ElementsAre(Pair(ElementsAre(), 5))); + + EXPECT_THAT(T, ElementsAre(Pair(ElementsAre(), 5))); + + EXPECT_EQ(T.countNodes(), 1u); +} + +TYPED_TEST(RadixTreeTypeTest, InsertEmptyTwice) { + using TreeType = RadixTree; + TreeType T; + T.emplace(this->make1(0), 5); + auto [It, IsNew] = T.emplace(this->make1(0), 6); + EXPECT_TRUE(!T.empty()); + EXPECT_EQ(T.size(), 1u); + EXPECT_TRUE(!IsNew); + const auto &[K, V] = *It; + EXPECT_TRUE(K.empty()); + EXPECT_EQ(5, V); + + EXPECT_THAT(T.find_prefixes(this->make1(0)), + ElementsAre(Pair(ElementsAre(), 5))); + + EXPECT_THAT(T.find_prefixes(this->make2(1)), + ElementsAre(Pair(ElementsAre(), 5))); + + EXPECT_THAT(T, ElementsAre(Pair(ElementsAre(), 5))); + + EXPECT_EQ(T.countNodes(), 1u); +} + +TYPED_TEST(RadixTreeTypeTest, InsertOne) { + using TreeType = RadixTree; + TreeType T; + auto [It, IsNew] = T.emplace(this->make1(1), 4); + EXPECT_TRUE(!T.empty()); + EXPECT_EQ(T.size(), 1u); + EXPECT_TRUE(IsNew); + const auto &[K, V] = *It; + EXPECT_THAT(K, RangeElementsAre(this->make1(1))); + EXPECT_EQ(4, V); + + EXPECT_THAT(T, ElementsAre(Pair(RangeElementsAre(this->make1(1)), 4))); + + EXPECT_THAT(T.find_prefixes(this->make1(1)), + ElementsAre(Pair(RangeElementsAre(this->make1(1)), 4))); + + EXPECT_THAT(T.find_prefixes(this->make1(2)), + ElementsAre(Pair(RangeElementsAre(this->make1(1)), 4))); + + EXPECT_EQ(T.countNodes(), 2u); +} + +TYPED_TEST(RadixTreeTypeTest, InsertOneTwice) { + using TreeType = RadixTree; + TreeType T; + T.emplace(this->make1(1), 4); + auto [It, IsNew] = T.emplace(this->make1(1), 4); + EXPECT_TRUE(!T.empty()); + EXPECT_EQ(T.size(), 1u); + EXPECT_TRUE(!IsNew); + + EXPECT_THAT(T, ElementsAre(Pair(RangeElementsAre(this->make1(1)), 4))); + EXPECT_EQ(T.countNodes(), 2u); +} + +TYPED_TEST(RadixTreeTypeTest, InsertSuperStrings) { + using TreeType = RadixTree; + TreeType T; + + for (size_t Len = 0; Len < 7; Len += 2) { + auto [It, IsNew] = T.emplace(this->make1(Len), Len); + EXPECT_TRUE(IsNew); + } + + EXPECT_THAT(T, + UnorderedElementsAre(Pair(RangeElementsAre(this->make1(0)), 0), + Pair(RangeElementsAre(this->make1(2)), 2), + Pair(RangeElementsAre(this->make1(4)), 4), + Pair(RangeElementsAre(this->make1(6)), 6))); + + EXPECT_THAT(T.find_prefixes(this->make1(0)), + UnorderedElementsAre(Pair(RangeElementsAre(this->make1(0)), 0))); + + EXPECT_THAT(T.find_prefixes(this->make1(3)), + UnorderedElementsAre(Pair(RangeElementsAre(this->make1(0)), 0), + Pair(RangeElementsAre(this->make1(2)), 2))); + + EXPECT_THAT(T.find_prefixes(this->make1(7)), + UnorderedElementsAre(Pair(RangeElementsAre(this->make1(0)), 0), + Pair(RangeElementsAre(this->make1(2)), 2), + Pair(RangeElementsAre(this->make1(4)), 4), + Pair(RangeElementsAre(this->make1(6)), 6))); + + EXPECT_EQ(T.countNodes(), 4u); +} + +TYPED_TEST(RadixTreeTypeTest, InsertSubStrings) { + using TreeType = RadixTree; + TreeType T; + + for (size_t Len = 0; Len < 7; Len += 2) { + auto [It, IsNew] = T.emplace(this->make1(7 - Len), 7 - Len); + EXPECT_TRUE(IsNew); + } + + EXPECT_THAT(T, + UnorderedElementsAre(Pair(RangeElementsAre(this->make1(1)), 1), + Pair(RangeElementsAre(this->make1(3)), 3), + Pair(RangeElementsAre(this->make1(5)), 5), + Pair(RangeElementsAre(this->make1(7)), 7))); + + EXPECT_THAT(T.find_prefixes(this->make1(0)), UnorderedElementsAre()); + + EXPECT_THAT(T.find_prefixes(this->make1(3)), + UnorderedElementsAre(Pair(RangeElementsAre(this->make1(1)), 1), + Pair(RangeElementsAre(this->make1(3)), 3))); + + EXPECT_THAT(T.find_prefixes(this->make1(6)), + UnorderedElementsAre(Pair(RangeElementsAre(this->make1(1)), 1), + Pair(RangeElementsAre(this->make1(3)), 3), + Pair(RangeElementsAre(this->make1(5)), 5))); + + EXPECT_EQ(T.countNodes(), 5u); +} + +TYPED_TEST(RadixTreeTypeTest, InsertVShape) { + using TreeType = RadixTree; + TreeType T; + + EXPECT_EQ(T.countNodes(), 1u); + T.emplace(this->make1(5), 15); + EXPECT_EQ(T.countNodes(), 2u); + T.emplace(this->make2(6), 26); + EXPECT_EQ(T.countNodes(), 4u); + T.emplace(this->make2(1), 21); + EXPECT_EQ(T.countNodes(), 5u); + + EXPECT_THAT(T, + UnorderedElementsAre(Pair(RangeElementsAre(this->make1(5)), 15), + Pair(RangeElementsAre(this->make2(6)), 26), + Pair(RangeElementsAre(this->make2(1)), 21))); + + EXPECT_THAT(T.find_prefixes(this->make1(7)), + UnorderedElementsAre(Pair(RangeElementsAre(this->make2(1)), 21), + Pair(RangeElementsAre(this->make1(5)), 15))); + + EXPECT_THAT(T.find_prefixes(this->make2(7)), + UnorderedElementsAre(Pair(RangeElementsAre(this->make2(1)), 21), + Pair(RangeElementsAre(this->make2(6)), 26))); + + EXPECT_EQ(T.countNodes(), 5u); +} + +} // namespace From 8955c8c77f6481954d34f1bf99d1a9bae4eb4096 Mon Sep 17 00:00:00 2001 From: Jordan Rupprecht Date: Fri, 24 Oct 2025 22:58:46 -0500 Subject: [PATCH 145/147] [ADT][NFC] Add missing #include (#165068) Added in #164524. Fails when using libc++ in a mode that prunes transitive headers. --- llvm/include/llvm/ADT/RadixTree.h | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/include/llvm/ADT/RadixTree.h b/llvm/include/llvm/ADT/RadixTree.h index d3c44e4e6345..a65acddf186b 100644 --- a/llvm/include/llvm/ADT/RadixTree.h +++ b/llvm/include/llvm/ADT/RadixTree.h @@ -22,6 +22,7 @@ #include #include #include +#include namespace llvm { From 5c7d083f9bcd066cfda6e87e84e2e70a75398607 Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Sat, 25 Oct 2025 15:46:28 -0700 Subject: [PATCH 146/147] [ADT] Remove #include in RadixTree.h (NFC) (#165115) RadixTree.h does not use anything from . --- llvm/include/llvm/ADT/RadixTree.h | 1 - 1 file changed, 1 deletion(-) diff --git a/llvm/include/llvm/ADT/RadixTree.h b/llvm/include/llvm/ADT/RadixTree.h index a65acddf186b..9e2ab9753d50 100644 --- a/llvm/include/llvm/ADT/RadixTree.h +++ b/llvm/include/llvm/ADT/RadixTree.h @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include From 722c9691a480134b35c7259bab1909a22cf294ca Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Mon, 27 Oct 2025 12:30:28 -0700 Subject: [PATCH 147/147] [RadixTree] Use std::optional for Node::Value (#165299) Don't rely on comparison to singular iterator, it's UB. Fixes bot crashes after https://github.com/llvm/llvm-project/pull/164524. --- llvm/include/llvm/ADT/RadixTree.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/llvm/include/llvm/ADT/RadixTree.h b/llvm/include/llvm/ADT/RadixTree.h index 9e2ab9753d50..87e2a3ebecc0 100644 --- a/llvm/include/llvm/ADT/RadixTree.h +++ b/llvm/include/llvm/ADT/RadixTree.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -92,7 +93,7 @@ template class RadixTree { /// If this node does not have a value (i.e., it's an internal node that /// only serves as a path to other values), this iterator will be equal /// to default constructed `ContainerType::iterator()`. - typename ContainerType::iterator Value; + std::optional Value; /// The first character of the Key. Used for fast child lookup. KeyValueType KeyFront; @@ -215,7 +216,7 @@ template class RadixTree { KeyConstIteratorType{}}; void findNextValid() { - while (Curr && Curr->Value == typename ContainerType::iterator()) + while (Curr && !Curr->Value.has_value()) advance(); } @@ -249,7 +250,7 @@ template class RadixTree { public: IteratorImpl() = default; - MappedType &operator*() const { return *Curr->Value; } + MappedType &operator*() const { return **Curr->Value; } IteratorImpl &operator++() { advance(); @@ -315,12 +316,12 @@ template class RadixTree { const value_type &NewValue = KeyValuePairs.emplace_front( std::move(Key), T(std::forward(Args)...)); Node &Node = findOrCreate(NewValue.first); - bool HasValue = Node.Value != typename ContainerType::iterator(); + bool HasValue = Node.Value.has_value(); if (!HasValue) Node.Value = KeyValuePairs.begin(); else KeyValuePairs.pop_front(); - return {Node.Value, !HasValue}; + return {*Node.Value, !HasValue}; } ///